Merge pull request #74 from alexheretic/fetch-releases-429
When fetching all releases handle 429 by backing off
Diff
CHANGELOG.md | 4 ++++
Cargo.lock | 271 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------
Cargo.toml | 11 ++++++-----
src/main.rs | 17 +++++++++++++----
src/providers/gitlab.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
5 files changed, 237 insertions(+), 125 deletions(-)
@@ -1,7 +1,11 @@
# Unreleased
- Fetch crate metadata concurrently.
- Handle missing invalid metadata non-fatally.
- Support env var `RUST_LOG` log filter configuration.
- Add info logs for release & metadata fetch latency.
- When fetching all releases handle 429 by backing off.
- Improve fetch error logging.
# v0.1.4
@@ -32,9 +32,9 @@
[[package]]
name = "ahash"
version = "0.8.7"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff"
dependencies = [
"cfg-if",
"getrandom",
@@ -54,9 +54,9 @@
[[package]]
name = "anstream"
version = "0.6.5"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6"
checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -68,9 +68,9 @@
[[package]]
name = "anstyle"
version = "1.0.4"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87"
checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc"
[[package]]
name = "anstyle-parse"
@@ -128,6 +128,17 @@
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backoff"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
dependencies = [
"getrandom",
"instant",
"rand",
]
[[package]]
name = "backtrace"
@@ -146,9 +157,9 @@
[[package]]
name = "base64"
version = "0.21.5"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64ct"
@@ -181,9 +192,9 @@
[[package]]
name = "bitflags"
version = "2.4.1"
version = "2.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
[[package]]
name = "block-buffer"
@@ -258,9 +269,9 @@
[[package]]
name = "cargo-platform"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d"
checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f"
dependencies = [
"serde",
]
@@ -316,9 +327,9 @@
[[package]]
name = "clap"
version = "4.4.13"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642"
checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f"
dependencies = [
"clap_builder",
"clap_derive",
@@ -326,9 +337,9 @@
[[package]]
name = "clap_builder"
version = "4.4.12"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9"
checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99"
dependencies = [
"anstream",
"anstyle",
@@ -339,9 +350,9 @@
[[package]]
name = "clap_derive"
version = "4.4.7"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47"
dependencies = [
"heck",
"proc-macro2",
@@ -351,9 +362,9 @@
[[package]]
name = "clap_lex"
version = "0.6.0"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
[[package]]
name = "colorchoice"
@@ -388,9 +399,9 @@
[[package]]
name = "crc32fast"
version = "1.3.2"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa"
dependencies = [
"cfg-if",
]
@@ -641,9 +652,9 @@
[[package]]
name = "getrandom"
version = "0.2.11"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
dependencies = [
"cfg-if",
"libc",
@@ -663,6 +674,7 @@
"anyhow",
"arrayvec",
"async-trait",
"backoff",
"bytes",
"cargo-platform",
"cargo_metadata",
@@ -699,9 +711,9 @@
[[package]]
name = "h2"
version = "0.3.22"
version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178"
checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
dependencies = [
"bytes",
"fnv",
@@ -730,9 +742,9 @@
[[package]]
name = "hermit-abi"
version = "0.3.3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd"
[[package]]
name = "hex"
@@ -834,9 +846,9 @@
[[package]]
name = "indexmap"
version = "2.1.0"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f"
checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
dependencies = [
"equivalent",
"hashbrown",
@@ -855,6 +867,15 @@
checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
dependencies = [
"generic-array",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
[[package]]
@@ -871,18 +892,18 @@
[[package]]
name = "jobserver"
version = "0.1.27"
version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d"
checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.66"
version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca"
checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee"
dependencies = [
"wasm-bindgen",
]
@@ -895,9 +916,9 @@
[[package]]
name = "libc"
version = "0.2.151"
version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "libredox"
@@ -905,7 +926,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
dependencies = [
"bitflags 2.4.1",
"bitflags 2.4.2",
"libc",
"redox_syscall",
]
@@ -924,9 +945,9 @@
[[package]]
name = "linux-raw-sys"
version = "0.4.12"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456"
checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
[[package]]
name = "lock_api"
@@ -943,6 +964,15 @@
version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "matchers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata 0.1.10",
]
[[package]]
name = "md5"
@@ -964,9 +994,9 @@
[[package]]
name = "miniz_oxide"
version = "0.7.1"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
dependencies = [
"adler",
]
@@ -1002,22 +1032,27 @@
"num-integer",
"num-traits",
]
[[package]]
name = "num-conv"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
name = "num-integer"
version = "0.1.45"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"autocfg",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.17"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a"
dependencies = [
"autocfg",
]
@@ -1164,9 +1199,9 @@
[[package]]
name = "pkg-config"
version = "0.3.28"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "powerfmt"
@@ -1182,9 +1217,9 @@
[[package]]
name = "proc-macro2"
version = "1.0.76"
version = "1.0.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c"
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
dependencies = [
"unicode-ident",
]
@@ -1250,26 +1285,41 @@
[[package]]
name = "regex"
version = "1.10.2"
version = "1.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
"regex-automata 0.4.5",
"regex-syntax 0.8.2",
]
[[package]]
name = "regex-automata"
version = "0.4.3"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
]
[[package]]
name = "regex-automata"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"regex-syntax 0.8.2",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
@@ -1279,9 +1329,9 @@
[[package]]
name = "reqwest"
version = "0.11.23"
version = "0.11.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41"
checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
dependencies = [
"base64",
"bytes",
@@ -1305,6 +1355,7 @@
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"system-configuration",
"tokio",
"tokio-rustls",
@@ -1339,11 +1390,11 @@
[[package]]
name = "rustix"
version = "0.38.28"
version = "0.38.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"
dependencies = [
"bitflags 2.4.1",
"bitflags 2.4.2",
"errno",
"libc",
"linux-raw-sys",
@@ -1423,18 +1474,18 @@
[[package]]
name = "serde"
version = "1.0.195"
version = "1.0.196"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02"
checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.195"
version = "1.0.196"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c"
checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
dependencies = [
"proc-macro2",
"quote",
@@ -1443,9 +1494,9 @@
[[package]]
name = "serde_json"
version = "1.0.111"
version = "1.0.113"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4"
checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
dependencies = [
"itoa",
"ryu",
@@ -1510,9 +1561,9 @@
[[package]]
name = "shlex"
version = "1.2.0"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook-registry"
@@ -1534,9 +1585,9 @@
[[package]]
name = "smallvec"
version = "1.11.2"
version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"
checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
[[package]]
name = "socket2"
@@ -1556,9 +1607,9 @@
[[package]]
name = "strsim"
version = "0.10.0"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
[[package]]
name = "subtle"
@@ -1578,6 +1629,12 @@
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "system-configuration"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1610,18 +1667,18 @@
[[package]]
name = "thiserror"
version = "1.0.56"
version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.56"
version = "1.0.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81"
dependencies = [
"proc-macro2",
"quote",
@@ -1707,11 +1764,12 @@
[[package]]
name = "time"
version = "0.3.31"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e"
checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749"
dependencies = [
"deranged",
"num-conv",
"powerfmt",
"serde",
"time-core",
@@ -1726,10 +1784,11 @@
[[package]]
name = "time-macros"
version = "0.2.16"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f"
checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774"
dependencies = [
"num-conv",
"time-core",
]
@@ -1750,9 +1809,9 @@
[[package]]
name = "tokio"
version = "1.35.1"
version = "1.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104"
checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
dependencies = [
"backtrace",
"bytes",
@@ -1877,10 +1936,14 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"matchers",
"nu-ansi-term",
"once_cell",
"regex",
"sharded-slab",
"smallvec",
"thread_local",
"tracing",
"tracing-core",
"tracing-log",
]
@@ -1899,9 +1962,9 @@
[[package]]
name = "unicode-bidi"
version = "0.3.14"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416"
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
[[package]]
name = "unicode-ident"
@@ -1962,9 +2025,9 @@
[[package]]
name = "uuid"
version = "1.6.1"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560"
checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a"
dependencies = [
"getrandom",
]
@@ -2014,9 +2077,9 @@
[[package]]
name = "wasm-bindgen"
version = "0.2.89"
version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e"
checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
@@ -2024,9 +2087,9 @@
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.89"
version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826"
checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b"
dependencies = [
"bumpalo",
"log",
@@ -2039,9 +2102,9 @@
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.39"
version = "0.4.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12"
checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97"
dependencies = [
"cfg-if",
"js-sys",
@@ -2051,9 +2114,9 @@
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.89"
version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2"
checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -2061,9 +2124,9 @@
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.89"
version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283"
checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66"
dependencies = [
"proc-macro2",
"quote",
@@ -2074,15 +2137,15 @@
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.89"
version = "0.2.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f"
checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838"
[[package]]
name = "web-sys"
version = "0.3.66"
version = "0.3.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f"
checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -2090,9 +2153,9 @@
[[package]]
name = "webpki-roots"
version = "0.25.3"
version = "0.25.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
[[package]]
name = "winapi"
@@ -10,33 +10,34 @@
anyhow = "1"
arrayvec = "0.7"
async-trait = "0.1"
backoff = "0.4"
bytes = "1.1"
cargo_metadata = "0.15"
cargo-platform = "0.1"
clap = { version = "4", features = ["derive", "cargo", "wrap_help"] }
futures = "0.3"
hex = "0.4"
itoa = "1.0"
indexmap = "2"
indoc = "2.0"
itoa = "1.0"
packfile = "0.1"
parse_link_header = "0.3"
parking_lot = "0.12"
parse_link_header = "0.3"
percent-encoding = "2.3"
reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
semver = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1"
shlex = "1.1"
time = { version = "0.3", features = ["serde"] }
tracing = "0.1"
tracing-subscriber = "0.3"
thrussh = "0.34"
thrussh-keys = "0.22"
thrussh-libsodium = "=0.2.1"
time = { version = "0.3", features = ["serde"] }
tokio = { version = "1.17", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
toml = "0.5"
tracing = "0.1"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
url = { version = "2.2", features = ["serde"] }
urlencoding = "2.1"
ustr = "0.10"
@@ -33,6 +33,7 @@
pin::Pin,
str::FromStr,
sync::Arc,
time::Instant,
};
use thrussh::{
server::{Auth, Session},
@@ -41,7 +42,7 @@
use thrussh_keys::key::PublicKey;
use tokio::sync::Semaphore;
use tokio_util::codec::{Decoder, Encoder as CodecEncoder};
use tracing::{debug, error, info, info_span, instrument, Instrument, Span};
use tracing::{error, info, info_span, instrument, trace, Instrument, Span};
use uuid::Uuid;
const AGENT: &str = concat!(
@@ -56,7 +57,11 @@
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let subscriber = tracing_subscriber::fmt();
let subscriber = tracing_subscriber::fmt().with_env_filter(
tracing_subscriber::EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::INFO.into())
.from_env_lossy(),
);
#[cfg(debug_assertions)]
let subscriber = subscriber.pretty();
subscriber.init();
@@ -311,13 +316,16 @@
packfile.insert(&[], "config.json", config_json)?;
let a = Instant::now();
let releases_by_crate = self.fetch_releases_by_crate().await?;
info!("Fetched crate releases in {:.1?}", a.elapsed());
let fetch_concurrency = Semaphore::new(PARALLEL_METADATA_FETCHES);
let mut metadata_fetches = FuturesOrdered::new();
let a = Instant::now();
for ((crate_path, crate_name), releases) in &releases_by_crate {
for release in releases {
metadata_fetches.push_back({
@@ -329,7 +337,7 @@
let version = &release.version;
async move {
let _guard = fetch_concurrency.acquire().await?;
debug!("Fetching metadata for {crate_name}-{version}");
trace!("Fetching metadata for {crate_name}-{version}");
Self::fetch_metadata(
gitlab, cache, crate_path, checksum, crate_name, version, &user,
)
@@ -367,6 +375,7 @@
buffer.get_mut().split().freeze(),
)?;
}
info!("Fetched crate metadata in {:.1?}", a.elapsed());
@@ -624,7 +633,7 @@
let res = fut.await;
if let Err(e) = &res {
error!("Error: {}", e);
error!("{e}");
}
res
@@ -1,19 +1,26 @@
#![allow(clippy::module_name_repetitions)]
#![allow(clippy::module_name_repetitions, clippy::blocks_in_conditions)]
use crate::{
config::{GitlabConfig, MetadataFormat},
providers::{Release, User},
};
use anyhow::Context;
use async_trait::async_trait;
use backoff::backoff::Backoff;
use futures::{stream::FuturesUnordered, StreamExt, TryStreamExt};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::{header, Certificate};
use serde::{Deserialize, Serialize};
use std::{borrow::Cow, sync::Arc};
use std::sync::Arc;
use time::{Duration, OffsetDateTime};
use tracing::{info_span, instrument, Instrument};
use tokio::sync::Semaphore;
use tracing::{debug, info_span, instrument, Instrument};
use url::Url;
const PARALLEL_PACKAGE_FILES_GETS: usize = 32;
pub struct Gitlab {
client: reqwest::Client,
base_url: Url,
@@ -171,6 +178,7 @@
uri
});
let fetch_concurrency = Semaphore::new(PARALLEL_PACKAGE_FILES_GETS);
let futures = FuturesUnordered::new();
while let Some(uri) = next_uri.take() {
@@ -178,7 +186,7 @@
self.client
.get(uri)
.user_or_admin_token(do_as, &self.admin_token)
.send()
.send_retry_429()
.await?,
)
.await?;
@@ -201,9 +209,12 @@
for release in res {
let this = Arc::clone(&self);
let do_as = Arc::clone(do_as);
let fetch_concurrency = &fetch_concurrency;
futures.push(
async move {
let _guard = fetch_concurrency.acquire().await?;
let (project, package) = {
let mut splitter = release.links.web_path.splitn(2, "/-/packages/");
match (splitter.next(), splitter.next()) {
@@ -227,7 +238,7 @@
utf8_percent_encode(package, NON_ALPHANUMERIC),
))
.user_or_admin_token(&do_as, &this.admin_token)
.send()
.send_retry_429()
.await?,
)
.await?
@@ -300,16 +311,17 @@
if resp.status().is_success() {
Ok(resp)
} else {
let resp: GitlabErrorResponse = resp.json().await?;
Err(anyhow::Error::msg(
resp.message
.or(resp.error)
.map_or_else(|| Cow::Borrowed("unknown error"), Cow::Owned),
))
let status = resp.status().as_u16();
let url = resp.url().clone();
let text = resp.text().await.unwrap_or_else(|_| "?".into());
let json: GitlabErrorResponse = serde_json::from_str(&text).unwrap_or_default();
let msg = json.message.or(json.error).unwrap_or(text);
anyhow::bail!("{url}: {status}: {msg}")
}
}
#[derive(Deserialize)]
#[derive(Default, Deserialize)]
pub struct GitlabErrorResponse {
message: Option<String>,
error: Option<String>,
@@ -386,6 +398,10 @@
fn private_token(self, token: &Option<String>) -> Self;
async fn send_retry_429(self) -> anyhow::Result<reqwest::Response>;
}
impl RequestBuilderExt for reqwest::RequestBuilder {
@@ -400,6 +416,25 @@
match token {
Some(token) => self.header("PRIVATE-TOKEN", token),
None => self,
}
}
async fn send_retry_429(self) -> anyhow::Result<reqwest::Response> {
let mut backoff = backoff::ExponentialBackoff::default();
loop {
let r = self
.try_clone()
.context("cannot retry request")?
.send()
.await?;
if r.status().as_u16() == 429 {
if let Some(wait) = backoff.next_backoff() {
debug!(url = %r.url(), "429: retrying after {wait:.1?}");
tokio::time::sleep(wait).await;
continue;
}
}
return Ok(r);
}
}
}