From 38fe070b54316eadc6033248a075905df1b3ceb5 Mon Sep 17 00:00:00 2001 From: jordan Date: Fri, 23 Feb 2024 10:22:54 +0000 Subject: [PATCH] Merge pull request #74 from alexheretic/fetch-releases-429 When fetching all releases handle 429 by backing off--- CHANGELOG.md | 4 ++++ Cargo.lock | 271 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------- Cargo.toml | 11 ++++++----- src/main.rs | 17 +++++++++++++---- src/providers/gitlab.rs | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 5 files changed, 237 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9280d70..a7990e5 100644 --- a/CHANGELOG.md +++ a/CHANGELOG.md @@ -1,7 +1,11 @@ # Unreleased - Fetch crate metadata concurrently. - Handle missing invalid metadata non-fatally. +- Support env var `RUST_LOG` log filter configuration. +- Add info logs for release & metadata fetch latency. +- When fetching all releases handle 429 by backing off. +- Improve fetch error logging. # v0.1.4 diff --git a/Cargo.lock b/Cargo.lock index 5c5dd1b..a62f164 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -32,9 +32,9 @@ [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "getrandom", @@ -54,9 +54,9 @@ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -68,9 +68,9 @@ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -128,6 +128,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom", + "instant", + "rand", +] [[package]] name = "backtrace" @@ -146,9 +157,9 @@ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64ct" @@ -181,9 +192,9 @@ [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -258,9 +269,9 @@ [[package]] name = "cargo-platform" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" dependencies = [ "serde", ] @@ -316,9 +327,9 @@ [[package]] name = "clap" -version = "4.4.13" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" +checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" dependencies = [ "clap_builder", "clap_derive", @@ -326,9 +337,9 @@ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" dependencies = [ "anstream", "anstyle", @@ -339,9 +350,9 @@ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", @@ -351,9 +362,9 @@ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -388,9 +399,9 @@ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -641,9 +652,9 @@ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -663,6 +674,7 @@ "anyhow", "arrayvec", "async-trait", + "backoff", "bytes", "cargo-platform", "cargo_metadata", @@ -699,9 +711,9 @@ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -730,9 +742,9 @@ [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hex" @@ -834,9 +846,9 @@ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown", @@ -855,6 +867,15 @@ checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ "generic-array", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", ] [[package]] @@ -871,18 +892,18 @@ [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -895,9 +916,9 @@ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libredox" @@ -905,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "redox_syscall", ] @@ -924,9 +945,9 @@ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -943,6 +964,15 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] [[package]] name = "md5" @@ -964,9 +994,9 @@ [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -1002,22 +1032,27 @@ "num-integer", "num-traits", ] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -1164,9 +1199,9 @@ [[package]] name = "pkg-config" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "powerfmt" @@ -1182,9 +1217,9 @@ [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1250,26 +1285,41 @@ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", ] [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.2", ] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" @@ -1279,9 +1329,9 @@ [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", @@ -1305,6 +1355,7 @@ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls", @@ -1339,11 +1390,11 @@ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -1423,18 +1474,18 @@ [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -1443,9 +1494,9 @@ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -1510,9 +1561,9 @@ [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" @@ -1534,9 +1585,9 @@ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" @@ -1556,9 +1607,9 @@ [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "subtle" @@ -1578,6 +1629,12 @@ ] [[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] name = "system-configuration" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1610,18 +1667,18 @@ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -1707,11 +1764,12 @@ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", + "num-conv", "powerfmt", "serde", "time-core", @@ -1726,10 +1784,11 @@ [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -1750,9 +1809,9 @@ [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -1877,10 +1936,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] @@ -1899,9 +1962,9 @@ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -1962,9 +2025,9 @@ [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", ] @@ -2014,9 +2077,9 @@ [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2024,9 +2087,9 @@ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -2039,9 +2102,9 @@ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -2051,9 +2114,9 @@ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2061,9 +2124,9 @@ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -2074,15 +2137,15 @@ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -2090,9 +2153,9 @@ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "winapi" diff --git a/Cargo.toml b/Cargo.toml index df60dac..51c248c 100644 --- a/Cargo.toml +++ a/Cargo.toml @@ -10,33 +10,34 @@ anyhow = "1" arrayvec = "0.7" async-trait = "0.1" +backoff = "0.4" bytes = "1.1" cargo_metadata = "0.15" cargo-platform = "0.1" clap = { version = "4", features = ["derive", "cargo", "wrap_help"] } futures = "0.3" hex = "0.4" -itoa = "1.0" indexmap = "2" indoc = "2.0" +itoa = "1.0" packfile = "0.1" -parse_link_header = "0.3" parking_lot = "0.12" +parse_link_header = "0.3" percent-encoding = "2.3" reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } semver = "1.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1" shlex = "1.1" -time = { version = "0.3", features = ["serde"] } -tracing = "0.1" -tracing-subscriber = "0.3" thrussh = "0.34" thrussh-keys = "0.22" thrussh-libsodium = "=0.2.1" # 0.2.2 causes dynamic linking by enabling use-pkg-config +time = { version = "0.3", features = ["serde"] } tokio = { version = "1.17", features = ["full"] } tokio-util = { version = "0.7", features = ["codec"] } toml = "0.5" +tracing = "0.1" +tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } url = { version = "2.2", features = ["serde"] } urlencoding = "2.1" ustr = "0.10" diff --git a/src/main.rs b/src/main.rs index 61a99be..a364429 100644 --- a/src/main.rs +++ a/src/main.rs @@ -33,6 +33,7 @@ pin::Pin, str::FromStr, sync::Arc, + time::Instant, }; use thrussh::{ server::{Auth, Session}, @@ -41,7 +42,7 @@ use thrussh_keys::key::PublicKey; use tokio::sync::Semaphore; use tokio_util::codec::{Decoder, Encoder as CodecEncoder}; -use tracing::{debug, error, info, info_span, instrument, Instrument, Span}; +use tracing::{error, info, info_span, instrument, trace, Instrument, Span}; use uuid::Uuid; const AGENT: &str = concat!( @@ -56,7 +57,11 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { - let subscriber = tracing_subscriber::fmt(); + let subscriber = tracing_subscriber::fmt().with_env_filter( + tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::INFO.into()) + .from_env_lossy(), + ); #[cfg(debug_assertions)] let subscriber = subscriber.pretty(); subscriber.init(); @@ -311,13 +316,16 @@ packfile.insert(&[], "config.json", config_json)?; // fetch the releases for every project within the given project + let a = Instant::now(); let releases_by_crate = self.fetch_releases_by_crate().await?; + info!("Fetched crate releases in {:.1?}", a.elapsed()); // fetch metadata concurrently // parses the `cargo metadata` stored in the release, which // should be stored according to MetadataFormat. let fetch_concurrency = Semaphore::new(PARALLEL_METADATA_FETCHES); let mut metadata_fetches = FuturesOrdered::new(); + let a = Instant::now(); for ((crate_path, crate_name), releases) in &releases_by_crate { for release in releases { metadata_fetches.push_back({ @@ -329,7 +337,7 @@ let version = &release.version; async move { let _guard = fetch_concurrency.acquire().await?; - debug!("Fetching metadata for {crate_name}-{version}"); + trace!("Fetching metadata for {crate_name}-{version}"); Self::fetch_metadata( gitlab, cache, crate_path, checksum, crate_name, version, &user, ) @@ -367,6 +375,7 @@ buffer.get_mut().split().freeze(), )?; } + info!("Fetched crate metadata in {:.1?}", a.elapsed()); // build a commit for all of our inserted files and build // into its lower-level `Vec` counter-part. @@ -624,7 +633,7 @@ let res = fut.await; if let Err(e) = &res { - error!("Error: {}", e); + error!("{e}"); } res diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 02cbf4c..ae3a14d 100644 --- a/src/providers/gitlab.rs +++ a/src/providers/gitlab.rs @@ -1,19 +1,26 @@ -#![allow(clippy::module_name_repetitions)] +// blocks_in_conditions: didn't work with `#[instrument...`` usage +#![allow(clippy::module_name_repetitions, clippy::blocks_in_conditions)] use crate::{ config::{GitlabConfig, MetadataFormat}, providers::{Release, User}, }; +use anyhow::Context; use async_trait::async_trait; +use backoff::backoff::Backoff; use futures::{stream::FuturesUnordered, StreamExt, TryStreamExt}; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use reqwest::{header, Certificate}; use serde::{Deserialize, Serialize}; -use std::{borrow::Cow, sync::Arc}; +use std::sync::Arc; use time::{Duration, OffsetDateTime}; -use tracing::{info_span, instrument, Instrument}; +use tokio::sync::Semaphore; +use tracing::{debug, info_span, instrument, Instrument}; use url::Url; +/// Number of `package_files` GETs to do in parallel. +const PARALLEL_PACKAGE_FILES_GETS: usize = 32; + pub struct Gitlab { client: reqwest::Client, base_url: Url, @@ -171,6 +178,7 @@ uri }); + let fetch_concurrency = Semaphore::new(PARALLEL_PACKAGE_FILES_GETS); let futures = FuturesUnordered::new(); while let Some(uri) = next_uri.take() { @@ -178,7 +186,7 @@ self.client .get(uri) .user_or_admin_token(do_as, &self.admin_token) - .send() + .send_retry_429() .await?, ) .await?; @@ -201,9 +209,12 @@ for release in res { let this = Arc::clone(&self); let do_as = Arc::clone(do_as); + let fetch_concurrency = &fetch_concurrency; futures.push( async move { + let _guard = fetch_concurrency.acquire().await?; + let (project, package) = { let mut splitter = release.links.web_path.splitn(2, "/-/packages/"); match (splitter.next(), splitter.next()) { @@ -227,7 +238,7 @@ utf8_percent_encode(package, NON_ALPHANUMERIC), )) .user_or_admin_token(&do_as, &this.admin_token) - .send() + .send_retry_429() .await?, ) .await? @@ -300,16 +311,17 @@ if resp.status().is_success() { Ok(resp) } else { - let resp: GitlabErrorResponse = resp.json().await?; - Err(anyhow::Error::msg( - resp.message - .or(resp.error) - .map_or_else(|| Cow::Borrowed("unknown error"), Cow::Owned), - )) + let status = resp.status().as_u16(); + let url = resp.url().clone(); + let text = resp.text().await.unwrap_or_else(|_| "?".into()); + let json: GitlabErrorResponse = serde_json::from_str(&text).unwrap_or_default(); + let msg = json.message.or(json.error).unwrap_or(text); + + anyhow::bail!("{url}: {status}: {msg}") } } -#[derive(Deserialize)] +#[derive(Default, Deserialize)] pub struct GitlabErrorResponse { message: Option, error: Option, @@ -386,6 +398,10 @@ /// Add given PRIVATE-TOKEN header. fn private_token(self, token: &Option) -> Self; + + /// [`reqwest::RequestBuilder::send`] send and retry 429 responses + /// backing off exponentially between trys. + async fn send_retry_429(self) -> anyhow::Result; } impl RequestBuilderExt for reqwest::RequestBuilder { @@ -400,6 +416,25 @@ match token { Some(token) => self.header("PRIVATE-TOKEN", token), None => self, + } + } + + async fn send_retry_429(self) -> anyhow::Result { + let mut backoff = backoff::ExponentialBackoff::default(); + loop { + let r = self + .try_clone() + .context("cannot retry request")? + .send() + .await?; + if r.status().as_u16() == 429 { + if let Some(wait) = backoff.next_backoff() { + debug!(url = %r.url(), "429: retrying after {wait:.1?}"); + tokio::time::sleep(wait).await; + continue; + } + } + return Ok(r); } } } -- rgit 0.1.3