From 6a50d80abbfcbdd844916769739dd432c8f4c80a Mon Sep 17 00:00:00 2001 From: Alex Butler Date: Tue, 14 Nov 2023 00:55:28 +0000 Subject: [PATCH] Support metadata-format = "json.zst" (#59) * Support metadata-format = "json.zst" * Use spawn_blocking for zstd decoding --- CHANGELOG.md | 4 ++++ Cargo.lock | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------ Cargo.toml | 1 + config.toml | 29 ++++++++++++++++++----------- src/config.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ src/providers/gitlab.rs | 21 ++++++++++++--------- 6 files changed, 131 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d01a9cd..02efe31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Unreleased +- Add optional `metadata-format` config. Options: `json` (default) & `json.zst`. + When the latter selected the server will fetch `metadata.json.zst` files. + # v0.1.3 - Add `ssl-cert` configuration value under `gitlab` to allow self-signed diff --git a/Cargo.lock b/Cargo.lock index e846a46..dd4d256 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856" dependencies = [ "libc", ] @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -640,9 +640,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -693,6 +693,7 @@ dependencies = [ "urlencoding", "ustr", "uuid", + "zstd", ] [[package]] @@ -756,9 +757,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -929,9 +930,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" @@ -1369,9 +1370,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] @@ -1539,9 +1540,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "socket2" @@ -1765,9 +1766,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -1784,9 +1785,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -1877,9 +1878,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ "log", "once_cell", @@ -1888,9 +1889,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -2245,3 +2246,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 89d50e6..2d95be4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ url = { version = "2.2", features = ["serde"] } urlencoding = "2.1" ustr = "0.10" uuid = { version = "1.1", features = ["v4"] } +zstd = "0.13" [profile.release] lto = "thin" diff --git a/config.toml b/config.toml index 137c0ea..a3299a0 100644 --- a/config.toml +++ b/config.toml @@ -1,18 +1,25 @@ -# socket address for the SSH server to listen on +## socket address for the SSH server to listen on listen-address = "[::]:2222" -# directory in which the generated private keys for the server -# should be stored +## directory in which the generated private keys for the server +## should be stored state-directory = "/var/lib/gitlab-cargo-shim" [gitlab] -# the base url of the gitlab instance +## the base url of the gitlab instance uri = "http://127.0.0.1:3000" -# a personal access token of an admin with permission to `sudo` as other -# users and create impersonation tokens. `sudo` is used to fetch all the -# packages the user can access, and the impersonation token is returned -# to the user to download packages -# -# May be omitted if clients are using their own personal access tokens. -admin-token = "personal-access-token" +## a personal access token of an admin with permission to `sudo` as other +## users and create impersonation tokens. `sudo` is used to fetch all the +## packages the user can access, and the impersonation token is returned +## to the user to download packages +## +## May be omitted if clients are using their own personal access tokens. +# admin-token = "personal-access-token" + +## metadata format to fetch from the package registry. +## Options: "json", "json.zst" +## +## The correct format must be available in the package registry for all +## packages. +# metadata-format = "json" diff --git a/src/config.rs b/src/config.rs index 234dd3b..0121075 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ #![allow(clippy::module_name_repetitions)] +use crate::providers::gitlab::handle_error; use clap::Parser; use serde::{de::DeserializeOwned, Deserialize}; use std::{io, net::SocketAddr, path::PathBuf, str::FromStr}; @@ -39,6 +40,9 @@ pub struct GitlabConfig { pub token_expiry: Duration, #[serde(default)] pub ssl_cert: Option, + /// Metadata format for fetching. + #[serde(default)] + pub metadata_format: MetadataFormat, } impl GitlabConfig { @@ -48,6 +52,45 @@ impl GitlabConfig { } } +/// Fetch format for package metadata. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum MetadataFormat { + /// Plain json. + /// + /// Fetches `metadata.json` files. + #[default] + Json, + /// Json compressed with zstd. + /// + /// Fetches `metadata.json.zst` files. + #[serde(rename = "json.zst")] + JsonZst, +} + +impl MetadataFormat { + #[must_use] + pub fn filename(self) -> &'static str { + match self { + Self::Json => "metadata.json", + Self::JsonZst => "metadata.json.zst", + } + } + + pub async fn decode(self, res: reqwest::Response) -> anyhow::Result { + match self { + Self::Json => Ok(handle_error(res).await?.json().await?), + Self::JsonZst => { + let body = handle_error(res).await?.bytes().await?; + tokio::task::spawn_blocking(move || { + Ok(serde_json::from_reader(zstd::Decoder::new(body.as_ref())?)?) + }) + .await? + } + } + } +} + pub fn from_toml_path(path: &str) -> Result { let contents = std::fs::read(path)?; toml::from_slice(&contents).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 919f79e..67afb4d 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -1,7 +1,7 @@ #![allow(clippy::module_name_repetitions)] use crate::{ - config::GitlabConfig, + config::{GitlabConfig, MetadataFormat}, providers::{Release, User}, }; use async_trait::async_trait; @@ -19,6 +19,7 @@ pub struct Gitlab { base_url: Url, token_expiry: Duration, ssl_cert: Option, + metadata_format: MetadataFormat, } impl Gitlab { @@ -46,6 +47,7 @@ impl Gitlab { base_url: config.uri.join("api/v4/")?, token_expiry: config.token_expiry, ssl_cert, + metadata_format: config.metadata_format, }) } @@ -288,16 +290,17 @@ impl super::PackageProvider for Gitlab { version: &str, do_as: &User, ) -> anyhow::Result { - let uri = self.base_url.join(&path.metadata_uri(version))?; + let fmt = self.metadata_format; + let url = self + .base_url + .join(&path.file_uri(fmt.filename(), version))?; + let client = match &do_as.token { None => self.client.clone(), Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)?, }; - Ok(handle_error(client.get(uri).send().await?) - .await? - .json() - .await?) + fmt.decode(client.get(url).send().await?).await } fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result { @@ -309,7 +312,7 @@ impl super::PackageProvider for Gitlab { } } -async fn handle_error(resp: reqwest::Response) -> Result { +pub async fn handle_error(resp: reqwest::Response) -> Result { if resp.status().is_success() { Ok(resp) } else { @@ -336,9 +339,9 @@ pub struct GitlabCratePath { impl GitlabCratePath { #[must_use] - pub fn metadata_uri(&self, version: &str) -> String { + pub fn file_uri(&self, file: &str, version: &str) -> String { format!( - "projects/{}/packages/generic/{}/{version}/metadata.json", + "projects/{}/packages/generic/{}/{version}/{file}", self.project, self.package_name ) } -- libgit2 1.7.2