🏡 index : ~doyle/gitlab-cargo-shim.git

author Alex Butler <alexheretic@gmail.com> 2023-11-14 0:55:28.0 +00:00:00
committer GitHub <noreply@github.com> 2023-11-14 0:55:28.0 +00:00:00
commit 
6a50d80abbfcbdd844916769739dd432c8f4c80a [patch]
tree 
bfbbe62eaf5aa23ed6b57a1f5010c9bca2c3887d
parent 
e45a3fde99d03f3cb8efbe4837d85fe5b8c6465a
download 
6a50d80abbfcbdd844916769739dd432c8f4c80a.tar.gz

Support metadata-format = "json.zst" (#59)

* Support metadata-format = "json.zst"

* Use spawn_blocking for zstd decoding

Diff

 CHANGELOG.md            |  4 +++-
 Cargo.lock              | 77 ++++++++++++++++++++++++++++++++++----------------
 Cargo.toml              |  1 +-
 config.toml             | 29 +++++++++++--------
 src/config.rs           | 43 ++++++++++++++++++++++++++++-
 src/providers/gitlab.rs | 21 ++++++++------
 6 files changed, 131 insertions(+), 44 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d01a9cd..02efe31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
# Unreleased
- Add optional `metadata-format` config. Options: `json` (default) & `json.zst`.
  When the latter selected the server will fetch `metadata.json.zst` files.

# v0.1.3

- Add `ssl-cert` configuration value under `gitlab` to allow self-signed
diff --git a/Cargo.lock b/Cargo.lock
index e846a46..dd4d256 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -281,9 +281,9 @@ dependencies = [

[[package]]
name = "cc"
version = "1.0.83"
version = "1.0.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
checksum = "0f8e7c90afad890484a21653d08b6e209ae34770fb5ee298f9c699fcc1e5c856"
dependencies = [
 "libc",
]
@@ -315,9 +315,9 @@ dependencies = [

[[package]]
name = "clap"
version = "4.4.7"
version = "4.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64"
dependencies = [
 "clap_builder",
 "clap_derive",
@@ -325,9 +325,9 @@ dependencies = [

[[package]]
name = "clap_builder"
version = "4.4.7"
version = "4.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc"
dependencies = [
 "anstream",
 "anstyle",
@@ -640,9 +640,9 @@ dependencies = [

[[package]]
name = "getrandom"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f"
dependencies = [
 "cfg-if",
 "libc",
@@ -693,6 +693,7 @@ dependencies = [
 "urlencoding",
 "ustr",
 "uuid",
 "zstd",
]

[[package]]
@@ -756,9 +757,9 @@ dependencies = [

[[package]]
name = "http"
version = "0.2.9"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb"
dependencies = [
 "bytes",
 "fnv",
@@ -929,9 +930,9 @@ dependencies = [

[[package]]
name = "linux-raw-sys"
version = "0.4.10"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829"

[[package]]
name = "lock_api"
@@ -1369,9 +1370,9 @@ dependencies = [

[[package]]
name = "rustls-pemfile"
version = "1.0.3"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
 "base64",
]
@@ -1539,9 +1540,9 @@ dependencies = [

[[package]]
name = "smallvec"
version = "1.11.1"
version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"
checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970"

[[package]]
name = "socket2"
@@ -1765,9 +1766,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"

[[package]]
name = "tokio"
version = "1.33.0"
version = "1.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653"
checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9"
dependencies = [
 "backtrace",
 "bytes",
@@ -1784,9 +1785,9 @@ dependencies = [

[[package]]
name = "tokio-macros"
version = "2.1.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
dependencies = [
 "proc-macro2",
 "quote",
@@ -1877,9 +1878,9 @@ dependencies = [

[[package]]
name = "tracing-log"
version = "0.1.4"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
 "log",
 "once_cell",
@@ -1888,9 +1889,9 @@ dependencies = [

[[package]]
name = "tracing-subscriber"
version = "0.3.17"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
 "nu-ansi-term",
 "sharded-slab",
@@ -2245,3 +2246,31 @@ dependencies = [
 "quote",
 "syn",
]

[[package]]
name = "zstd"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110"
dependencies = [
 "zstd-safe",
]

[[package]]
name = "zstd-safe"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e"
dependencies = [
 "zstd-sys",
]

[[package]]
name = "zstd-sys"
version = "2.0.9+zstd.1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656"
dependencies = [
 "cc",
 "pkg-config",
]
diff --git a/Cargo.toml b/Cargo.toml
index 89d50e6..2d95be4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -41,6 +41,7 @@ url = { version = "2.2", features = ["serde"] }
urlencoding = "2.1"
ustr = "0.10"
uuid = { version = "1.1", features = ["v4"] }
zstd = "0.13"

[profile.release]
lto = "thin"
diff --git a/config.toml b/config.toml
index 137c0ea..a3299a0 100644
--- a/config.toml
+++ b/config.toml
@@ -1,18 +1,25 @@
# socket address for the SSH server to listen on
## socket address for the SSH server to listen on
listen-address = "[::]:2222"

# directory in which the generated private keys for the server
# should be stored
## directory in which the generated private keys for the server
## should be stored
state-directory = "/var/lib/gitlab-cargo-shim"

[gitlab]
# the base url of the gitlab instance
## the base url of the gitlab instance
uri = "http://127.0.0.1:3000"

# a personal access token of an admin with permission to `sudo` as other
# users and create impersonation tokens. `sudo` is used to fetch all the
# packages the user can access, and the impersonation token is returned
# to the user to download packages
#
# May be omitted if clients are using their own personal access tokens.
admin-token = "personal-access-token"
## a personal access token of an admin with permission to `sudo` as other
## users and create impersonation tokens. `sudo` is used to fetch all the
## packages the user can access, and the impersonation token is returned
## to the user to download packages
##
## May be omitted if clients are using their own personal access tokens.
# admin-token = "personal-access-token"

## metadata format to fetch from the package registry.
## Options: "json", "json.zst"
##
## The correct format must be available in the package registry for all
## packages.
# metadata-format = "json"
diff --git a/src/config.rs b/src/config.rs
index 234dd3b..0121075 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,5 +1,6 @@
#![allow(clippy::module_name_repetitions)]

use crate::providers::gitlab::handle_error;
use clap::Parser;
use serde::{de::DeserializeOwned, Deserialize};
use std::{io, net::SocketAddr, path::PathBuf, str::FromStr};
@@ -39,6 +40,9 @@ pub struct GitlabConfig {
    pub token_expiry: Duration,
    #[serde(default)]
    pub ssl_cert: Option<String>,
    /// Metadata format for fetching.
    #[serde(default)]
    pub metadata_format: MetadataFormat,
}

impl GitlabConfig {
@@ -48,6 +52,45 @@ impl GitlabConfig {
    }
}

/// Fetch format for package metadata.
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum MetadataFormat {
    /// Plain json.
    ///
    /// Fetches `metadata.json` files.
    #[default]
    Json,
    /// Json compressed with zstd.
    ///
    /// Fetches `metadata.json.zst` files.
    #[serde(rename = "json.zst")]
    JsonZst,
}

impl MetadataFormat {
    #[must_use]
    pub fn filename(self) -> &'static str {
        match self {
            Self::Json => "metadata.json",
            Self::JsonZst => "metadata.json.zst",
        }
    }

    pub async fn decode(self, res: reqwest::Response) -> anyhow::Result<cargo_metadata::Metadata> {
        match self {
            Self::Json => Ok(handle_error(res).await?.json().await?),
            Self::JsonZst => {
                let body = handle_error(res).await?.bytes().await?;
                tokio::task::spawn_blocking(move || {
                    Ok(serde_json::from_reader(zstd::Decoder::new(body.as_ref())?)?)
                })
                .await?
            }
        }
    }
}

pub fn from_toml_path<T: DeserializeOwned>(path: &str) -> Result<T, std::io::Error> {
    let contents = std::fs::read(path)?;
    toml::from_slice(&contents).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs
index 919f79e..67afb4d 100644
--- a/src/providers/gitlab.rs
+++ b/src/providers/gitlab.rs
@@ -1,7 +1,7 @@
#![allow(clippy::module_name_repetitions)]

use crate::{
    config::GitlabConfig,
    config::{GitlabConfig, MetadataFormat},
    providers::{Release, User},
};
use async_trait::async_trait;
@@ -19,6 +19,7 @@ pub struct Gitlab {
    base_url: Url,
    token_expiry: Duration,
    ssl_cert: Option<Certificate>,
    metadata_format: MetadataFormat,
}

impl Gitlab {
@@ -46,6 +47,7 @@ impl Gitlab {
            base_url: config.uri.join("api/v4/")?,
            token_expiry: config.token_expiry,
            ssl_cert,
            metadata_format: config.metadata_format,
        })
    }

@@ -288,16 +290,17 @@ impl super::PackageProvider for Gitlab {
        version: &str,
        do_as: &User,
    ) -> anyhow::Result<cargo_metadata::Metadata> {
        let uri = self.base_url.join(&path.metadata_uri(version))?;
        let fmt = self.metadata_format;
        let url = self
            .base_url
            .join(&path.file_uri(fmt.filename(), version))?;

        let client = match &do_as.token {
            None => self.client.clone(),
            Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)?,
        };

        Ok(handle_error(client.get(uri).send().await?)
            .await?
            .json()
            .await?)
        fmt.decode(client.get(url).send().await?).await
    }

    fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result<String> {
@@ -309,7 +312,7 @@ impl super::PackageProvider for Gitlab {
    }
}

async fn handle_error(resp: reqwest::Response) -> Result<reqwest::Response, anyhow::Error> {
pub async fn handle_error(resp: reqwest::Response) -> Result<reqwest::Response, anyhow::Error> {
    if resp.status().is_success() {
        Ok(resp)
    } else {
@@ -336,9 +339,9 @@ pub struct GitlabCratePath {

impl GitlabCratePath {
    #[must_use]
    pub fn metadata_uri(&self, version: &str) -> String {
    pub fn file_uri(&self, file: &str, version: &str) -> String {
        format!(
            "projects/{}/packages/generic/{}/{version}/metadata.json",
            "projects/{}/packages/generic/{}/{version}/{file}",
            self.project, self.package_name
        )
    }