🏡 index : ~doyle/gitlab-cargo-shim.git

bb203c01a9c931b6032978c0df751c51a85c584d [patch]

2b96ad24ef8def0653103c3983798a42967783e3

f06a5be6b3f617513c5c02e676cb878a6638a468

bb203c01a9c931b6032978c0df751c51a85c584d.tar.gz

Refer to projects directly rather than groups so we don't have to rely on gitlab/cargo patches

Diff

 Cargo.lock              |  7 +++++-
 Cargo.toml              |  1 +-
 README.md               |  4 +--
 src/main.rs             | 67 ++++++++++++++++++++------------------------------
 src/metadata.rs         |  6 +---
 src/providers/gitlab.rs | 67 +++++++++-----------------------------------------
 src/providers/mod.rs    | 19 ++------------
 7 files changed, 56 insertions(+), 115 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5c6db58..f664d61 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -596,6 +596,7 @@ dependencies = [
 "tracing",
 "tracing-subscriber",
 "url",
 "urlencoding",
 "ustr",
 "uuid",
]
@@ -1838,6 +1839,12 @@ dependencies = [
]

[[package]]
name = "urlencoding"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821"

[[package]]
name = "ustr"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 9c54bbf..09e0fc8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -40,6 +40,7 @@ tokio = { version = "1.17", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
toml = "0.5"
url = { version = "2.2", features = ["serde"] }
urlencoding = "2.1"
ustr = "0.8"
uuid = { version = "1.0.0-alpha.1", features = ["v4"] }

diff --git a/README.md b/README.md
index 3211701..4eea3ec 100644
--- a/README.md
+++ b/README.md
@@ -26,11 +26,11 @@ accordingly.
```toml
# .cargo/config.toml
[registries]
my-gitlab-group = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group" }
my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project" }

# Cargo.toml
[dependencies]
my-crate = { version = "0.1", registry = "my-gitlab-group" }
my-crate = { version = "0.1", registry = "my-gitlab-project" }
```

In your CI build, setup a `before_script` step to replace the connection string
diff --git a/src/main.rs b/src/main.rs
index 54ca331..cd6cb4f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -10,14 +10,14 @@ pub mod util;

use crate::{
    config::Args,
    metadata::{CargoIndexCrateMetadata, CargoIndexCrateMetadataWithDlOverride},
    metadata::{CargoConfig, CargoIndexCrateMetadata},
    protocol::{
        codec::{Encoder, GitCodec},
        high_level::GitRepository,
        low_level::{HashOutput, PackFileEntry},
        packet_line::PktLine,
    },
    providers::{gitlab::Gitlab, Group, PackageProvider, Release, ReleaseName, User, UserProvider},
    providers::{gitlab::Gitlab, PackageProvider, Release, ReleaseName, User, UserProvider},
    util::get_crate_folder,
};
use anyhow::anyhow;
@@ -136,7 +136,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::server:
            codec: GitCodec::default(),
            gitlab: Arc::clone(&self.gitlab),
            user: None,
            group: None,
            project: None,
            input_bytes: BytesMut::new(),
            output_bytes: BytesMut::new(),
            is_git_protocol_v2: false,
@@ -151,7 +151,7 @@ pub struct Handler<U: UserProvider + PackageProvider + Send + Sync + 'static> {
    codec: GitCodec,
    gitlab: Arc<U>,
    user: Option<Arc<User>>,
    group: Option<Group>,
    project: Option<Arc<str>>,
    // fetcher_future: Option<JoinHandle<anyhow::Result<Vec<Release>>>>,
    input_bytes: BytesMut,
    output_bytes: BytesMut,
@@ -170,10 +170,10 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
            .ok_or_else(|| anyhow::anyhow!("no user set"))
    }

    fn group(&self) -> anyhow::Result<&Group> {
        self.group
            .as_ref()
            .ok_or_else(|| anyhow::anyhow!("no group set"))
    fn project(&self) -> anyhow::Result<&str> {
        self.project
            .as_deref()
            .ok_or_else(|| anyhow::anyhow!("no project set"))
    }

    /// Writes a Git packet line response to the buffer, this should only
@@ -190,19 +190,19 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
        );
    }

    /// Fetches all the releases from the provider for the given group
    /// Fetches all the releases from the provider for the given project
    /// and groups them by crate.
    #[instrument(skip(self), err)]
    async fn fetch_releases_by_crate(
        &self,
    ) -> anyhow::Result<IndexMap<(U::CratePath, ReleaseName), Vec<Release>>> {
        let user = self.user()?;
        let group = self.group()?;
        let project = self.project()?;

        let mut res = IndexMap::new();

        for (path, release) in Arc::clone(&self.gitlab)
            .fetch_releases_for_group(group, user)
            .fetch_releases_for_project(project, user)
            .await?
        {
            res.entry((path, Arc::clone(&release.name)))
@@ -282,18 +282,22 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
        // create the high-level packfile generator
        let mut packfile = GitRepository::default();

        let project = self.project()?;

        // fetch the impersonation token for the user we'll embed
        // the `dl` string.
        let token = self.gitlab.fetch_token_for_user(self.user()?).await?;

        // generate the config for the user, containing the download
        // url template from gitlab and the impersonation token embedded
        let config_json = Bytes::from_static(b"{}");
        let config_json = Bytes::from(serde_json::to_vec(&CargoConfig {
            dl: self.gitlab.cargo_dl_uri(project, &token)?,
        })?);

        // write config.json to the root of the repo
        packfile.insert(&[], "config.json".into(), config_json)?;

        // fetch the releases for every project within the given group
        // fetch the releases for every project within the given project
        let releases_by_crate = self.fetch_releases_by_crate().await?;

        // a reusable buffer for writing the metadata json blobs out to
@@ -315,12 +319,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {

                // each crates file in the index is a metadata blob for
                // each version separated by a newline
                buffer.extend_from_slice(&serde_json::to_vec(
                    &CargoIndexCrateMetadataWithDlOverride {
                        meta: &meta,
                        dl: &self.gitlab.cargo_dl_uri(crate_path, version, &token)?,
                    },
                )?);
                buffer.extend_from_slice(&serde_json::to_vec(&*meta)?);
                buffer.put_u8(b'\n');
            }

@@ -544,30 +543,20 @@ impl<'a, U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::ser
                anyhow::bail!("not git-upload-pack");
            }

            // parse the requested group from the given path (the argument
            // parse the requested project from the given path (the argument
            // given to `git-upload-pack`)
            if let Some(group) = args.next().filter(|v| v.as_str() != "/") {
                let user = self.user()?;
                let group = group.trim_start_matches('/').trim_end_matches('/');

                match Arc::clone(&self.gitlab).fetch_group(group, user).await {
                    Ok(v) => self.group = Some(v),
                    Err(e) => {
                        session.extended_data(channel, 1, CryptoVec::from_slice(format!(indoc::indoc! {"
                            \r\nGitLab returned an error when attempting to query for group `{}` as `{}`:

                                {}

                            The group might not exist or you may not have permission to view it.\r\n
                        "}, group, user.username, e).as_bytes()));
                        session.close(channel);
                    }
                }
            let arg = args.next();
            if let Some(project) = arg.as_deref()
                .filter(|v| *v != "/")
                .map(|project| project.trim_start_matches('/').trim_end_matches('/'))
                .filter(|project| project.contains('/'))
            {
                self.project = Some(Arc::from(project.to_string()));
            } else {
                session.extended_data(channel, 1, CryptoVec::from_slice(indoc::indoc! {b"
                    \r\nNo group was given in the path part of the SSH URI. A GitLab group should be defined in your .cargo/config.toml as follows:
                    \r\nNo project was given in the path part of the SSH URI. A GitLab group and project should be defined in your .cargo/config.toml as follows:
                        [registries]
                        chartered = {{ index = \"ssh://domain.to.registry.com/my-group\" }}\r\n
                        my-project = {{ index = \"ssh://domain.to.registry.com/my-group/my-project\" }}\r\n
                "}));
                session.close(channel);
            }
diff --git a/src/metadata.rs b/src/metadata.rs
index ea187e9..b6f39c7 100644
--- a/src/metadata.rs
+++ b/src/metadata.rs
@@ -49,10 +49,8 @@ pub fn transform(
}

#[derive(Serialize, Debug)]
pub struct CargoIndexCrateMetadataWithDlOverride<'a> {
    #[serde(flatten)]
    pub meta: &'a CargoIndexCrateMetadata,
    pub dl: &'a str,
pub struct CargoConfig {
    pub dl: String,
}

#[derive(Serialize, Deserialize, Debug)]
diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs
index 17a2ce8..790315a 100644
--- a/src/providers/gitlab.rs
+++ b/src/providers/gitlab.rs
@@ -1,7 +1,7 @@
#![allow(clippy::module_name_repetitions)]

use crate::config::GitlabConfig;
use crate::providers::{Group, Release, User};
use crate::providers::{Release, User};
use async_trait::async_trait;
use futures::{stream::FuturesUnordered, StreamExt, TryStreamExt};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
@@ -112,34 +112,16 @@ impl super::UserProvider for Gitlab {
impl super::PackageProvider for Gitlab {
    type CratePath = Arc<GitlabCratePath>;

    #[instrument(skip(self), err)]
    async fn fetch_group(&self, group: &str, do_as: &User) -> anyhow::Result<Group> {
        let mut url = self
            .base_url
            .join("groups/")?
            .join(&utf8_percent_encode(group, NON_ALPHANUMERIC).to_string())?;
        url.query_pairs_mut()
            .append_pair("sudo", itoa::Buffer::new().format(do_as.id));

        let req = handle_error(self.client.get(url).send().await?)
            .await?
            .json::<GitlabGroupResponse>()
            .await?
            .into();

        Ok(req)
    }

    #[instrument(skip(self), err)]
    async fn fetch_releases_for_group(
    async fn fetch_releases_for_project(
        self: Arc<Self>,
        group: &Group,
        project: &str,
        do_as: &User,
    ) -> anyhow::Result<Vec<(Self::CratePath, Release)>> {
        let mut next_uri = Some({
            let mut uri = self
                .base_url
                .join(&format!("groups/{}/packages", group.id,))?;
            let mut uri = self.base_url.join(&format!(
                "projects/{}/packages",
                urlencoding::encode(project)
            ))?;
            {
                let mut query = uri.query_pairs_mut();
                query.append_pair("per_page", itoa::Buffer::new().format(100u16));
@@ -244,20 +226,12 @@ impl super::PackageProvider for Gitlab {
            .await?)
    }

    fn cargo_dl_uri(
        &self,
        path: &Self::CratePath,
        version: &str,
        token: &str,
    ) -> anyhow::Result<String> {
        Ok(format!(
            "{uri}projects/{project}/packages/generic/{crate_name}/{version}/{crate_name}-{version}.crate?private_token={token}",
            uri = self.base_url,
            project = path.project,
            crate_name = path.package_name,
            version = version,
            token = token,
        ))
    fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result<String> {
        let uri = self
            .base_url
            .join("projects/")?
            .join(&format!("{}/", urlencoding::encode(project)))?;
        Ok(format!("{uri}packages/generic/{{crate}}/{{version}}/{{crate}}-{{version}}.crate?private_token={token}"))
    }
}

@@ -280,21 +254,6 @@ pub struct GitlabErrorResponse {
    error: Option<String>,
}

#[derive(Deserialize)]
pub struct GitlabGroupResponse {
    id: u64,
    name: String,
}

impl From<GitlabGroupResponse> for Group {
    fn from(v: GitlabGroupResponse) -> Self {
        Self {
            id: v.id,
            name: v.name,
        }
    }
}

#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct GitlabCratePath {
    project: String,
diff --git a/src/providers/mod.rs b/src/providers/mod.rs
index 271043c..031e63e 100644
--- a/src/providers/mod.rs
+++ b/src/providers/mod.rs
@@ -21,11 +21,9 @@ pub trait PackageProvider {
    /// figure out the path of a package.
    type CratePath: std::fmt::Debug + Send + std::hash::Hash + Clone + Eq + PartialEq + Send + Sync;

    async fn fetch_group(&self, group: &str, do_as: &User) -> anyhow::Result<Group>;

    async fn fetch_releases_for_group(
    async fn fetch_releases_for_project(
        self: Arc<Self>,
        group: &Group,
        project: &str,
        do_as: &User,
    ) -> anyhow::Result<Vec<(Self::CratePath, Release)>>;

@@ -35,12 +33,7 @@ pub trait PackageProvider {
        version: &str,
    ) -> anyhow::Result<cargo_metadata::Metadata>;

    fn cargo_dl_uri(
        &self,
        path: &Self::CratePath,
        version: &str,
        token: &str,
    ) -> anyhow::Result<String>;
    fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result<String>;
}

#[derive(Debug, Clone)]
@@ -49,12 +42,6 @@ pub struct User {
    pub username: String,
}

#[derive(Debug, Clone)]
pub struct Group {
    pub id: u64,
    pub name: String,
}

pub type ReleaseName = Arc<str>;

#[derive(Debug)]