From bb203c01a9c931b6032978c0df751c51a85c584d Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Mon, 4 Jul 2022 21:10:14 +0100 Subject: [PATCH] Refer to projects directly rather than groups so we don't have to rely on gitlab/cargo patches --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + README.md | 4 ++-- src/main.rs | 67 ++++++++++++++++++++++++++++--------------------------------------- src/metadata.rs | 6 ++---- src/providers/gitlab.rs | 67 +++++++++++++------------------------------------------------------ src/providers/mod.rs | 19 +++---------------- 7 files changed, 56 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c6db58..f664d61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,6 +596,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "urlencoding", "ustr", "uuid", ] @@ -1838,6 +1839,12 @@ dependencies = [ ] [[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + +[[package]] name = "ustr" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index 9c54bbf..09e0fc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ tokio = { version = "1.17", features = ["full"] } tokio-util = { version = "0.7", features = ["codec"] } toml = "0.5" url = { version = "2.2", features = ["serde"] } +urlencoding = "2.1" ustr = "0.8" uuid = { version = "1.0.0-alpha.1", features = ["v4"] } diff --git a/README.md b/README.md index 3211701..4eea3ec 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,11 @@ accordingly. ```toml # .cargo/config.toml [registries] -my-gitlab-group = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group" } +my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project" } # Cargo.toml [dependencies] -my-crate = { version = "0.1", registry = "my-gitlab-group" } +my-crate = { version = "0.1", registry = "my-gitlab-project" } ``` In your CI build, setup a `before_script` step to replace the connection string diff --git a/src/main.rs b/src/main.rs index 54ca331..cd6cb4f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,14 +10,14 @@ pub mod util; use crate::{ config::Args, - metadata::{CargoIndexCrateMetadata, CargoIndexCrateMetadataWithDlOverride}, + metadata::{CargoConfig, CargoIndexCrateMetadata}, protocol::{ codec::{Encoder, GitCodec}, high_level::GitRepository, low_level::{HashOutput, PackFileEntry}, packet_line::PktLine, }, - providers::{gitlab::Gitlab, Group, PackageProvider, Release, ReleaseName, User, UserProvider}, + providers::{gitlab::Gitlab, PackageProvider, Release, ReleaseName, User, UserProvider}, util::get_crate_folder, }; use anyhow::anyhow; @@ -136,7 +136,7 @@ impl thrussh::server: codec: GitCodec::default(), gitlab: Arc::clone(&self.gitlab), user: None, - group: None, + project: None, input_bytes: BytesMut::new(), output_bytes: BytesMut::new(), is_git_protocol_v2: false, @@ -151,7 +151,7 @@ pub struct Handler { codec: GitCodec, gitlab: Arc, user: Option>, - group: Option, + project: Option>, // fetcher_future: Option>>>, input_bytes: BytesMut, output_bytes: BytesMut, @@ -170,10 +170,10 @@ impl Handler { .ok_or_else(|| anyhow::anyhow!("no user set")) } - fn group(&self) -> anyhow::Result<&Group> { - self.group - .as_ref() - .ok_or_else(|| anyhow::anyhow!("no group set")) + fn project(&self) -> anyhow::Result<&str> { + self.project + .as_deref() + .ok_or_else(|| anyhow::anyhow!("no project set")) } /// Writes a Git packet line response to the buffer, this should only @@ -190,19 +190,19 @@ impl Handler { ); } - /// Fetches all the releases from the provider for the given group + /// Fetches all the releases from the provider for the given project /// and groups them by crate. #[instrument(skip(self), err)] async fn fetch_releases_by_crate( &self, ) -> anyhow::Result>> { let user = self.user()?; - let group = self.group()?; + let project = self.project()?; let mut res = IndexMap::new(); for (path, release) in Arc::clone(&self.gitlab) - .fetch_releases_for_group(group, user) + .fetch_releases_for_project(project, user) .await? { res.entry((path, Arc::clone(&release.name))) @@ -282,18 +282,22 @@ impl Handler { // create the high-level packfile generator let mut packfile = GitRepository::default(); + let project = self.project()?; + // fetch the impersonation token for the user we'll embed // the `dl` string. let token = self.gitlab.fetch_token_for_user(self.user()?).await?; // generate the config for the user, containing the download // url template from gitlab and the impersonation token embedded - let config_json = Bytes::from_static(b"{}"); + let config_json = Bytes::from(serde_json::to_vec(&CargoConfig { + dl: self.gitlab.cargo_dl_uri(project, &token)?, + })?); // write config.json to the root of the repo packfile.insert(&[], "config.json".into(), config_json)?; - // fetch the releases for every project within the given group + // fetch the releases for every project within the given project let releases_by_crate = self.fetch_releases_by_crate().await?; // a reusable buffer for writing the metadata json blobs out to @@ -315,12 +319,7 @@ impl Handler { // each crates file in the index is a metadata blob for // each version separated by a newline - buffer.extend_from_slice(&serde_json::to_vec( - &CargoIndexCrateMetadataWithDlOverride { - meta: &meta, - dl: &self.gitlab.cargo_dl_uri(crate_path, version, &token)?, - }, - )?); + buffer.extend_from_slice(&serde_json::to_vec(&*meta)?); buffer.put_u8(b'\n'); } @@ -544,30 +543,20 @@ impl<'a, U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::ser anyhow::bail!("not git-upload-pack"); } - // parse the requested group from the given path (the argument + // parse the requested project from the given path (the argument // given to `git-upload-pack`) - if let Some(group) = args.next().filter(|v| v.as_str() != "/") { - let user = self.user()?; - let group = group.trim_start_matches('/').trim_end_matches('/'); - - match Arc::clone(&self.gitlab).fetch_group(group, user).await { - Ok(v) => self.group = Some(v), - Err(e) => { - session.extended_data(channel, 1, CryptoVec::from_slice(format!(indoc::indoc! {" - \r\nGitLab returned an error when attempting to query for group `{}` as `{}`: - - {} - - The group might not exist or you may not have permission to view it.\r\n - "}, group, user.username, e).as_bytes())); - session.close(channel); - } - } + let arg = args.next(); + if let Some(project) = arg.as_deref() + .filter(|v| *v != "/") + .map(|project| project.trim_start_matches('/').trim_end_matches('/')) + .filter(|project| project.contains('/')) + { + self.project = Some(Arc::from(project.to_string())); } else { session.extended_data(channel, 1, CryptoVec::from_slice(indoc::indoc! {b" - \r\nNo group was given in the path part of the SSH URI. A GitLab group should be defined in your .cargo/config.toml as follows: + \r\nNo project was given in the path part of the SSH URI. A GitLab group and project should be defined in your .cargo/config.toml as follows: [registries] - chartered = {{ index = \"ssh://domain.to.registry.com/my-group\" }}\r\n + my-project = {{ index = \"ssh://domain.to.registry.com/my-group/my-project\" }}\r\n "})); session.close(channel); } diff --git a/src/metadata.rs b/src/metadata.rs index ea187e9..b6f39c7 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -49,10 +49,8 @@ pub fn transform( } #[derive(Serialize, Debug)] -pub struct CargoIndexCrateMetadataWithDlOverride<'a> { - #[serde(flatten)] - pub meta: &'a CargoIndexCrateMetadata, - pub dl: &'a str, +pub struct CargoConfig { + pub dl: String, } #[derive(Serialize, Deserialize, Debug)] diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 17a2ce8..790315a 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -1,7 +1,7 @@ #![allow(clippy::module_name_repetitions)] use crate::config::GitlabConfig; -use crate::providers::{Group, Release, User}; +use crate::providers::{Release, User}; use async_trait::async_trait; use futures::{stream::FuturesUnordered, StreamExt, TryStreamExt}; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; @@ -112,34 +112,16 @@ impl super::UserProvider for Gitlab { impl super::PackageProvider for Gitlab { type CratePath = Arc; - #[instrument(skip(self), err)] - async fn fetch_group(&self, group: &str, do_as: &User) -> anyhow::Result { - let mut url = self - .base_url - .join("groups/")? - .join(&utf8_percent_encode(group, NON_ALPHANUMERIC).to_string())?; - url.query_pairs_mut() - .append_pair("sudo", itoa::Buffer::new().format(do_as.id)); - - let req = handle_error(self.client.get(url).send().await?) - .await? - .json::() - .await? - .into(); - - Ok(req) - } - - #[instrument(skip(self), err)] - async fn fetch_releases_for_group( + async fn fetch_releases_for_project( self: Arc, - group: &Group, + project: &str, do_as: &User, ) -> anyhow::Result> { let mut next_uri = Some({ - let mut uri = self - .base_url - .join(&format!("groups/{}/packages", group.id,))?; + let mut uri = self.base_url.join(&format!( + "projects/{}/packages", + urlencoding::encode(project) + ))?; { let mut query = uri.query_pairs_mut(); query.append_pair("per_page", itoa::Buffer::new().format(100u16)); @@ -244,20 +226,12 @@ impl super::PackageProvider for Gitlab { .await?) } - fn cargo_dl_uri( - &self, - path: &Self::CratePath, - version: &str, - token: &str, - ) -> anyhow::Result { - Ok(format!( - "{uri}projects/{project}/packages/generic/{crate_name}/{version}/{crate_name}-{version}.crate?private_token={token}", - uri = self.base_url, - project = path.project, - crate_name = path.package_name, - version = version, - token = token, - )) + fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result { + let uri = self + .base_url + .join("projects/")? + .join(&format!("{}/", urlencoding::encode(project)))?; + Ok(format!("{uri}packages/generic/{{crate}}/{{version}}/{{crate}}-{{version}}.crate?private_token={token}")) } } @@ -280,21 +254,6 @@ pub struct GitlabErrorResponse { error: Option, } -#[derive(Deserialize)] -pub struct GitlabGroupResponse { - id: u64, - name: String, -} - -impl From for Group { - fn from(v: GitlabGroupResponse) -> Self { - Self { - id: v.id, - name: v.name, - } - } -} - #[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct GitlabCratePath { project: String, diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 271043c..031e63e 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -21,11 +21,9 @@ pub trait PackageProvider { /// figure out the path of a package. type CratePath: std::fmt::Debug + Send + std::hash::Hash + Clone + Eq + PartialEq + Send + Sync; - async fn fetch_group(&self, group: &str, do_as: &User) -> anyhow::Result; - - async fn fetch_releases_for_group( + async fn fetch_releases_for_project( self: Arc, - group: &Group, + project: &str, do_as: &User, ) -> anyhow::Result>; @@ -35,12 +33,7 @@ pub trait PackageProvider { version: &str, ) -> anyhow::Result; - fn cargo_dl_uri( - &self, - path: &Self::CratePath, - version: &str, - token: &str, - ) -> anyhow::Result; + fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result; } #[derive(Debug, Clone)] @@ -49,12 +42,6 @@ pub struct User { pub username: String, } -#[derive(Debug, Clone)] -pub struct Group { - pub id: u64, - pub name: String, -} - pub type ReleaseName = Arc; #[derive(Debug)] -- libgit2 1.7.2