From 208565205aae59eaedb66da9b32f868b232b8408 Mon Sep 17 00:00:00 2001 From: Morian Sonnet Date: Thu, 09 Nov 2023 10:44:39 +0100 Subject: [PATCH] New feature: Allow authentication by personal tokens (#53) * Correct usage of cargo-get in .gitlab-ci snippet * Add note reg. incompatibility of cargo and thrussh * README: Add .ssh/config for personal token usage * README: Newer cargo versions do not allow URL-embedded passwords * Refactor: Add function to build client with token * Allow authentication with personal token * Keep personal token in User * Optionally use personal token for reading packages * Optionally use personal token for DL urls * README: Add usage of personal token authentication * Consider only packages with type generic --------- Co-authored-by: Morian Sonnet --- README.md | 21 +++++++++++++++++---- src/main.rs | 9 ++++++--- src/providers/gitlab.rs | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------- src/providers/mod.rs | 4 +++- 4 files changed, 97 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 7dcf5f5..f1b9c15 100644 --- a/README.md +++ a/README.md @@ -6,18 +6,27 @@ Access controls work like they do in GitLab, builds are scoped to users - if they don't have permission to the dependency they can't build it, it's that simple. -Users are identified by their SSH keys from GitLab when connecting to the server and an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Builds will insert their token as a username to the SSH server and the shim will use that to call the GitLab API. +Users are either identified by their SSH keys from GitLab when connecting to the server or by an Gitlab personal-token. If no token is given, an [impersonation token][imp-token] will be generated for that run in order to pull available versions. Doing so requires ad admin personal token. -To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml` and `Cargo.toml` accordingly. +To publish run `cargo package` and push the resulting `.crate` file to the GitLab package repository with a semver-compatible version string, to consume the package configure your `.cargo/config.toml`, `Cargo.toml` and, optionally, `.ssh/config` accordingly. +At time of writing, `libssh2`, which `cargo` implicitly uses for communicating with the registry by SSH, is incompatible with rust's `thrussh`, due to non-overlapping ciphers. Hence, activating `net.git-fetch-with-cli` is necessary. + ```toml # .cargo/config.toml [registries] -my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project" } +my-gitlab-project = { index = "ssh://gitlab-cargo-shim.local/my-gitlab-group/my-gitlab-project/" } +[net] +git-fetch-with-cli = true # Cargo.toml [dependencies] my-crate = { version = "0.1", registry = "my-gitlab-project" } +``` +```ssh-config +# .ssh/config (only if authentication by personal token is requires) +Host gitlab-cargo-shim.local + User personal-token: ``` In your CI build, setup a `before_script` step to replace the connection string with one containing the CI token: @@ -34,13 +43,13 @@ ```yaml release-crate: - image: rust:1.62 + image: rust:latest stage: deploy only: # release when a tag is pushed - tags before_script: - cargo install cargo-get - - export CRATE_NAME=$(cargo get --name) CRATE_VERSION=$(cargo get version) + - export CRATE_NAME=$(cargo-get package.name) CRATE_VERSION=$(cargo-get package.version) - export CRATE_FILE=${CRATE_NAME}-${CRATE_VERSION}.crate script: - cargo package @@ -54,4 +63,4 @@ [gitlab-package-registry]: https://docs.gitlab.com/ee/user/packages/package_registry/index.html [imp-token]: https://docs.gitlab.com/ee/api/index.html#impersonation-tokens [envvar]: https://doc.rust-lang.org/cargo/reference/registries.html#using-an-alternate-registry -[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml+[example-configuration]: https://github.com/w4/gitlab-cargo-shim/blob/main/config.toml diff --git a/src/main.rs b/src/main.rs index 031e252..fd9d34e 100644 --- a/src/main.rs +++ a/src/main.rs @@ -247,7 +247,7 @@ // fetch metadata from the provider let metadata = Arc::clone(&self.gitlab) - .fetch_metadata_for_release(path, crate_version) + .fetch_metadata_for_release(path, crate_version, self.user()?) .await?; // transform the `cargo metadata` output to the cargo index @@ -291,7 +291,10 @@ // fetch the impersonation token for the user we'll embed // the `dl` string. - let token = self.gitlab.fetch_token_for_user(self.user()?).await?; + let token = match &self.user()?.token { + None => self.gitlab.fetch_token_for_user(self.user()?).await?, + Some(token) => token.clone(), + }; // generate the config for the user, containing the download // url template from gitlab and the impersonation token embedded @@ -408,7 +411,7 @@ info!( "Successfully authenticated for GitLab user `{}` by {}", &user.username, - if by_ssh_key { "SSH Key" } else { "Build Token" }, + if by_ssh_key { "SSH Key" } else { "Build or Personal Token" }, ); self.user = Some(Arc::new(user)); self.finished_auth(Auth::Accept).await diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 3faf752..9751442 100644 --- a/src/providers/gitlab.rs +++ a/src/providers/gitlab.rs @@ -11,6 +11,7 @@ use time::{Duration, OffsetDateTime}; use tracing::{info_span, instrument, Instrument}; use url::Url; +use std::str::FromStr; pub struct Gitlab { client: reqwest::Client, @@ -46,6 +47,19 @@ ssl_cert, }) } + + pub fn build_client_with_token(&self, token_field: &str, token: &str) -> anyhow::Result { + let mut headers = header::HeaderMap::new(); + headers.insert( + header::HeaderName::from_str(token_field)?, + header::HeaderValue::from_str(token)?, + ); + let mut client_builder = reqwest::ClientBuilder::new().default_headers(headers); + if let Some(cert) = &self.ssl_cert { + client_builder = client_builder.add_root_certificate(cert.clone()); + } + Ok(client_builder.build()?) + } } #[async_trait] @@ -60,29 +74,43 @@ return Ok(None); }; - if username == "gitlab-ci-token" { + if username == "gitlab-ci-token" || username == "personal-token" { // we're purposely not using `self.client` here as we don't // want to use our admin token for this request but still want to use any ssl cert provided. - let mut client_builder = reqwest::Client::builder(); - if let Some(cert) = &self.ssl_cert { - client_builder = client_builder.add_root_certificate(cert.clone()); + let client = self.build_client_with_token(if username == "gitlab-ci-token" { "JOB-TOKEN" } else { "PRIVATE-TOKEN" }, password); + if username == "gitlab-ci-token" { + let res: GitlabJobResponse = handle_error( + client? + .get(self.base_url.join("job/")?) + .send() + .await?, + ) + .await? + .json() + .await?; + + Ok(Some(User { + id: res.user.id, + username: res.user.username, + ..Default::default() + })) + } else { + let res: GitlabUserResponse = handle_error( + client? + .get(self.base_url.join("user/")?) + .send() + .await?, + ) + .await? + .json() + .await?; + + Ok(Some(User { + id: res.id, + username: res.username, + token: Some(password.to_string()), + })) } - let client = client_builder.build(); - let res: GitlabJobResponse = handle_error( - client? - .get(self.base_url.join("job/")?) - .header("JOB-TOKEN", password) - .send() - .await?, - ) - .await? - .json() - .await?; - - Ok(Some(User { - id: res.user.id, - username: res.user.username, - })) } else { Ok(None) } @@ -101,6 +129,7 @@ Ok(res.user.map(|u| User { id: u.id, username: u.username, + ..Default::default() })) } @@ -149,15 +178,23 @@ query.append_pair("per_page", itoa::Buffer::new().format(100u16)); query.append_pair("pagination", "keyset"); query.append_pair("sort", "asc"); - query.append_pair("sudo", itoa::Buffer::new().format(do_as.id)); + if do_as.token.is_none() { + query.append_pair("sudo", itoa::Buffer::new().format(do_as.id)); + } } uri }); let futures = FuturesUnordered::new(); + + let client = match &do_as.token { + None => self.client.clone(), + Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)? + }; + let client = Arc::new(client); while let Some(uri) = next_uri.take() { - let res = handle_error(self.client.get(uri).send().await?).await?; + let res = handle_error(client.get(uri).send().await?).await?; if let Some(link_header) = res.headers().get(header::LINK) { let mut link_header = parse_link_header::parse_with_rel(link_header.to_str()?)?; @@ -167,10 +204,15 @@ } } - let res: Vec = res.json().await?; + let res: Vec = res.json::>() + .await? + .into_iter() + .filter(|release| release.package_type == "generic") + .collect(); for release in res { let this = Arc::clone(&self); + let client = Arc::clone(&client); futures.push(tokio::spawn( async move { @@ -189,7 +231,7 @@ }); let package_files: Vec = handle_error( - this.client + client .get(format!( "{}/projects/{}/packages/{}/package_files", this.base_url, @@ -239,10 +281,15 @@ &self, path: &Self::CratePath, version: &str, + do_as: &User, ) -> anyhow::Result { let uri = self.base_url.join(&path.metadata_uri(version))?; + let client = match &do_as.token { + None => self.client.clone(), + Some(token) => self.build_client_with_token("PRIVATE-TOKEN", token)? + }; - Ok(handle_error(self.client.get(uri).send().await?) + Ok(handle_error(client.get(uri).send().await?) .await? .json() .await?) @@ -315,6 +362,7 @@ pub id: u64, pub name: String, pub version: String, + pub package_type: String, #[serde(rename = "_links")] pub links: GitlabPackageLinksResponse, } diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 031e63e..079e720 100644 --- a/src/providers/mod.rs +++ a/src/providers/mod.rs @@ -31,15 +31,17 @@ &self, path: &Self::CratePath, version: &str, + do_as: &User, ) -> anyhow::Result; fn cargo_dl_uri(&self, project: &str, token: &str) -> anyhow::Result; } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct User { pub id: u64, pub username: String, + pub token: Option, } pub type ReleaseName = Arc; -- rgit 0.1.3