Less allocations in the hot path
Diff
Cargo.lock | 66 ++++++++++++++++++++++++++++++--
Cargo.toml | 5 ++-
README.md | 2 +-
src/config.rs | 3 +-
src/main.rs | 26 ++++++-------
src/metadata.rs | 31 ++++++++-------
src/protocol/high_level.rs | 13 +++---
src/protocol/low_level.rs | 29 ++++++++------
src/providers/gitlab.rs | 96 +++++++++++++++++++++++------------------------
src/providers/mod.rs | 6 ++-
src/util.rs | 78 ++++++++++++++++++++++++++++++++++----
11 files changed, 249 insertions(+), 106 deletions(-)
@@ -22,6 +22,17 @@ dependencies = [
]
[[package]]
name = "ahash"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
"getrandom",
"once_cell",
"version_check",
]
[[package]]
name = "aho-corasick"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -46,6 +57,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0"
[[package]]
name = "arrayvec"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "async-trait"
version = "0.1.52"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -549,8 +566,10 @@ name = "gitlab-cargo-shim"
version = "0.1.0"
dependencies = [
"anyhow",
"arrayvec",
"async-trait",
"bytes",
"cargo-platform",
"cargo_metadata",
"clap",
"flate2",
@@ -559,10 +578,11 @@ dependencies = [
"indexmap",
"indoc",
"itoa",
"parking_lot",
"parking_lot 0.12.0",
"parse_link_header",
"percent-encoding",
"reqwest",
"semver",
"serde",
"serde_json",
"sha1",
@@ -575,6 +595,8 @@ dependencies = [
"toml",
"tracing",
"tracing-subscriber",
"url",
"ustr",
"uuid",
]
@@ -995,12 +1017,37 @@ dependencies = [
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.5",
]
[[package]]
name = "parking_lot"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.1",
]
[[package]]
name = "parking_lot_core"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall",
"smallvec",
"winapi",
]
[[package]]
@@ -1595,7 +1642,7 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
"parking_lot",
"parking_lot 0.12.0",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -1785,6 +1832,19 @@ dependencies = [
"idna",
"matches",
"percent-encoding",
"serde",
]
[[package]]
name = "ustr"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbd539d8973e229b9d04f15d36e6a8f8d8f85f946b366f06bb001aaed3fa9dd9"
dependencies = [
"ahash",
"byteorder",
"lazy_static",
"parking_lot 0.11.2",
]
[[package]]
@@ -10,9 +10,11 @@ authors = [
[dependencies]
anyhow = "1"
arrayvec = "0.7"
async-trait = "0.1"
bytes = "1.1"
cargo_metadata = "0.14"
cargo-platform = "0.1"
clap = { version = "3.1", features = ["derive", "cargo"] }
flate2 = "1.0"
futures = "0.3"
@@ -24,6 +26,7 @@ parse_link_header = "0.3"
parking_lot = "0.12"
percent-encoding = "2.1"
reqwest = { version = "0.11", features = ["json"] }
semver = "1.0"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1"
sha1 = "0.10"
@@ -36,4 +39,6 @@ time = "0.3"
tokio = { version = "1.17", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
toml = "0.5"
url = { version = "2.2", features = ["serde"] }
ustr = "0.8"
uuid = { version = "1.0.0-alpha.1", features = ["v4"] }
@@ -43,7 +43,7 @@ before_script:
To release your package from CI, add a new pipeline step:
```
```yaml
stage:
- release-crate
@@ -3,6 +3,7 @@
use clap::Parser;
use serde::{de::DeserializeOwned, Deserialize};
use std::path::PathBuf;
use url::Url;
#[derive(Parser)]
#[clap(version = clap::crate_version!(), author = clap::crate_authors!())]
@@ -21,7 +22,7 @@ pub struct Config {
#[derive(Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct GitlabConfig {
pub uri: String,
pub uri: Url,
pub admin_token: String,
}
@@ -17,7 +17,7 @@ use crate::{
low_level::{HashOutput, PackFileEntry},
packet_line::PktLine,
},
providers::{gitlab::Gitlab, Group, PackageProvider, Release, User, UserProvider},
providers::{gitlab::Gitlab, Group, PackageProvider, Release, ReleaseName, User, UserProvider},
util::get_crate_folder,
};
use anyhow::anyhow;
@@ -147,7 +147,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::server:
pub struct Handler<U: UserProvider + PackageProvider + Send + Sync + 'static> {
codec: GitCodec,
gitlab: Arc<U>,
user: Option<User>,
user: Option<Arc<User>>,
group: Option<Group>,
input_bytes: BytesMut,
@@ -161,7 +161,7 @@ pub struct Handler<U: UserProvider + PackageProvider + Send + Sync + 'static> {
}
impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
fn user(&self) -> anyhow::Result<&User> {
fn user(&self) -> anyhow::Result<&Arc<User>> {
self.user.as_ref().ok_or(anyhow::anyhow!("no user set"))
}
@@ -187,7 +187,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
async fn fetch_releases_by_crate(
&self,
) -> anyhow::Result<IndexMap<(U::CratePath, String), Vec<Release>>> {
) -> anyhow::Result<IndexMap<(U::CratePath, ReleaseName), Vec<Release>>> {
let user = self.user()?;
let group = self.group()?;
@@ -197,7 +197,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
.fetch_releases_for_group(group, user)
.await?
{
res.entry((path, release.name.clone()))
res.entry((path, Arc::clone(&release.name)))
.or_insert_with(Vec::new)
.push(release);
}
@@ -266,7 +266,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
if let Some(packfile_cache) = &self.packfile_cache {
return Ok(packfile_cache.clone());
return Ok(Arc::clone(packfile_cache));
}
@@ -282,11 +282,11 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
let config_json = Bytes::from(serde_json::to_vec(&CargoConfig {
dl: self.gitlab.cargo_dl_uri(group, &token),
dl: self.gitlab.cargo_dl_uri(group, &token)?,
})?);
packfile.insert(vec![], "config.json".to_string(), config_json)?;
packfile.insert(&[], "config.json".into(), config_json)?;
let releases_by_crate = self.fetch_releases_by_crate().await?;
@@ -316,8 +316,8 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
packfile.insert(
get_crate_folder(crate_name),
crate_name.to_string(),
&get_crate_folder(crate_name),
Arc::clone(crate_name).into(),
buffer.split().freeze(),
)?;
}
@@ -396,7 +396,7 @@ impl<'a, U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::ser
&user.username,
if by_ssh_key { "SSH Key" } else { "Build Token" },
);
self.user = Some(user);
self.user = Some(Arc::new(user));
self.finished_auth(Auth::Accept).await
} else {
info!("Public key rejected");
@@ -481,11 +481,11 @@ impl<'a, U: UserProvider + PackageProvider + Send + Sync + 'static> thrussh::ser
let span = info_span!(parent: &self.span, "shell_request");
Box::pin(capture_errors(async move {
let username = self.user()?.username.clone();
let user = Arc::clone(self.user()?);
write!(
&mut self.output_bytes,
"Hi there, {}! You've successfully authenticated, but {} does not provide shell access.\r\n",
username,
user.username,
env!("CARGO_PKG_NAME")
)?;
info!("Shell requested, dropping connection");
@@ -1,8 +1,10 @@
#![allow(clippy::module_name_repetitions)]
use cargo_metadata::Package;
use cargo_metadata::{DependencyKind, Package};
use cargo_platform::Platform;
use semver::{Version, VersionReq};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::{borrow::Cow, collections::HashMap};
@@ -20,21 +22,22 @@ pub fn transform(
Some(CargoIndexCrateMetadata {
name: package.name,
vers: package.version.to_string(),
vers: package.version,
deps: package
.dependencies
.into_iter()
.map(|v| CargoIndexCrateMetadataDependency {
name: v.name,
req: v.req.to_string(),
req: v.req,
features: v.features,
optional: v.optional,
default_features: v.uses_default_features,
target: v.target.map(|v| v.to_string()),
kind: v.kind.to_string(),
registry: Some(v.registry.unwrap_or_else(|| {
"https://github.com/rust-lang/crates.io-index.git".to_string()
})),
target: v.target,
kind: v.kind,
registry: Some(v.registry.map_or(
Cow::Borrowed("https://github.com/rust-lang/crates.io-index.git"),
Cow::Owned,
)),
package: v.rename,
})
.collect(),
@@ -53,7 +56,7 @@ pub struct CargoConfig {
#[derive(Serialize, Deserialize, Debug)]
pub struct CargoIndexCrateMetadata {
name: String,
vers: String,
vers: Version,
deps: Vec<CargoIndexCrateMetadataDependency>,
cksum: String,
features: HashMap<String, Vec<String>>,
@@ -64,12 +67,12 @@ pub struct CargoIndexCrateMetadata {
#[derive(Serialize, Deserialize, Debug)]
pub struct CargoIndexCrateMetadataDependency {
name: String,
req: String,
req: VersionReq,
features: Vec<String>,
optional: bool,
default_features: bool,
target: Option<String>,
kind: String,
registry: Option<String>,
target: Option<Platform>,
kind: DependencyKind,
registry: Option<Cow<'static, str>>,
package: Option<String>,
}
@@ -6,6 +6,7 @@
use crate::util::ArcOrCowStr;
use bytes::Bytes;
use indexmap::IndexMap;
@@ -34,8 +35,8 @@ impl GitRepository {
pub fn insert(
&mut self,
path: Vec<String>,
file: String,
path: &[&'static str],
file: ArcOrCowStr,
content: Bytes,
) -> Result<(), anyhow::Error> {
@@ -48,7 +49,7 @@ impl GitRepository {
for part in path {
let tree_item = directory
.0
.entry(part)
.entry((*part).into())
.or_insert_with(|| Box::new(TreeItem::Tree(Tree::default())));
if let TreeItem::Tree(d) = tree_item.as_mut() {
@@ -98,7 +99,7 @@ impl GitRepository {
let commit = PackFileEntry::Commit(Commit {
tree: tree_hash,
author: commit_user.clone(),
author: commit_user,
committer: commit_user,
message,
});
@@ -116,7 +117,7 @@ impl GitRepository {
#[derive(Default, Debug)]
struct Tree(IndexMap<String, Box<TreeItem>>);
struct Tree(IndexMap<ArcOrCowStr, Box<TreeItem>>);
impl Tree {
@@ -132,7 +133,7 @@ impl Tree {
tree.push(match *item {
TreeItem::Blob(hash) => LowLevelTreeItem {
kind: TreeItemKind::File,
sort_name: name.clone(),
sort_name: name.to_string(),
name,
hash,
},
@@ -1,7 +1,12 @@
use crate::util::ArcOrCowStr;
use bytes::{BufMut, Bytes, BytesMut};
use flate2::{write::ZlibEncoder, Compression};
use sha1::Digest;
use std::{convert::TryInto, fmt::Write, io::Write as IoWrite};
use std::{
convert::TryInto,
fmt::{Display, Formatter, Write},
io::Write as IoWrite,
};
pub type HashOutput = [u8; 20];
@@ -73,8 +78,8 @@ impl Commit {
out.extend_from_slice(&tree_hex);
out.write_char('\n')?;
writeln!(out, "author {}", self.author.encode())?;
writeln!(out, "committer {}", self.committer.encode())?;
writeln!(out, "author {}", self.author)?;
writeln!(out, "committer {}", self.committer)?;
write!(out, "\n{}", self.message)?;
Ok(())
@@ -91,24 +96,26 @@ impl Commit {
}
}
#[derive(Clone, Debug)]
#[derive(Clone, Copy, Debug)]
pub struct CommitUserInfo {
pub name: &'static str,
pub email: &'static str,
pub time: time::OffsetDateTime,
}
impl CommitUserInfo {
fn encode(&self) -> String {
format!(
impl Display for CommitUserInfo {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{} <{}> {} +0000",
self.name,
self.email,
self.time.unix_timestamp()
)
}
}
impl CommitUserInfo {
#[must_use]
pub fn size(&self) -> usize {
let timestamp_len = itoa::Buffer::new().format(self.time.unix_timestamp()).len();
@@ -138,10 +145,10 @@ impl TreeItemKind {
}
}
#[derive(Debug, Clone)]
#[derive(Debug)]
pub struct TreeItem {
pub kind: TreeItemKind,
pub name: String,
pub name: ArcOrCowStr,
pub hash: HashOutput,
pub sort_name: String,
}
@@ -161,7 +168,7 @@ impl TreeItem {
}
}
#[derive(Debug, Clone)] #[derive(Debug)] pub enum PackFileEntry {
@@ -7,13 +7,13 @@ use futures::{stream::FuturesUnordered, StreamExt, TryStreamExt};
use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::header;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
use std::sync::Arc;
use std::{borrow::Cow, sync::Arc};
use tracing::Instrument;
use url::Url;
pub struct Gitlab {
client: reqwest::Client,
base_url: String,
base_url: Url,
}
impl Gitlab {
@@ -28,7 +28,7 @@ impl Gitlab {
client: reqwest::ClientBuilder::new()
.default_headers(headers)
.build()?,
base_url: format!("{}/api/v4", config.uri),
base_url: config.uri.join("api/v4/")?,
})
}
}
@@ -48,7 +48,7 @@ impl super::UserProvider for Gitlab {
if username == "gitlab-ci-token" {
let res: GitlabJobResponse = handle_error(
self.client
.get(format!("{}/job", self.base_url))
.get(self.base_url.join("job/")?)
.header("JOB-TOKEN", password)
.send()
.await?,
@@ -67,19 +67,14 @@ impl super::UserProvider for Gitlab {
}
async fn find_user_by_ssh_key(&self, fingerprint: &str) -> anyhow::Result<Option<User>> {
let res: GitlabSshKeyLookupResponse = handle_error(
self.client
.get(format!(
"{}/keys?fingerprint={}",
self.base_url,
utf8_percent_encode(fingerprint, NON_ALPHANUMERIC)
))
.send()
.await?,
)
.await?
.json()
.await?;
let mut url = self.base_url.join("keys")?;
url.query_pairs_mut()
.append_pair("fingerprint", fingerprint);
let res: GitlabSshKeyLookupResponse = handle_error(self.client.get(url).send().await?)
.await?
.json()
.await?;
Ok(res.user.map(|u| User {
id: u.id,
username: u.username,
@@ -89,10 +84,10 @@ impl super::UserProvider for Gitlab {
async fn fetch_token_for_user(&self, user: &User) -> anyhow::Result<String> {
let impersonation_token: GitlabImpersonationTokenResponse = handle_error(
self.client
.post(format!(
"{}/users/{}/impersonation_tokens",
self.base_url, user.id
))
.post(
self.base_url
.join(&format!("users/{}/impersonation_tokens", user.id))?,
)
.json(&GitlabImpersonationTokenRequest {
name: env!("CARGO_PKG_NAME"),
scopes: vec!["api"],
@@ -113,14 +108,14 @@ impl super::PackageProvider for Gitlab {
type CratePath = Arc<GitlabCratePath>;
async fn fetch_group(self: Arc<Self>, group: &str, do_as: &User) -> anyhow::Result<Group> {
let uri = format!(
"{}/groups/{}?sudo={}",
self.base_url,
utf8_percent_encode(group, NON_ALPHANUMERIC),
do_as.id
);
let req = handle_error(self.client.get(uri).send().await?)
let mut url = self
.base_url
.join("groups/")?
.join(&utf8_percent_encode(group, NON_ALPHANUMERIC).to_string())?;
url.query_pairs_mut()
.append_pair("sudo", itoa::Buffer::new().format(do_as.id));
let req = handle_error(self.client.get(url).send().await?)
.await?
.json::<GitlabGroupResponse>()
.await?
@@ -134,12 +129,19 @@ impl super::PackageProvider for Gitlab {
group: &Group,
do_as: &User,
) -> anyhow::Result<Vec<(Self::CratePath, Release)>> {
let mut next_uri = Some(format!(
"{}/groups/{}/packages?per_page=100&pagination=keyset&sort=asc&sudo={}",
self.base_url,
utf8_percent_encode(&group.name, NON_ALPHANUMERIC),
do_as.id
));
let mut next_uri = Some({
let mut uri = self
.base_url
.join(&format!("groups/{}/packages", group.id,))?;
{
let mut query = uri.query_pairs_mut();
query.append_pair("per_page", itoa::Buffer::new().format(100u16));
query.append_pair("pagination", "keyset");
query.append_pair("sort", "asc");
query.append_pair("sudo", itoa::Buffer::new().format(do_as.id));
}
uri
});
let futures = FuturesUnordered::new();
@@ -150,14 +152,14 @@ impl super::PackageProvider for Gitlab {
let mut link_header = parse_link_header::parse_with_rel(link_header.to_str()?)?;
if let Some(next) = link_header.remove("next") {
next_uri = Some(next.raw_uri);
next_uri = Some(next.raw_uri.parse()?);
}
}
let res: Vec<GitlabPackageResponse> = res.json().await?;
for release in res {
let this = self.clone();
let this = Arc::clone(&self);
futures.push(tokio::spawn(
async move {
@@ -201,7 +203,7 @@ impl super::PackageProvider for Gitlab {
(
Arc::clone(&package_path),
Release {
name: release.name,
name: Arc::from(release.name),
version: release.version,
checksum: package_file.file_sha256,
},
@@ -226,7 +228,7 @@ impl super::PackageProvider for Gitlab {
path: &Self::CratePath,
version: &str,
) -> anyhow::Result<cargo_metadata::Metadata> {
let uri = format!("{}{}", self.base_url, path.metadata_uri(version),);
let uri = self.base_url.join(&path.metadata_uri(version))?;
Ok(handle_error(self.client.get(uri).send().await?)
.await?
@@ -234,12 +236,12 @@ impl super::PackageProvider for Gitlab {
.await?)
}
fn cargo_dl_uri(&self, group: &Group, token: &str) -> String {
format!(
"{}/groups/{}/packages/generic/{{sha256-checksum}}/{{crate}}-{{version}}.crate?private_token={token}",
self.base_url,
group.id,
)
fn cargo_dl_uri(&self, group: &Group, token: &str) -> anyhow::Result<String> {
let uri = self
.base_url
.join("groups/")?
.join(&format!("{}/", group.id))?;
Ok(format!("{uri}packages/generic/{{sha256-checksum}}/{{crate}}-{{version}}.crate?private_token={token}"))
}
}
@@ -287,7 +289,7 @@ impl GitlabCratePath {
#[must_use]
pub fn metadata_uri(&self, version: &str) -> String {
format!(
"/projects/{}/packages/generic/{}/{version}/metadata.json",
"projects/{}/packages/generic/{}/{version}/metadata.json",
self.project, self.package_name
)
}
@@ -35,7 +35,7 @@ pub trait PackageProvider {
version: &str,
) -> anyhow::Result<cargo_metadata::Metadata>;
fn cargo_dl_uri(&self, group: &Group, token: &str) -> String;
fn cargo_dl_uri(&self, group: &Group, token: &str) -> anyhow::Result<String>;
}
#[derive(Debug, Clone)]
@@ -50,9 +50,11 @@ pub struct Group {
pub name: String,
}
pub type ReleaseName = Arc<str>;
#[derive(Debug)]
pub struct Release {
pub name: String,
pub name: ReleaseName,
pub version: String,
pub checksum: String,
}
@@ -1,3 +1,12 @@
use arrayvec::ArrayVec;
use std::{
borrow::Cow,
fmt::{Debug, Display, Formatter},
ops::Deref,
sync::Arc,
};
use ustr::ustr;
#[must_use]
pub fn format_fingerprint(fingerprint: &str) -> String {
format!("SHA256:{}", fingerprint)
@@ -5,22 +14,75 @@ pub fn format_fingerprint(fingerprint: &str) -> String {
#[must_use]
pub fn get_crate_folder(crate_name: &str) -> Vec<String> {
let mut folders = Vec::new();
pub fn get_crate_folder(crate_name: &str) -> ArrayVec<&'static str, 2> {
let mut folders = ArrayVec::new();
match crate_name.len() {
0 => {}
1 => folders.push("1".to_string()),
2 => folders.push("2".to_string()),
3 => folders.push("3".to_string()),
1 => folders.push("1"),
2 => folders.push("2"),
3 => folders.push("3"),
_ => {
folders.push(crate_name[..2].to_string());
folders.push(crate_name[2..4].to_string());
folders.push(ustr(&crate_name[..2]).as_str());
folders.push(ustr(&crate_name[2..4]).as_str());
}
}
folders
}
#[derive(Debug, Hash, PartialEq, Eq)]
pub enum ArcOrCowStr {
Arc(Arc<str>),
Cow(Cow<'static, str>),
}
impl From<Arc<str>> for ArcOrCowStr {
fn from(v: Arc<str>) -> Self {
Self::Arc(v)
}
}
impl From<Cow<'static, str>> for ArcOrCowStr {
fn from(v: Cow<'static, str>) -> Self {
Self::Cow(v)
}
}
impl From<&'static str> for ArcOrCowStr {
fn from(v: &'static str) -> Self {
Self::Cow(Cow::Borrowed(v))
}
}
impl From<String> for ArcOrCowStr {
fn from(v: String) -> Self {
Self::Cow(Cow::Owned(v))
}
}
impl AsRef<str> for ArcOrCowStr {
fn as_ref(&self) -> &str {
match self {
Self::Arc(v) => v.as_ref(),
Self::Cow(v) => v.as_ref(),
}
}
}
impl Deref for ArcOrCowStr {
type Target = str;
fn deref(&self) -> &Self::Target {
self.as_ref()
}
}
impl Display for ArcOrCowStr {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&**self, f)
}
}