🏡 index : ~doyle/gitlab-cargo-shim.git

author Jordan Doyle <jordan@doyle.la> 2022-07-04 14:05:35.0 +00:00:00
committer Jordan Doyle <jordan@doyle.la> 2022-07-04 14:05:35.0 +00:00:00
commit 
6a73ed05d28a43b7e32220bdf4117aa45b421444 [patch]
tree 
a0b47426744dbb7ef07aad33696a7627d1ac8665
parent 
f06a5be6b3f617513c5c02e676cb878a6638a468
download 
6a73ed05d28a43b7e32220bdf4117aa45b421444.tar.gz

Implement HTTP redirect server instead of relying on Cargo/Gitlab changes




Diff


 Cargo.lock                       | 139 ++++++++++++++++++++++++++++++++++++++--
 Cargo.toml                       |   8 +-
 config.toml                      |   5 +-
 src/config.rs                    |   3 +-
 src/http_server/endpoints/dl.rs  |   3 +-
 src/http_server/endpoints/mod.rs |   1 +-
 src/http_server/layer/logging.rs | 109 +++++++++++++++++++++++++++++++-
 src/http_server/layer/mod.rs     |   1 +-
 src/http_server/mod.rs           |  13 ++++-
 src/main.rs                      |  48 +++++++++-----
 src/metadata.rs                  |   6 +--
 src/providers/gitlab.rs          |   1 +-
 src/providers/mod.rs             |   1 +-
 13 files changed, 309 insertions(+), 29 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5c6db58..7c365e9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -91,6 +91,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

[[package]]
name = "axum"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a"
dependencies = [
 "async-trait",
 "axum-core",
 "bitflags",
 "bytes",
 "futures-util",
 "http",
 "http-body",
 "hyper",
 "itoa",
 "matchit",
 "memchr",
 "mime",
 "percent-encoding",
 "pin-project-lite",
 "serde",
 "serde_json",
 "serde_urlencoded",
 "sync_wrapper",
 "tokio",
 "tower",
 "tower-http",
 "tower-layer",
 "tower-service",
]

[[package]]
name = "axum-core"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bdc19781b16e32f8a7200368a336fa4509d4b72ef15dd4e41df5290855ee1e6"
dependencies = [
 "async-trait",
 "bytes",
 "futures-util",
 "http",
 "http-body",
 "mime",
]

[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -568,6 +613,7 @@ dependencies = [
 "anyhow",
 "arrayvec",
 "async-trait",
 "axum",
 "bytes",
 "cargo-platform",
 "cargo_metadata",
@@ -578,9 +624,11 @@ dependencies = [
 "indexmap",
 "indoc",
 "itoa",
 "once_cell",
 "parking_lot 0.12.0",
 "parse_link_header",
 "percent-encoding",
 "regex",
 "reqwest",
 "semver",
 "serde",
@@ -593,6 +641,8 @@ dependencies = [
 "tokio",
 "tokio-util 0.7.0",
 "toml",
 "tower-layer",
 "tower-service",
 "tracing",
 "tracing-subscriber",
 "url",
@@ -679,6 +729,12 @@ dependencies = [
]

[[package]]
name = "http-range-header"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"

[[package]]
name = "httparse"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -836,6 +892,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"

[[package]]
name = "matchit"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"

[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -964,9 +1026,9 @@ dependencies = [

[[package]]
name = "once_cell"
version = "1.10.0"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
checksum = "ac8b1a9b2518dc799a2271eff1688707eb315f0d4697aa6b0871369ca4c4da55"

[[package]]
name = "opaque-debug"
@@ -1106,6 +1168,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"

[[package]]
name = "pin-project"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260"
dependencies = [
 "pin-project-internal",
]

[[package]]
name = "pin-project-internal"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74"
dependencies = [
 "proc-macro2",
 "quote",
 "syn",
]

[[package]]
name = "pin-project-lite"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1483,6 +1565,12 @@ dependencies = [
]

[[package]]
name = "sync_wrapper"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"

[[package]]
name = "tempfile"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1722,6 +1810,48 @@ dependencies = [
]

[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
 "futures-core",
 "futures-util",
 "pin-project",
 "pin-project-lite",
 "tokio",
 "tokio-util 0.7.0",
 "tower-layer",
 "tower-service",
 "tracing",
]

[[package]]
name = "tower-http"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae"
dependencies = [
 "bitflags",
 "bytes",
 "futures-core",
 "futures-util",
 "http",
 "http-body",
 "http-range-header",
 "pin-project-lite",
 "tower",
 "tower-layer",
 "tower-service",
]

[[package]]
name = "tower-layer"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"

[[package]]
name = "tower-service"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1734,6 +1864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
dependencies = [
 "cfg-if",
 "log",
 "pin-project-lite",
 "tracing-attributes",
 "tracing-core",
@@ -1851,9 +1982,9 @@ dependencies = [

[[package]]
name = "uuid"
version = "1.0.0-alpha.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb3ab47baa004111b323696c6eaa2752e7356f7f77cf6b6dc7a2087368ce1ca4"
checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
dependencies = [
 "getrandom",
]
diff --git a/Cargo.toml b/Cargo.toml
index 9c54bbf..c70915b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,6 +12,7 @@ authors = [
anyhow = "1"
arrayvec = "0.7"
async-trait = "0.1"
axum = "0.5"
bytes = "1.1"
cargo_metadata = "0.14"
cargo-platform = "0.1"
@@ -22,9 +23,11 @@ hex = "0.4"
itoa = "1.0"
indexmap = "1.8"
indoc = "1.0"
once_cell = "1.12"
parse_link_header = "0.3"
parking_lot = "0.12"
percent-encoding = "2.1"
regex = "1.5"
reqwest = { version = "0.11", features = ["json"] }
semver = "1.0"
serde = { version = "1.0", features = ["derive"] }
@@ -39,7 +42,8 @@ time = "0.3"
tokio = { version = "1.17", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
toml = "0.5"
tower-layer = "0.3"
tower-service = "0.3"
url = { version = "2.2", features = ["serde"] }
ustr = "0.8"
uuid = { version = "1.0.0-alpha.1", features = ["v4"] }

uuid = { version = "1.1", features = ["v4"] }
diff --git a/config.toml b/config.toml
index 0830a09..284f406 100644
--- a/config.toml
+++ b/config.toml
@@ -1,5 +1,8 @@
# socket address for the SSH server to listen on
listen-address = "[::]:2222"
ssh-listen-address = "[::]:2222"

# socket address for the HTTP redirect server to listen on
http-listen-address = "[::]:2280"

# directory in which the generated private keys for the server
# should be stored
diff --git a/src/config.rs b/src/config.rs
index 5518d2d..fb358c5 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -15,7 +15,8 @@ pub struct Args {
#[derive(Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Config {
    pub listen_address: SocketAddr,
    pub ssh_listen_address: SocketAddr,
    pub http_listen_address: SocketAddr,
    pub state_directory: PathBuf,
    pub gitlab: GitlabConfig,
}
diff --git a/src/http_server/endpoints/dl.rs b/src/http_server/endpoints/dl.rs
new file mode 100644
index 0000000..afb6b6f
--- /dev/null
+++ b/src/http_server/endpoints/dl.rs
@@ -0,0 +1,3 @@
pub async fn get() -> &'static str {
    "hello world!"
}
diff --git a/src/http_server/endpoints/mod.rs b/src/http_server/endpoints/mod.rs
new file mode 100644
index 0000000..1a92f12
--- /dev/null
+++ b/src/http_server/endpoints/mod.rs
@@ -0,0 +1 @@
pub mod dl;
diff --git a/src/http_server/layer/logging.rs b/src/http_server/layer/logging.rs
new file mode 100644
index 0000000..3e640e8
--- /dev/null
+++ b/src/http_server/layer/logging.rs
@@ -0,0 +1,109 @@
//! Logs each and every request out in a format similar to that of Apache's logs.

use axum::{
    extract::{self, FromRequest, RequestParts},
    http::{Request, Response},
};
use futures::future::BoxFuture;
use once_cell::sync::Lazy;
use regex::Regex;
use std::{
    fmt::Debug,
    task::{Context, Poll},
};
use tower_service::Service;
use tracing::{error, info, Instrument};
use uuid::Uuid;

pub trait GenericError: std::error::Error + Debug + Send + Sync {}

#[derive(Clone)]
pub struct LoggingMiddleware<S>(pub S);

impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for LoggingMiddleware<S>
where
    S: Service<Request<ReqBody>, Response = Response<ResBody>, Error = std::convert::Infallible>
        + Clone
        + Send
        + 'static,
    S::Future: Send + 'static,
    S::Response: Default + Debug,
    ReqBody: Send + Debug + 'static,
    ResBody: Default + Send + 'static,
{
    type Response = S::Response;
    type Error = S::Error;
    type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;

    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
        self.0.poll_ready(cx)
    }

    fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
        // best practice is to clone the inner service like this
        // see https://github.com/tower-rs/tower/issues/547 for details
        let clone = self.0.clone();
        let mut inner = std::mem::replace(&mut self.0, clone);

        let request_id = Uuid::new_v4();
        let span = tracing::info_span!("web", "request_id" = request_id.to_string().as_str());

        Box::pin(async move {
            let start = std::time::Instant::now();
            let user_agent = req.headers().get(axum::http::header::USER_AGENT).cloned();
            let method = req.method().clone();
            let uri = replace_sensitive_path(req.uri().path());

            let mut req = RequestParts::new(req);
            let socket_addr = extract::ConnectInfo::<std::net::SocketAddr>::from_request(&mut req)
                .await
                .map_or_else(|_| "0.0.0.0:0".parse().unwrap(), |v| v.0);

            // this is infallible because of the type of S::Error
            let response = inner.call(req.try_into_request().unwrap()).await?;

            if response.status().is_server_error() {
                error!(
                    "{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"",
                    ip = socket_addr,
                    method = method,
                    uri = uri,
                    status = response.status().as_u16(),
                    duration = start.elapsed(),
                    user_agent = user_agent
                        .as_ref()
                        .and_then(|v| v.to_str().ok())
                        .unwrap_or("unknown"),
                    error = match response.extensions().get::<Box<dyn GenericError>>() {
                        Some(e) => Err(e),
                        None => Ok(()),
                    }
                );
            } else {
                info!(
                    "{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"",
                    ip = socket_addr,
                    method = method,
                    uri = uri,
                    status = response.status().as_u16(),
                    duration = start.elapsed(),
                    user_agent = user_agent
                        .as_ref()
                        .and_then(|v| v.to_str().ok())
                        .unwrap_or("unknown"),
                    error = match response.extensions().get::<Box<dyn GenericError>>() {
                        Some(e) => Err(e),
                        None => Ok(()),
                    }
                );
            }

            Ok(response)
        }.instrument(span))
    }
}

fn replace_sensitive_path(uri: &str) -> String {
    static SENSITIVE_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^/a/(.*?)/").unwrap());
    SENSITIVE_REGEX.replace(uri, "/a/[snip]/").into_owned()
}
diff --git a/src/http_server/layer/mod.rs b/src/http_server/layer/mod.rs
new file mode 100644
index 0000000..31348d2
--- /dev/null
+++ b/src/http_server/layer/mod.rs
@@ -0,0 +1 @@
pub mod logging;
diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs
new file mode 100644
index 0000000..4bef88e
--- /dev/null
+++ b/src/http_server/mod.rs
@@ -0,0 +1,13 @@
mod endpoints;
mod layer;

use crate::http_server::layer::logging::LoggingMiddleware;
use axum::{routing::get, Router};
use tower_layer::layer_fn;

#[must_use]
pub fn build_http_router() -> Router {
    Router::new()
        .route("/dl", get(endpoints::dl::get))
        .layer(layer_fn(LoggingMiddleware))
}
diff --git a/src/main.rs b/src/main.rs
index 54ca331..49d6a6a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,14 +3,16 @@

pub mod config;
pub mod git_command_handlers;
pub mod http_server;
pub mod metadata;
pub mod protocol;
pub mod providers;
pub mod util;

use crate::http_server::build_http_router;
use crate::{
    config::Args,
    metadata::{CargoIndexCrateMetadata, CargoIndexCrateMetadataWithDlOverride},
    metadata::{CargoConfig, CargoIndexCrateMetadata},
    protocol::{
        codec::{Encoder, GitCodec},
        high_level::GitRepository,
@@ -98,15 +100,30 @@ async fn main() -> anyhow::Result<()> {

    let gitlab = Arc::new(Gitlab::new(&args.config.gitlab)?);

    thrussh::server::run(
        thrussh_config,
        &args.config.listen_address.to_string(),
        Server {
            gitlab,
            metadata_cache: MetadataCache::default(),
        },
    )
    .await?;
    let ssh_server_fut = async {
        thrussh::server::run(
            thrussh_config,
            &args.config.ssh_listen_address.to_string(),
            Server {
                gitlab: gitlab.clone(),
                metadata_cache: MetadataCache::default(),
            },
        )
        .await
        .map_err(|e| anyhow::Error::from(e).context("ssh server startup failure"))
    };

    let http_server_fut = async {
        axum::Server::bind(&args.config.http_listen_address)
            .serve(
                build_http_router().into_make_service_with_connect_info::<std::net::SocketAddr>(),
            )
            .await
            .map_err(|e| anyhow::Error::from(e).context("http server startup failure"))
    };

    tokio::try_join!(ssh_server_fut, http_server_fut)?;

    Ok(())
}

@@ -288,7 +305,9 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {

        // generate the config for the user, containing the download
        // url template from gitlab and the impersonation token embedded
        let config_json = Bytes::from_static(b"{}");
        let config_json = Bytes::from(serde_json::to_vec(&CargoConfig {
            dl: format!("http://127.0.0.1:2280/dl/{{crate}}/{{version}}?token={token}"),
        })?);

        // write config.json to the root of the repo
        packfile.insert(&[], "config.json".into(), config_json)?;
@@ -315,12 +334,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {

                // each crates file in the index is a metadata blob for
                // each version separated by a newline
                buffer.extend_from_slice(&serde_json::to_vec(
                    &CargoIndexCrateMetadataWithDlOverride {
                        meta: &meta,
                        dl: &self.gitlab.cargo_dl_uri(crate_path, version, &token)?,
                    },
                )?);
                buffer.extend_from_slice(&serde_json::to_vec(&*meta)?);
                buffer.put_u8(b'\n');
            }

diff --git a/src/metadata.rs b/src/metadata.rs
index ea187e9..b6f39c7 100644
--- a/src/metadata.rs
+++ b/src/metadata.rs
@@ -49,10 +49,8 @@ pub fn transform(
}

#[derive(Serialize, Debug)]
pub struct CargoIndexCrateMetadataWithDlOverride<'a> {
    #[serde(flatten)]
    pub meta: &'a CargoIndexCrateMetadata,
    pub dl: &'a str,
pub struct CargoConfig {
    pub dl: String,
}

#[derive(Serialize, Deserialize, Debug)]
diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs
index 17a2ce8..8b446d8 100644
--- a/src/providers/gitlab.rs
+++ b/src/providers/gitlab.rs
@@ -246,6 +246,7 @@ impl super::PackageProvider for Gitlab {

    fn cargo_dl_uri(
        &self,
        group: &str,
        path: &Self::CratePath,
        version: &str,
        token: &str,
diff --git a/src/providers/mod.rs b/src/providers/mod.rs
index 271043c..3b1c3a6 100644
--- a/src/providers/mod.rs
+++ b/src/providers/mod.rs
@@ -37,6 +37,7 @@ pub trait PackageProvider {

    fn cargo_dl_uri(
        &self,
        group: &str,
        path: &Self::CratePath,
        version: &str,
        token: &str,