From 6a73ed05d28a43b7e32220bdf4117aa45b421444 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Mon, 4 Jul 2022 15:05:35 +0100 Subject: [PATCH] Implement HTTP redirect server instead of relying on Cargo/Gitlab changes --- Cargo.lock | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 8 ++++++-- config.toml | 5 ++++- src/config.rs | 3 ++- src/http_server/endpoints/dl.rs | 3 +++ src/http_server/endpoints/mod.rs | 1 + src/http_server/layer/logging.rs | 109 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/http_server/layer/mod.rs | 1 + src/http_server/mod.rs | 13 +++++++++++++ src/main.rs | 48 +++++++++++++++++++++++++++++++----------------- src/metadata.rs | 6 ++---- src/providers/gitlab.rs | 1 + src/providers/mod.rs | 1 + 13 files changed, 309 insertions(+), 29 deletions(-) create mode 100644 src/http_server/endpoints/dl.rs create mode 100644 src/http_server/endpoints/mod.rs create mode 100644 src/http_server/layer/logging.rs create mode 100644 src/http_server/layer/mod.rs create mode 100644 src/http_server/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 5c6db58..7c365e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] +name = "axum" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bdc19781b16e32f8a7200368a336fa4509d4b72ef15dd4e41df5290855ee1e6" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", +] + +[[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -568,6 +613,7 @@ dependencies = [ "anyhow", "arrayvec", "async-trait", + "axum", "bytes", "cargo-platform", "cargo_metadata", @@ -578,9 +624,11 @@ dependencies = [ "indexmap", "indoc", "itoa", + "once_cell", "parking_lot 0.12.0", "parse_link_header", "percent-encoding", + "regex", "reqwest", "semver", "serde", @@ -593,6 +641,8 @@ dependencies = [ "tokio", "tokio-util 0.7.0", "toml", + "tower-layer", + "tower-service", "tracing", "tracing-subscriber", "url", @@ -679,6 +729,12 @@ dependencies = [ ] [[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] name = "httparse" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -836,6 +892,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + +[[package]] name = "md5" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -964,9 +1026,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "ac8b1a9b2518dc799a2271eff1688707eb315f0d4697aa6b0871369ca4c4da55" [[package]] name = "opaque-debug" @@ -1106,6 +1168,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] +name = "pin-project" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "pin-project-lite" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1483,6 +1565,12 @@ dependencies = [ ] [[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + +[[package]] name = "tempfile" version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1722,6 +1810,48 @@ dependencies = [ ] [[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tokio-util 0.7.0", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] name = "tower-service" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1734,6 +1864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1851,9 +1982,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.0.0-alpha.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb3ab47baa004111b323696c6eaa2752e7356f7f77cf6b6dc7a2087368ce1ca4" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" dependencies = [ "getrandom", ] diff --git a/Cargo.toml b/Cargo.toml index 9c54bbf..c70915b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ authors = [ anyhow = "1" arrayvec = "0.7" async-trait = "0.1" +axum = "0.5" bytes = "1.1" cargo_metadata = "0.14" cargo-platform = "0.1" @@ -22,9 +23,11 @@ hex = "0.4" itoa = "1.0" indexmap = "1.8" indoc = "1.0" +once_cell = "1.12" parse_link_header = "0.3" parking_lot = "0.12" percent-encoding = "2.1" +regex = "1.5" reqwest = { version = "0.11", features = ["json"] } semver = "1.0" serde = { version = "1.0", features = ["derive"] } @@ -39,7 +42,8 @@ time = "0.3" tokio = { version = "1.17", features = ["full"] } tokio-util = { version = "0.7", features = ["codec"] } toml = "0.5" +tower-layer = "0.3" +tower-service = "0.3" url = { version = "2.2", features = ["serde"] } ustr = "0.8" -uuid = { version = "1.0.0-alpha.1", features = ["v4"] } - +uuid = { version = "1.1", features = ["v4"] } diff --git a/config.toml b/config.toml index 0830a09..284f406 100644 --- a/config.toml +++ b/config.toml @@ -1,5 +1,8 @@ # socket address for the SSH server to listen on -listen-address = "[::]:2222" +ssh-listen-address = "[::]:2222" + +# socket address for the HTTP redirect server to listen on +http-listen-address = "[::]:2280" # directory in which the generated private keys for the server # should be stored diff --git a/src/config.rs b/src/config.rs index 5518d2d..fb358c5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -15,7 +15,8 @@ pub struct Args { #[derive(Deserialize)] #[serde(rename_all = "kebab-case")] pub struct Config { - pub listen_address: SocketAddr, + pub ssh_listen_address: SocketAddr, + pub http_listen_address: SocketAddr, pub state_directory: PathBuf, pub gitlab: GitlabConfig, } diff --git a/src/http_server/endpoints/dl.rs b/src/http_server/endpoints/dl.rs new file mode 100644 index 0000000..afb6b6f --- /dev/null +++ b/src/http_server/endpoints/dl.rs @@ -0,0 +1,3 @@ +pub async fn get() -> &'static str { + "hello world!" +} diff --git a/src/http_server/endpoints/mod.rs b/src/http_server/endpoints/mod.rs new file mode 100644 index 0000000..1a92f12 --- /dev/null +++ b/src/http_server/endpoints/mod.rs @@ -0,0 +1 @@ +pub mod dl; diff --git a/src/http_server/layer/logging.rs b/src/http_server/layer/logging.rs new file mode 100644 index 0000000..3e640e8 --- /dev/null +++ b/src/http_server/layer/logging.rs @@ -0,0 +1,109 @@ +//! Logs each and every request out in a format similar to that of Apache's logs. + +use axum::{ + extract::{self, FromRequest, RequestParts}, + http::{Request, Response}, +}; +use futures::future::BoxFuture; +use once_cell::sync::Lazy; +use regex::Regex; +use std::{ + fmt::Debug, + task::{Context, Poll}, +}; +use tower_service::Service; +use tracing::{error, info, Instrument}; +use uuid::Uuid; + +pub trait GenericError: std::error::Error + Debug + Send + Sync {} + +#[derive(Clone)] +pub struct LoggingMiddleware(pub S); + +impl Service> for LoggingMiddleware +where + S: Service, Response = Response, Error = std::convert::Infallible> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + S::Response: Default + Debug, + ReqBody: Send + Debug + 'static, + ResBody: Default + Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.0.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + // best practice is to clone the inner service like this + // see https://github.com/tower-rs/tower/issues/547 for details + let clone = self.0.clone(); + let mut inner = std::mem::replace(&mut self.0, clone); + + let request_id = Uuid::new_v4(); + let span = tracing::info_span!("web", "request_id" = request_id.to_string().as_str()); + + Box::pin(async move { + let start = std::time::Instant::now(); + let user_agent = req.headers().get(axum::http::header::USER_AGENT).cloned(); + let method = req.method().clone(); + let uri = replace_sensitive_path(req.uri().path()); + + let mut req = RequestParts::new(req); + let socket_addr = extract::ConnectInfo::::from_request(&mut req) + .await + .map_or_else(|_| "0.0.0.0:0".parse().unwrap(), |v| v.0); + + // this is infallible because of the type of S::Error + let response = inner.call(req.try_into_request().unwrap()).await?; + + if response.status().is_server_error() { + error!( + "{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"", + ip = socket_addr, + method = method, + uri = uri, + status = response.status().as_u16(), + duration = start.elapsed(), + user_agent = user_agent + .as_ref() + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown"), + error = match response.extensions().get::>() { + Some(e) => Err(e), + None => Ok(()), + } + ); + } else { + info!( + "{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"", + ip = socket_addr, + method = method, + uri = uri, + status = response.status().as_u16(), + duration = start.elapsed(), + user_agent = user_agent + .as_ref() + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown"), + error = match response.extensions().get::>() { + Some(e) => Err(e), + None => Ok(()), + } + ); + } + + Ok(response) + }.instrument(span)) + } +} + +fn replace_sensitive_path(uri: &str) -> String { + static SENSITIVE_REGEX: Lazy = Lazy::new(|| Regex::new(r"^/a/(.*?)/").unwrap()); + SENSITIVE_REGEX.replace(uri, "/a/[snip]/").into_owned() +} diff --git a/src/http_server/layer/mod.rs b/src/http_server/layer/mod.rs new file mode 100644 index 0000000..31348d2 --- /dev/null +++ b/src/http_server/layer/mod.rs @@ -0,0 +1 @@ +pub mod logging; diff --git a/src/http_server/mod.rs b/src/http_server/mod.rs new file mode 100644 index 0000000..4bef88e --- /dev/null +++ b/src/http_server/mod.rs @@ -0,0 +1,13 @@ +mod endpoints; +mod layer; + +use crate::http_server::layer::logging::LoggingMiddleware; +use axum::{routing::get, Router}; +use tower_layer::layer_fn; + +#[must_use] +pub fn build_http_router() -> Router { + Router::new() + .route("/dl", get(endpoints::dl::get)) + .layer(layer_fn(LoggingMiddleware)) +} diff --git a/src/main.rs b/src/main.rs index 54ca331..49d6a6a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,14 +3,16 @@ pub mod config; pub mod git_command_handlers; +pub mod http_server; pub mod metadata; pub mod protocol; pub mod providers; pub mod util; +use crate::http_server::build_http_router; use crate::{ config::Args, - metadata::{CargoIndexCrateMetadata, CargoIndexCrateMetadataWithDlOverride}, + metadata::{CargoConfig, CargoIndexCrateMetadata}, protocol::{ codec::{Encoder, GitCodec}, high_level::GitRepository, @@ -98,15 +100,30 @@ async fn main() -> anyhow::Result<()> { let gitlab = Arc::new(Gitlab::new(&args.config.gitlab)?); - thrussh::server::run( - thrussh_config, - &args.config.listen_address.to_string(), - Server { - gitlab, - metadata_cache: MetadataCache::default(), - }, - ) - .await?; + let ssh_server_fut = async { + thrussh::server::run( + thrussh_config, + &args.config.ssh_listen_address.to_string(), + Server { + gitlab: gitlab.clone(), + metadata_cache: MetadataCache::default(), + }, + ) + .await + .map_err(|e| anyhow::Error::from(e).context("ssh server startup failure")) + }; + + let http_server_fut = async { + axum::Server::bind(&args.config.http_listen_address) + .serve( + build_http_router().into_make_service_with_connect_info::(), + ) + .await + .map_err(|e| anyhow::Error::from(e).context("http server startup failure")) + }; + + tokio::try_join!(ssh_server_fut, http_server_fut)?; + Ok(()) } @@ -288,7 +305,9 @@ impl Handler { // generate the config for the user, containing the download // url template from gitlab and the impersonation token embedded - let config_json = Bytes::from_static(b"{}"); + let config_json = Bytes::from(serde_json::to_vec(&CargoConfig { + dl: format!("http://127.0.0.1:2280/dl/{{crate}}/{{version}}?token={token}"), + })?); // write config.json to the root of the repo packfile.insert(&[], "config.json".into(), config_json)?; @@ -315,12 +334,7 @@ impl Handler { // each crates file in the index is a metadata blob for // each version separated by a newline - buffer.extend_from_slice(&serde_json::to_vec( - &CargoIndexCrateMetadataWithDlOverride { - meta: &meta, - dl: &self.gitlab.cargo_dl_uri(crate_path, version, &token)?, - }, - )?); + buffer.extend_from_slice(&serde_json::to_vec(&*meta)?); buffer.put_u8(b'\n'); } diff --git a/src/metadata.rs b/src/metadata.rs index ea187e9..b6f39c7 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -49,10 +49,8 @@ pub fn transform( } #[derive(Serialize, Debug)] -pub struct CargoIndexCrateMetadataWithDlOverride<'a> { - #[serde(flatten)] - pub meta: &'a CargoIndexCrateMetadata, - pub dl: &'a str, +pub struct CargoConfig { + pub dl: String, } #[derive(Serialize, Deserialize, Debug)] diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index 17a2ce8..8b446d8 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -246,6 +246,7 @@ impl super::PackageProvider for Gitlab { fn cargo_dl_uri( &self, + group: &str, path: &Self::CratePath, version: &str, token: &str, diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 271043c..3b1c3a6 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -37,6 +37,7 @@ pub trait PackageProvider { fn cargo_dl_uri( &self, + group: &str, path: &Self::CratePath, version: &str, token: &str, -- libgit2 1.7.2