Implement HTTP redirect server instead of relying on Cargo/Gitlab changes
Diff
Cargo.lock | 139 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Cargo.toml | 8 ++++++--
config.toml | 5 ++++-
src/config.rs | 3 ++-
src/main.rs | 48 +++++++++++++++++++++++++++++++++++-------------
src/metadata.rs | 6 ++----
src/http_server/mod.rs | 13 +++++++++++++
src/providers/gitlab.rs | 1 +
src/providers/mod.rs | 1 +
src/http_server/endpoints/dl.rs | 3 +++
src/http_server/endpoints/mod.rs | 1 +
src/http_server/layer/logging.rs | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/http_server/layer/mod.rs | 1 +
13 files changed, 309 insertions(+), 29 deletions(-)
@@ -91,6 +91,51 @@
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "axum"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4af7447fc1214c1f3a1ace861d0216a6c8bb13965b64bbad9650f375b67689a"
dependencies = [
"async-trait",
"axum-core",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tower",
"tower-http",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bdc19781b16e32f8a7200368a336fa4509d4b72ef15dd4e41df5290855ee1e6"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
]
[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -568,6 +613,7 @@
"anyhow",
"arrayvec",
"async-trait",
"axum",
"bytes",
"cargo-platform",
"cargo_metadata",
@@ -578,9 +624,11 @@
"indexmap",
"indoc",
"itoa",
"once_cell",
"parking_lot 0.12.0",
"parse_link_header",
"percent-encoding",
"regex",
"reqwest",
"semver",
"serde",
@@ -593,6 +641,8 @@
"tokio",
"tokio-util 0.7.0",
"toml",
"tower-layer",
"tower-service",
"tracing",
"tracing-subscriber",
"url",
@@ -677,6 +727,12 @@
"http",
"pin-project-lite",
]
[[package]]
name = "http-range-header"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29"
[[package]]
name = "httparse"
@@ -834,6 +890,12 @@
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "matchit"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
[[package]]
name = "md5"
@@ -964,9 +1026,9 @@
[[package]]
name = "once_cell"
version = "1.10.0"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
checksum = "ac8b1a9b2518dc799a2271eff1688707eb315f0d4697aa6b0871369ca4c4da55"
[[package]]
name = "opaque-debug"
@@ -1104,6 +1166,26 @@
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78203e83c48cffbe01e4a2d35d566ca4de445d79a85372fc64e378bfc812a260"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "710faf75e1b33345361201d36d04e98ac1ed8909151a017ed384700836104c74"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pin-project-lite"
@@ -1481,6 +1563,12 @@
"quote",
"unicode-xid",
]
[[package]]
name = "sync_wrapper"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8"
[[package]]
name = "tempfile"
@@ -1719,7 +1807,49 @@
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
dependencies = [
"serde",
]
[[package]]
name = "tower"
version = "0.4.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
dependencies = [
"futures-core",
"futures-util",
"pin-project",
"pin-project-lite",
"tokio",
"tokio-util 0.7.0",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-http"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae"
dependencies = [
"bitflags",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
"http-range-header",
"pin-project-lite",
"tower",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
[[package]]
name = "tower-service"
@@ -1734,6 +1864,7 @@
checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f"
dependencies = [
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -1851,9 +1982,9 @@
[[package]]
name = "uuid"
version = "1.0.0-alpha.1"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb3ab47baa004111b323696c6eaa2752e7356f7f77cf6b6dc7a2087368ce1ca4"
checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
dependencies = [
"getrandom",
]
@@ -12,6 +12,7 @@
anyhow = "1"
arrayvec = "0.7"
async-trait = "0.1"
axum = "0.5"
bytes = "1.1"
cargo_metadata = "0.14"
cargo-platform = "0.1"
@@ -22,9 +23,11 @@
itoa = "1.0"
indexmap = "1.8"
indoc = "1.0"
once_cell = "1.12"
parse_link_header = "0.3"
parking_lot = "0.12"
percent-encoding = "2.1"
regex = "1.5"
reqwest = { version = "0.11", features = ["json"] }
semver = "1.0"
serde = { version = "1.0", features = ["derive"] }
@@ -39,7 +42,8 @@
tokio = { version = "1.17", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
toml = "0.5"
tower-layer = "0.3"
tower-service = "0.3"
url = { version = "2.2", features = ["serde"] }
ustr = "0.8"
uuid = { version = "1.0.0-alpha.1", features = ["v4"] }
uuid = { version = "1.1", features = ["v4"] }
@@ -1,5 +1,8 @@
listen-address = "[::]:2222"
ssh-listen-address = "[::]:2222"
http-listen-address = "[::]:2280"
@@ -15,7 +15,8 @@
#[derive(Deserialize)]
#[serde(rename_all = "kebab-case")]
pub struct Config {
pub listen_address: SocketAddr,
pub ssh_listen_address: SocketAddr,
pub http_listen_address: SocketAddr,
pub state_directory: PathBuf,
pub gitlab: GitlabConfig,
}
@@ -1,16 +1,18 @@
#![deny(clippy::pedantic)]
#![allow(clippy::missing_errors_doc)]
pub mod config;
pub mod git_command_handlers;
pub mod http_server;
pub mod metadata;
pub mod protocol;
pub mod providers;
pub mod util;
use crate::http_server::build_http_router;
use crate::{
config::Args,
metadata::{CargoIndexCrateMetadata, CargoIndexCrateMetadataWithDlOverride},
metadata::{CargoConfig, CargoIndexCrateMetadata},
protocol::{
codec::{Encoder, GitCodec},
high_level::GitRepository,
@@ -98,15 +100,30 @@
let gitlab = Arc::new(Gitlab::new(&args.config.gitlab)?);
thrussh::server::run(
thrussh_config,
&args.config.listen_address.to_string(),
Server {
gitlab,
metadata_cache: MetadataCache::default(),
},
)
.await?;
let ssh_server_fut = async {
thrussh::server::run(
thrussh_config,
&args.config.ssh_listen_address.to_string(),
Server {
gitlab: gitlab.clone(),
metadata_cache: MetadataCache::default(),
},
)
.await
.map_err(|e| anyhow::Error::from(e).context("ssh server startup failure"))
};
let http_server_fut = async {
axum::Server::bind(&args.config.http_listen_address)
.serve(
build_http_router().into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.await
.map_err(|e| anyhow::Error::from(e).context("http server startup failure"))
};
tokio::try_join!(ssh_server_fut, http_server_fut)?;
Ok(())
}
@@ -288,7 +305,9 @@
let config_json = Bytes::from_static(b"{}");
let config_json = Bytes::from(serde_json::to_vec(&CargoConfig {
dl: format!("http://127.0.0.1:2280/dl/{{crate}}/{{version}}?token={token}"),
})?);
packfile.insert(&[], "config.json".into(), config_json)?;
@@ -315,12 +334,7 @@
buffer.extend_from_slice(&serde_json::to_vec(
&CargoIndexCrateMetadataWithDlOverride {
meta: &meta,
dl: &self.gitlab.cargo_dl_uri(crate_path, version, &token)?,
},
)?);
buffer.extend_from_slice(&serde_json::to_vec(&*meta)?);
buffer.put_u8(b'\n');
}
@@ -49,10 +49,8 @@
}
#[derive(Serialize, Debug)]
pub struct CargoIndexCrateMetadataWithDlOverride<'a> {
#[serde(flatten)]
pub meta: &'a CargoIndexCrateMetadata,
pub dl: &'a str,
pub struct CargoConfig {
pub dl: String,
}
#[derive(Serialize, Deserialize, Debug)]
@@ -1,0 +1,13 @@
mod endpoints;
mod layer;
use crate::http_server::layer::logging::LoggingMiddleware;
use axum::{routing::get, Router};
use tower_layer::layer_fn;
#[must_use]
pub fn build_http_router() -> Router {
Router::new()
.route("/dl", get(endpoints::dl::get))
.layer(layer_fn(LoggingMiddleware))
}
@@ -246,6 +246,7 @@
fn cargo_dl_uri(
&self,
group: &str,
path: &Self::CratePath,
version: &str,
token: &str,
@@ -37,6 +37,7 @@
fn cargo_dl_uri(
&self,
group: &str,
path: &Self::CratePath,
version: &str,
token: &str,
@@ -1,0 +1,3 @@
pub async fn get() -> &'static str {
"hello world!"
}
@@ -1,0 +1,1 @@
pub mod dl;
@@ -1,0 +1,109 @@
use axum::{
extract::{self, FromRequest, RequestParts},
http::{Request, Response},
};
use futures::future::BoxFuture;
use once_cell::sync::Lazy;
use regex::Regex;
use std::{
fmt::Debug,
task::{Context, Poll},
};
use tower_service::Service;
use tracing::{error, info, Instrument};
use uuid::Uuid;
pub trait GenericError: std::error::Error + Debug + Send + Sync {}
#[derive(Clone)]
pub struct LoggingMiddleware<S>(pub S);
impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for LoggingMiddleware<S>
where
S: Service<Request<ReqBody>, Response = Response<ResBody>, Error = std::convert::Infallible>
+ Clone
+ Send
+ 'static,
S::Future: Send + 'static,
S::Response: Default + Debug,
ReqBody: Send + Debug + 'static,
ResBody: Default + Send + 'static,
{
type Response = S::Response;
type Error = S::Error;
type Future = BoxFuture<'static, Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.0.poll_ready(cx)
}
fn call(&mut self, req: Request<ReqBody>) -> Self::Future {
let clone = self.0.clone();
let mut inner = std::mem::replace(&mut self.0, clone);
let request_id = Uuid::new_v4();
let span = tracing::info_span!("web", "request_id" = request_id.to_string().as_str());
Box::pin(async move {
let start = std::time::Instant::now();
let user_agent = req.headers().get(axum::http::header::USER_AGENT).cloned();
let method = req.method().clone();
let uri = replace_sensitive_path(req.uri().path());
let mut req = RequestParts::new(req);
let socket_addr = extract::ConnectInfo::<std::net::SocketAddr>::from_request(&mut req)
.await
.map_or_else(|_| "0.0.0.0:0".parse().unwrap(), |v| v.0);
let response = inner.call(req.try_into_request().unwrap()).await?;
if response.status().is_server_error() {
error!(
"{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"",
ip = socket_addr,
method = method,
uri = uri,
status = response.status().as_u16(),
duration = start.elapsed(),
user_agent = user_agent
.as_ref()
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown"),
error = match response.extensions().get::<Box<dyn GenericError>>() {
Some(e) => Err(e),
None => Ok(()),
}
);
} else {
info!(
"{ip} - \"{method} {uri}\" {status} {duration:?} \"{user_agent}\" \"{error:?}\"",
ip = socket_addr,
method = method,
uri = uri,
status = response.status().as_u16(),
duration = start.elapsed(),
user_agent = user_agent
.as_ref()
.and_then(|v| v.to_str().ok())
.unwrap_or("unknown"),
error = match response.extensions().get::<Box<dyn GenericError>>() {
Some(e) => Err(e),
None => Ok(()),
}
);
}
Ok(response)
}.instrument(span))
}
}
fn replace_sensitive_path(uri: &str) -> String {
static SENSITIVE_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"^/a/(.*?)/").unwrap());
SENSITIVE_REGEX.replace(uri, "/a/[snip]/").into_owned()
}
@@ -1,0 +1,1 @@
pub mod logging;