configurable filesystem (s3, local)
Diff
Cargo.lock | 493 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
chartered-fs/Cargo.toml | 8 +++++++-
chartered-web/Cargo.toml | 1 +
chartered-fs/src/lib.rs | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
chartered-web/src/config.rs | 12 +++++++++++-
chartered-web/src/main.rs | 8 +++++---
book/src/getting-started/installation.md | 14 +++++++-------
chartered-web/src/endpoints/cargo_api/download.rs | 41 ++++++++++++++++++++++++++++++++++++-----
chartered-web/src/endpoints/cargo_api/publish.rs | 24 ++++++++++++++++--------
9 files changed, 698 insertions(+), 106 deletions(-)
@@ -90,6 +90,175 @@
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "aws-auth"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-types",
"pin-project",
"smithy-async",
"smithy-http",
"tokio",
"tracing",
"zeroize",
]
[[package]]
name = "aws-config"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-http",
"aws-hyper",
"aws-sdk-sts",
"aws-types",
"bytes",
"http",
"smithy-async",
"smithy-client",
"smithy-http",
"smithy-http-tower",
"smithy-json",
"smithy-types",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "aws-endpoint"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-types",
"http",
"regex",
"smithy-http",
]
[[package]]
name = "aws-http"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-types",
"http",
"lazy_static",
"smithy-http",
"smithy-types",
"thiserror",
]
[[package]]
name = "aws-hyper"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-auth",
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"bytes",
"fastrand",
"http",
"http-body",
"hyper",
"hyper-rustls",
"pin-project",
"smithy-client",
"smithy-http",
"smithy-http-tower",
"smithy-types",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "aws-sdk-s3"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-auth",
"aws-endpoint",
"aws-http",
"aws-hyper",
"aws-sig-auth",
"aws-sigv4",
"aws-types",
"bytes",
"http",
"md5",
"smithy-client",
"smithy-eventstream",
"smithy-http",
"smithy-types",
"smithy-xml",
"tower",
]
[[package]]
name = "aws-sdk-sts"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-auth",
"aws-endpoint",
"aws-http",
"aws-hyper",
"aws-sig-auth",
"aws-types",
"bytes",
"http",
"smithy-client",
"smithy-http",
"smithy-query",
"smithy-types",
"smithy-xml",
]
[[package]]
name = "aws-sig-auth"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"aws-sigv4",
"aws-types",
"http",
"smithy-eventstream",
"smithy-http",
"thiserror",
"tracing",
]
[[package]]
name = "aws-sigv4"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"bytes",
"chrono",
"form_urlencoded",
"hex",
"http",
"percent-encoding",
"ring",
"smithy-eventstream",
"tracing",
]
[[package]]
name = "aws-types"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"rustc_version",
"smithy-async",
"smithy-types",
"zeroize",
]
[[package]]
name = "axum"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -131,9 +300,9 @@
[[package]]
name = "bcrypt-pbkdf"
version = "0.6.2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c38c03b9506bd92bf1ef50665a81eda156f615438f7654bffba58907e6149d7"
checksum = "12621b8e87feb183a6e5dbb315e49026b2229c4398797ee0ae2d1bc00aef41b9"
dependencies = [
"blowfish",
"crypto-mac",
@@ -222,6 +391,16 @@
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "bytes-utils"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54"
dependencies = [
"bytes",
"either",
]
[[package]]
name = "cc"
@@ -294,8 +473,14 @@
version = "0.1.0"
dependencies = [
"async-trait",
"aws-config",
"aws-sdk-s3",
"bytes",
"http",
"itertools",
"md5",
"serde",
"thiserror",
"tokio",
"url",
"uuid",
@@ -359,6 +544,7 @@
"headers",
"hex",
"nom",
"nom-bytes",
"once_cell",
"openid",
"rand",
@@ -401,9 +587,9 @@
[[package]]
name = "clap"
version = "3.0.0-beta.4"
version = "3.0.0-beta.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcd70aa5597dbc42f7217a543f9ef2768b2ef823ba29036072d30e1d88e98406"
checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63"
dependencies = [
"atty",
"bitflags",
@@ -414,14 +600,14 @@
"strsim",
"termcolor",
"textwrap",
"vec_map",
"unicase",
]
[[package]]
name = "clap_derive"
version = "3.0.0-beta.4"
version = "3.0.0-beta.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b5bb0d655624a0b8770d1c178fb8ffcb1f91cc722cb08f451e3dc72465421ac"
checksum = "8b15c6b4f786ffb6192ffe65a36855bc1fc2444bcd0945ae16748dcd6ed7d0d3"
dependencies = [
"heck",
"proc-macro-error",
@@ -503,6 +689,15 @@
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "ct-logs"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
dependencies = [
"sct",
]
[[package]]
@@ -623,14 +818,23 @@
[[package]]
name = "encoding_rs"
version = "0.8.28"
version = "0.8.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746"
dependencies = [
"cfg-if",
]
[[package]]
name = "fastrand"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e"
dependencies = [
"instant",
]
[[package]]
name = "flate2"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -837,18 +1041,18 @@
[[package]]
name = "headers"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855"
checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0"
dependencies = [
"base64",
"bitflags",
"bytes",
"headers-core",
"http",
"httpdate",
"mime",
"sha-1",
"time",
]
[[package]]
@@ -950,6 +1154,23 @@
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [
"ct-logs",
"futures-util",
"hyper",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"webpki",
]
[[package]]
@@ -1003,9 +1224,9 @@
[[package]]
name = "instant"
version = "0.1.11"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
@@ -1057,9 +1278,9 @@
[[package]]
name = "libc"
version = "0.2.103"
version = "0.2.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6"
checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
[[package]]
name = "libsodium-sys"
@@ -1174,9 +1395,9 @@
[[package]]
name = "mio"
version = "0.7.13"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
dependencies = [
"libc",
"log",
@@ -1221,6 +1442,15 @@
"memchr",
"minimal-lexical",
"version_check",
]
[[package]]
name = "nom-bytes"
version = "0.1.0"
source = "git+https://github.com/w4/nom-bytes#2ede4dc22f1c303a2377c556d1a3b3f42464a0e7"
dependencies = [
"bytes",
"nom",
]
[[package]]
@@ -1404,9 +1634,12 @@
[[package]]
name = "os_str_bytes"
version = "3.1.0"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d"
checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799"
dependencies = [
"memchr",
]
[[package]]
name = "parking_lot"
@@ -1688,9 +1921,9 @@
[[package]]
name = "reqwest"
version = "0.11.5"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51c732d463dd300362ffb44b7b125f299c23d2990411a4253824630ebc7467fb"
checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280"
dependencies = [
"base64",
"bytes",
@@ -1734,6 +1967,40 @@
"untrusted",
"web-sys",
"winapi",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "rustls"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
dependencies = [
"base64",
"log",
"ring",
"sct",
"webpki",
]
[[package]]
name = "rustls-native-certs"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
"openssl-probe",
"rustls",
"schannel",
"security-framework",
]
[[package]]
@@ -1775,6 +2042,16 @@
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "security-framework"
@@ -1798,6 +2075,12 @@
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
[[package]]
name = "serde"
@@ -1904,6 +2187,120 @@
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
[[package]]
name = "smithy-async"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"pin-project-lite",
"tokio",
]
[[package]]
name = "smithy-client"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"bytes",
"fastrand",
"http",
"http-body",
"hyper",
"hyper-rustls",
"lazy_static",
"pin-project",
"pin-project-lite",
"smithy-async",
"smithy-http",
"smithy-http-tower",
"smithy-types",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "smithy-eventstream"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"bytes",
"crc32fast",
"smithy-types",
]
[[package]]
name = "smithy-http"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"bytes",
"bytes-utils",
"futures-core",
"http",
"http-body",
"hyper",
"percent-encoding",
"pin-project",
"smithy-eventstream",
"smithy-types",
"thiserror",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "smithy-http-tower"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"bytes",
"http",
"http-body",
"pin-project",
"smithy-http",
"tower",
"tracing",
]
[[package]]
name = "smithy-json"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"smithy-types",
]
[[package]]
name = "smithy-query"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"smithy-types",
"urlencoding",
]
[[package]]
name = "smithy-types"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"chrono",
"itoa",
"num-integer",
"ryu",
]
[[package]]
name = "smithy-xml"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
"thiserror",
"xmlparser",
]
[[package]]
name = "socket2"
@@ -2142,7 +2539,18 @@
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
"rustls",
"tokio",
"webpki",
]
[[package]]
@@ -2327,6 +2735,15 @@
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
@@ -2395,6 +2812,12 @@
"percent-encoding",
"serde",
]
[[package]]
name = "urlencoding"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"
[[package]]
name = "uuid"
@@ -2450,12 +2873,6 @@
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
@@ -2564,6 +2981,16 @@
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
"ring",
"untrusted",
]
[[package]]
@@ -2605,6 +3032,12 @@
dependencies = [
"winapi",
]
[[package]]
name = "xmlparser"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"
[[package]]
name = "yasna"
@@ -2618,6 +3051,6 @@
[[package]]
name = "zeroize"
version = "1.3.0"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd"
checksum = "bf68b08513768deaa790264a7fac27a58cbf2705cfcdc9448362229217d7e970"
@@ -7,8 +7,14 @@
[dependencies]
async-trait = "0.1"
itertools = "*"
aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-config" }
aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-sdk-s3" }
bytes = "1.1"
http = "0.2"
itertools = "0.10"
md5 = "0.7.0"
serde = { version = "1", features = ["derive"] }
thiserror = "1.0"
tokio = { version = "1", features = ["fs", "io-util"] }
url = "2"
uuid = { version = "0.8", features = ["v4", "serde"] }
@@ -21,6 +21,7 @@
headers = "0.3"
hex = "0.4"
nom = "7"
nom-bytes = { git = "https://github.com/w4/nom-bytes" }
once_cell = "1.8"
openid = "0.9"
rand = "0.8"
@@ -1,50 +1,91 @@
#![deny(clippy::pedantic)]
#![deny(rust_2018_idioms)]
#![allow(clippy::missing_errors_doc)]
use std::path::PathBuf;
use std::{path::PathBuf, time::Duration};
use async_trait::async_trait;
use aws_sdk_s3::error::{GetObjectError, PutObjectError};
use aws_sdk_s3::{
model::ObjectCannedAcl, presigning::config::PresigningConfig, ByteStream, SdkError,
};
use bytes::Bytes;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncWriteExt},
};
#[derive(Debug)]
pub enum FS {
S3 {
host: String,
bucket: String,
path: String,
},
Local {
path: PathBuf,
},
#[derive(Debug, Error)]
pub enum Error {
#[error("failed to parse filesystem uri: {0}")]
UriParse(#[from] url::ParseError),
#[error("unknown filesystem kind (expected `s3` or `file`)")]
UnknownFileSystemKind,
#[error("failed to insert object to s3: {0}")]
S3Put(#[from] SdkError<PutObjectError>),
#[error("failed to get object from s3: {0}")]
S3Get(#[from] SdkError<GetObjectError>),
#[error("i/o failure: {0}")]
Io(#[from] std::io::Error),
#[error("failed to parse uuid: {0}")]
UuidParse(#[from] uuid::Error),
#[error("path missing from uri")]
MissingPath,
#[error("host missing from uri")]
MissingHost,
#[error("bucket missing from uri")]
MissingBucket,
#[error("invalid aws presigning config: {0}")]
AwsPresigningConfig(#[from] aws_sdk_s3::presigning::config::Error),
}
impl std::str::FromStr for FS {
type Err = url::ParseError;
#[derive(Debug)]
pub enum FileSystem {
S3(S3),
Local(Local),
}
fn from_str(s: &str) -> Result<Self, Self::Err> {
impl FileSystem {
pub async fn from_str(s: &str) -> Result<Self, Error> {
let uri = url::Url::parse(s)?;
Ok(match uri.scheme() {
"s3" => {
let mut path = uri.path_segments().unwrap();
Self::S3 {
host: uri.host().unwrap().to_string(),
bucket: path.next().unwrap().to_string(),
let shared_config = aws_config::load_from_env().await;
let client = aws_sdk_s3::Client::new(&shared_config);
let mut path = uri.path_segments().ok_or(Error::MissingPath)?;
Self::S3(S3 {
host: uri.host().ok_or(Error::MissingHost)?.to_string(),
bucket: path.next().ok_or(Error::MissingBucket)?.to_string(),
path: Itertools::intersperse(path, "/").collect(),
}
}
"file" => {
panic!("{:#?}", uri);
client,
})
}
_ => panic!("na"),
"file" => Self::Local(Local {
path: uri.to_file_path().map_err(|()| Error::MissingPath)?,
}),
_ => return Err(Error::UnknownFileSystemKind),
})
}
pub async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
match self {
Self::S3(v) => v.read(file_ref).await,
Self::Local(v) => v.read(file_ref).await,
}
}
pub async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
match self {
Self::S3(v) => v.write(data).await,
Self::Local(v) => v.write(data).await,
}
}
}
#[derive(Debug, Serialize, Deserialize)]
@@ -63,15 +104,13 @@
}
impl std::str::FromStr for FileSystemKind {
type Err = std::io::Error;
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
fn from_str(s: &str) -> Result<Self, Error> {
match s {
"local" => Ok(Self::Local),
_ => Err(std::io::Error::new(
std::io::ErrorKind::Other,
"unknown filesystemkind",
)),
"s3" => Ok(Self::S3),
_ => Err(Error::UnknownFileSystemKind),
}
}
}
@@ -89,13 +128,12 @@
}
impl std::str::FromStr for FileReference {
type Err = std::io::Error;
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let mut split = s.splitn(2, ':');
let file_system = FileSystemKind::from_str(split.next().unwrap_or_default())?;
let reference = uuid::Uuid::from_str(split.next().unwrap_or_default())
.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
let reference = uuid::Uuid::from_str(split.next().unwrap_or_default())?;
Ok(FileReference {
file_system,
reference,
@@ -103,12 +141,18 @@
}
}
#[derive(Debug, PartialEq, Eq)]
pub enum FilePointer {
Content(Vec<u8>),
Redirect(http::Uri),
}
#[async_trait]
pub trait FileSystem {
pub trait FileSystemIo {
const KIND: FileSystemKind;
async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error>;
async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error>;
async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error>;
async fn write(&self, data: Bytes) -> Result<FileReference, Error>;
#[must_use]
fn create_ref() -> FileReference {
@@ -119,26 +163,73 @@
}
}
pub struct Local;
#[derive(Debug)]
pub struct Local {
pub path: PathBuf,
}
#[async_trait]
impl FileSystem for Local {
impl FileSystemIo for Local {
const KIND: FileSystemKind = FileSystemKind::Local;
async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error> {
let mut file = File::open(format!("/tmp/{}", file_ref.reference)).await?;
async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
let path = self.path.join(file_ref.reference.to_string());
let mut file = File::open(path).await?;
let mut contents = vec![];
file.read_to_end(&mut contents).await?;
Ok(FilePointer::Content(contents))
}
async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
let file_ref = Self::create_ref();
let path = self.path.join(file_ref.reference.to_string());
Ok(contents)
let mut file = File::create(path).await?;
file.write_all(&data).await?;
Ok(file_ref)
}
}
#[derive(Debug)]
pub struct S3 {
host: String,
bucket: String,
path: String,
client: aws_sdk_s3::Client,
}
#[async_trait]
impl FileSystemIo for S3 {
const KIND: FileSystemKind = FileSystemKind::S3;
async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
Ok(FilePointer::Redirect(
self.client
.get_object()
.key(format!("{}/{}", self.path, file_ref.reference.to_string()))
.bucket(&self.bucket)
.presigned(PresigningConfig::expires_in(Duration::from_secs(600))?)
.await?
.uri()
.clone(),
))
}
async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error> {
async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
let file_ref = Self::create_ref();
let mut file = File::create(format!("/tmp/{}", file_ref.reference)).await?;
file.write_all(data).await?;
self.client
.put_object()
.key(format!("{}/{}", self.path, file_ref.reference.to_string()))
.content_md5(format!("{:x}", md5::compute(&data)))
.body(ByteStream::new(data.into()))
.bucket(&self.bucket)
.acl(ObjectCannedAcl::Private)
.send()
.await?;
Ok(file_ref)
}
@@ -146,20 +237,32 @@
#[cfg(test)]
mod tests {
use super::FileSystem;
use super::{FilePointer, FileSystem, FileSystemIo};
use bytes::Bytes;
#[tokio::test]
#[allow(clippy::pedantic)]
async fn parse_filesystem() {
assert!(matches!(
FileSystem::from_str("file:///tmp/chartered").await,
Ok(FileSystem::Local(inner)) if inner.path.to_str().unwrap() == "/tmp/chartered"
));
}
#[tokio::test]
#[allow(clippy::pedantic)]
async fn local() {
let fs = super::Local;
let file_ref = fs.write(b"abcdef").await.unwrap();
assert_eq!(fs.read(file_ref).await.unwrap(), b"abcdef");
let fs = super::Local {
path: "/tmp".into(),
};
let file_ref = fs.write(Bytes::from_static(b"abcdef")).await.unwrap();
assert_eq!(
fs.read(file_ref).await.unwrap(),
FilePointer::Content(Vec::from(b"abcdef".as_ref()))
);
}
}
@@ -1,4 +1,5 @@
use chacha20poly1305::Key as ChaCha20Poly1305Key;
use chartered_fs::FileSystem;
use openid::DiscoveredClient;
use serde::{de::Error as SerdeDeError, Deserialize};
use std::collections::HashMap;
@@ -8,19 +9,28 @@
pub enum Error {
#[error("Error discovering OpenID provider: {0}")]
OpenId(#[from] openid::error::Error),
#[error("Failed to create file system handle: {0}")]
Fs(#[from] Box<chartered_fs::Error>),
}
pub type OidcClients = HashMap<String, DiscoveredClient>;
#[derive(Deserialize, Default, Debug)]
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct Config {
pub storage_uri: String,
pub auth: AuthConfig,
#[serde(deserialize_with = "deserialize_encryption_key")]
pub encryption_key: ChaCha20Poly1305Key,
}
impl Config {
pub async fn get_file_system(&self) -> Result<FileSystem, Error> {
Ok(FileSystem::from_str(&self.storage_uri)
.await
.map_err(Box::new)?)
}
pub async fn create_oidc_clients(&self) -> Result<OidcClients, Error> {
Ok(futures::future::try_join_all(
self.auth
@@ -11,15 +11,14 @@
http::{header, Method},
AddExtensionLayer, Router,
};
use clap::Clap;
use clap::Parser;
use std::path::PathBuf;
use std::sync::Arc;
use tower::ServiceBuilder;
use tower_http::cors::{Any, CorsLayer};
#[derive(Clap)]
#[derive(Parser)]
#[clap(version = clap::crate_version!(), author = clap::crate_authors!())]
#[clap(setting = clap::AppSettings::ColoredHelp)]
pub struct Opts {
#[clap(short, long, parse(from_occurrences))]
verbose: i32,
@@ -107,6 +106,9 @@
.layer(AddExtensionLayer::new(pool))
.layer(AddExtensionLayer::new(Arc::new(
config.create_oidc_clients().await.unwrap(),
)))
.layer(AddExtensionLayer::new(Arc::new(
config.get_file_system().await.unwrap(),
)))
.layer(AddExtensionLayer::new(Arc::new(config)));
@@ -28,22 +28,22 @@
Using the recommended setup, S3 & PostgreSQL:
```toml
bind-address = "127.0.0.1:8080" # hint: use a different port for each service
database-uri = "postgres://user:password@localhost/chartered"
bind_address = "127.0.0.1:8080" # hint: use a different port for each service
database_uri = "postgres://user:password@localhost/chartered"
# the below configuration options should only be set for chartered-web
crate-store = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/"
frontend-url = "https://my.instance.chart.rs" # this is used for CORS
storage_uri = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/"
frontend_url = "https://my.instance.chart.rs" # this is used for CORS
# if unset defaults to *
```
Or, using the defaults of `chartered-web` as an example:
```toml
bind-address = "127.0.0.1:8899"
database-uri = "sqlite://chartered.db"
bind_address = "127.0.0.1:8899"
database_uri = "sqlite://chartered.db"
crate-store = "file:///tmp/chartered"
storage_uri = "file:///tmp/chartered"
```
These configuration files can be passed into each binary using the `-c` CLI argument.
@@ -1,6 +1,12 @@
use axum::extract;
use axum::{
body::{Full, HttpBody},
extract,
http::Response,
response::{IntoResponse, Redirect},
};
use bytes::Bytes;
use chartered_db::{crates::Crate, users::User, ConnectionPool};
use chartered_fs::FileSystem;
use chartered_fs::{FilePointer, FileSystem};
use std::{str::FromStr, sync::Arc};
use thiserror::Error;
@@ -8,8 +14,8 @@
pub enum Error {
#[error("{0}")]
Database(#[from] chartered_db::Error),
#[error("Failed to fetch crate file")]
File(#[from] std::io::Error),
#[error("Failed to fetch crate file: {0}")]
File(#[from] Box<chartered_fs::Error>),
#[error("The requested version does not exist for the crate")]
NoVersion,
}
@@ -28,6 +34,23 @@
define_error_response!(Error);
pub enum ResponseOrRedirect {
Response(Vec<u8>),
Redirect(Redirect),
}
impl IntoResponse for ResponseOrRedirect {
type Body = Full<Bytes>;
type BodyError = <Self::Body as HttpBody>::Error;
fn into_response(self) -> Response<Self::Body> {
match self {
Self::Response(v) => v.into_response(),
Self::Redirect(v) => v.into_response().map(|_| Full::from(Bytes::new())),
}
}
}
pub async fn handle(
extract::Path((_session_key, organisation, name, version)): extract::Path<(
String,
@@ -37,7 +60,8 @@
)>,
extract::Extension(db): extract::Extension<ConnectionPool>,
extract::Extension(user): extract::Extension<Arc<User>>,
) -> Result<Vec<u8>, Error> {
extract::Extension(fs): extract::Extension<Arc<FileSystem>>,
) -> Result<ResponseOrRedirect, Error> {
let crate_with_permissions =
Arc::new(Crate::find_by_name(db.clone(), user.id, organisation, name).await?);
@@ -56,5 +80,10 @@
let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap();
Ok(chartered_fs::Local.read(file_ref).await?)
let res = fs.read(file_ref).await.map_err(Box::new)?;
match res {
FilePointer::Redirect(uri) => Ok(ResponseOrRedirect::Redirect(Redirect::to(uri))),
FilePointer::Content(content) => Ok(ResponseOrRedirect::Response(content)),
}
}
@@ -1,8 +1,9 @@
use axum::extract;
use bytes::Bytes;
use chartered_db::{crates::Crate, users::User, ConnectionPool};
use chartered_fs::FileSystem;
use chartered_types::cargo::{CrateDependency, CrateFeatures, CrateVersion};
use nom_bytes::BytesWrapper;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{borrow::Cow, convert::TryInto, sync::Arc};
@@ -18,6 +19,8 @@
MetadataParse,
#[error("expected a valid crate name to start with a letter, contain only letters, numbers, hyphens, or underscores and have at most 64 characters ")]
InvalidCrateName,
#[error("Failed to push crate file to storage: {0}")]
File(#[from] Box<chartered_fs::Error>),
}
impl Error {
@@ -29,6 +32,7 @@
Self::JsonParse(_) | Self::MetadataParse | Self::InvalidCrateName => {
StatusCode::BAD_REQUEST
}
Self::File(_) => StatusCode::INTERNAL_SERVER_ERROR,
}
}
}
@@ -67,11 +71,11 @@
extract::Path((_session_key, organisation)): extract::Path<(String, String)>,
extract::Extension(db): extract::Extension<ConnectionPool>,
extract::Extension(user): extract::Extension<Arc<User>>,
extract::Extension(fs): extract::Extension<Arc<FileSystem>>,
body: Bytes,
) -> Result<axum::response::Json<PublishCrateResponse>, Error> {
let (_, (metadata_bytes, crate_bytes)) =
parse(body.as_ref()).map_err(|_| Error::MetadataParse)?;
let metadata: Metadata<'_> = serde_json::from_slice(metadata_bytes)?;
let (_, (metadata_bytes, crate_bytes)) = parse(body).map_err(|_| Error::MetadataParse)?;
let metadata: Metadata<'_> = serde_json::from_slice(&metadata_bytes)?;
if !validate_crate_name(&metadata.inner.name) {
return Err(Error::InvalidCrateName);
@@ -100,14 +104,16 @@
Err(e) => return Err(e.into()),
};
let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap();
let checksum = hex::encode(Sha256::digest(&crate_bytes));
let file_ref = fs.write(crate_bytes).await.map_err(Box::new)?;
crate_with_permissions
.publish_version(
db,
user,
file_ref,
hex::encode(Sha256::digest(crate_bytes)),
checksum,
metadata_bytes.len().try_into().unwrap(),
metadata.inner.into(),
metadata.meta,
@@ -117,12 +123,14 @@
Ok(axum::response::Json(PublishCrateResponse::default()))
}
fn parse(body: &[u8]) -> nom::IResult<&[u8], (&[u8], &[u8])> {
fn parse(body: impl Into<BytesWrapper>) -> nom::IResult<BytesWrapper, (Bytes, Bytes)> {
use nom::{bytes::complete::take, combinator::map_res};
use std::array::TryFromSliceError;
let body = body.into();
let u32_from_le_bytes =
|b: &[u8]| Ok::<_, TryFromSliceError>(u32::from_le_bytes(b.try_into()?));
|b: BytesWrapper| Ok::<_, TryFromSliceError>(u32::from_le_bytes((&b[..]).try_into()?));
let mut read_u32 = map_res(take(4_usize), u32_from_le_bytes);
let (rest, metadata_length) = read_u32(body)?;
@@ -130,7 +138,7 @@
let (rest, crate_length) = read_u32(rest)?;
let (rest, crate_bytes) = take(crate_length)(rest)?;
Ok((rest, (metadata_bytes, crate_bytes)))
Ok((rest, (metadata_bytes.into(), crate_bytes.into())))
}
#[allow(dead_code)]