From 5a7e80647cff641510a4239cea2c7fcc87661da1 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Tue, 19 Oct 2021 01:48:52 +0100 Subject: [PATCH] configurable filesystem (s3, local) --- Cargo.lock | 493 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- chartered-fs/Cargo.toml | 8 +++++++- chartered-web/Cargo.toml | 1 + chartered-fs/src/lib.rs | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- chartered-web/src/config.rs | 12 +++++++++++- chartered-web/src/main.rs | 8 +++++--- book/src/getting-started/installation.md | 14 +++++++------- chartered-web/src/endpoints/cargo_api/download.rs | 41 ++++++++++++++++++++++++++++++++++++----- chartered-web/src/endpoints/cargo_api/publish.rs | 24 ++++++++++++++++-------- 9 files changed, 698 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a4c758..3ea4769 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -90,6 +90,175 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] +name = "aws-auth" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-types", + "pin-project", + "smithy-async", + "smithy-http", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-config" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-http", + "aws-hyper", + "aws-sdk-sts", + "aws-types", + "bytes", + "http", + "smithy-async", + "smithy-client", + "smithy-http", + "smithy-http-tower", + "smithy-json", + "smithy-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-endpoint" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-types", + "http", + "regex", + "smithy-http", +] + +[[package]] +name = "aws-http" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-types", + "http", + "lazy_static", + "smithy-http", + "smithy-types", + "thiserror", +] + +[[package]] +name = "aws-hyper" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-auth", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "pin-project", + "smithy-client", + "smithy-http", + "smithy-http-tower", + "smithy-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-auth", + "aws-endpoint", + "aws-http", + "aws-hyper", + "aws-sig-auth", + "aws-sigv4", + "aws-types", + "bytes", + "http", + "md5", + "smithy-client", + "smithy-eventstream", + "smithy-http", + "smithy-types", + "smithy-xml", + "tower", +] + +[[package]] +name = "aws-sdk-sts" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-auth", + "aws-endpoint", + "aws-http", + "aws-hyper", + "aws-sig-auth", + "aws-types", + "bytes", + "http", + "smithy-client", + "smithy-http", + "smithy-query", + "smithy-types", + "smithy-xml", +] + +[[package]] +name = "aws-sig-auth" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "aws-sigv4", + "aws-types", + "http", + "smithy-eventstream", + "smithy-http", + "thiserror", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "bytes", + "chrono", + "form_urlencoded", + "hex", + "http", + "percent-encoding", + "ring", + "smithy-eventstream", + "tracing", +] + +[[package]] +name = "aws-types" +version = "0.0.21-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "rustc_version", + "smithy-async", + "smithy-types", + "zeroize", +] + +[[package]] name = "axum" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -131,9 +300,9 @@ [[package]] name = "bcrypt-pbkdf" -version = "0.6.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c38c03b9506bd92bf1ef50665a81eda156f615438f7654bffba58907e6149d7" +checksum = "12621b8e87feb183a6e5dbb315e49026b2229c4398797ee0ae2d1bc00aef41b9" dependencies = [ "blowfish", "crypto-mac", @@ -222,6 +391,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "bytes-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54" +dependencies = [ + "bytes", + "either", +] [[package]] name = "cc" @@ -294,8 +473,14 @@ version = "0.1.0" dependencies = [ "async-trait", + "aws-config", + "aws-sdk-s3", + "bytes", + "http", "itertools", + "md5", "serde", + "thiserror", "tokio", "url", "uuid", @@ -359,6 +544,7 @@ "headers", "hex", "nom", + "nom-bytes", "once_cell", "openid", "rand", @@ -401,9 +587,9 @@ [[package]] name = "clap" -version = "3.0.0-beta.4" +version = "3.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcd70aa5597dbc42f7217a543f9ef2768b2ef823ba29036072d30e1d88e98406" +checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63" dependencies = [ "atty", "bitflags", @@ -414,14 +600,14 @@ "strsim", "termcolor", "textwrap", - "vec_map", + "unicase", ] [[package]] name = "clap_derive" -version = "3.0.0-beta.4" +version = "3.0.0-beta.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5bb0d655624a0b8770d1c178fb8ffcb1f91cc722cb08f451e3dc72465421ac" +checksum = "8b15c6b4f786ffb6192ffe65a36855bc1fc2444bcd0945ae16748dcd6ed7d0d3" dependencies = [ "heck", "proc-macro-error", @@ -503,6 +689,15 @@ dependencies = [ "libc", "winapi", +] + +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct", ] [[package]] @@ -623,14 +818,23 @@ [[package]] name = "encoding_rs" -version = "0.8.28" +version = "0.8.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746" dependencies = [ "cfg-if", ] [[package]] +name = "fastrand" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e" +dependencies = [ + "instant", +] + +[[package]] name = "flate2" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -837,18 +1041,18 @@ [[package]] name = "headers" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855" +checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" dependencies = [ "base64", "bitflags", "bytes", "headers-core", "http", + "httpdate", "mime", "sha-1", - "time", ] [[package]] @@ -950,6 +1154,23 @@ "tower-service", "tracing", "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "webpki", ] [[package]] @@ -1003,9 +1224,9 @@ [[package]] name = "instant" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", ] @@ -1057,9 +1278,9 @@ [[package]] name = "libc" -version = "0.2.103" +version = "0.2.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6" +checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce" [[package]] name = "libsodium-sys" @@ -1174,9 +1395,9 @@ [[package]] name = "mio" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" dependencies = [ "libc", "log", @@ -1221,6 +1442,15 @@ "memchr", "minimal-lexical", "version_check", +] + +[[package]] +name = "nom-bytes" +version = "0.1.0" +source = "git+https://github.com/w4/nom-bytes#2ede4dc22f1c303a2377c556d1a3b3f42464a0e7" +dependencies = [ + "bytes", + "nom", ] [[package]] @@ -1404,9 +1634,12 @@ [[package]] name = "os_str_bytes" -version = "3.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d" +checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799" +dependencies = [ + "memchr", +] [[package]] name = "parking_lot" @@ -1688,9 +1921,9 @@ [[package]] name = "reqwest" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51c732d463dd300362ffb44b7b125f299c23d2990411a4253824630ebc7467fb" +checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280" dependencies = [ "base64", "bytes", @@ -1734,6 +1967,40 @@ "untrusted", "web-sys", "winapi", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls", + "schannel", + "security-framework", ] [[package]] @@ -1775,6 +2042,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] [[package]] name = "security-framework" @@ -1798,6 +2075,12 @@ "core-foundation-sys", "libc", ] + +[[package]] +name = "semver" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" [[package]] name = "serde" @@ -1904,6 +2187,120 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "smithy-async" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "smithy-client" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "bytes", + "fastrand", + "http", + "http-body", + "hyper", + "hyper-rustls", + "lazy_static", + "pin-project", + "pin-project-lite", + "smithy-async", + "smithy-http", + "smithy-http-tower", + "smithy-types", + "tokio", + "tower", + "tracing", +] + +[[package]] +name = "smithy-eventstream" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "bytes", + "crc32fast", + "smithy-types", +] + +[[package]] +name = "smithy-http" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "percent-encoding", + "pin-project", + "smithy-eventstream", + "smithy-types", + "thiserror", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "smithy-http-tower" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "bytes", + "http", + "http-body", + "pin-project", + "smithy-http", + "tower", + "tracing", +] + +[[package]] +name = "smithy-json" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "smithy-types", +] + +[[package]] +name = "smithy-query" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "smithy-types", + "urlencoding", +] + +[[package]] +name = "smithy-types" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "chrono", + "itoa", + "num-integer", + "ryu", +] + +[[package]] +name = "smithy-xml" +version = "0.26.0-alpha" +source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873" +dependencies = [ + "thiserror", + "xmlparser", +] [[package]] name = "socket2" @@ -2142,7 +2539,18 @@ checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", "tokio", + "webpki", ] [[package]] @@ -2327,6 +2735,15 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] [[package]] name = "unicode-bidi" @@ -2395,6 +2812,12 @@ "percent-encoding", "serde", ] + +[[package]] +name = "urlencoding" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb" [[package]] name = "uuid" @@ -2450,12 +2873,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "vec_map" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" @@ -2564,6 +2981,16 @@ dependencies = [ "js-sys", "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", ] [[package]] @@ -2605,6 +3032,12 @@ dependencies = [ "winapi", ] + +[[package]] +name = "xmlparser" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8" [[package]] name = "yasna" @@ -2618,6 +3051,6 @@ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "bf68b08513768deaa790264a7fac27a58cbf2705cfcdc9448362229217d7e970" diff --git a/chartered-fs/Cargo.toml b/chartered-fs/Cargo.toml index 3709d6e..0a8947f 100644 --- a/chartered-fs/Cargo.toml +++ a/chartered-fs/Cargo.toml @@ -7,8 +7,14 @@ [dependencies] async-trait = "0.1" -itertools = "*" +aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-config" } +aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-sdk-s3" } +bytes = "1.1" +http = "0.2" +itertools = "0.10" +md5 = "0.7.0" serde = { version = "1", features = ["derive"] } +thiserror = "1.0" tokio = { version = "1", features = ["fs", "io-util"] } url = "2" uuid = { version = "0.8", features = ["v4", "serde"] } diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml index 9988378..1f7479f 100644 --- a/chartered-web/Cargo.toml +++ a/chartered-web/Cargo.toml @@ -21,6 +21,7 @@ headers = "0.3" hex = "0.4" nom = "7" +nom-bytes = { git = "https://github.com/w4/nom-bytes" } once_cell = "1.8" openid = "0.9" rand = "0.8" diff --git a/chartered-fs/src/lib.rs b/chartered-fs/src/lib.rs index 77f6c4c..e4f426d 100644 --- a/chartered-fs/src/lib.rs +++ a/chartered-fs/src/lib.rs @@ -1,50 +1,91 @@ #![deny(clippy::pedantic)] #![deny(rust_2018_idioms)] +#![allow(clippy::missing_errors_doc)] -use std::path::PathBuf; +use std::{path::PathBuf, time::Duration}; use async_trait::async_trait; +use aws_sdk_s3::error::{GetObjectError, PutObjectError}; +use aws_sdk_s3::{ + model::ObjectCannedAcl, presigning::config::PresigningConfig, ByteStream, SdkError, +}; +use bytes::Bytes; use itertools::Itertools; use serde::{Deserialize, Serialize}; +use thiserror::Error; use tokio::{ fs::File, io::{AsyncReadExt, AsyncWriteExt}, }; -#[derive(Debug)] -pub enum FS { - S3 { - host: String, - bucket: String, - path: String, - }, - Local { - path: PathBuf, - }, +#[derive(Debug, Error)] +pub enum Error { + #[error("failed to parse filesystem uri: {0}")] + UriParse(#[from] url::ParseError), + #[error("unknown filesystem kind (expected `s3` or `file`)")] + UnknownFileSystemKind, + #[error("failed to insert object to s3: {0}")] + S3Put(#[from] SdkError), + #[error("failed to get object from s3: {0}")] + S3Get(#[from] SdkError), + #[error("i/o failure: {0}")] + Io(#[from] std::io::Error), + #[error("failed to parse uuid: {0}")] + UuidParse(#[from] uuid::Error), + #[error("path missing from uri")] + MissingPath, + #[error("host missing from uri")] + MissingHost, + #[error("bucket missing from uri")] + MissingBucket, + #[error("invalid aws presigning config: {0}")] + AwsPresigningConfig(#[from] aws_sdk_s3::presigning::config::Error), } -impl std::str::FromStr for FS { - type Err = url::ParseError; +#[derive(Debug)] +pub enum FileSystem { + S3(S3), + Local(Local), +} - fn from_str(s: &str) -> Result { +impl FileSystem { + pub async fn from_str(s: &str) -> Result { let uri = url::Url::parse(s)?; Ok(match uri.scheme() { "s3" => { - let mut path = uri.path_segments().unwrap(); - - Self::S3 { - host: uri.host().unwrap().to_string(), - bucket: path.next().unwrap().to_string(), + let shared_config = aws_config::load_from_env().await; + let client = aws_sdk_s3::Client::new(&shared_config); + + let mut path = uri.path_segments().ok_or(Error::MissingPath)?; + + Self::S3(S3 { + host: uri.host().ok_or(Error::MissingHost)?.to_string(), + bucket: path.next().ok_or(Error::MissingBucket)?.to_string(), path: Itertools::intersperse(path, "/").collect(), - } - } - "file" => { - panic!("{:#?}", uri); + client, + }) } - _ => panic!("na"), + "file" => Self::Local(Local { + path: uri.to_file_path().map_err(|()| Error::MissingPath)?, + }), + _ => return Err(Error::UnknownFileSystemKind), }) } + + pub async fn read(&self, file_ref: FileReference) -> Result { + match self { + Self::S3(v) => v.read(file_ref).await, + Self::Local(v) => v.read(file_ref).await, + } + } + + pub async fn write(&self, data: Bytes) -> Result { + match self { + Self::S3(v) => v.write(data).await, + Self::Local(v) => v.write(data).await, + } + } } #[derive(Debug, Serialize, Deserialize)] @@ -63,15 +104,13 @@ } impl std::str::FromStr for FileSystemKind { - type Err = std::io::Error; + type Err = Error; - fn from_str(s: &str) -> Result { + fn from_str(s: &str) -> Result { match s { "local" => Ok(Self::Local), - _ => Err(std::io::Error::new( - std::io::ErrorKind::Other, - "unknown filesystemkind", - )), + "s3" => Ok(Self::S3), + _ => Err(Error::UnknownFileSystemKind), } } } @@ -89,13 +128,12 @@ } impl std::str::FromStr for FileReference { - type Err = std::io::Error; + type Err = Error; fn from_str(s: &str) -> Result { let mut split = s.splitn(2, ':'); let file_system = FileSystemKind::from_str(split.next().unwrap_or_default())?; - let reference = uuid::Uuid::from_str(split.next().unwrap_or_default()) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + let reference = uuid::Uuid::from_str(split.next().unwrap_or_default())?; Ok(FileReference { file_system, reference, @@ -103,12 +141,18 @@ } } +#[derive(Debug, PartialEq, Eq)] +pub enum FilePointer { + Content(Vec), + Redirect(http::Uri), +} + #[async_trait] -pub trait FileSystem { +pub trait FileSystemIo { const KIND: FileSystemKind; - async fn read(&self, file_ref: FileReference) -> Result, std::io::Error>; - async fn write(&self, data: &[u8]) -> Result; + async fn read(&self, file_ref: FileReference) -> Result; + async fn write(&self, data: Bytes) -> Result; #[must_use] fn create_ref() -> FileReference { @@ -119,26 +163,73 @@ } } -pub struct Local; +#[derive(Debug)] +pub struct Local { + pub path: PathBuf, +} #[async_trait] -impl FileSystem for Local { +impl FileSystemIo for Local { const KIND: FileSystemKind = FileSystemKind::Local; - async fn read(&self, file_ref: FileReference) -> Result, std::io::Error> { - let mut file = File::open(format!("/tmp/{}", file_ref.reference)).await?; + async fn read(&self, file_ref: FileReference) -> Result { + let path = self.path.join(file_ref.reference.to_string()); + let mut file = File::open(path).await?; let mut contents = vec![]; file.read_to_end(&mut contents).await?; + + Ok(FilePointer::Content(contents)) + } + + async fn write(&self, data: Bytes) -> Result { + let file_ref = Self::create_ref(); + let path = self.path.join(file_ref.reference.to_string()); - Ok(contents) + let mut file = File::create(path).await?; + file.write_all(&data).await?; + + Ok(file_ref) + } +} + +#[derive(Debug)] +pub struct S3 { + host: String, + bucket: String, + path: String, + client: aws_sdk_s3::Client, +} + +#[async_trait] +impl FileSystemIo for S3 { + const KIND: FileSystemKind = FileSystemKind::S3; + + async fn read(&self, file_ref: FileReference) -> Result { + Ok(FilePointer::Redirect( + self.client + .get_object() + .key(format!("{}/{}", self.path, file_ref.reference.to_string())) + .bucket(&self.bucket) + .presigned(PresigningConfig::expires_in(Duration::from_secs(600))?) + .await? + .uri() + .clone(), + )) } - async fn write(&self, data: &[u8]) -> Result { + async fn write(&self, data: Bytes) -> Result { let file_ref = Self::create_ref(); - let mut file = File::create(format!("/tmp/{}", file_ref.reference)).await?; - file.write_all(data).await?; + self.client + .put_object() + .key(format!("{}/{}", self.path, file_ref.reference.to_string())) + .content_md5(format!("{:x}", md5::compute(&data))) + .body(ByteStream::new(data.into())) + .bucket(&self.bucket) + .acl(ObjectCannedAcl::Private) + .send() + .await?; Ok(file_ref) } @@ -146,20 +237,32 @@ #[cfg(test)] mod tests { - use super::FileSystem; + use super::{FilePointer, FileSystem, FileSystemIo}; + use bytes::Bytes; #[tokio::test] #[allow(clippy::pedantic)] async fn parse_filesystem() { - // panic!("{:#?}", FS::from_str("s3://10.0.64.101:9000/my-bucket/my-location")); - // FS::from_str("file:///tmp/chartered"); + // assert!(matches!( + // FileSystem::from_str("s3://10.0.64.101:9000/my-bucket/my-location"), + // Ok(FileSystem::S3(inner)) if inner.host == "10.0.64.101" && inner.bucket == "my-bucket" && inner.path == "my-location" + // )); + assert!(matches!( + FileSystem::from_str("file:///tmp/chartered").await, + Ok(FileSystem::Local(inner)) if inner.path.to_str().unwrap() == "/tmp/chartered" + )); } #[tokio::test] #[allow(clippy::pedantic)] async fn local() { - let fs = super::Local; - let file_ref = fs.write(b"abcdef").await.unwrap(); - assert_eq!(fs.read(file_ref).await.unwrap(), b"abcdef"); + let fs = super::Local { + path: "/tmp".into(), + }; + let file_ref = fs.write(Bytes::from_static(b"abcdef")).await.unwrap(); + assert_eq!( + fs.read(file_ref).await.unwrap(), + FilePointer::Content(Vec::from(b"abcdef".as_ref())) + ); } } diff --git a/chartered-web/src/config.rs b/chartered-web/src/config.rs index 0a222c0..09be38c 100644 --- a/chartered-web/src/config.rs +++ a/chartered-web/src/config.rs @@ -1,4 +1,5 @@ use chacha20poly1305::Key as ChaCha20Poly1305Key; +use chartered_fs::FileSystem; use openid::DiscoveredClient; use serde::{de::Error as SerdeDeError, Deserialize}; use std::collections::HashMap; @@ -8,19 +9,28 @@ pub enum Error { #[error("Error discovering OpenID provider: {0}")] OpenId(#[from] openid::error::Error), + #[error("Failed to create file system handle: {0}")] + Fs(#[from] Box), } pub type OidcClients = HashMap; -#[derive(Deserialize, Default, Debug)] +#[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct Config { + pub storage_uri: String, pub auth: AuthConfig, #[serde(deserialize_with = "deserialize_encryption_key")] pub encryption_key: ChaCha20Poly1305Key, } impl Config { + pub async fn get_file_system(&self) -> Result { + Ok(FileSystem::from_str(&self.storage_uri) + .await + .map_err(Box::new)?) + } + pub async fn create_oidc_clients(&self) -> Result { Ok(futures::future::try_join_all( self.auth diff --git a/chartered-web/src/main.rs b/chartered-web/src/main.rs index 7480aa4..8269267 100644 --- a/chartered-web/src/main.rs +++ a/chartered-web/src/main.rs @@ -11,15 +11,14 @@ http::{header, Method}, AddExtensionLayer, Router, }; -use clap::Clap; +use clap::Parser; use std::path::PathBuf; use std::sync::Arc; use tower::ServiceBuilder; use tower_http::cors::{Any, CorsLayer}; -#[derive(Clap)] +#[derive(Parser)] #[clap(version = clap::crate_version!(), author = clap::crate_authors!())] -#[clap(setting = clap::AppSettings::ColoredHelp)] pub struct Opts { #[clap(short, long, parse(from_occurrences))] verbose: i32, @@ -107,6 +106,9 @@ .layer(AddExtensionLayer::new(pool)) .layer(AddExtensionLayer::new(Arc::new( config.create_oidc_clients().await.unwrap(), + ))) + .layer(AddExtensionLayer::new(Arc::new( + config.get_file_system().await.unwrap(), ))) .layer(AddExtensionLayer::new(Arc::new(config))); diff --git a/book/src/getting-started/installation.md b/book/src/getting-started/installation.md index 8d7d77c..a47c960 100644 --- a/book/src/getting-started/installation.md +++ a/book/src/getting-started/installation.md @@ -28,22 +28,22 @@ Using the recommended setup, S3 & PostgreSQL: ```toml -bind-address = "127.0.0.1:8080" # hint: use a different port for each service -database-uri = "postgres://user:password@localhost/chartered" +bind_address = "127.0.0.1:8080" # hint: use a different port for each service +database_uri = "postgres://user:password@localhost/chartered" # the below configuration options should only be set for chartered-web -crate-store = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/" -frontend-url = "https://my.instance.chart.rs" # this is used for CORS +storage_uri = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/" +frontend_url = "https://my.instance.chart.rs" # this is used for CORS # if unset defaults to * ``` Or, using the defaults of `chartered-web` as an example: ```toml -bind-address = "127.0.0.1:8899" -database-uri = "sqlite://chartered.db" +bind_address = "127.0.0.1:8899" +database_uri = "sqlite://chartered.db" -crate-store = "file:///tmp/chartered" +storage_uri = "file:///tmp/chartered" ``` These configuration files can be passed into each binary using the `-c` CLI argument. diff --git a/chartered-web/src/endpoints/cargo_api/download.rs b/chartered-web/src/endpoints/cargo_api/download.rs index b26c2d7..858fdf2 100644 --- a/chartered-web/src/endpoints/cargo_api/download.rs +++ a/chartered-web/src/endpoints/cargo_api/download.rs @@ -1,6 +1,12 @@ -use axum::extract; +use axum::{ + body::{Full, HttpBody}, + extract, + http::Response, + response::{IntoResponse, Redirect}, +}; +use bytes::Bytes; use chartered_db::{crates::Crate, users::User, ConnectionPool}; -use chartered_fs::FileSystem; +use chartered_fs::{FilePointer, FileSystem}; use std::{str::FromStr, sync::Arc}; use thiserror::Error; @@ -8,8 +14,8 @@ pub enum Error { #[error("{0}")] Database(#[from] chartered_db::Error), - #[error("Failed to fetch crate file")] - File(#[from] std::io::Error), + #[error("Failed to fetch crate file: {0}")] + File(#[from] Box), #[error("The requested version does not exist for the crate")] NoVersion, } @@ -28,6 +34,23 @@ define_error_response!(Error); +pub enum ResponseOrRedirect { + Response(Vec), + Redirect(Redirect), +} + +impl IntoResponse for ResponseOrRedirect { + type Body = Full; + type BodyError = ::Error; + + fn into_response(self) -> Response { + match self { + Self::Response(v) => v.into_response(), + Self::Redirect(v) => v.into_response().map(|_| Full::from(Bytes::new())), + } + } +} + pub async fn handle( extract::Path((_session_key, organisation, name, version)): extract::Path<( String, @@ -37,7 +60,8 @@ )>, extract::Extension(db): extract::Extension, extract::Extension(user): extract::Extension>, -) -> Result, Error> { + extract::Extension(fs): extract::Extension>, +) -> Result { let crate_with_permissions = Arc::new(Crate::find_by_name(db.clone(), user.id, organisation, name).await?); @@ -56,5 +80,10 @@ let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap(); - Ok(chartered_fs::Local.read(file_ref).await?) + let res = fs.read(file_ref).await.map_err(Box::new)?; + + match res { + FilePointer::Redirect(uri) => Ok(ResponseOrRedirect::Redirect(Redirect::to(uri))), + FilePointer::Content(content) => Ok(ResponseOrRedirect::Response(content)), + } } diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs index 80e73d1..ef56d46 100644 --- a/chartered-web/src/endpoints/cargo_api/publish.rs +++ a/chartered-web/src/endpoints/cargo_api/publish.rs @@ -1,8 +1,9 @@ use axum::extract; use bytes::Bytes; use chartered_db::{crates::Crate, users::User, ConnectionPool}; use chartered_fs::FileSystem; use chartered_types::cargo::{CrateDependency, CrateFeatures, CrateVersion}; +use nom_bytes::BytesWrapper; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{borrow::Cow, convert::TryInto, sync::Arc}; @@ -18,6 +19,8 @@ MetadataParse, #[error("expected a valid crate name to start with a letter, contain only letters, numbers, hyphens, or underscores and have at most 64 characters ")] InvalidCrateName, + #[error("Failed to push crate file to storage: {0}")] + File(#[from] Box), } impl Error { @@ -29,6 +32,7 @@ Self::JsonParse(_) | Self::MetadataParse | Self::InvalidCrateName => { StatusCode::BAD_REQUEST } + Self::File(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -67,11 +71,11 @@ extract::Path((_session_key, organisation)): extract::Path<(String, String)>, extract::Extension(db): extract::Extension, extract::Extension(user): extract::Extension>, + extract::Extension(fs): extract::Extension>, body: Bytes, ) -> Result, Error> { - let (_, (metadata_bytes, crate_bytes)) = - parse(body.as_ref()).map_err(|_| Error::MetadataParse)?; - let metadata: Metadata<'_> = serde_json::from_slice(metadata_bytes)?; + let (_, (metadata_bytes, crate_bytes)) = parse(body).map_err(|_| Error::MetadataParse)?; + let metadata: Metadata<'_> = serde_json::from_slice(&metadata_bytes)?; if !validate_crate_name(&metadata.inner.name) { return Err(Error::InvalidCrateName); @@ -100,14 +104,16 @@ Err(e) => return Err(e.into()), }; - let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap(); + let checksum = hex::encode(Sha256::digest(&crate_bytes)); + let file_ref = fs.write(crate_bytes).await.map_err(Box::new)?; + crate_with_permissions .publish_version( db, user, file_ref, - hex::encode(Sha256::digest(crate_bytes)), + checksum, metadata_bytes.len().try_into().unwrap(), metadata.inner.into(), metadata.meta, @@ -117,12 +123,14 @@ Ok(axum::response::Json(PublishCrateResponse::default())) } -fn parse(body: &[u8]) -> nom::IResult<&[u8], (&[u8], &[u8])> { +fn parse(body: impl Into) -> nom::IResult { use nom::{bytes::complete::take, combinator::map_res}; use std::array::TryFromSliceError; + + let body = body.into(); let u32_from_le_bytes = - |b: &[u8]| Ok::<_, TryFromSliceError>(u32::from_le_bytes(b.try_into()?)); + |b: BytesWrapper| Ok::<_, TryFromSliceError>(u32::from_le_bytes((&b[..]).try_into()?)); let mut read_u32 = map_res(take(4_usize), u32_from_le_bytes); let (rest, metadata_length) = read_u32(body)?; @@ -130,7 +138,7 @@ let (rest, crate_length) = read_u32(rest)?; let (rest, crate_bytes) = take(crate_length)(rest)?; - Ok((rest, (metadata_bytes, crate_bytes))) + Ok((rest, (metadata_bytes.into(), crate_bytes.into()))) } #[allow(dead_code)] // a lot of these need checking/validating -- rgit 0.1.3