🏡 index : ~doyle/chartered.git

author Jordan Doyle <jordan@doyle.la> 2021-10-19 1:48:52.0 +01:00:00
committer Jordan Doyle <jordan@doyle.la> 2021-10-19 9:11:26.0 +01:00:00
commit
5a7e80647cff641510a4239cea2c7fcc87661da1 [patch]
tree
db7c53f5fb2a5a7b1f1a43770476130e58b1d724
parent
f22899783aa0ea9d35b02508fe996943caefe7e1
download
5a7e80647cff641510a4239cea2c7fcc87661da1.tar.gz

configurable filesystem (s3, local)



Diff

 Cargo.lock                                        | 493 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 chartered-fs/Cargo.toml                           |   8 +++++++-
 chartered-web/Cargo.toml                          |   1 +
 chartered-fs/src/lib.rs                           | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 chartered-web/src/config.rs                       |  12 +++++++++++-
 chartered-web/src/main.rs                         |   8 +++++---
 book/src/getting-started/installation.md          |  14 +++++++-------
 chartered-web/src/endpoints/cargo_api/download.rs |  41 ++++++++++++++++++++++++++++++++++++-----
 chartered-web/src/endpoints/cargo_api/publish.rs  |  24 ++++++++++++++++--------
 9 files changed, 698 insertions(+), 106 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 7a4c758..3ea4769 100644
--- a/Cargo.lock
+++ a/Cargo.lock
@@ -90,6 +90,175 @@
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"

[[package]]
name = "aws-auth"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-types",
 "pin-project",
 "smithy-async",
 "smithy-http",
 "tokio",
 "tracing",
 "zeroize",
]

[[package]]
name = "aws-config"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-http",
 "aws-hyper",
 "aws-sdk-sts",
 "aws-types",
 "bytes",
 "http",
 "smithy-async",
 "smithy-client",
 "smithy-http",
 "smithy-http-tower",
 "smithy-json",
 "smithy-types",
 "tokio",
 "tower",
 "tracing",
]

[[package]]
name = "aws-endpoint"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-types",
 "http",
 "regex",
 "smithy-http",
]

[[package]]
name = "aws-http"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-types",
 "http",
 "lazy_static",
 "smithy-http",
 "smithy-types",
 "thiserror",
]

[[package]]
name = "aws-hyper"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-auth",
 "aws-endpoint",
 "aws-http",
 "aws-sig-auth",
 "bytes",
 "fastrand",
 "http",
 "http-body",
 "hyper",
 "hyper-rustls",
 "pin-project",
 "smithy-client",
 "smithy-http",
 "smithy-http-tower",
 "smithy-types",
 "tokio",
 "tower",
 "tracing",
]

[[package]]
name = "aws-sdk-s3"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-auth",
 "aws-endpoint",
 "aws-http",
 "aws-hyper",
 "aws-sig-auth",
 "aws-sigv4",
 "aws-types",
 "bytes",
 "http",
 "md5",
 "smithy-client",
 "smithy-eventstream",
 "smithy-http",
 "smithy-types",
 "smithy-xml",
 "tower",
]

[[package]]
name = "aws-sdk-sts"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-auth",
 "aws-endpoint",
 "aws-http",
 "aws-hyper",
 "aws-sig-auth",
 "aws-types",
 "bytes",
 "http",
 "smithy-client",
 "smithy-http",
 "smithy-query",
 "smithy-types",
 "smithy-xml",
]

[[package]]
name = "aws-sig-auth"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "aws-sigv4",
 "aws-types",
 "http",
 "smithy-eventstream",
 "smithy-http",
 "thiserror",
 "tracing",
]

[[package]]
name = "aws-sigv4"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "bytes",
 "chrono",
 "form_urlencoded",
 "hex",
 "http",
 "percent-encoding",
 "ring",
 "smithy-eventstream",
 "tracing",
]

[[package]]
name = "aws-types"
version = "0.0.21-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "rustc_version",
 "smithy-async",
 "smithy-types",
 "zeroize",
]

[[package]]
name = "axum"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -131,9 +300,9 @@

[[package]]
name = "bcrypt-pbkdf"
version = "0.6.2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c38c03b9506bd92bf1ef50665a81eda156f615438f7654bffba58907e6149d7"
checksum = "12621b8e87feb183a6e5dbb315e49026b2229c4398797ee0ae2d1bc00aef41b9"
dependencies = [
 "blowfish",
 "crypto-mac",
@@ -222,6 +391,16 @@
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"

[[package]]
name = "bytes-utils"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54"
dependencies = [
 "bytes",
 "either",
]

[[package]]
name = "cc"
@@ -294,8 +473,14 @@
version = "0.1.0"
dependencies = [
 "async-trait",
 "aws-config",
 "aws-sdk-s3",
 "bytes",
 "http",
 "itertools",
 "md5",
 "serde",
 "thiserror",
 "tokio",
 "url",
 "uuid",
@@ -359,6 +544,7 @@
 "headers",
 "hex",
 "nom",
 "nom-bytes",
 "once_cell",
 "openid",
 "rand",
@@ -401,9 +587,9 @@

[[package]]
name = "clap"
version = "3.0.0-beta.4"
version = "3.0.0-beta.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcd70aa5597dbc42f7217a543f9ef2768b2ef823ba29036072d30e1d88e98406"
checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63"
dependencies = [
 "atty",
 "bitflags",
@@ -414,14 +600,14 @@
 "strsim",
 "termcolor",
 "textwrap",
 "vec_map",
 "unicase",
]

[[package]]
name = "clap_derive"
version = "3.0.0-beta.4"
version = "3.0.0-beta.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b5bb0d655624a0b8770d1c178fb8ffcb1f91cc722cb08f451e3dc72465421ac"
checksum = "8b15c6b4f786ffb6192ffe65a36855bc1fc2444bcd0945ae16748dcd6ed7d0d3"
dependencies = [
 "heck",
 "proc-macro-error",
@@ -503,6 +689,15 @@
dependencies = [
 "libc",
 "winapi",
]

[[package]]
name = "ct-logs"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
dependencies = [
 "sct",
]

[[package]]
@@ -623,14 +818,23 @@

[[package]]
name = "encoding_rs"
version = "0.8.28"
version = "0.8.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
checksum = "a74ea89a0a1b98f6332de42c95baff457ada66d1cb4030f9ff151b2041a1c746"
dependencies = [
 "cfg-if",
]

[[package]]
name = "fastrand"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b394ed3d285a429378d3b384b9eb1285267e7df4b166df24b7a6939a04dc392e"
dependencies = [
 "instant",
]

[[package]]
name = "flate2"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -837,18 +1041,18 @@

[[package]]
name = "headers"
version = "0.3.4"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0b7591fb62902706ae8e7aaff416b1b0fa2c0fd0878b46dc13baa3712d8a855"
checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0"
dependencies = [
 "base64",
 "bitflags",
 "bytes",
 "headers-core",
 "http",
 "httpdate",
 "mime",
 "sha-1",
 "time",
]

[[package]]
@@ -950,6 +1154,23 @@
 "tower-service",
 "tracing",
 "want",
]

[[package]]
name = "hyper-rustls"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [
 "ct-logs",
 "futures-util",
 "hyper",
 "log",
 "rustls",
 "rustls-native-certs",
 "tokio",
 "tokio-rustls",
 "webpki",
]

[[package]]
@@ -1003,9 +1224,9 @@

[[package]]
name = "instant"
version = "0.1.11"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
 "cfg-if",
]
@@ -1057,9 +1278,9 @@

[[package]]
name = "libc"
version = "0.2.103"
version = "0.2.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6"
checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"

[[package]]
name = "libsodium-sys"
@@ -1174,9 +1395,9 @@

[[package]]
name = "mio"
version = "0.7.13"
version = "0.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc"
dependencies = [
 "libc",
 "log",
@@ -1221,6 +1442,15 @@
 "memchr",
 "minimal-lexical",
 "version_check",
]

[[package]]
name = "nom-bytes"
version = "0.1.0"
source = "git+https://github.com/w4/nom-bytes#2ede4dc22f1c303a2377c556d1a3b3f42464a0e7"
dependencies = [
 "bytes",
 "nom",
]

[[package]]
@@ -1404,9 +1634,12 @@

[[package]]
name = "os_str_bytes"
version = "3.1.0"
version = "4.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6acbef58a60fe69ab50510a55bc8cdd4d6cf2283d27ad338f54cb52747a9cf2d"
checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799"
dependencies = [
 "memchr",
]

[[package]]
name = "parking_lot"
@@ -1688,9 +1921,9 @@

[[package]]
name = "reqwest"
version = "0.11.5"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51c732d463dd300362ffb44b7b125f299c23d2990411a4253824630ebc7467fb"
checksum = "66d2927ca2f685faf0fc620ac4834690d29e7abb153add10f5812eef20b5e280"
dependencies = [
 "base64",
 "bytes",
@@ -1734,6 +1967,40 @@
 "untrusted",
 "web-sys",
 "winapi",
]

[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
 "semver",
]

[[package]]
name = "rustls"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
dependencies = [
 "base64",
 "log",
 "ring",
 "sct",
 "webpki",
]

[[package]]
name = "rustls-native-certs"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
 "openssl-probe",
 "rustls",
 "schannel",
 "security-framework",
]

[[package]]
@@ -1775,6 +2042,16 @@
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

[[package]]
name = "sct"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
dependencies = [
 "ring",
 "untrusted",
]

[[package]]
name = "security-framework"
@@ -1798,6 +2075,12 @@
 "core-foundation-sys",
 "libc",
]

[[package]]
name = "semver"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"

[[package]]
name = "serde"
@@ -1904,6 +2187,120 @@
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"

[[package]]
name = "smithy-async"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "pin-project-lite",
 "tokio",
]

[[package]]
name = "smithy-client"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "bytes",
 "fastrand",
 "http",
 "http-body",
 "hyper",
 "hyper-rustls",
 "lazy_static",
 "pin-project",
 "pin-project-lite",
 "smithy-async",
 "smithy-http",
 "smithy-http-tower",
 "smithy-types",
 "tokio",
 "tower",
 "tracing",
]

[[package]]
name = "smithy-eventstream"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "bytes",
 "crc32fast",
 "smithy-types",
]

[[package]]
name = "smithy-http"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "bytes",
 "bytes-utils",
 "futures-core",
 "http",
 "http-body",
 "hyper",
 "percent-encoding",
 "pin-project",
 "smithy-eventstream",
 "smithy-types",
 "thiserror",
 "tokio",
 "tokio-util",
 "tracing",
]

[[package]]
name = "smithy-http-tower"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "bytes",
 "http",
 "http-body",
 "pin-project",
 "smithy-http",
 "tower",
 "tracing",
]

[[package]]
name = "smithy-json"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "smithy-types",
]

[[package]]
name = "smithy-query"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "smithy-types",
 "urlencoding",
]

[[package]]
name = "smithy-types"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "chrono",
 "itoa",
 "num-integer",
 "ryu",
]

[[package]]
name = "smithy-xml"
version = "0.26.0-alpha"
source = "git+https://github.com/awslabs/aws-sdk-rust?tag=v0.0.21-alpha#d02a7f0ddc19463ed645641d6758819247348873"
dependencies = [
 "thiserror",
 "xmlparser",
]

[[package]]
name = "socket2"
@@ -2142,7 +2539,18 @@
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
 "native-tls",
 "tokio",
]

[[package]]
name = "tokio-rustls"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
 "rustls",
 "tokio",
 "webpki",
]

[[package]]
@@ -2327,6 +2735,15 @@
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"

[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
 "version_check",
]

[[package]]
name = "unicode-bidi"
@@ -2395,6 +2812,12 @@
 "percent-encoding",
 "serde",
]

[[package]]
name = "urlencoding"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"

[[package]]
name = "uuid"
@@ -2450,12 +2873,6 @@
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"

[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"

[[package]]
name = "version_check"
@@ -2564,6 +2981,16 @@
dependencies = [
 "js-sys",
 "wasm-bindgen",
]

[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
 "ring",
 "untrusted",
]

[[package]]
@@ -2605,6 +3032,12 @@
dependencies = [
 "winapi",
]

[[package]]
name = "xmlparser"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"

[[package]]
name = "yasna"
@@ -2618,6 +3051,6 @@

[[package]]
name = "zeroize"
version = "1.3.0"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd"
checksum = "bf68b08513768deaa790264a7fac27a58cbf2705cfcdc9448362229217d7e970"
diff --git a/chartered-fs/Cargo.toml b/chartered-fs/Cargo.toml
index 3709d6e..0a8947f 100644
--- a/chartered-fs/Cargo.toml
+++ a/chartered-fs/Cargo.toml
@@ -7,8 +7,14 @@

[dependencies]
async-trait = "0.1"
itertools = "*"
aws-config = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-config" }
aws-sdk-s3 = { git = "https://github.com/awslabs/aws-sdk-rust", tag = "v0.0.21-alpha", package = "aws-sdk-s3" }
bytes = "1.1"
http = "0.2"
itertools = "0.10"
md5 = "0.7.0"
serde = { version = "1", features = ["derive"] }
thiserror = "1.0"
tokio = { version = "1", features = ["fs", "io-util"] }
url = "2"
uuid = { version = "0.8", features = ["v4", "serde"] }
diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml
index 9988378..1f7479f 100644
--- a/chartered-web/Cargo.toml
+++ a/chartered-web/Cargo.toml
@@ -21,6 +21,7 @@
headers = "0.3"
hex = "0.4"
nom = "7"
nom-bytes = { git = "https://github.com/w4/nom-bytes" }
once_cell = "1.8"
openid = "0.9"
rand = "0.8"
diff --git a/chartered-fs/src/lib.rs b/chartered-fs/src/lib.rs
index 77f6c4c..e4f426d 100644
--- a/chartered-fs/src/lib.rs
+++ a/chartered-fs/src/lib.rs
@@ -1,50 +1,91 @@
#![deny(clippy::pedantic)]
#![deny(rust_2018_idioms)]
#![allow(clippy::missing_errors_doc)]

use std::path::PathBuf;
use std::{path::PathBuf, time::Duration};

use async_trait::async_trait;
use aws_sdk_s3::error::{GetObjectError, PutObjectError};
use aws_sdk_s3::{
    model::ObjectCannedAcl, presigning::config::PresigningConfig, ByteStream, SdkError,
};
use bytes::Bytes;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::{
    fs::File,
    io::{AsyncReadExt, AsyncWriteExt},
};

#[derive(Debug)]
pub enum FS {
    S3 {
        host: String,
        bucket: String,
        path: String,
    },
    Local {
        path: PathBuf,
    },
#[derive(Debug, Error)]
pub enum Error {
    #[error("failed to parse filesystem uri: {0}")]
    UriParse(#[from] url::ParseError),
    #[error("unknown filesystem kind (expected `s3` or `file`)")]
    UnknownFileSystemKind,
    #[error("failed to insert object to s3: {0}")]
    S3Put(#[from] SdkError<PutObjectError>),
    #[error("failed to get object from s3: {0}")]
    S3Get(#[from] SdkError<GetObjectError>),
    #[error("i/o failure: {0}")]
    Io(#[from] std::io::Error),
    #[error("failed to parse uuid: {0}")]
    UuidParse(#[from] uuid::Error),
    #[error("path missing from uri")]
    MissingPath,
    #[error("host missing from uri")]
    MissingHost,
    #[error("bucket missing from uri")]
    MissingBucket,
    #[error("invalid aws presigning config: {0}")]
    AwsPresigningConfig(#[from] aws_sdk_s3::presigning::config::Error),
}

impl std::str::FromStr for FS {
    type Err = url::ParseError;
#[derive(Debug)]
pub enum FileSystem {
    S3(S3),
    Local(Local),
}

    fn from_str(s: &str) -> Result<Self, Self::Err> {
impl FileSystem {
    pub async fn from_str(s: &str) -> Result<Self, Error> {
        let uri = url::Url::parse(s)?;

        Ok(match uri.scheme() {
            "s3" => {
                let mut path = uri.path_segments().unwrap();

                Self::S3 {
                    host: uri.host().unwrap().to_string(),
                    bucket: path.next().unwrap().to_string(),
                let shared_config = aws_config::load_from_env().await;
                let client = aws_sdk_s3::Client::new(&shared_config);

                let mut path = uri.path_segments().ok_or(Error::MissingPath)?;

                Self::S3(S3 {
                    host: uri.host().ok_or(Error::MissingHost)?.to_string(),
                    bucket: path.next().ok_or(Error::MissingBucket)?.to_string(),
                    path: Itertools::intersperse(path, "/").collect(),
                }
            }
            "file" => {
                panic!("{:#?}", uri);
                    client,
                })
            }
            _ => panic!("na"),
            "file" => Self::Local(Local {
                path: uri.to_file_path().map_err(|()| Error::MissingPath)?,
            }),
            _ => return Err(Error::UnknownFileSystemKind),
        })
    }

    pub async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
        match self {
            Self::S3(v) => v.read(file_ref).await,
            Self::Local(v) => v.read(file_ref).await,
        }
    }

    pub async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
        match self {
            Self::S3(v) => v.write(data).await,
            Self::Local(v) => v.write(data).await,
        }
    }
}

#[derive(Debug, Serialize, Deserialize)]
@@ -63,15 +104,13 @@
}

impl std::str::FromStr for FileSystemKind {
    type Err = std::io::Error;
    type Err = Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
    fn from_str(s: &str) -> Result<Self, Error> {
        match s {
            "local" => Ok(Self::Local),
            _ => Err(std::io::Error::new(
                std::io::ErrorKind::Other,
                "unknown filesystemkind",
            )),
            "s3" => Ok(Self::S3),
            _ => Err(Error::UnknownFileSystemKind),
        }
    }
}
@@ -89,13 +128,12 @@
}

impl std::str::FromStr for FileReference {
    type Err = std::io::Error;
    type Err = Error;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let mut split = s.splitn(2, ':');
        let file_system = FileSystemKind::from_str(split.next().unwrap_or_default())?;
        let reference = uuid::Uuid::from_str(split.next().unwrap_or_default())
            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
        let reference = uuid::Uuid::from_str(split.next().unwrap_or_default())?;
        Ok(FileReference {
            file_system,
            reference,
@@ -103,12 +141,18 @@
    }
}

#[derive(Debug, PartialEq, Eq)]
pub enum FilePointer {
    Content(Vec<u8>),
    Redirect(http::Uri),
}

#[async_trait]
pub trait FileSystem {
pub trait FileSystemIo {
    const KIND: FileSystemKind;

    async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error>;
    async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error>;
    async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error>;
    async fn write(&self, data: Bytes) -> Result<FileReference, Error>;

    #[must_use]
    fn create_ref() -> FileReference {
@@ -119,26 +163,73 @@
    }
}

pub struct Local;
#[derive(Debug)]
pub struct Local {
    pub path: PathBuf,
}

#[async_trait]
impl FileSystem for Local {
impl FileSystemIo for Local {
    const KIND: FileSystemKind = FileSystemKind::Local;

    async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error> {
        let mut file = File::open(format!("/tmp/{}", file_ref.reference)).await?;
    async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
        let path = self.path.join(file_ref.reference.to_string());
        let mut file = File::open(path).await?;

        let mut contents = vec![];
        file.read_to_end(&mut contents).await?;

        Ok(FilePointer::Content(contents))
    }

    async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
        let file_ref = Self::create_ref();
        let path = self.path.join(file_ref.reference.to_string());

        Ok(contents)
        let mut file = File::create(path).await?;
        file.write_all(&data).await?;

        Ok(file_ref)
    }
}

#[derive(Debug)]
pub struct S3 {
    host: String,
    bucket: String,
    path: String,
    client: aws_sdk_s3::Client,
}

#[async_trait]
impl FileSystemIo for S3 {
    const KIND: FileSystemKind = FileSystemKind::S3;

    async fn read(&self, file_ref: FileReference) -> Result<FilePointer, Error> {
        Ok(FilePointer::Redirect(
            self.client
                .get_object()
                .key(format!("{}/{}", self.path, file_ref.reference.to_string()))
                .bucket(&self.bucket)
                .presigned(PresigningConfig::expires_in(Duration::from_secs(600))?)
                .await?
                .uri()
                .clone(),
        ))
    }

    async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error> {
    async fn write(&self, data: Bytes) -> Result<FileReference, Error> {
        let file_ref = Self::create_ref();

        let mut file = File::create(format!("/tmp/{}", file_ref.reference)).await?;
        file.write_all(data).await?;
        self.client
            .put_object()
            .key(format!("{}/{}", self.path, file_ref.reference.to_string()))
            .content_md5(format!("{:x}", md5::compute(&data)))
            .body(ByteStream::new(data.into()))
            .bucket(&self.bucket)
            .acl(ObjectCannedAcl::Private)
            .send()
            .await?;

        Ok(file_ref)
    }
@@ -146,20 +237,32 @@

#[cfg(test)]
mod tests {
    use super::FileSystem;
    use super::{FilePointer, FileSystem, FileSystemIo};
    use bytes::Bytes;

    #[tokio::test]
    #[allow(clippy::pedantic)]
    async fn parse_filesystem() {
        // panic!("{:#?}", FS::from_str("s3://10.0.64.101:9000/my-bucket/my-location"));
        // FS::from_str("file:///tmp/chartered");
        // assert!(matches!(
        //     FileSystem::from_str("s3://10.0.64.101:9000/my-bucket/my-location"),
        //     Ok(FileSystem::S3(inner)) if inner.host == "10.0.64.101" && inner.bucket == "my-bucket" && inner.path == "my-location"
        // ));
        assert!(matches!(
            FileSystem::from_str("file:///tmp/chartered").await,
            Ok(FileSystem::Local(inner)) if inner.path.to_str().unwrap() == "/tmp/chartered"
        ));
    }

    #[tokio::test]
    #[allow(clippy::pedantic)]
    async fn local() {
        let fs = super::Local;
        let file_ref = fs.write(b"abcdef").await.unwrap();
        assert_eq!(fs.read(file_ref).await.unwrap(), b"abcdef");
        let fs = super::Local {
            path: "/tmp".into(),
        };
        let file_ref = fs.write(Bytes::from_static(b"abcdef")).await.unwrap();
        assert_eq!(
            fs.read(file_ref).await.unwrap(),
            FilePointer::Content(Vec::from(b"abcdef".as_ref()))
        );
    }
}
diff --git a/chartered-web/src/config.rs b/chartered-web/src/config.rs
index 0a222c0..09be38c 100644
--- a/chartered-web/src/config.rs
+++ a/chartered-web/src/config.rs
@@ -1,4 +1,5 @@
use chacha20poly1305::Key as ChaCha20Poly1305Key;
use chartered_fs::FileSystem;
use openid::DiscoveredClient;
use serde::{de::Error as SerdeDeError, Deserialize};
use std::collections::HashMap;
@@ -8,19 +9,28 @@
pub enum Error {
    #[error("Error discovering OpenID provider: {0}")]
    OpenId(#[from] openid::error::Error),
    #[error("Failed to create file system handle: {0}")]
    Fs(#[from] Box<chartered_fs::Error>),
}

pub type OidcClients = HashMap<String, DiscoveredClient>;

#[derive(Deserialize, Default, Debug)]
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub storage_uri: String,
    pub auth: AuthConfig,
    #[serde(deserialize_with = "deserialize_encryption_key")]
    pub encryption_key: ChaCha20Poly1305Key,
}

impl Config {
    pub async fn get_file_system(&self) -> Result<FileSystem, Error> {
        Ok(FileSystem::from_str(&self.storage_uri)
            .await
            .map_err(Box::new)?)
    }

    pub async fn create_oidc_clients(&self) -> Result<OidcClients, Error> {
        Ok(futures::future::try_join_all(
            self.auth
diff --git a/chartered-web/src/main.rs b/chartered-web/src/main.rs
index 7480aa4..8269267 100644
--- a/chartered-web/src/main.rs
+++ a/chartered-web/src/main.rs
@@ -11,15 +11,14 @@
    http::{header, Method},
    AddExtensionLayer, Router,
};
use clap::Clap;
use clap::Parser;
use std::path::PathBuf;
use std::sync::Arc;
use tower::ServiceBuilder;
use tower_http::cors::{Any, CorsLayer};

#[derive(Clap)]
#[derive(Parser)]
#[clap(version = clap::crate_version!(), author = clap::crate_authors!())]
#[clap(setting = clap::AppSettings::ColoredHelp)]
pub struct Opts {
    #[clap(short, long, parse(from_occurrences))]
    verbose: i32,
@@ -107,6 +106,9 @@
        .layer(AddExtensionLayer::new(pool))
        .layer(AddExtensionLayer::new(Arc::new(
            config.create_oidc_clients().await.unwrap(),
        )))
        .layer(AddExtensionLayer::new(Arc::new(
            config.get_file_system().await.unwrap(),
        )))
        .layer(AddExtensionLayer::new(Arc::new(config)));

diff --git a/book/src/getting-started/installation.md b/book/src/getting-started/installation.md
index 8d7d77c..a47c960 100644
--- a/book/src/getting-started/installation.md
+++ a/book/src/getting-started/installation.md
@@ -28,22 +28,22 @@
Using the recommended setup, S3 & PostgreSQL:

```toml

bind-address = "127.0.0.1:8080" # hint: use a different port for each service
database-uri = "postgres://user:password@localhost/chartered"
bind_address = "127.0.0.1:8080" # hint: use a different port for each service
database_uri = "postgres://user:password@localhost/chartered"

# the below configuration options should only be set for chartered-web
crate-store  = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/"
frontend-url = "https://my.instance.chart.rs" # this is used for CORS
storage_uri  = "s3://s3-eu-west-1.amazonaws.com/my-cool-crate-store/"
frontend_url = "https://my.instance.chart.rs" # this is used for CORS
                                              # if unset defaults to *

```


Or, using the defaults of `chartered-web` as an example:

```toml

bind-address = "127.0.0.1:8899"
database-uri = "sqlite://chartered.db"
bind_address = "127.0.0.1:8899"
database_uri = "sqlite://chartered.db"

crate-store  = "file:///tmp/chartered"
storage_uri  = "file:///tmp/chartered"
```


These configuration files can be passed into each binary using the `-c` CLI argument.
diff --git a/chartered-web/src/endpoints/cargo_api/download.rs b/chartered-web/src/endpoints/cargo_api/download.rs
index b26c2d7..858fdf2 100644
--- a/chartered-web/src/endpoints/cargo_api/download.rs
+++ a/chartered-web/src/endpoints/cargo_api/download.rs
@@ -1,6 +1,12 @@
use axum::extract;
use axum::{
    body::{Full, HttpBody},
    extract,
    http::Response,
    response::{IntoResponse, Redirect},
};
use bytes::Bytes;
use chartered_db::{crates::Crate, users::User, ConnectionPool};
use chartered_fs::FileSystem;
use chartered_fs::{FilePointer, FileSystem};
use std::{str::FromStr, sync::Arc};
use thiserror::Error;

@@ -8,8 +14,8 @@
pub enum Error {
    #[error("{0}")]
    Database(#[from] chartered_db::Error),
    #[error("Failed to fetch crate file")]
    File(#[from] std::io::Error),
    #[error("Failed to fetch crate file: {0}")]
    File(#[from] Box<chartered_fs::Error>),
    #[error("The requested version does not exist for the crate")]
    NoVersion,
}
@@ -28,6 +34,23 @@

define_error_response!(Error);

pub enum ResponseOrRedirect {
    Response(Vec<u8>),
    Redirect(Redirect),
}

impl IntoResponse for ResponseOrRedirect {
    type Body = Full<Bytes>;
    type BodyError = <Self::Body as HttpBody>::Error;

    fn into_response(self) -> Response<Self::Body> {
        match self {
            Self::Response(v) => v.into_response(),
            Self::Redirect(v) => v.into_response().map(|_| Full::from(Bytes::new())),
        }
    }
}

pub async fn handle(
    extract::Path((_session_key, organisation, name, version)): extract::Path<(
        String,
@@ -37,7 +60,8 @@
    )>,
    extract::Extension(db): extract::Extension<ConnectionPool>,
    extract::Extension(user): extract::Extension<Arc<User>>,
) -> Result<Vec<u8>, Error> {
    extract::Extension(fs): extract::Extension<Arc<FileSystem>>,
) -> Result<ResponseOrRedirect, Error> {
    let crate_with_permissions =
        Arc::new(Crate::find_by_name(db.clone(), user.id, organisation, name).await?);

@@ -56,5 +80,10 @@

    let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap();

    Ok(chartered_fs::Local.read(file_ref).await?)
    let res = fs.read(file_ref).await.map_err(Box::new)?;

    match res {
        FilePointer::Redirect(uri) => Ok(ResponseOrRedirect::Redirect(Redirect::to(uri))),
        FilePointer::Content(content) => Ok(ResponseOrRedirect::Response(content)),
    }
}
diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs
index 80e73d1..ef56d46 100644
--- a/chartered-web/src/endpoints/cargo_api/publish.rs
+++ a/chartered-web/src/endpoints/cargo_api/publish.rs
@@ -1,8 +1,9 @@
use axum::extract;
use bytes::Bytes;
use chartered_db::{crates::Crate, users::User, ConnectionPool};
use chartered_fs::FileSystem;
use chartered_types::cargo::{CrateDependency, CrateFeatures, CrateVersion};
use nom_bytes::BytesWrapper;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{borrow::Cow, convert::TryInto, sync::Arc};
@@ -18,6 +19,8 @@
    MetadataParse,
    #[error("expected a valid crate name to start with a letter, contain only letters, numbers, hyphens, or underscores and have at most 64 characters ")]
    InvalidCrateName,
    #[error("Failed to push crate file to storage: {0}")]
    File(#[from] Box<chartered_fs::Error>),
}

impl Error {
@@ -29,6 +32,7 @@
            Self::JsonParse(_) | Self::MetadataParse | Self::InvalidCrateName => {
                StatusCode::BAD_REQUEST
            }
            Self::File(_) => StatusCode::INTERNAL_SERVER_ERROR,
        }
    }
}
@@ -67,11 +71,11 @@
    extract::Path((_session_key, organisation)): extract::Path<(String, String)>,
    extract::Extension(db): extract::Extension<ConnectionPool>,
    extract::Extension(user): extract::Extension<Arc<User>>,
    extract::Extension(fs): extract::Extension<Arc<FileSystem>>,
    body: Bytes,
) -> Result<axum::response::Json<PublishCrateResponse>, Error> {
    let (_, (metadata_bytes, crate_bytes)) =
        parse(body.as_ref()).map_err(|_| Error::MetadataParse)?;
    let metadata: Metadata<'_> = serde_json::from_slice(metadata_bytes)?;
    let (_, (metadata_bytes, crate_bytes)) = parse(body).map_err(|_| Error::MetadataParse)?;
    let metadata: Metadata<'_> = serde_json::from_slice(&metadata_bytes)?;

    if !validate_crate_name(&metadata.inner.name) {
        return Err(Error::InvalidCrateName);
@@ -100,14 +104,16 @@
        Err(e) => return Err(e.into()),
    };

    let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap();
    let checksum = hex::encode(Sha256::digest(&crate_bytes));

    let file_ref = fs.write(crate_bytes).await.map_err(Box::new)?;

    crate_with_permissions
        .publish_version(
            db,
            user,
            file_ref,
            hex::encode(Sha256::digest(crate_bytes)),
            checksum,
            metadata_bytes.len().try_into().unwrap(),
            metadata.inner.into(),
            metadata.meta,
@@ -117,12 +123,14 @@
    Ok(axum::response::Json(PublishCrateResponse::default()))
}

fn parse(body: &[u8]) -> nom::IResult<&[u8], (&[u8], &[u8])> {
fn parse(body: impl Into<BytesWrapper>) -> nom::IResult<BytesWrapper, (Bytes, Bytes)> {
    use nom::{bytes::complete::take, combinator::map_res};
    use std::array::TryFromSliceError;

    let body = body.into();

    let u32_from_le_bytes =
        |b: &[u8]| Ok::<_, TryFromSliceError>(u32::from_le_bytes(b.try_into()?));
        |b: BytesWrapper| Ok::<_, TryFromSliceError>(u32::from_le_bytes((&b[..]).try_into()?));
    let mut read_u32 = map_res(take(4_usize), u32_from_le_bytes);

    let (rest, metadata_length) = read_u32(body)?;
@@ -130,7 +138,7 @@
    let (rest, crate_length) = read_u32(rest)?;
    let (rest, crate_bytes) = take(crate_length)(rest)?;

    Ok((rest, (metadata_bytes, crate_bytes)))
    Ok((rest, (metadata_bytes.into(), crate_bytes.into())))
}

#[allow(dead_code)] // a lot of these need checking/validating