From 603d7a9c72ee401ba80b53c3de7651a57ad73039 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Wed, 01 Sep 2021 02:08:52 +0100 Subject: [PATCH] basic crate pushing (no metadata, validation, error checking) --- Cargo.lock | 26 ++++++++++++++++++++++++++ chartered-db/Cargo.toml | 2 ++ chartered-fs/Cargo.toml | 3 +++ chartered-web/Cargo.toml | 5 +++++ chartered-db/src/lib.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++------- chartered-db/src/schema.rs | 5 +---- chartered-fs/src/lib.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ chartered-git/src/main.rs | 4 ++-- chartered-web/src/main.rs | 14 ++++++++++---- chartered-web/src/middleware/auth.rs | 12 ++++++------ chartered-web/src/endpoints/cargo_api/mod.rs | 2 ++ chartered-web/src/endpoints/cargo_api/publish.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 226 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be6c069..5fed164 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -186,6 +186,7 @@ name = "chartered-db" version = "0.1.0" dependencies = [ + "chartered-fs", "diesel", "dotenv", "tokio", @@ -228,8 +229,13 @@ version = "0.1.0" dependencies = [ "axum", + "bytes", "chartered-db", + "chartered-fs", "futures", + "nom", + "serde", + "serde_json", "tokio", "tower", "tower-http", @@ -739,6 +745,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "minimal-lexical" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6595bb28ed34f43c3fe088e48f6cfb2e033cab45f25a5384d5fdf564fbc8c4b2" [[package]] name = "miniz_oxide" @@ -770,6 +782,17 @@ checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ "winapi", +] + +[[package]] +name = "nom" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffd9d26838a953b4af82cbeb9f1592c6798916983959be223a7124e992742c1" +dependencies = [ + "memchr", + "minimal-lexical", + "version_check", ] [[package]] @@ -1084,6 +1107,9 @@ version = "1.0.129" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" diff --git a/chartered-db/Cargo.toml b/chartered-db/Cargo.toml index b6a463c..4b4f1f2 100644 --- a/chartered-db/Cargo.toml +++ a/chartered-db/Cargo.toml @@ -6,6 +6,8 @@ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +chartered-fs = { path = "../chartered-fs" } + diesel = { version = "1", features = ["sqlite", "r2d2"] } tokio = { version = "1" } dotenv = "0.15" diff --git a/chartered-fs/Cargo.toml b/chartered-fs/Cargo.toml index c3fa827..c11a4b7 100644 --- a/chartered-fs/Cargo.toml +++ a/chartered-fs/Cargo.toml @@ -6,3 +6,6 @@ # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +async-trait = "0.1" +tokio = { version = "1", features = ["fs"] } +uuid = { version = "0.8", features = ["v4"] } diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml index 34b4f34..0803b55 100644 --- a/chartered-web/Cargo.toml +++ a/chartered-web/Cargo.toml @@ -7,9 +7,14 @@ [dependencies] chartered-db = { path = "../chartered-db" } +chartered-fs = { path = "../chartered-fs" } axum = "0.2" +bytes = "1" futures = "0.3" +nom = "7" +serde = { version = "1", features = ["derive"] } +serde_json = "1" tokio = { version = "1", features = ["full"] } tower = { version = "0.4", features = ["util", "filter"] } tower-http = { version = "0.1", features = ["trace"] } diff --git a/chartered-db/src/lib.rs b/chartered-db/src/lib.rs index ca673ad..4e28847 100644 --- a/chartered-db/src/lib.rs +++ a/chartered-db/src/lib.rs @@ -1,20 +1,15 @@ +pub mod schema; + #[macro_use] extern crate diesel; - -pub mod schema; +use diesel::{Associations, Identifiable, Queryable, insert_into, insert_or_ignore_into, prelude::*, r2d2::{ConnectionManager, Pool}}; +use schema::{crate_versions, crates}; use std::sync::Arc; -use self::diesel::prelude::*; -use diesel::{ - r2d2::{ConnectionManager, Pool}, - Associations, Identifiable, Queryable, -}; +pub type ConnectionPool = Arc>>; -use schema::crate_versions; -use schema::crates; - -pub fn init() -> Arc>> { +pub fn init() -> ConnectionPool { Arc::new(Pool::new(ConnectionManager::new("chartered.db")).unwrap()) } @@ -34,10 +29,7 @@ yanked: bool, } -pub async fn get_crate_versions( - conn: Arc>>, - crate_name: String, -) -> Vec { +pub async fn get_crate_versions(conn: ConnectionPool, crate_name: String) -> Vec { use crate::schema::crates::dsl::*; tokio::task::spawn_blocking(move || { @@ -52,6 +44,40 @@ .expect("no crate versions"); selected_crate_versions + }) + .await + .unwrap() +} + +pub async fn publish_crate( + conn: ConnectionPool, + crate_name: String, + version_string: String, + file_identifier: chartered_fs::FileReference, +) { + use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; + + tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + insert_or_ignore_into(crates) + .values(name.eq(&crate_name)) + .execute(&conn) + .unwrap(); + + let selected_crate = crates + .filter(name.eq(crate_name)) + .first::(&conn) + .unwrap(); + + insert_into(crate_versions) + .values(( + crate_id.eq(selected_crate.id), + version.eq(version_string), + filesystem_object.eq(file_identifier.to_string()), + )) + .execute(&conn) + .unwrap(); }) .await .unwrap() diff --git a/chartered-db/src/schema.rs b/chartered-db/src/schema.rs index 31bea00..ce4ffe3 100644 --- a/chartered-db/src/schema.rs +++ a/chartered-db/src/schema.rs @@ -17,7 +17,4 @@ joinable!(crate_versions -> crates (crate_id)); -allow_tables_to_appear_in_same_query!( - crate_versions, - crates, -); +allow_tables_to_appear_in_same_query!(crate_versions, crates,); diff --git a/chartered-fs/src/lib.rs b/chartered-fs/src/lib.rs index 31e1bb2..1c63ea9 100644 --- a/chartered-fs/src/lib.rs +++ a/chartered-fs/src/lib.rs @@ -1,3 +1,46 @@ +use async_trait::async_trait; +use tokio::{ + fs::File, + io::{AsyncReadExt, AsyncWriteExt}, +}; + +pub struct FileReference(uuid::Uuid); + +impl std::fmt::Display for FileReference { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[async_trait] +pub trait FileSystem { + async fn read(&self, file_ref: FileReference) -> Result, std::io::Error>; + async fn write(&self, data: &[u8]) -> Result; +} + +pub struct Local; + +#[async_trait] +impl FileSystem for Local { + async fn read(&self, file_ref: FileReference) -> Result, std::io::Error> { + let mut file = File::open(format!("/tmp/{}", file_ref.0)).await?; + + let mut contents = vec![]; + file.read_to_end(&mut contents).await?; + + Ok(contents) + } + + async fn write(&self, data: &[u8]) -> Result { + let uuid = uuid::Uuid::new_v4(); + + let mut file = File::create(format!("/tmp/{}", uuid)).await?; + file.write_all(data).await?; + + Ok(FileReference(uuid)) + } +} + #[cfg(test)] mod tests { #[test] diff --git a/chartered-git/src/main.rs b/chartered-git/src/main.rs index fb358e6..48bf033 100644 --- a/chartered-git/src/main.rs +++ a/chartered-git/src/main.rs @@ -172,8 +172,8 @@ let test_crate_file = PackFileEntry::Blob(br#"{"name":"charteredtest","vers":"1.0.0","deps":[],"cksum":"7b821735f0211fd00032a9892d1bf2323c9d05d9c59b9303eb382f5ec1898bfc","features":{},"yanked":false,"links":null}"#); let config_file = PackFileEntry::Blob( br#"{ - "dl": "http://127.0.0.1:8888/api/v1/crates", - "api": "http://127.0.0.1:8888" + "dl": "http://127.0.0.1:8888/a/abc/api/v1/crates", + "api": "http://127.0.0.1:8888/a/abc" }"#, ); diff --git a/chartered-web/src/main.rs b/chartered-web/src/main.rs index 1765168..2d85377 100644 --- a/chartered-web/src/main.rs +++ a/chartered-web/src/main.rs @@ -28,7 +28,7 @@ #[tokio::main] async fn main() { let api_authenticated = axum_box_after_every_route!(Router::new() - .route("/crates/new", put(hello_world)) + .route("/crates/new", put(endpoints::cargo_api::publish)) .route("/crates/search", get(hello_world)) .route("/crates/:crate/owners", get(hello_world)) .route("/crates/:crate/owners", put(hello_world)) @@ -55,9 +55,15 @@ let pool = chartered_db::init(); let app = Router::new() - .route("/", get(|| async move { - format!("{:#?}", chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await) - })) + .route( + "/", + get(|| async move { + format!( + "{:#?}", + chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await + ) + }), + ) .nest("/a/:key/api/v1", api_authenticated) .layer(middleware_stack); diff --git a/chartered-web/src/middleware/auth.rs b/chartered-web/src/middleware/auth.rs index a6a6154..e37cf0a 100644 --- a/chartered-web/src/middleware/auth.rs +++ a/chartered-web/src/middleware/auth.rs @@ -28,12 +28,12 @@ let mut inner = std::mem::replace(&mut self.0, clone); Box::pin(async move { - if true { - return Ok(Response::builder() - .status(StatusCode::UNAUTHORIZED) - .body(ResBody::default()) - .unwrap()); - } + // if true { + // return Ok(Response::builder() + // .status(StatusCode::UNAUTHORIZED) + // .body(ResBody::default()) + // .unwrap()); + // } let res: Response = inner.call(req).await?; diff --git a/chartered-web/src/endpoints/cargo_api/mod.rs b/chartered-web/src/endpoints/cargo_api/mod.rs index 4ca25db..113c927 100644 --- a/chartered-web/src/endpoints/cargo_api/mod.rs +++ a/chartered-web/src/endpoints/cargo_api/mod.rs @@ -1,3 +1,5 @@ mod download; +mod publish; pub use download::handle as download; +pub use publish::handle as publish; diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs new file mode 100644 index 0000000..d3b8335 100644 --- /dev/null +++ a/chartered-web/src/endpoints/cargo_api/publish.rs @@ -1,0 +1,85 @@ +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use std::convert::TryInto; + +#[derive(Serialize, Debug, Default)] +pub struct PublishCrateResponse { + warnings: PublishCrateResponseWarnings, +} + +#[derive(Serialize, Debug, Default)] +pub struct PublishCrateResponseWarnings { + invalid_categories: Vec, + invalid_badges: Vec, + other: Vec, +} + +pub async fn handle(body: Bytes) -> axum::response::Json { + use chartered_fs::FileSystem; + + let (_, (metadata_bytes, crate_bytes)) = parse(body.as_ref()).unwrap(); + + let metadata: Metadata = serde_json::from_slice(metadata_bytes).unwrap(); + + let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap(); + + chartered_db::publish_crate( + chartered_db::init(), + metadata.name.to_string(), + metadata.vers.to_string(), + file_ref, + ) + .await; + + axum::response::Json(PublishCrateResponse::default()) +} + +fn parse(body: &[u8]) -> nom::IResult<&[u8], (&[u8], &[u8])> { + use nom::{bytes::complete::take, combinator::map_res}; + use std::array::TryFromSliceError; + + let u32_from_le_bytes = + |b: &[u8]| Ok::<_, TryFromSliceError>(u32::from_le_bytes(b.try_into()?)); + let mut read_u32 = map_res(take(4usize), u32_from_le_bytes); + + let (rest, metadata_length) = read_u32(body)?; + let (rest, metadata_bytes) = take(metadata_length)(rest)?; + let (rest, crate_length) = read_u32(rest)?; + let (rest, crate_bytes) = take(crate_length)(rest)?; + + Ok((rest, (metadata_bytes, crate_bytes))) +} + +#[derive(Deserialize, Debug)] +pub struct Metadata<'a> { + name: &'a str, + vers: &'a str, + deps: Vec>, + features: std::collections::HashMap<&'a str, Vec<&'a str>>, + authors: Vec<&'a str>, + description: Option<&'a str>, + documentation: Option<&'a str>, + homepage: Option<&'a str>, + readme: Option<&'a str>, + readme_file: Option<&'a str>, + keywords: Vec<&'a str>, + categories: Vec<&'a str>, + license: Option<&'a str>, + license_file: Option<&'a str>, + repository: Option<&'a str>, + links: Option<&'a str>, +} + +#[derive(Deserialize, Debug)] +pub struct MetadataDependency<'a> { + name: &'a str, + version_req: &'a str, + features: Vec<&'a str>, + optional: bool, + default_features: bool, + target: Option<&'a str>, + kind: &'a str, // 'dev', 'build', or 'normal' + registry: &'a str, + explicit_name_in_toml: Option<&'a str>, +} + -- rgit 0.1.3