From 35bab4d4b5fc2e469442dde60f96ff9b1e6684cc Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Mon, 06 Sep 2021 23:19:43 +0100 Subject: [PATCH] Crate downloading works --- Cargo.lock | 4 ++++ chartered-fs/Cargo.toml | 8 ++++++-- chartered-web/Cargo.toml | 2 ++ chartered-db/src/lib.rs | 28 ++++++++++++++++++++++++++++ chartered-db/src/schema.rs | 6 +++++- chartered-fs/src/lib.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- chartered-git/src/main.rs | 2 +- migrations/2021-08-31-214501_create_crates_table/up.sql | 4 +--- chartered-web/src/endpoints/cargo_api/download.rs | 16 +++++++++++++++- chartered-web/src/endpoints/cargo_api/publish.rs | 3 ++- 10 files changed, 134 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9779f0a..aa663ef 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -198,6 +198,7 @@ version = "0.1.0" dependencies = [ "async-trait", + "serde", "tokio", "uuid", ] @@ -237,9 +238,11 @@ "chartered-db", "chartered-fs", "futures", + "hex", "nom", "serde", "serde_json", + "sha2", "tokio", "tower", "tower-http", @@ -1509,6 +1512,7 @@ checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom", + "serde", ] [[package]] diff --git a/chartered-fs/Cargo.toml b/chartered-fs/Cargo.toml index c11a4b7..5467201 100644 --- a/chartered-fs/Cargo.toml +++ a/chartered-fs/Cargo.toml @@ -7,5 +7,9 @@ [dependencies] async-trait = "0.1" -tokio = { version = "1", features = ["fs"] } -uuid = { version = "0.8", features = ["v4"] } +serde = { version = "1", features = ["derive"] } +tokio = { version = "1", features = ["fs", "io-util"] } +uuid = { version = "0.8", features = ["v4", "serde"] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros"] }diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml index 0803b55..92a92a0 100644 --- a/chartered-web/Cargo.toml +++ a/chartered-web/Cargo.toml @@ -12,9 +12,11 @@ axum = "0.2" bytes = "1" futures = "0.3" +hex = "0.4" nom = "7" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.9" tokio = { version = "1", features = ["full"] } tower = { version = "0.4", features = ["util", "filter"] } tower-http = { version = "0.1", features = ["trace"] } diff --git a/chartered-db/src/lib.rs b/chartered-db/src/lib.rs index c95e1a2..a8794bb 100644 --- a/chartered-db/src/lib.rs +++ a/chartered-db/src/lib.rs @@ -33,6 +33,7 @@ pub version: String, pub filesystem_object: String, pub yanked: bool, + pub checksum: String, } pub async fn get_crate_versions(conn: ConnectionPool, crate_name: String) -> Vec { @@ -54,6 +55,31 @@ .unwrap() } +pub async fn get_specific_crate_version( + conn: ConnectionPool, + crate_name: String, + crate_version: String, +) -> Option { + use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; + + tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + let selected_crate = crates + .filter(name.eq(crate_name)) + .first::(&conn) + .expect("no crate"); + + CrateVersion::belonging_to(&selected_crate) + .filter(version.eq(crate_version)) + .get_result::(&conn) + .optional() + .expect("no crate version") + }) + .await + .unwrap() +} + pub async fn get_crates(conn: ConnectionPool) -> HashMap> { tokio::task::spawn_blocking(move || { let conn = conn.get().unwrap(); @@ -74,6 +100,7 @@ crate_name: String, version_string: String, file_identifier: chartered_fs::FileReference, + file_checksum: String, ) { use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; @@ -95,6 +122,7 @@ crate_id.eq(selected_crate.id), version.eq(version_string), filesystem_object.eq(file_identifier.to_string()), + checksum.eq(file_checksum), )) .execute(&conn) .unwrap(); diff --git a/chartered-db/src/schema.rs b/chartered-db/src/schema.rs index ce4ffe3..ab84f14 100644 --- a/chartered-db/src/schema.rs +++ a/chartered-db/src/schema.rs @@ -5,6 +5,7 @@ version -> Text, filesystem_object -> Text, yanked -> Bool, + checksum -> Text, } } @@ -17,4 +18,7 @@ joinable!(crate_versions -> crates (crate_id)); -allow_tables_to_appear_in_same_query!(crate_versions, crates,); +allow_tables_to_appear_in_same_query!( + crate_versions, + crates, +); diff --git a/chartered-fs/src/lib.rs b/chartered-fs/src/lib.rs index 1c63ea9..cbbd41f 100644 --- a/chartered-fs/src/lib.rs +++ a/chartered-fs/src/lib.rs @@ -1,29 +1,87 @@ use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use tokio::{ fs::File, io::{AsyncReadExt, AsyncWriteExt}, }; -pub struct FileReference(uuid::Uuid); +#[derive(Debug, Serialize, Deserialize)] +pub enum FileSystemKind { + Local, +} + +impl std::fmt::Display for FileSystemKind { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Local => f.write_str("local"), + } + } +} + +impl std::str::FromStr for FileSystemKind { + type Err = std::io::Error; + + fn from_str(s: &str) -> Result { + match s { + "local" => Ok(Self::Local), + _ => Err(std::io::Error::new( + std::io::ErrorKind::Other, + "unknown filesystemkind", + )), + } + } +} +#[derive(Debug, Serialize, Deserialize)] +pub struct FileReference { + file_system: FileSystemKind, + reference: uuid::Uuid, +} + impl std::fmt::Display for FileReference { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{}", self.0) + write!(f, "{}:{}", self.file_system, self.reference) + } +} + +impl std::str::FromStr for FileReference { + type Err = std::io::Error; + + fn from_str(s: &str) -> Result { + let mut split = s.splitn(2, ':'); + let file_system = FileSystemKind::from_str(split.next().unwrap_or_default())?; + let reference = uuid::Uuid::from_str(split.next().unwrap_or_default()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + Ok(FileReference { + file_system, + reference, + }) } } #[async_trait] pub trait FileSystem { + const KIND: FileSystemKind; + async fn read(&self, file_ref: FileReference) -> Result, std::io::Error>; async fn write(&self, data: &[u8]) -> Result; + + fn create_ref() -> FileReference { + FileReference { + file_system: Self::KIND, + reference: uuid::Uuid::new_v4(), + } + } } pub struct Local; #[async_trait] impl FileSystem for Local { + const KIND: FileSystemKind = FileSystemKind::Local; + async fn read(&self, file_ref: FileReference) -> Result, std::io::Error> { - let mut file = File::open(format!("/tmp/{}", file_ref.0)).await?; + let mut file = File::open(format!("/tmp/{}", file_ref.reference)).await?; let mut contents = vec![]; file.read_to_end(&mut contents).await?; @@ -32,19 +90,23 @@ } async fn write(&self, data: &[u8]) -> Result { - let uuid = uuid::Uuid::new_v4(); + let file_ref = Self::create_ref(); - let mut file = File::create(format!("/tmp/{}", uuid)).await?; + let mut file = File::create(format!("/tmp/{}", file_ref.reference)).await?; file.write_all(data).await?; - Ok(FileReference(uuid)) + Ok(file_ref) } } #[cfg(test)] mod tests { - #[test] - fn it_works() { - assert_eq!(2 + 2, 4); + use super::FileSystem; + + #[tokio::test] + async fn local() { + let fs = super::Local; + let file_ref = fs.write(b"abcdef").await.unwrap(); + assert_eq!(fs.read(file_ref).await.unwrap(), b"abcdef"); } } diff --git a/chartered-git/src/main.rs b/chartered-git/src/main.rs index 71cafd5..5c96d91 100644 --- a/chartered-git/src/main.rs +++ a/chartered-git/src/main.rs @@ -297,7 +297,7 @@ name: &crate_def.name, vers: &version.version, deps: &[], - cksum: "cool-checksum-dude", + cksum: &version.checksum, features: BTreeMap::new(), yanked: version.yanked, links: None, diff --git a/migrations/2021-08-31-214501_create_crates_table/up.sql b/migrations/2021-08-31-214501_create_crates_table/up.sql index 3c766a0..a1f3ae1 100644 --- a/migrations/2021-08-31-214501_create_crates_table/up.sql +++ a/migrations/2021-08-31-214501_create_crates_table/up.sql @@ -9,9 +9,7 @@ version VARCHAR(255) NOT NULL, filesystem_object VARCHAR(255) NOT NULL, yanked BOOLEAN NOT NULL DEFAULT FALSE, + checksum VARCHAR(255) NOT NULL, UNIQUE (crate_id, version), FOREIGN KEY (crate_id) REFERENCES crates (id) ); - -INSERT INTO crates VALUES (1, "cool-test-crate"); -INSERT INTO crate_versions VALUES (1, 1, "1.0.0", "cool-object", FALSE);diff --git a/chartered-web/src/endpoints/cargo_api/download.rs b/chartered-web/src/endpoints/cargo_api/download.rs index a03b66a..c72773b 100644 --- a/chartered-web/src/endpoints/cargo_api/download.rs +++ a/chartered-web/src/endpoints/cargo_api/download.rs @@ -1,3 +1,15 @@ -pub async fn handle() -> &'static str { - "download" +use axum::extract; +use chartered_fs::FileSystem; +use std::str::FromStr; + +pub async fn handle( + extract::Path((_api_key, name, version)): extract::Path<(String, String, String)>, +) -> Vec { + let version = chartered_db::get_specific_crate_version(chartered_db::init(), name, version) + .await + .unwrap(); + + let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap(); + + chartered_fs::Local.read(file_ref).await.unwrap() } diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs index d3b8335..de08d6a 100644 --- a/chartered-web/src/endpoints/cargo_api/publish.rs +++ a/chartered-web/src/endpoints/cargo_api/publish.rs @@ -16,6 +16,7 @@ pub async fn handle(body: Bytes) -> axum::response::Json { use chartered_fs::FileSystem; + use sha2::{Digest, Sha256}; let (_, (metadata_bytes, crate_bytes)) = parse(body.as_ref()).unwrap(); @@ -28,6 +29,7 @@ metadata.name.to_string(), metadata.vers.to_string(), file_ref, + hex::encode(Sha256::digest(crate_bytes)), ) .await; @@ -82,4 +84,3 @@ registry: &'a str, explicit_name_in_toml: Option<&'a str>, } - -- rgit 0.1.3