From df2c19ea2fcc0efddbda43cb08e6fbd0cf288ffd Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Tue, 07 Sep 2021 13:09:17 +0100 Subject: [PATCH] Refactor database code & introduce user tables --- Cargo.lock | 16 ++++++++++++++++ chartered-db/Cargo.toml | 4 +++- chartered-web/Cargo.toml | 3 +++ chartered-db/src/crates.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ chartered-db/src/lib.rs | 144 +++++++++++++++++++++++--------------------------------------------------------- chartered-db/src/schema.rs | 40 ++++++++++++++++++++++++++++++++++++++++ chartered-db/src/users.rs | 33 +++++++++++++++++++++++++++++++++ chartered-git/src/main.rs | 6 ++++-- chartered-web/src/main.rs | 21 ++++++++------------- migrations/2021-08-31-214501_create_crates_table/down.sql | 4 ++++ migrations/2021-08-31-214501_create_crates_table/up.sql | 29 +++++++++++++++++++++++++++++ chartered-web/src/endpoints/mod.rs | 40 ++++++++++++++++++++++++++++++++++++++++ chartered-web/src/endpoints/cargo_api/download.rs | 26 +++++++++++++++++++++----- chartered-web/src/endpoints/cargo_api/publish.rs | 11 ++++++++--- 14 files changed, 348 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa663ef..c1ff201 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -188,8 +188,10 @@ dependencies = [ "chartered-fs", "diesel", + "displaydoc", "dotenv", "itertools", + "thiserror", "tokio", ] @@ -237,12 +239,15 @@ "bytes", "chartered-db", "chartered-fs", + "env_logger", "futures", "hex", + "log", "nom", "serde", "serde_json", "sha2", + "thiserror", "tokio", "tower", "tower-http", @@ -394,6 +399,17 @@ "libc", "redox_users", "winapi", +] + +[[package]] +name = "displaydoc" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] diff --git a/chartered-db/Cargo.toml b/chartered-db/Cargo.toml index 19c613d..f22b0af 100644 --- a/chartered-db/Cargo.toml +++ a/chartered-db/Cargo.toml @@ -9,6 +9,8 @@ chartered-fs = { path = "../chartered-fs" } diesel = { version = "1", features = ["sqlite", "r2d2"] } +displaydoc = "0.2" itertools = "0.10" -tokio = { version = "1" } +thiserror = "1" +tokio = "1" dotenv = "0.15" diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml index 92a92a0..a664606 100644 --- a/chartered-web/Cargo.toml +++ a/chartered-web/Cargo.toml @@ -11,12 +11,15 @@ axum = "0.2" bytes = "1" +env_logger = "0.9" futures = "0.3" hex = "0.4" +log = "0.4" nom = "7" serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.9" +thiserror = "1" tokio = { version = "1", features = ["full"] } tower = { version = "0.4", features = ["util", "filter"] } tower-http = { version = "0.1", features = ["trace"] } diff --git a/chartered-db/src/crates.rs b/chartered-db/src/crates.rs new file mode 100644 index 0000000..2187d6b 100644 --- /dev/null +++ a/chartered-db/src/crates.rs @@ -1,0 +1,120 @@ +use super::{ + schema::{crate_versions, crates}, + ConnectionPool, Result, +}; +use diesel::{ + insert_into, insert_or_ignore_into, prelude::*, Associations, Identifiable, Queryable, +}; +use itertools::Itertools; +use std::{collections::HashMap, sync::Arc}; + +#[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)] +pub struct Crate { + pub id: i32, + pub name: String, +} + +impl Crate { + pub async fn all_with_versions( + conn: ConnectionPool, + ) -> Result>> { + tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + let crate_versions = crates::table + .inner_join(crate_versions::table) + .load(&conn)?; + + Ok(crate_versions.into_iter().into_grouping_map().collect()) + }) + .await? + } + + pub async fn find_by_name(conn: ConnectionPool, crate_name: String) -> Result> { + use crate::schema::crates::dsl::*; + + Ok(tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + crates + .filter(name.eq(crate_name)) + .first::(&conn) + .optional() + }) + .await??) + } + + pub async fn versions(self: Arc, conn: ConnectionPool) -> Result> { + Ok(tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + CrateVersion::belonging_to(&*self).load::(&conn) + }) + .await??) + } + + pub async fn version( + self: Arc, + conn: ConnectionPool, + crate_version: String, + ) -> Result> { + use crate::schema::crate_versions::*; + + Ok(tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + CrateVersion::belonging_to(&*self) + .filter(version.eq(crate_version)) + .get_result::(&conn) + .optional() + }) + .await??) + } +} + +#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] +#[belongs_to(Crate)] +pub struct CrateVersion { + pub id: i32, + pub crate_id: i32, + pub version: String, + pub filesystem_object: String, + pub yanked: bool, + pub checksum: String, +} + +pub async fn publish_crate( + conn: ConnectionPool, + crate_name: String, + version_string: String, + file_identifier: chartered_fs::FileReference, + file_checksum: String, +) { + use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; + + tokio::task::spawn_blocking(move || { + let conn = conn.get().unwrap(); + + insert_or_ignore_into(crates) + .values(name.eq(&crate_name)) + .execute(&conn) + .unwrap(); + + let selected_crate = crates + .filter(name.eq(crate_name)) + .first::(&conn) + .unwrap(); + + insert_into(crate_versions) + .values(( + crate_id.eq(selected_crate.id), + version.eq(version_string), + filesystem_object.eq(file_identifier.to_string()), + checksum.eq(file_checksum), + )) + .execute(&conn) + .unwrap(); + }) + .await + .unwrap() +} diff --git a/chartered-db/src/lib.rs b/chartered-db/src/lib.rs index a8794bb..cadecd3 100644 --- a/chartered-db/src/lib.rs +++ a/chartered-db/src/lib.rs @@ -1,134 +1,30 @@ +pub mod crates; pub mod schema; +pub mod users; #[macro_use] extern crate diesel; -use diesel::{ - insert_into, insert_or_ignore_into, - prelude::*, - r2d2::{ConnectionManager, Pool}, - Associations, Identifiable, Queryable, -}; -use itertools::Itertools; -use schema::{crate_versions, crates}; -use std::{collections::HashMap, sync::Arc}; +use diesel::r2d2::{ConnectionManager, Pool}; +use displaydoc::Display; +use std::sync::Arc; +use thiserror::Error; pub type ConnectionPool = Arc>>; - -pub fn init() -> ConnectionPool { - Arc::new(Pool::new(ConnectionManager::new("chartered.db")).unwrap()) -} - -#[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)] -pub struct Crate { - pub id: i32, - pub name: String, -} - -#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] -#[belongs_to(Crate)] -pub struct CrateVersion { - pub id: i32, - pub crate_id: i32, - pub version: String, - pub filesystem_object: String, - pub yanked: bool, - pub checksum: String, -} - -pub async fn get_crate_versions(conn: ConnectionPool, crate_name: String) -> Vec { - use crate::schema::crates::dsl::*; - - tokio::task::spawn_blocking(move || { - let conn = conn.get().unwrap(); - - let selected_crate = crates - .filter(name.eq(crate_name)) - .first::(&conn) - .expect("no crate"); - - CrateVersion::belonging_to(&selected_crate) - .load::(&conn) - .expect("no crate versions") - }) - .await - .unwrap() -} - -pub async fn get_specific_crate_version( - conn: ConnectionPool, - crate_name: String, - crate_version: String, -) -> Option { - use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; - - tokio::task::spawn_blocking(move || { - let conn = conn.get().unwrap(); - - let selected_crate = crates - .filter(name.eq(crate_name)) - .first::(&conn) - .expect("no crate"); - - CrateVersion::belonging_to(&selected_crate) - .filter(version.eq(crate_version)) - .get_result::(&conn) - .optional() - .expect("no crate version") - }) - .await - .unwrap() -} - -pub async fn get_crates(conn: ConnectionPool) -> HashMap> { - tokio::task::spawn_blocking(move || { - let conn = conn.get().unwrap(); - - let crate_versions = crates::table - .inner_join(crate_versions::table) - .load(&conn) - .unwrap(); - - crate_versions.into_iter().into_grouping_map().collect() - }) - .await - .unwrap() -} - -pub async fn publish_crate( - conn: ConnectionPool, - crate_name: String, - version_string: String, - file_identifier: chartered_fs::FileReference, - file_checksum: String, -) { - use crate::schema::{crate_versions::dsl::*, crates::dsl::*}; - - tokio::task::spawn_blocking(move || { - let conn = conn.get().unwrap(); - - insert_or_ignore_into(crates) - .values(name.eq(&crate_name)) - .execute(&conn) - .unwrap(); - - let selected_crate = crates - .filter(name.eq(crate_name)) - .first::(&conn) - .unwrap(); - - insert_into(crate_versions) - .values(( - crate_id.eq(selected_crate.id), - version.eq(version_string), - filesystem_object.eq(file_identifier.to_string()), - checksum.eq(file_checksum), - )) - .execute(&conn) - .unwrap(); - }) - .await - .unwrap() +pub type Result = std::result::Result; + +pub fn init() -> Result { + Ok(Arc::new(Pool::new(ConnectionManager::new("chartered.db"))?)) +} + +#[derive(Error, Display, Debug)] +pub enum Error { + /// Failed to initialise to database connection pool: `{0}` + Connection(#[from] diesel::r2d2::PoolError), + /// Failed to run query: `{0}` + Query(#[from] diesel::result::Error), + /// Failed to complete query task: `{0}` + TaskJoin(#[from] tokio::task::JoinError), } #[cfg(test)] diff --git a/chartered-db/src/schema.rs b/chartered-db/src/schema.rs index ab84f14..fb96e7e 100644 --- a/chartered-db/src/schema.rs +++ a/chartered-db/src/schema.rs @@ -16,9 +16,49 @@ } } +table! { + user_api_keys (id) { + id -> Integer, + user_id -> Integer, + api_key -> Text, + } +} + +table! { + user_crate_permissions (id) { + id -> Integer, + user_id -> Integer, + crate_id -> Integer, + permissions -> Integer, + } +} + +table! { + user_ssh_keys (id) { + id -> Integer, + user_id -> Integer, + ssh_key -> Binary, + } +} + +table! { + users (id) { + id -> Integer, + username -> Integer, + } +} + joinable!(crate_versions -> crates (crate_id)); +joinable!(user_api_keys -> users (user_id)); +joinable!(user_crate_permissions -> crates (crate_id)); +joinable!(user_crate_permissions -> users (user_id)); +joinable!(user_ssh_keys -> users (user_id)); allow_tables_to_appear_in_same_query!( crate_versions, crates, + user_api_keys, + user_crate_permissions, + user_ssh_keys, + users, ); diff --git a/chartered-db/src/users.rs b/chartered-db/src/users.rs new file mode 100644 index 0000000..f9c0bc0 100644 --- /dev/null +++ a/chartered-db/src/users.rs @@ -1,0 +1,33 @@ +use super::schema::{user_api_keys, user_crate_permissions, user_ssh_keys, users}; +use diesel::{Associations, Identifiable, Queryable}; + +#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)] +pub struct User { + id: i32, + username: String, +} + +#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)] +#[belongs_to(User)] +pub struct UserApiKey { + id: i32, + user_id: i32, + api_key: String, +} + +#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)] +#[belongs_to(User)] +pub struct UserCratePermission { + id: i32, + user_id: i32, + crate_id: i32, + permissions: i32, +} + +#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)] +#[belongs_to(User)] +pub struct UserSshKey { + id: i32, + user_id: i32, + ssh_key: Vec, +} diff --git a/chartered-git/src/main.rs b/chartered-git/src/main.rs index 5c96d91..d0fb4ef 100644 --- a/chartered-git/src/main.rs +++ a/chartered-git/src/main.rs @@ -30,7 +30,7 @@ let config = Arc::new(config); let server = Server { - db: chartered_db::init(), + db: chartered_db::init().unwrap(), }; thrussh::server::run(config, "127.0.0.1:2233", server) @@ -280,10 +280,12 @@ async fn fetch_tree( db: chartered_db::ConnectionPool, ) -> BTreeMap<[u8; 2], BTreeMap<[u8; 2], BTreeMap>> { + use chartered_db::crates::Crate; + let mut tree: BTreeMap<[u8; 2], BTreeMap<[u8; 2], BTreeMap>> = BTreeMap::new(); // todo: handle files with 1/2/3 characters - for (crate_def, versions) in chartered_db::get_crates(db).await { + for (crate_def, versions) in Crate::all_with_versions(db).await.unwrap() { let mut name_chars = crate_def.name.as_bytes().iter(); let first_dir = [*name_chars.next().unwrap(), *name_chars.next().unwrap()]; let second_dir = [*name_chars.next().unwrap(), *name_chars.next().unwrap()]; diff --git a/chartered-web/src/main.rs b/chartered-web/src/main.rs index 2d85377..46d41ff 100644 --- a/chartered-web/src/main.rs +++ a/chartered-web/src/main.rs @@ -1,9 +1,9 @@ mod endpoints; mod middleware; use axum::{ handler::{delete, get, put}, - Router, + AddExtensionLayer, Router, }; use tower::{filter::AsyncFilterLayer, ServiceBuilder}; @@ -27,6 +27,10 @@ #[tokio::main] async fn main() { + env_logger::init(); + + let pool = chartered_db::init().unwrap(); + let api_authenticated = axum_box_after_every_route!(Router::new() .route("/crates/new", put(endpoints::cargo_api::publish)) .route("/crates/search", get(hello_world)) @@ -43,7 +47,8 @@ ServiceBuilder::new() .layer_fn(middleware::auth::AuthMiddleware) .into_inner(), - ); + ) + .layer(AddExtensionLayer::new(pool)); let middleware_stack = ServiceBuilder::new() .layer(AsyncFilterLayer::new(|req| async { @@ -52,18 +57,8 @@ })) .into_inner(); - let pool = chartered_db::init(); - let app = Router::new() - .route( - "/", - get(|| async move { - format!( - "{:#?}", - chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await - ) - }), - ) + .route("/", get(hello_world)) .nest("/a/:key/api/v1", api_authenticated) .layer(middleware_stack); diff --git a/migrations/2021-08-31-214501_create_crates_table/down.sql b/migrations/2021-08-31-214501_create_crates_table/down.sql index d012c2a..075945e 100644 --- a/migrations/2021-08-31-214501_create_crates_table/down.sql +++ a/migrations/2021-08-31-214501_create_crates_table/down.sql @@ -1,2 +1,6 @@ DROP TABLE crates; DROP TABLE crate_versions; +DROP TABLE users; +DROP TABLE user_ssh_keys; +DROP TABLE user_api_keys; +DROP TABLE user_crate_permissions; diff --git a/migrations/2021-08-31-214501_create_crates_table/up.sql b/migrations/2021-08-31-214501_create_crates_table/up.sql index a1f3ae1..4d27fb8 100644 --- a/migrations/2021-08-31-214501_create_crates_table/up.sql +++ a/migrations/2021-08-31-214501_create_crates_table/up.sql @@ -13,3 +13,32 @@ UNIQUE (crate_id, version), FOREIGN KEY (crate_id) REFERENCES crates (id) ); + +CREATE TABLE users ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + username INTEGER NOT NULL UNIQUE +); + +CREATE TABLE user_ssh_keys ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + ssh_key BLOB NOT NULL, + FOREIGN KEY (user_id) REFERENCES users (id) +); + +CREATE TABLE user_api_keys ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + api_key VARCHAR(255) NOT NULL UNIQUE, + FOREIGN KEY (user_id) REFERENCES users (id) +); + +CREATE TABLE user_crate_permissions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + user_id INTEGER NOT NULL, + crate_id INTEGER NOT NULL, + permissions INTEGER NOT NULL, + UNIQUE (user_id, crate_id), + FOREIGN KEY (user_id) REFERENCES users (id) + FOREIGN KEY (crate_id) REFERENCES crates (id) +); diff --git a/chartered-web/src/endpoints/mod.rs b/chartered-web/src/endpoints/mod.rs index 168c1af..b853cc8 100644 --- a/chartered-web/src/endpoints/mod.rs +++ a/chartered-web/src/endpoints/mod.rs @@ -1,1 +1,41 @@ +macro_rules! define_error { + ($($kind:ident$(($inner_name:ident: $inner:ty))? => $status:ident / $public_text:expr,)*) => { + #[derive(thiserror::Error, Debug)] + pub enum Error { + $($kind$((#[from] $inner))?),* + } + + /// a (web-safe) explanation of the error, this shouldn't reveal internal details that + /// may be sensitive + impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + $(Self::$kind$(($inner_name))? => f.write_str($public_text)),* + } + } + } + + impl axum::response::IntoResponse for Error { + type Body = axum::body::Body; + type BodyError = ::Error; + + fn into_response(self) -> axum::http::Response { + log::error!("Failed to handle request: {:?}", self); + + let (status, body) = match self { + $(Self::$kind$(($inner_name))? => ( + axum::http::StatusCode::$status, + $public_text.into(), + )),* + }; + + axum::http::Response::builder() + .status(status) + .body(body) + .unwrap() + } + } + }; +} + pub mod cargo_api; diff --git a/chartered-web/src/endpoints/cargo_api/download.rs b/chartered-web/src/endpoints/cargo_api/download.rs index c72773b..828a864 100644 --- a/chartered-web/src/endpoints/cargo_api/download.rs +++ a/chartered-web/src/endpoints/cargo_api/download.rs @@ -1,15 +1,29 @@ use axum::extract; +use chartered_db::{crates::Crate, ConnectionPool}; use chartered_fs::FileSystem; -use std::str::FromStr; +use std::{str::FromStr, sync::Arc}; +define_error!( + Database(_e: chartered_db::Error) => INTERNAL_SERVER_ERROR / "Failed to query database", + File(_e: std::io::Error) => INTERNAL_SERVER_ERROR / "Failed to fetch crate file", + NoVersion => NOT_FOUND / "That requested version does not exist for the crate", + NoCrate => NOT_FOUND / "The requested crate does not exist", +); + pub async fn handle( extract::Path((_api_key, name, version)): extract::Path<(String, String, String)>, -) -> Vec { - let version = chartered_db::get_specific_crate_version(chartered_db::init(), name, version) - .await - .unwrap(); + extract::Extension(db): extract::Extension, +) -> Result, Error> { + let c = Crate::find_by_name(db.clone(), name) + .await? + .ok_or(Error::NoCrate)?; + + let version = Arc::new(c) + .version(db, version) + .await? + .ok_or(Error::NoVersion)?; let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap(); - chartered_fs::Local.read(file_ref).await.unwrap() + Ok(chartered_fs::Local.read(file_ref).await?) } diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs index de08d6a..18d677c 100644 --- a/chartered-web/src/endpoints/cargo_api/publish.rs +++ a/chartered-web/src/endpoints/cargo_api/publish.rs @@ -1,4 +1,6 @@ +use axum::extract; use bytes::Bytes; +use chartered_db::ConnectionPool; use serde::{Deserialize, Serialize}; use std::convert::TryInto; @@ -14,7 +16,10 @@ other: Vec, } -pub async fn handle(body: Bytes) -> axum::response::Json { +pub async fn handle( + extract::Extension(db): extract::Extension, + body: Bytes, +) -> axum::response::Json { use chartered_fs::FileSystem; use sha2::{Digest, Sha256}; @@ -24,8 +29,8 @@ let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap(); - chartered_db::publish_crate( - chartered_db::init(), + chartered_db::crates::publish_crate( + db, metadata.name.to_string(), metadata.vers.to_string(), file_ref, -- rgit 0.1.3