Refactor database code & introduce user tables
Diff
Cargo.lock | 16 ++++++++++++++++
chartered-db/Cargo.toml | 4 +++-
chartered-web/Cargo.toml | 3 +++
chartered-db/src/crates.rs | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
chartered-db/src/lib.rs | 144 +++++++++++++++++++++++---------------------------------------------------------
chartered-db/src/schema.rs | 40 ++++++++++++++++++++++++++++++++++++++++
chartered-db/src/users.rs | 33 +++++++++++++++++++++++++++++++++
chartered-git/src/main.rs | 6 ++++--
chartered-web/src/main.rs | 21 ++++++++-------------
migrations/2021-08-31-214501_create_crates_table/down.sql | 4 ++++
migrations/2021-08-31-214501_create_crates_table/up.sql | 29 +++++++++++++++++++++++++++++
chartered-web/src/endpoints/mod.rs | 40 ++++++++++++++++++++++++++++++++++++++++
chartered-web/src/endpoints/cargo_api/download.rs | 26 +++++++++++++++++++++-----
chartered-web/src/endpoints/cargo_api/publish.rs | 11 ++++++++---
14 files changed, 348 insertions(+), 149 deletions(-)
@@ -188,8 +188,10 @@
dependencies = [
"chartered-fs",
"diesel",
"displaydoc",
"dotenv",
"itertools",
"thiserror",
"tokio",
]
@@ -237,12 +239,15 @@
"bytes",
"chartered-db",
"chartered-fs",
"env_logger",
"futures",
"hex",
"log",
"nom",
"serde",
"serde_json",
"sha2",
"thiserror",
"tokio",
"tower",
"tower-http",
@@ -394,6 +399,17 @@
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "displaydoc"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -9,6 +9,8 @@
chartered-fs = { path = "../chartered-fs" }
diesel = { version = "1", features = ["sqlite", "r2d2"] }
displaydoc = "0.2"
itertools = "0.10"
tokio = { version = "1" }
thiserror = "1"
tokio = "1"
dotenv = "0.15"
@@ -11,12 +11,15 @@
axum = "0.2"
bytes = "1"
env_logger = "0.9"
futures = "0.3"
hex = "0.4"
log = "0.4"
nom = "7"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.9"
thiserror = "1"
tokio = { version = "1", features = ["full"] }
tower = { version = "0.4", features = ["util", "filter"] }
tower-http = { version = "0.1", features = ["trace"] }
@@ -1,0 +1,120 @@
use super::{
schema::{crate_versions, crates},
ConnectionPool, Result,
};
use diesel::{
insert_into, insert_or_ignore_into, prelude::*, Associations, Identifiable, Queryable,
};
use itertools::Itertools;
use std::{collections::HashMap, sync::Arc};
#[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)]
pub struct Crate {
pub id: i32,
pub name: String,
}
impl Crate {
pub async fn all_with_versions(
conn: ConnectionPool,
) -> Result<HashMap<Crate, Vec<CrateVersion>>> {
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
let crate_versions = crates::table
.inner_join(crate_versions::table)
.load(&conn)?;
Ok(crate_versions.into_iter().into_grouping_map().collect())
})
.await?
}
pub async fn find_by_name(conn: ConnectionPool, crate_name: String) -> Result<Option<Self>> {
use crate::schema::crates::dsl::*;
Ok(tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.optional()
})
.await??)
}
pub async fn versions(self: Arc<Self>, conn: ConnectionPool) -> Result<Vec<CrateVersion>> {
Ok(tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
CrateVersion::belonging_to(&*self).load::<CrateVersion>(&conn)
})
.await??)
}
pub async fn version(
self: Arc<Self>,
conn: ConnectionPool,
crate_version: String,
) -> Result<Option<CrateVersion>> {
use crate::schema::crate_versions::*;
Ok(tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
CrateVersion::belonging_to(&*self)
.filter(version.eq(crate_version))
.get_result::<CrateVersion>(&conn)
.optional()
})
.await??)
}
}
#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)]
#[belongs_to(Crate)]
pub struct CrateVersion {
pub id: i32,
pub crate_id: i32,
pub version: String,
pub filesystem_object: String,
pub yanked: bool,
pub checksum: String,
}
pub async fn publish_crate(
conn: ConnectionPool,
crate_name: String,
version_string: String,
file_identifier: chartered_fs::FileReference,
file_checksum: String,
) {
use crate::schema::{crate_versions::dsl::*, crates::dsl::*};
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
insert_or_ignore_into(crates)
.values(name.eq(&crate_name))
.execute(&conn)
.unwrap();
let selected_crate = crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.unwrap();
insert_into(crate_versions)
.values((
crate_id.eq(selected_crate.id),
version.eq(version_string),
filesystem_object.eq(file_identifier.to_string()),
checksum.eq(file_checksum),
))
.execute(&conn)
.unwrap();
})
.await
.unwrap()
}
@@ -1,134 +1,30 @@
pub mod crates;
pub mod schema;
pub mod users;
#[macro_use]
extern crate diesel;
use diesel::{
insert_into, insert_or_ignore_into,
prelude::*,
r2d2::{ConnectionManager, Pool},
Associations, Identifiable, Queryable,
};
use itertools::Itertools;
use schema::{crate_versions, crates};
use std::{collections::HashMap, sync::Arc};
use diesel::r2d2::{ConnectionManager, Pool};
use displaydoc::Display;
use std::sync::Arc;
use thiserror::Error;
pub type ConnectionPool = Arc<Pool<ConnectionManager<diesel::SqliteConnection>>>;
pub fn init() -> ConnectionPool {
Arc::new(Pool::new(ConnectionManager::new("chartered.db")).unwrap())
}
#[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)]
pub struct Crate {
pub id: i32,
pub name: String,
}
#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)]
#[belongs_to(Crate)]
pub struct CrateVersion {
pub id: i32,
pub crate_id: i32,
pub version: String,
pub filesystem_object: String,
pub yanked: bool,
pub checksum: String,
}
pub async fn get_crate_versions(conn: ConnectionPool, crate_name: String) -> Vec<CrateVersion> {
use crate::schema::crates::dsl::*;
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
let selected_crate = crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.expect("no crate");
CrateVersion::belonging_to(&selected_crate)
.load::<CrateVersion>(&conn)
.expect("no crate versions")
})
.await
.unwrap()
}
pub async fn get_specific_crate_version(
conn: ConnectionPool,
crate_name: String,
crate_version: String,
) -> Option<CrateVersion> {
use crate::schema::{crate_versions::dsl::*, crates::dsl::*};
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
let selected_crate = crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.expect("no crate");
CrateVersion::belonging_to(&selected_crate)
.filter(version.eq(crate_version))
.get_result::<CrateVersion>(&conn)
.optional()
.expect("no crate version")
})
.await
.unwrap()
}
pub async fn get_crates(conn: ConnectionPool) -> HashMap<Crate, Vec<CrateVersion>> {
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
let crate_versions = crates::table
.inner_join(crate_versions::table)
.load(&conn)
.unwrap();
crate_versions.into_iter().into_grouping_map().collect()
})
.await
.unwrap()
}
pub async fn publish_crate(
conn: ConnectionPool,
crate_name: String,
version_string: String,
file_identifier: chartered_fs::FileReference,
file_checksum: String,
) {
use crate::schema::{crate_versions::dsl::*, crates::dsl::*};
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
insert_or_ignore_into(crates)
.values(name.eq(&crate_name))
.execute(&conn)
.unwrap();
let selected_crate = crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.unwrap();
insert_into(crate_versions)
.values((
crate_id.eq(selected_crate.id),
version.eq(version_string),
filesystem_object.eq(file_identifier.to_string()),
checksum.eq(file_checksum),
))
.execute(&conn)
.unwrap();
})
.await
.unwrap()
pub type Result<T> = std::result::Result<T, Error>;
pub fn init() -> Result<ConnectionPool> {
Ok(Arc::new(Pool::new(ConnectionManager::new("chartered.db"))?))
}
#[derive(Error, Display, Debug)]
pub enum Error {
Connection(#[from] diesel::r2d2::PoolError),
Query(#[from] diesel::result::Error),
TaskJoin(#[from] tokio::task::JoinError),
}
#[cfg(test)]
@@ -16,9 +16,49 @@
}
}
table! {
user_api_keys (id) {
id -> Integer,
user_id -> Integer,
api_key -> Text,
}
}
table! {
user_crate_permissions (id) {
id -> Integer,
user_id -> Integer,
crate_id -> Integer,
permissions -> Integer,
}
}
table! {
user_ssh_keys (id) {
id -> Integer,
user_id -> Integer,
ssh_key -> Binary,
}
}
table! {
users (id) {
id -> Integer,
username -> Integer,
}
}
joinable!(crate_versions -> crates (crate_id));
joinable!(user_api_keys -> users (user_id));
joinable!(user_crate_permissions -> crates (crate_id));
joinable!(user_crate_permissions -> users (user_id));
joinable!(user_ssh_keys -> users (user_id));
allow_tables_to_appear_in_same_query!(
crate_versions,
crates,
user_api_keys,
user_crate_permissions,
user_ssh_keys,
users,
);
@@ -1,0 +1,33 @@
use super::schema::{user_api_keys, user_crate_permissions, user_ssh_keys, users};
use diesel::{Associations, Identifiable, Queryable};
#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)]
pub struct User {
id: i32,
username: String,
}
#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)]
#[belongs_to(User)]
pub struct UserApiKey {
id: i32,
user_id: i32,
api_key: String,
}
#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)]
#[belongs_to(User)]
pub struct UserCratePermission {
id: i32,
user_id: i32,
crate_id: i32,
permissions: i32,
}
#[derive(Identifiable, Queryable, Associations, PartialEq, Eq, Hash, Debug)]
#[belongs_to(User)]
pub struct UserSshKey {
id: i32,
user_id: i32,
ssh_key: Vec<u8>,
}
@@ -30,7 +30,7 @@
let config = Arc::new(config);
let server = Server {
db: chartered_db::init(),
db: chartered_db::init().unwrap(),
};
thrussh::server::run(config, "127.0.0.1:2233", server)
@@ -280,10 +280,12 @@
async fn fetch_tree(
db: chartered_db::ConnectionPool,
) -> BTreeMap<[u8; 2], BTreeMap<[u8; 2], BTreeMap<String, String>>> {
use chartered_db::crates::Crate;
let mut tree: BTreeMap<[u8; 2], BTreeMap<[u8; 2], BTreeMap<String, String>>> = BTreeMap::new();
for (crate_def, versions) in chartered_db::get_crates(db).await {
for (crate_def, versions) in Crate::all_with_versions(db).await.unwrap() {
let mut name_chars = crate_def.name.as_bytes().iter();
let first_dir = [*name_chars.next().unwrap(), *name_chars.next().unwrap()];
let second_dir = [*name_chars.next().unwrap(), *name_chars.next().unwrap()];
@@ -1,9 +1,9 @@
mod endpoints;
mod middleware;
use axum::{
handler::{delete, get, put},
Router,
AddExtensionLayer, Router,
};
use tower::{filter::AsyncFilterLayer, ServiceBuilder};
@@ -27,6 +27,10 @@
#[tokio::main]
async fn main() {
env_logger::init();
let pool = chartered_db::init().unwrap();
let api_authenticated = axum_box_after_every_route!(Router::new()
.route("/crates/new", put(endpoints::cargo_api::publish))
.route("/crates/search", get(hello_world))
@@ -43,7 +47,8 @@
ServiceBuilder::new()
.layer_fn(middleware::auth::AuthMiddleware)
.into_inner(),
);
)
.layer(AddExtensionLayer::new(pool));
let middleware_stack = ServiceBuilder::new()
.layer(AsyncFilterLayer::new(|req| async {
@@ -52,18 +57,8 @@
}))
.into_inner();
let pool = chartered_db::init();
let app = Router::new()
.route(
"/",
get(|| async move {
format!(
"{:#?}",
chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await
)
}),
)
.route("/", get(hello_world))
.nest("/a/:key/api/v1", api_authenticated)
.layer(middleware_stack);
@@ -1,2 +1,6 @@
DROP TABLE crates;
DROP TABLE crate_versions;
DROP TABLE users;
DROP TABLE user_ssh_keys;
DROP TABLE user_api_keys;
DROP TABLE user_crate_permissions;
@@ -13,3 +13,32 @@
UNIQUE (crate_id, version),
FOREIGN KEY (crate_id) REFERENCES crates (id)
);
CREATE TABLE users (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
username INTEGER NOT NULL UNIQUE
);
CREATE TABLE user_ssh_keys (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
ssh_key BLOB NOT NULL,
FOREIGN KEY (user_id) REFERENCES users (id)
);
CREATE TABLE user_api_keys (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
api_key VARCHAR(255) NOT NULL UNIQUE,
FOREIGN KEY (user_id) REFERENCES users (id)
);
CREATE TABLE user_crate_permissions (
id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
crate_id INTEGER NOT NULL,
permissions INTEGER NOT NULL,
UNIQUE (user_id, crate_id),
FOREIGN KEY (user_id) REFERENCES users (id)
FOREIGN KEY (crate_id) REFERENCES crates (id)
);
@@ -1,1 +1,41 @@
macro_rules! define_error {
($($kind:ident$(($inner_name:ident: $inner:ty))? => $status:ident / $public_text:expr,)*) => {
#[derive(thiserror::Error, Debug)]
pub enum Error {
$($kind$((#[from] $inner))?),*
}
impl std::fmt::Display for Error {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
$(Self::$kind$(($inner_name))? => f.write_str($public_text)),*
}
}
}
impl axum::response::IntoResponse for Error {
type Body = axum::body::Body;
type BodyError = <Self::Body as axum::body::HttpBody>::Error;
fn into_response(self) -> axum::http::Response<Self::Body> {
log::error!("Failed to handle request: {:?}", self);
let (status, body) = match self {
$(Self::$kind$(($inner_name))? => (
axum::http::StatusCode::$status,
$public_text.into(),
)),*
};
axum::http::Response::builder()
.status(status)
.body(body)
.unwrap()
}
}
};
}
pub mod cargo_api;
@@ -1,15 +1,29 @@
use axum::extract;
use chartered_db::{crates::Crate, ConnectionPool};
use chartered_fs::FileSystem;
use std::str::FromStr;
use std::{str::FromStr, sync::Arc};
define_error!(
Database(_e: chartered_db::Error) => INTERNAL_SERVER_ERROR / "Failed to query database",
File(_e: std::io::Error) => INTERNAL_SERVER_ERROR / "Failed to fetch crate file",
NoVersion => NOT_FOUND / "That requested version does not exist for the crate",
NoCrate => NOT_FOUND / "The requested crate does not exist",
);
pub async fn handle(
extract::Path((_api_key, name, version)): extract::Path<(String, String, String)>,
) -> Vec<u8> {
let version = chartered_db::get_specific_crate_version(chartered_db::init(), name, version)
.await
.unwrap();
extract::Extension(db): extract::Extension<ConnectionPool>,
) -> Result<Vec<u8>, Error> {
let c = Crate::find_by_name(db.clone(), name)
.await?
.ok_or(Error::NoCrate)?;
let version = Arc::new(c)
.version(db, version)
.await?
.ok_or(Error::NoVersion)?;
let file_ref = chartered_fs::FileReference::from_str(&version.filesystem_object).unwrap();
chartered_fs::Local.read(file_ref).await.unwrap()
Ok(chartered_fs::Local.read(file_ref).await?)
}
@@ -1,4 +1,6 @@
use axum::extract;
use bytes::Bytes;
use chartered_db::ConnectionPool;
use serde::{Deserialize, Serialize};
use std::convert::TryInto;
@@ -14,7 +16,10 @@
other: Vec<String>,
}
pub async fn handle(body: Bytes) -> axum::response::Json<PublishCrateResponse> {
pub async fn handle(
extract::Extension(db): extract::Extension<ConnectionPool>,
body: Bytes,
) -> axum::response::Json<PublishCrateResponse> {
use chartered_fs::FileSystem;
use sha2::{Digest, Sha256};
@@ -24,8 +29,8 @@
let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap();
chartered_db::publish_crate(
chartered_db::init(),
chartered_db::crates::publish_crate(
db,
metadata.name.to_string(),
metadata.vers.to_string(),
file_ref,