basic crate pushing (no metadata, validation, error checking)
Diff
Cargo.lock | 26 ++++++++++++++++++++++++++
chartered-db/Cargo.toml | 2 ++
chartered-fs/Cargo.toml | 3 +++
chartered-web/Cargo.toml | 5 +++++
chartered-db/src/lib.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++-------
chartered-db/src/schema.rs | 5 +----
chartered-fs/src/lib.rs | 43 +++++++++++++++++++++++++++++++++++++++++++
chartered-git/src/main.rs | 4 ++--
chartered-web/src/main.rs | 14 ++++++++++----
chartered-web/src/middleware/auth.rs | 12 ++++++------
chartered-web/src/endpoints/cargo_api/mod.rs | 2 ++
chartered-web/src/endpoints/cargo_api/publish.rs | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
12 files changed, 226 insertions(+), 31 deletions(-)
@@ -186,6 +186,7 @@
name = "chartered-db"
version = "0.1.0"
dependencies = [
"chartered-fs",
"diesel",
"dotenv",
"tokio",
@@ -228,8 +229,13 @@
version = "0.1.0"
dependencies = [
"axum",
"bytes",
"chartered-db",
"chartered-fs",
"futures",
"nom",
"serde",
"serde_json",
"tokio",
"tower",
"tower-http",
@@ -739,6 +745,12 @@
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "minimal-lexical"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6595bb28ed34f43c3fe088e48f6cfb2e033cab45f25a5384d5fdf564fbc8c4b2"
[[package]]
name = "miniz_oxide"
@@ -770,6 +782,17 @@
checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
dependencies = [
"winapi",
]
[[package]]
name = "nom"
version = "7.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffd9d26838a953b4af82cbeb9f1592c6798916983959be223a7124e992742c1"
dependencies = [
"memchr",
"minimal-lexical",
"version_check",
]
[[package]]
@@ -1084,6 +1107,9 @@
version = "1.0.129"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
@@ -6,6 +6,8 @@
[dependencies]
chartered-fs = { path = "../chartered-fs" }
diesel = { version = "1", features = ["sqlite", "r2d2"] }
tokio = { version = "1" }
dotenv = "0.15"
@@ -6,3 +6,6 @@
[dependencies]
async-trait = "0.1"
tokio = { version = "1", features = ["fs"] }
uuid = { version = "0.8", features = ["v4"] }
@@ -7,9 +7,14 @@
[dependencies]
chartered-db = { path = "../chartered-db" }
chartered-fs = { path = "../chartered-fs" }
axum = "0.2"
bytes = "1"
futures = "0.3"
nom = "7"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
tokio = { version = "1", features = ["full"] }
tower = { version = "0.4", features = ["util", "filter"] }
tower-http = { version = "0.1", features = ["trace"] }
@@ -1,20 +1,15 @@
pub mod schema;
#[macro_use]
extern crate diesel;
pub mod schema;
use diesel::{Associations, Identifiable, Queryable, insert_into, insert_or_ignore_into, prelude::*, r2d2::{ConnectionManager, Pool}};
use schema::{crate_versions, crates};
use std::sync::Arc;
use self::diesel::prelude::*;
use diesel::{
r2d2::{ConnectionManager, Pool},
Associations, Identifiable, Queryable,
};
pub type ConnectionPool = Arc<Pool<ConnectionManager<diesel::SqliteConnection>>>;
use schema::crate_versions;
use schema::crates;
pub fn init() -> Arc<Pool<ConnectionManager<diesel::SqliteConnection>>> {
pub fn init() -> ConnectionPool {
Arc::new(Pool::new(ConnectionManager::new("chartered.db")).unwrap())
}
@@ -34,10 +29,7 @@
yanked: bool,
}
pub async fn get_crate_versions(
conn: Arc<Pool<ConnectionManager<diesel::SqliteConnection>>>,
crate_name: String,
) -> Vec<CrateVersion> {
pub async fn get_crate_versions(conn: ConnectionPool, crate_name: String) -> Vec<CrateVersion> {
use crate::schema::crates::dsl::*;
tokio::task::spawn_blocking(move || {
@@ -52,6 +44,40 @@
.expect("no crate versions");
selected_crate_versions
})
.await
.unwrap()
}
pub async fn publish_crate(
conn: ConnectionPool,
crate_name: String,
version_string: String,
file_identifier: chartered_fs::FileReference,
) {
use crate::schema::{crate_versions::dsl::*, crates::dsl::*};
tokio::task::spawn_blocking(move || {
let conn = conn.get().unwrap();
insert_or_ignore_into(crates)
.values(name.eq(&crate_name))
.execute(&conn)
.unwrap();
let selected_crate = crates
.filter(name.eq(crate_name))
.first::<Crate>(&conn)
.unwrap();
insert_into(crate_versions)
.values((
crate_id.eq(selected_crate.id),
version.eq(version_string),
filesystem_object.eq(file_identifier.to_string()),
))
.execute(&conn)
.unwrap();
})
.await
.unwrap()
@@ -17,7 +17,4 @@
joinable!(crate_versions -> crates (crate_id));
allow_tables_to_appear_in_same_query!(
crate_versions,
crates,
);
allow_tables_to_appear_in_same_query!(crate_versions, crates,);
@@ -1,3 +1,46 @@
use async_trait::async_trait;
use tokio::{
fs::File,
io::{AsyncReadExt, AsyncWriteExt},
};
pub struct FileReference(uuid::Uuid);
impl std::fmt::Display for FileReference {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
#[async_trait]
pub trait FileSystem {
async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error>;
async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error>;
}
pub struct Local;
#[async_trait]
impl FileSystem for Local {
async fn read(&self, file_ref: FileReference) -> Result<Vec<u8>, std::io::Error> {
let mut file = File::open(format!("/tmp/{}", file_ref.0)).await?;
let mut contents = vec![];
file.read_to_end(&mut contents).await?;
Ok(contents)
}
async fn write(&self, data: &[u8]) -> Result<FileReference, std::io::Error> {
let uuid = uuid::Uuid::new_v4();
let mut file = File::create(format!("/tmp/{}", uuid)).await?;
file.write_all(data).await?;
Ok(FileReference(uuid))
}
}
#[cfg(test)]
mod tests {
#[test]
@@ -172,8 +172,8 @@
let test_crate_file = PackFileEntry::Blob(br#"{"name":"charteredtest","vers":"1.0.0","deps":[],"cksum":"7b821735f0211fd00032a9892d1bf2323c9d05d9c59b9303eb382f5ec1898bfc","features":{},"yanked":false,"links":null}"#);
let config_file = PackFileEntry::Blob(
br#"{
"dl": "http://127.0.0.1:8888/api/v1/crates",
"api": "http://127.0.0.1:8888"
"dl": "http://127.0.0.1:8888/a/abc/api/v1/crates",
"api": "http://127.0.0.1:8888/a/abc"
}"#,
);
@@ -28,7 +28,7 @@
#[tokio::main]
async fn main() {
let api_authenticated = axum_box_after_every_route!(Router::new()
.route("/crates/new", put(hello_world))
.route("/crates/new", put(endpoints::cargo_api::publish))
.route("/crates/search", get(hello_world))
.route("/crates/:crate/owners", get(hello_world))
.route("/crates/:crate/owners", put(hello_world))
@@ -55,9 +55,15 @@
let pool = chartered_db::init();
let app = Router::new()
.route("/", get(|| async move {
format!("{:#?}", chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await)
}))
.route(
"/",
get(|| async move {
format!(
"{:#?}",
chartered_db::get_crate_versions(pool, "cool-test-crate".to_string()).await
)
}),
)
.nest("/a/:key/api/v1", api_authenticated)
.layer(middleware_stack);
@@ -28,12 +28,12 @@
let mut inner = std::mem::replace(&mut self.0, clone);
Box::pin(async move {
if true {
return Ok(Response::builder()
.status(StatusCode::UNAUTHORIZED)
.body(ResBody::default())
.unwrap());
}
let res: Response<ResBody> = inner.call(req).await?;
@@ -1,3 +1,5 @@
mod download;
mod publish;
pub use download::handle as download;
pub use publish::handle as publish;
@@ -1,0 +1,85 @@
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use std::convert::TryInto;
#[derive(Serialize, Debug, Default)]
pub struct PublishCrateResponse {
warnings: PublishCrateResponseWarnings,
}
#[derive(Serialize, Debug, Default)]
pub struct PublishCrateResponseWarnings {
invalid_categories: Vec<String>,
invalid_badges: Vec<String>,
other: Vec<String>,
}
pub async fn handle(body: Bytes) -> axum::response::Json<PublishCrateResponse> {
use chartered_fs::FileSystem;
let (_, (metadata_bytes, crate_bytes)) = parse(body.as_ref()).unwrap();
let metadata: Metadata = serde_json::from_slice(metadata_bytes).unwrap();
let file_ref = chartered_fs::Local.write(crate_bytes).await.unwrap();
chartered_db::publish_crate(
chartered_db::init(),
metadata.name.to_string(),
metadata.vers.to_string(),
file_ref,
)
.await;
axum::response::Json(PublishCrateResponse::default())
}
fn parse(body: &[u8]) -> nom::IResult<&[u8], (&[u8], &[u8])> {
use nom::{bytes::complete::take, combinator::map_res};
use std::array::TryFromSliceError;
let u32_from_le_bytes =
|b: &[u8]| Ok::<_, TryFromSliceError>(u32::from_le_bytes(b.try_into()?));
let mut read_u32 = map_res(take(4usize), u32_from_le_bytes);
let (rest, metadata_length) = read_u32(body)?;
let (rest, metadata_bytes) = take(metadata_length)(rest)?;
let (rest, crate_length) = read_u32(rest)?;
let (rest, crate_bytes) = take(crate_length)(rest)?;
Ok((rest, (metadata_bytes, crate_bytes)))
}
#[derive(Deserialize, Debug)]
pub struct Metadata<'a> {
name: &'a str,
vers: &'a str,
deps: Vec<MetadataDependency<'a>>,
features: std::collections::HashMap<&'a str, Vec<&'a str>>,
authors: Vec<&'a str>,
description: Option<&'a str>,
documentation: Option<&'a str>,
homepage: Option<&'a str>,
readme: Option<&'a str>,
readme_file: Option<&'a str>,
keywords: Vec<&'a str>,
categories: Vec<&'a str>,
license: Option<&'a str>,
license_file: Option<&'a str>,
repository: Option<&'a str>,
links: Option<&'a str>,
}
#[derive(Deserialize, Debug)]
pub struct MetadataDependency<'a> {
name: &'a str,
version_req: &'a str,
features: Vec<&'a str>,
optional: bool,
default_features: bool,
target: Option<&'a str>,
kind: &'a str,
registry: &'a str,
explicit_name_in_toml: Option<&'a str>,
}