From e0fa194583615e821dc82b72276352394f4a8443 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Sun, 12 Sep 2021 17:32:22 +0100 Subject: [PATCH] Implement support for features/dependencies in the publish endpoint --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 1 + chartered-db/Cargo.toml | 3 +++ chartered-git/Cargo.toml | 1 + chartered-types/Cargo.toml | 9 +++++++++ chartered-types/README.md | 6 ++++++ chartered-web/Cargo.toml | 1 + chartered-db/src/crates.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- chartered-db/src/lib.rs | 28 ++++++++++++++++++++++++++++ chartered-db/src/schema.rs | 3 +++ chartered-git/src/main.rs | 21 +++++++++------------ chartered-types/src/cargo.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ chartered-types/src/lib.rs | 1 + migrations/2021-08-31-214501_create_crates_table/up.sql | 3 +++ chartered-web/src/endpoints/mod.rs | 2 ++ chartered-web/src/endpoints/cargo_api/publish.rs | 26 +++++--------------------- 16 files changed, 213 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 418337d..03cf7b7 100644 --- a/Cargo.lock +++ a/Cargo.lock @@ -188,12 +188,15 @@ dependencies = [ "bitflags", "chartered-fs", + "chartered-types", "chrono", "diesel", "displaydoc", "dotenv", "itertools", "rand", + "serde", + "serde_json", "thiserror", "tokio", ] @@ -216,6 +219,7 @@ "async-trait", "bytes", "chartered-db", + "chartered-types", "chrono", "const-sha1", "crc", @@ -232,6 +236,13 @@ "thrussh-keys", "tokio", "tokio-util", +] + +[[package]] +name = "chartered-types" +version = "0.1.0" +dependencies = [ + "serde", ] [[package]] @@ -242,6 +253,7 @@ "bytes", "chartered-db", "chartered-fs", + "chartered-types", "env_logger", "futures", "hex", diff --git a/Cargo.toml b/Cargo.toml index 0add9bb..a3d1f14 100644 --- a/Cargo.toml +++ a/Cargo.toml @@ -1,7 +1,8 @@ [workspace] members = [ "chartered-git", "chartered-web", "chartered-fs", "chartered-db", + "chartered-types", ] diff --git a/chartered-db/Cargo.toml b/chartered-db/Cargo.toml index e5dbda2..d864b79 100644 --- a/chartered-db/Cargo.toml +++ a/chartered-db/Cargo.toml @@ -7,6 +7,7 @@ [dependencies] chartered-fs = { path = "../chartered-fs" } +chartered-types = { path = "../chartered-types" } bitflags = "1" chrono = "0.4" @@ -14,6 +15,8 @@ displaydoc = "0.2" itertools = "0.10" rand = "0.8" +serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1" tokio = "1" dotenv = "0.15" diff --git a/chartered-git/Cargo.toml b/chartered-git/Cargo.toml index 97cdfa5..ce2e466 100644 --- a/chartered-git/Cargo.toml +++ a/chartered-git/Cargo.toml @@ -7,6 +7,7 @@ [dependencies] chartered-db = { path = "../chartered-db" } +chartered-types = { path = "../chartered-types" } anyhow = "1" async-trait = "0" diff --git a/chartered-types/Cargo.toml b/chartered-types/Cargo.toml new file mode 100644 index 0000000..a5b8707 100644 --- /dev/null +++ a/chartered-types/Cargo.toml @@ -1,0 +1,9 @@ +[package] +name = "chartered-types" +version = "0.1.0" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde = { version = "1", features = ["derive"] } diff --git a/chartered-types/README.md b/chartered-types/README.md new file mode 100644 index 0000000..ea2b537 100644 --- /dev/null +++ a/chartered-types/README.md @@ -1,0 +1,6 @@ +# chartered-types (library) + +Shared types that don't particularly belong in any one crate. + +For example, types that are created by the user's `cargo` and also consumed +by it. diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml index a664606..7fe1c91 100644 --- a/chartered-web/Cargo.toml +++ a/chartered-web/Cargo.toml @@ -8,6 +8,7 @@ [dependencies] chartered-db = { path = "../chartered-db" } chartered-fs = { path = "../chartered-fs" } +chartered-types = { path = "../chartered-types" } axum = "0.2" bytes = "1" diff --git a/chartered-db/src/crates.rs b/chartered-db/src/crates.rs index de2e7df..49fb933 100644 --- a/chartered-db/src/crates.rs +++ a/chartered-db/src/crates.rs @@ -1,9 +1,10 @@ use super::{ schema::{crate_versions, crates}, BitwiseExpressionMethods, ConnectionPool, Result, }; use diesel::{insert_into, prelude::*, Associations, Identifiable, Queryable}; use itertools::Itertools; +use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; #[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)] @@ -12,10 +13,49 @@ pub name: String, } +#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] +#[belongs_to(Crate)] +pub struct CrateVersion<'a> { + pub id: i32, + pub crate_id: i32, + pub version: String, + pub filesystem_object: String, + pub yanked: bool, + pub checksum: String, + pub dependencies: CrateDependencies<'a>, + pub features: CrateFeatures, + pub links: Option, +} + +impl<'a> CrateVersion<'a> { + #[must_use] + pub fn into_cargo_format(self, crate_: &'a Crate) -> chartered_types::cargo::CrateVersion<'a> { + chartered_types::cargo::CrateVersion { + name: crate_.name.as_str().into(), + vers: self.version.into(), + deps: self.dependencies.0, + features: self.features.0, + links: self.links.map(Into::into), + } + } +} + +#[derive(Serialize, Deserialize, FromSqlRow, AsExpression, Debug, Clone, PartialEq, Eq)] +#[sql_type = "diesel::sql_types::Blob"] +pub struct CrateDependencies<'a>(pub Vec>); + +derive_diesel_json!(CrateDependencies<'a>); + +#[derive(Serialize, Deserialize, FromSqlRow, AsExpression, Debug, Clone, PartialEq, Eq)] +#[sql_type = "diesel::sql_types::Blob"] +pub struct CrateFeatures(pub chartered_types::cargo::CrateFeatures); + +derive_diesel_json!(CrateFeatures); + impl Crate { pub async fn all_with_versions( conn: ConnectionPool, - ) -> Result>> { + ) -> Result>>> { tokio::task::spawn_blocking(move || { let conn = conn.get()?; @@ -31,7 +71,7 @@ pub async fn all_visible_with_versions( conn: ConnectionPool, given_user_id: i32, - ) -> Result>> { + ) -> Result>>> { tokio::task::spawn_blocking(move || { let conn = conn.get()?; @@ -66,7 +106,10 @@ .await? } - pub async fn versions(self: Arc, conn: ConnectionPool) -> Result> { + pub async fn versions( + self: Arc, + conn: ConnectionPool, + ) -> Result>> { tokio::task::spawn_blocking(move || { let conn = conn.get()?; @@ -79,7 +122,7 @@ self: Arc, conn: ConnectionPool, crate_version: String, - ) -> Result> { + ) -> Result>> { use crate::schema::crate_versions::version; tokio::task::spawn_blocking(move || { @@ -117,12 +160,13 @@ pub async fn publish_version( self: Arc, conn: ConnectionPool, - version_string: String, file_identifier: chartered_fs::FileReference, file_checksum: String, + given: chartered_types::cargo::CrateVersion<'static>, ) -> Result<()> { use crate::schema::crate_versions::dsl::{ - checksum, crate_id, crate_versions, filesystem_object, version, + checksum, crate_id, crate_versions, dependencies, features, filesystem_object, links, + version, }; tokio::task::spawn_blocking(move || { @@ -131,9 +175,12 @@ insert_into(crate_versions) .values(( crate_id.eq(self.id), - version.eq(version_string), filesystem_object.eq(file_identifier.to_string()), checksum.eq(file_checksum), + version.eq(given.vers), + dependencies.eq(CrateDependencies(given.deps)), + features.eq(CrateFeatures(given.features)), + links.eq(given.links), )) .execute(&conn)?; @@ -143,13 +190,14 @@ } } -#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)] -#[belongs_to(Crate)] -pub struct CrateVersion { - pub id: i32, - pub crate_id: i32, - pub version: String, - pub filesystem_object: String, - pub yanked: bool, - pub checksum: String, +impl<'a> From>> for CrateDependencies<'a> { + fn from(o: Vec>) -> Self { + Self(o) + } +} + +impl<'a> From for CrateFeatures { + fn from(o: chartered_types::cargo::CrateFeatures) -> Self { + Self(o) + } } diff --git a/chartered-db/src/lib.rs b/chartered-db/src/lib.rs index 59d0a71..d81889b 100644 --- a/chartered-db/src/lib.rs +++ a/chartered-db/src/lib.rs @@ -1,6 +1,34 @@ #![deny(clippy::pedantic)] #![allow(clippy::missing_errors_doc)] +macro_rules! derive_diesel_json { + ($typ:ident$(<$lt:lifetime>)?) => { + impl<$($lt, )?B: diesel::backend::Backend> + diesel::deserialize::FromSql for $typ$(<$lt>)? + where + Vec: diesel::deserialize::FromSql, + { + fn from_sql(bytes: Option<&B::RawValue>) -> diesel::deserialize::Result { + let bytes = >::from_sql(bytes)?; // todo: we either have to allocate or deal with a raw pointer... + serde_json::from_slice(&bytes).map_err(|_| "Invalid Json".into()) + } + } + + impl<$($lt, )?B: diesel::backend::Backend> diesel::serialize::ToSql + for $typ$(<$lt>)? + { + fn to_sql( + &self, + out: &mut diesel::serialize::Output, + ) -> diesel::serialize::Result { + serde_json::to_writer(out, self) + .map(|_| diesel::serialize::IsNull::No) + .map_err(Into::into) + } + } + }; +} + pub mod crates; pub mod schema; pub mod users; diff --git a/chartered-db/src/schema.rs b/chartered-db/src/schema.rs index 92e9f46..8657413 100644 --- a/chartered-db/src/schema.rs +++ a/chartered-db/src/schema.rs @@ -6,6 +6,9 @@ filesystem_object -> Text, yanked -> Bool, checksum -> Text, + dependencies -> Binary, + features -> Binary, + links -> Nullable, } } diff --git a/chartered-git/src/main.rs b/chartered-git/src/main.rs index d044c54..966f915 100644 --- a/chartered-git/src/main.rs +++ a/chartered-git/src/main.rs @@ -336,13 +336,10 @@ #[derive(serde::Serialize)] pub struct CrateFileEntry<'a> { - name: &'a str, - vers: &'a str, - deps: &'a [&'a str], + #[serde(flatten)] + inner: &'a chartered_types::cargo::CrateVersion<'a>, cksum: &'a str, - features: BTreeMap>, yanked: bool, - links: Option<()>, } pub type TwoCharTree = BTreeMap<[u8; 2], T>; @@ -366,14 +363,14 @@ let mut file = String::new(); for version in versions { + let cksum = version.checksum.clone(); + let yanked = version.yanked; + let version = version.into_cargo_format(&crate_def); + let entry = CrateFileEntry { - name: &crate_def.name, - vers: &version.version, - deps: &[], - cksum: &version.checksum, - features: BTreeMap::new(), - yanked: version.yanked, - links: None, + inner: &version, + cksum: &cksum, + yanked, }; file.push_str(&serde_json::to_string(&entry).unwrap()); diff --git a/chartered-types/src/cargo.rs b/chartered-types/src/cargo.rs new file mode 100644 index 0000000..52e9148 100644 --- /dev/null +++ a/chartered-types/src/cargo.rs @@ -1,0 +1,65 @@ +//! 'Raw' types that are passed by `cargo publish` and also consumed via +//! cargo when pulling. These are just inserted into the database as-is. + +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, collections::BTreeMap}; + +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +pub struct CrateVersion<'a> { + #[serde(borrow)] + pub name: Cow<'a, str>, + #[serde(borrow)] + pub vers: Cow<'a, str>, + pub deps: Vec>, + pub features: CrateFeatures, + #[serde(borrow)] + pub links: Option>, +} + +impl CrateVersion<'_> { + pub fn into_owned(self) -> CrateVersion<'static> { + CrateVersion { + name: Cow::Owned(self.name.into_owned()), + vers: Cow::Owned(self.vers.into_owned()), + deps: self.deps.into_iter().map(|v| v.into_owned()).collect(), + features: self.features, + links: self.links.map(|v| Cow::Owned(v.into_owned())), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CrateDependency<'a> { + pub name: Cow<'a, str>, + pub version_req: Cow<'a, str>, // needs to be: https://github.com/steveklabnik/semver#requirements + pub features: Vec>, + pub optional: bool, + pub default_features: bool, + pub target: Option>, // a string such as "cfg(windows)" + pub kind: Cow<'a, str>, // dev, build or normal + pub registry: Option>, + pub package: Option>, +} + +impl CrateDependency<'_> { + pub fn into_owned(self) -> CrateDependency<'static> { + CrateDependency { + name: Cow::Owned(self.name.into_owned()), + version_req: Cow::Owned(self.version_req.into_owned()), + features: self + .features + .into_iter() + .map(|v| Cow::Owned(v.into_owned())) + .collect(), + optional: self.optional, + default_features: self.default_features, + target: self.target.map(|v| Cow::Owned(v.into_owned())), + kind: Cow::Owned(self.kind.into_owned()), + registry: self.registry.map(|v| Cow::Owned(v.into_owned())), + package: self.package.map(|v| Cow::Owned(v.into_owned())), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct CrateFeatures(pub BTreeMap>); diff --git a/chartered-types/src/lib.rs b/chartered-types/src/lib.rs new file mode 100644 index 0000000..fbe9fcc 100644 --- /dev/null +++ a/chartered-types/src/lib.rs @@ -1,0 +1,1 @@ +pub mod cargo; diff --git a/migrations/2021-08-31-214501_create_crates_table/up.sql b/migrations/2021-08-31-214501_create_crates_table/up.sql index 1c3640a..57c59ca 100644 --- a/migrations/2021-08-31-214501_create_crates_table/up.sql +++ a/migrations/2021-08-31-214501_create_crates_table/up.sql @@ -10,6 +10,9 @@ filesystem_object VARCHAR(255) NOT NULL, yanked BOOLEAN NOT NULL DEFAULT FALSE, checksum VARCHAR(255) NOT NULL, + dependencies BLOB NOT NULL, + features BLOB NOT NULL, + links VARCHAR(255), UNIQUE (crate_id, version), FOREIGN KEY (crate_id) REFERENCES crates (id) ); diff --git a/chartered-web/src/endpoints/mod.rs b/chartered-web/src/endpoints/mod.rs index 0a7a829..e6ed334 100644 --- a/chartered-web/src/endpoints/mod.rs +++ a/chartered-web/src/endpoints/mod.rs @@ -10,6 +10,8 @@ type BodyError = ::Error; fn into_response(self) -> axum::http::Response { + eprintln!("error: {:?}", self); + let body = serde_json::to_vec(&crate::endpoints::ErrorResponse { error: self.to_string(), }) diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs index b58ec33..1e99537 100644 --- a/chartered-web/src/endpoints/cargo_api/publish.rs +++ a/chartered-web/src/endpoints/cargo_api/publish.rs @@ -19,7 +19,7 @@ NoCrate, #[error("You don't have permission to publish versions for this crate")] NoPermission, - #[error("Invalid JSON from client")] + #[error("Invalid JSON from client: {0}")] JsonParse(#[from] serde_json::Error), #[error("Invalid body")] MetadataParse, @@ -61,7 +61,7 @@ parse(body.as_ref()).map_err(|_| Error::MetadataParse)?; let metadata: Metadata = serde_json::from_slice(metadata_bytes)?; - let crate_ = Crate::find_by_name(db.clone(), metadata.name.to_string()) + let crate_ = Crate::find_by_name(db.clone(), metadata.inner.name.to_string()) .await? .ok_or(Error::NoCrate) .map(std::sync::Arc::new)?; @@ -76,9 +76,9 @@ crate_ .publish_version( db, - metadata.vers.to_string(), file_ref, hex::encode(Sha256::digest(crate_bytes)), + metadata.inner.into_owned(), ) .await?; @@ -103,10 +103,6 @@ #[derive(Deserialize, Debug)] pub struct Metadata<'a> { - name: &'a str, - vers: &'a str, - deps: Vec>, - features: std::collections::HashMap<&'a str, Vec<&'a str>>, authors: Vec<&'a str>, description: Option<&'a str>, documentation: Option<&'a str>, @@ -118,18 +114,6 @@ license: Option<&'a str>, license_file: Option<&'a str>, repository: Option<&'a str>, - links: Option<&'a str>, -} - -#[derive(Deserialize, Debug)] -pub struct MetadataDependency<'a> { - name: &'a str, - version_req: &'a str, - features: Vec<&'a str>, - optional: bool, - default_features: bool, - target: Option<&'a str>, - kind: &'a str, // 'dev', 'build', or 'normal' - registry: &'a str, - explicit_name_in_toml: Option<&'a str>, + #[serde(flatten)] + inner: chartered_types::cargo::CrateVersion<'a>, } -- rgit 0.1.3