🏡 index : ~doyle/chartered.git

author Jordan Doyle <jordan@doyle.la> 2021-09-12 17:32:22.0 +01:00:00
committer Jordan Doyle <jordan@doyle.la> 2021-09-12 17:32:22.0 +01:00:00
commit
e0fa194583615e821dc82b72276352394f4a8443 [patch]
tree
590d25b8b09458c8f79c111f0601a4ef02e00466
parent
f4a3e07b9e82bd3a07eecbb8bd880f2011decac7
download
e0fa194583615e821dc82b72276352394f4a8443.tar.gz

Implement support for features/dependencies in the publish endpoint



Diff

 Cargo.lock                                              | 12 ++++++++++++
 Cargo.toml                                              |  1 +
 chartered-db/Cargo.toml                                 |  3 +++
 chartered-git/Cargo.toml                                |  1 +
 chartered-types/Cargo.toml                              |  9 +++++++++
 chartered-types/README.md                               |  6 ++++++
 chartered-web/Cargo.toml                                |  1 +
 chartered-db/src/crates.rs                              | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 chartered-db/src/lib.rs                                 | 28 ++++++++++++++++++++++++++++
 chartered-db/src/schema.rs                              |  3 +++
 chartered-git/src/main.rs                               | 21 +++++++++------------
 chartered-types/src/cargo.rs                            | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 chartered-types/src/lib.rs                              |  1 +
 migrations/2021-08-31-214501_create_crates_table/up.sql |  3 +++
 chartered-web/src/endpoints/mod.rs                      |  2 ++
 chartered-web/src/endpoints/cargo_api/publish.rs        | 26 +++++---------------------
 16 files changed, 213 insertions(+), 49 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 418337d..03cf7b7 100644
--- a/Cargo.lock
+++ a/Cargo.lock
@@ -188,12 +188,15 @@
dependencies = [
 "bitflags",
 "chartered-fs",
 "chartered-types",
 "chrono",
 "diesel",
 "displaydoc",
 "dotenv",
 "itertools",
 "rand",
 "serde",
 "serde_json",
 "thiserror",
 "tokio",
]
@@ -216,6 +219,7 @@
 "async-trait",
 "bytes",
 "chartered-db",
 "chartered-types",
 "chrono",
 "const-sha1",
 "crc",
@@ -232,6 +236,13 @@
 "thrussh-keys",
 "tokio",
 "tokio-util",
]

[[package]]
name = "chartered-types"
version = "0.1.0"
dependencies = [
 "serde",
]

[[package]]
@@ -242,6 +253,7 @@
 "bytes",
 "chartered-db",
 "chartered-fs",
 "chartered-types",
 "env_logger",
 "futures",
 "hex",
diff --git a/Cargo.toml b/Cargo.toml
index 0add9bb..a3d1f14 100644
--- a/Cargo.toml
+++ a/Cargo.toml
@@ -1,7 +1,8 @@
[workspace]
members = [
    "chartered-git",
    "chartered-web",
    "chartered-fs",
    "chartered-db",
    "chartered-types",
]
diff --git a/chartered-db/Cargo.toml b/chartered-db/Cargo.toml
index e5dbda2..d864b79 100644
--- a/chartered-db/Cargo.toml
+++ a/chartered-db/Cargo.toml
@@ -7,6 +7,7 @@

[dependencies]
chartered-fs = { path = "../chartered-fs" }
chartered-types = { path = "../chartered-types" }

bitflags = "1"
chrono = "0.4"
@@ -14,6 +15,8 @@
displaydoc = "0.2"
itertools = "0.10"
rand = "0.8"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
thiserror = "1"
tokio = "1"
dotenv = "0.15"
diff --git a/chartered-git/Cargo.toml b/chartered-git/Cargo.toml
index 97cdfa5..ce2e466 100644
--- a/chartered-git/Cargo.toml
+++ a/chartered-git/Cargo.toml
@@ -7,6 +7,7 @@

[dependencies]
chartered-db = { path = "../chartered-db" }
chartered-types = { path = "../chartered-types" }

anyhow = "1"
async-trait = "0"
diff --git a/chartered-types/Cargo.toml b/chartered-types/Cargo.toml
new file mode 100644
index 0000000..a5b8707 100644
--- /dev/null
+++ a/chartered-types/Cargo.toml
@@ -1,0 +1,9 @@
[package]
name = "chartered-types"
version = "0.1.0"
edition = "2018"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
serde = { version = "1", features = ["derive"] }
diff --git a/chartered-types/README.md b/chartered-types/README.md
new file mode 100644
index 0000000..ea2b537 100644
--- /dev/null
+++ a/chartered-types/README.md
@@ -1,0 +1,6 @@
# chartered-types (library)

Shared types that don't particularly belong in any one crate.

For example, types that are created by the user's `cargo` and also consumed
by it.
diff --git a/chartered-web/Cargo.toml b/chartered-web/Cargo.toml
index a664606..7fe1c91 100644
--- a/chartered-web/Cargo.toml
+++ a/chartered-web/Cargo.toml
@@ -8,6 +8,7 @@
[dependencies]
chartered-db = { path = "../chartered-db" }
chartered-fs = { path = "../chartered-fs" }
chartered-types = { path = "../chartered-types" }

axum = "0.2"
bytes = "1"
diff --git a/chartered-db/src/crates.rs b/chartered-db/src/crates.rs
index de2e7df..49fb933 100644
--- a/chartered-db/src/crates.rs
+++ a/chartered-db/src/crates.rs
@@ -1,9 +1,10 @@
use super::{
    schema::{crate_versions, crates},
    BitwiseExpressionMethods, ConnectionPool, Result,
};
use diesel::{insert_into, prelude::*, Associations, Identifiable, Queryable};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, sync::Arc};

#[derive(Identifiable, Queryable, PartialEq, Eq, Hash, Debug)]
@@ -12,10 +13,49 @@
    pub name: String,
}

#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)]
#[belongs_to(Crate)]
pub struct CrateVersion<'a> {
    pub id: i32,
    pub crate_id: i32,
    pub version: String,
    pub filesystem_object: String,
    pub yanked: bool,
    pub checksum: String,
    pub dependencies: CrateDependencies<'a>,
    pub features: CrateFeatures,
    pub links: Option<String>,
}

impl<'a> CrateVersion<'a> {
    #[must_use]
    pub fn into_cargo_format(self, crate_: &'a Crate) -> chartered_types::cargo::CrateVersion<'a> {
        chartered_types::cargo::CrateVersion {
            name: crate_.name.as_str().into(),
            vers: self.version.into(),
            deps: self.dependencies.0,
            features: self.features.0,
            links: self.links.map(Into::into),
        }
    }
}

#[derive(Serialize, Deserialize, FromSqlRow, AsExpression, Debug, Clone, PartialEq, Eq)]
#[sql_type = "diesel::sql_types::Blob"]
pub struct CrateDependencies<'a>(pub Vec<chartered_types::cargo::CrateDependency<'a>>);

derive_diesel_json!(CrateDependencies<'a>);

#[derive(Serialize, Deserialize, FromSqlRow, AsExpression, Debug, Clone, PartialEq, Eq)]
#[sql_type = "diesel::sql_types::Blob"]
pub struct CrateFeatures(pub chartered_types::cargo::CrateFeatures);

derive_diesel_json!(CrateFeatures);

impl Crate {
    pub async fn all_with_versions(
        conn: ConnectionPool,
    ) -> Result<HashMap<Crate, Vec<CrateVersion>>> {
    ) -> Result<HashMap<Crate, Vec<CrateVersion<'static>>>> {
        tokio::task::spawn_blocking(move || {
            let conn = conn.get()?;

@@ -31,7 +71,7 @@
    pub async fn all_visible_with_versions(
        conn: ConnectionPool,
        given_user_id: i32,
    ) -> Result<HashMap<Crate, Vec<CrateVersion>>> {
    ) -> Result<HashMap<Crate, Vec<CrateVersion<'static>>>> {
        tokio::task::spawn_blocking(move || {
            let conn = conn.get()?;

@@ -66,7 +106,10 @@
        .await?
    }

    pub async fn versions(self: Arc<Self>, conn: ConnectionPool) -> Result<Vec<CrateVersion>> {
    pub async fn versions(
        self: Arc<Self>,
        conn: ConnectionPool,
    ) -> Result<Vec<CrateVersion<'static>>> {
        tokio::task::spawn_blocking(move || {
            let conn = conn.get()?;

@@ -79,7 +122,7 @@
        self: Arc<Self>,
        conn: ConnectionPool,
        crate_version: String,
    ) -> Result<Option<CrateVersion>> {
    ) -> Result<Option<CrateVersion<'static>>> {
        use crate::schema::crate_versions::version;

        tokio::task::spawn_blocking(move || {
@@ -117,12 +160,13 @@
    pub async fn publish_version(
        self: Arc<Self>,
        conn: ConnectionPool,
        version_string: String,
        file_identifier: chartered_fs::FileReference,
        file_checksum: String,
        given: chartered_types::cargo::CrateVersion<'static>,
    ) -> Result<()> {
        use crate::schema::crate_versions::dsl::{
            checksum, crate_id, crate_versions, filesystem_object, version,
            checksum, crate_id, crate_versions, dependencies, features, filesystem_object, links,
            version,
        };

        tokio::task::spawn_blocking(move || {
@@ -131,9 +175,12 @@
            insert_into(crate_versions)
                .values((
                    crate_id.eq(self.id),
                    version.eq(version_string),
                    filesystem_object.eq(file_identifier.to_string()),
                    checksum.eq(file_checksum),
                    version.eq(given.vers),
                    dependencies.eq(CrateDependencies(given.deps)),
                    features.eq(CrateFeatures(given.features)),
                    links.eq(given.links),
                ))
                .execute(&conn)?;

@@ -143,13 +190,14 @@
    }
}

#[derive(Identifiable, Queryable, Associations, PartialEq, Debug)]
#[belongs_to(Crate)]
pub struct CrateVersion {
    pub id: i32,
    pub crate_id: i32,
    pub version: String,
    pub filesystem_object: String,
    pub yanked: bool,
    pub checksum: String,
impl<'a> From<Vec<chartered_types::cargo::CrateDependency<'a>>> for CrateDependencies<'a> {
    fn from(o: Vec<chartered_types::cargo::CrateDependency<'a>>) -> Self {
        Self(o)
    }
}

impl<'a> From<chartered_types::cargo::CrateFeatures> for CrateFeatures {
    fn from(o: chartered_types::cargo::CrateFeatures) -> Self {
        Self(o)
    }
}
diff --git a/chartered-db/src/lib.rs b/chartered-db/src/lib.rs
index 59d0a71..d81889b 100644
--- a/chartered-db/src/lib.rs
+++ a/chartered-db/src/lib.rs
@@ -1,6 +1,34 @@
#![deny(clippy::pedantic)]
#![allow(clippy::missing_errors_doc)]

macro_rules! derive_diesel_json {
    ($typ:ident$(<$lt:lifetime>)?) => {
        impl<$($lt, )?B: diesel::backend::Backend>
            diesel::deserialize::FromSql<diesel::sql_types::Blob, B> for $typ$(<$lt>)?
        where
            Vec<u8>: diesel::deserialize::FromSql<diesel::sql_types::Blob, B>,
        {
            fn from_sql(bytes: Option<&B::RawValue>) -> diesel::deserialize::Result<Self> {
                let bytes = <Vec<u8>>::from_sql(bytes)?; // todo: we either have to allocate or deal with a raw pointer...
                serde_json::from_slice(&bytes).map_err(|_| "Invalid Json".into())
            }
        }

        impl<$($lt, )?B: diesel::backend::Backend> diesel::serialize::ToSql<diesel::sql_types::Blob, B>
            for $typ$(<$lt>)?
        {
            fn to_sql<W: std::io::Write>(
                &self,
                out: &mut diesel::serialize::Output<W, B>,
            ) -> diesel::serialize::Result {
                serde_json::to_writer(out, self)
                    .map(|_| diesel::serialize::IsNull::No)
                    .map_err(Into::into)
            }
        }
    };
}

pub mod crates;
pub mod schema;
pub mod users;
diff --git a/chartered-db/src/schema.rs b/chartered-db/src/schema.rs
index 92e9f46..8657413 100644
--- a/chartered-db/src/schema.rs
+++ a/chartered-db/src/schema.rs
@@ -6,6 +6,9 @@
        filesystem_object -> Text,
        yanked -> Bool,
        checksum -> Text,
        dependencies -> Binary,
        features -> Binary,
        links -> Nullable<Text>,
    }
}

diff --git a/chartered-git/src/main.rs b/chartered-git/src/main.rs
index d044c54..966f915 100644
--- a/chartered-git/src/main.rs
+++ a/chartered-git/src/main.rs
@@ -336,13 +336,10 @@

#[derive(serde::Serialize)]
pub struct CrateFileEntry<'a> {
    name: &'a str,
    vers: &'a str,
    deps: &'a [&'a str],
    #[serde(flatten)]
    inner: &'a chartered_types::cargo::CrateVersion<'a>,
    cksum: &'a str,
    features: BTreeMap<String, Vec<String>>,
    yanked: bool,
    links: Option<()>,
}

pub type TwoCharTree<T> = BTreeMap<[u8; 2], T>;
@@ -366,14 +363,14 @@

        let mut file = String::new();
        for version in versions {
            let cksum = version.checksum.clone();
            let yanked = version.yanked;
            let version = version.into_cargo_format(&crate_def);

            let entry = CrateFileEntry {
                name: &crate_def.name,
                vers: &version.version,
                deps: &[],
                cksum: &version.checksum,
                features: BTreeMap::new(),
                yanked: version.yanked,
                links: None,
                inner: &version,
                cksum: &cksum,
                yanked,
            };

            file.push_str(&serde_json::to_string(&entry).unwrap());
diff --git a/chartered-types/src/cargo.rs b/chartered-types/src/cargo.rs
new file mode 100644
index 0000000..52e9148 100644
--- /dev/null
+++ a/chartered-types/src/cargo.rs
@@ -1,0 +1,65 @@
//! 'Raw' types that are passed by `cargo publish` and also consumed via

//! cargo when pulling. These are just inserted into the database as-is.


use serde::{Deserialize, Serialize};
use std::{borrow::Cow, collections::BTreeMap};

#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
pub struct CrateVersion<'a> {
    #[serde(borrow)]
    pub name: Cow<'a, str>,
    #[serde(borrow)]
    pub vers: Cow<'a, str>,
    pub deps: Vec<CrateDependency<'a>>,
    pub features: CrateFeatures,
    #[serde(borrow)]
    pub links: Option<Cow<'a, str>>,
}

impl CrateVersion<'_> {
    pub fn into_owned(self) -> CrateVersion<'static> {
        CrateVersion {
            name: Cow::Owned(self.name.into_owned()),
            vers: Cow::Owned(self.vers.into_owned()),
            deps: self.deps.into_iter().map(|v| v.into_owned()).collect(),
            features: self.features,
            links: self.links.map(|v| Cow::Owned(v.into_owned())),
        }
    }
}

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct CrateDependency<'a> {
    pub name: Cow<'a, str>,
    pub version_req: Cow<'a, str>, // needs to be: https://github.com/steveklabnik/semver#requirements
    pub features: Vec<Cow<'a, str>>,
    pub optional: bool,
    pub default_features: bool,
    pub target: Option<Cow<'a, str>>, // a string such as "cfg(windows)"
    pub kind: Cow<'a, str>,           // dev, build or normal
    pub registry: Option<Cow<'a, str>>,
    pub package: Option<Cow<'a, str>>,
}

impl CrateDependency<'_> {
    pub fn into_owned(self) -> CrateDependency<'static> {
        CrateDependency {
            name: Cow::Owned(self.name.into_owned()),
            version_req: Cow::Owned(self.version_req.into_owned()),
            features: self
                .features
                .into_iter()
                .map(|v| Cow::Owned(v.into_owned()))
                .collect(),
            optional: self.optional,
            default_features: self.default_features,
            target: self.target.map(|v| Cow::Owned(v.into_owned())),
            kind: Cow::Owned(self.kind.into_owned()),
            registry: self.registry.map(|v| Cow::Owned(v.into_owned())),
            package: self.package.map(|v| Cow::Owned(v.into_owned())),
        }
    }
}

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
pub struct CrateFeatures(pub BTreeMap<String, Vec<String>>);
diff --git a/chartered-types/src/lib.rs b/chartered-types/src/lib.rs
new file mode 100644
index 0000000..fbe9fcc 100644
--- /dev/null
+++ a/chartered-types/src/lib.rs
@@ -1,0 +1,1 @@
pub mod cargo;
diff --git a/migrations/2021-08-31-214501_create_crates_table/up.sql b/migrations/2021-08-31-214501_create_crates_table/up.sql
index 1c3640a..57c59ca 100644
--- a/migrations/2021-08-31-214501_create_crates_table/up.sql
+++ a/migrations/2021-08-31-214501_create_crates_table/up.sql
@@ -10,6 +10,9 @@
    filesystem_object VARCHAR(255) NOT NULL,
    yanked BOOLEAN NOT NULL DEFAULT FALSE,
    checksum VARCHAR(255) NOT NULL,
    dependencies BLOB NOT NULL,
    features BLOB NOT NULL,
    links VARCHAR(255),
    UNIQUE (crate_id, version),
    FOREIGN KEY (crate_id) REFERENCES crates (id)
);
diff --git a/chartered-web/src/endpoints/mod.rs b/chartered-web/src/endpoints/mod.rs
index 0a7a829..e6ed334 100644
--- a/chartered-web/src/endpoints/mod.rs
+++ a/chartered-web/src/endpoints/mod.rs
@@ -10,6 +10,8 @@
            type BodyError = <Self::Body as axum::body::HttpBody>::Error;

            fn into_response(self) -> axum::http::Response<Self::Body> {
                eprintln!("error: {:?}", self);

                let body = serde_json::to_vec(&crate::endpoints::ErrorResponse {
                    error: self.to_string(),
                })
diff --git a/chartered-web/src/endpoints/cargo_api/publish.rs b/chartered-web/src/endpoints/cargo_api/publish.rs
index b58ec33..1e99537 100644
--- a/chartered-web/src/endpoints/cargo_api/publish.rs
+++ a/chartered-web/src/endpoints/cargo_api/publish.rs
@@ -19,7 +19,7 @@
    NoCrate,
    #[error("You don't have permission to publish versions for this crate")]
    NoPermission,
    #[error("Invalid JSON from client")]
    #[error("Invalid JSON from client: {0}")]
    JsonParse(#[from] serde_json::Error),
    #[error("Invalid body")]
    MetadataParse,
@@ -61,7 +61,7 @@
        parse(body.as_ref()).map_err(|_| Error::MetadataParse)?;
    let metadata: Metadata = serde_json::from_slice(metadata_bytes)?;

    let crate_ = Crate::find_by_name(db.clone(), metadata.name.to_string())
    let crate_ = Crate::find_by_name(db.clone(), metadata.inner.name.to_string())
        .await?
        .ok_or(Error::NoCrate)
        .map(std::sync::Arc::new)?;
@@ -76,9 +76,9 @@
    crate_
        .publish_version(
            db,
            metadata.vers.to_string(),
            file_ref,
            hex::encode(Sha256::digest(crate_bytes)),
            metadata.inner.into_owned(),
        )
        .await?;

@@ -103,10 +103,6 @@

#[derive(Deserialize, Debug)]
pub struct Metadata<'a> {
    name: &'a str,
    vers: &'a str,
    deps: Vec<MetadataDependency<'a>>,
    features: std::collections::HashMap<&'a str, Vec<&'a str>>,
    authors: Vec<&'a str>,
    description: Option<&'a str>,
    documentation: Option<&'a str>,
@@ -118,18 +114,6 @@
    license: Option<&'a str>,
    license_file: Option<&'a str>,
    repository: Option<&'a str>,
    links: Option<&'a str>,
}

#[derive(Deserialize, Debug)]
pub struct MetadataDependency<'a> {
    name: &'a str,
    version_req: &'a str,
    features: Vec<&'a str>,
    optional: bool,
    default_features: bool,
    target: Option<&'a str>,
    kind: &'a str, // 'dev', 'build', or 'normal'
    registry: &'a str,
    explicit_name_in_toml: Option<&'a str>,
    #[serde(flatten)]
    inner: chartered_types::cargo::CrateVersion<'a>,
}