Move to rkyv from bincode
Diff
Cargo.lock | 133 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
Cargo.toml | 3 ++-
src/git.rs | 11 +++++++----
src/main.rs | 6 ++++--
templates/index.html | 6 +++---
tree-sitter-grammar-repository/build.rs | 26 +++++++++++++-------------
src/database/indexer.rs | 44 ++++++++++++++++++++++++--------------------
src/methods/filters.rs | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++
src/methods/index.rs | 1 +
src/database/schema/commit.rs | 121 ++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------
src/database/schema/mod.rs | 2 +-
src/database/schema/repository.rs | 115 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
src/database/schema/tag.rs | 33 ++++++++++++++++++++-------------
src/methods/repo/refs.rs | 19 ++++++++++++++-----
src/methods/repo/summary.rs | 19 ++++++++++++++-----
templates/repo/macros/refs.html | 6 +++---
16 files changed, 384 insertions(+), 214 deletions(-)
@@ -197,15 +197,6 @@
]
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.69.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -262,6 +253,29 @@
version = "3.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
[[package]]
name = "bytecheck"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c8f430744b23b54ad15161fcbc22d82a29b73eacbe425fea23ec822600bc6f"
dependencies = [
"bytecheck_derive",
"ptr_meta",
"rancor",
"simdutf8",
]
[[package]]
name = "bytecheck_derive"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523363cbe1df49b68215efdf500b103ac3b0fb4836aed6d15689a076eadb8fff"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "byteorder"
@@ -1711,6 +1725,16 @@
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kanal"
version = "0.1.0-pre8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05d55519627edaf7fd0f29981f6dc03fb52df3f5b257130eb8d0bf2801ea1d7"
dependencies = [
"futures-core",
"lock_api",
]
[[package]]
@@ -1925,6 +1949,26 @@
"thiserror",
"triomphe",
"uuid",
]
[[package]]
name = "munge"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64142d38c84badf60abf06ff9bd80ad2174306a5b11bd4706535090a30a419df"
dependencies = [
"munge_macro",
]
[[package]]
name = "munge_macro"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bb5c1d8184f13f7d0ccbeeca0def2f9a181bce2624302793005f5ca8aa62e5e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -2143,6 +2187,26 @@
]
[[package]]
name = "ptr_meta"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90"
dependencies = [
"ptr_meta_derive",
]
[[package]]
name = "ptr_meta_derive"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "quanta"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2164,6 +2228,15 @@
checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rancor"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947"
dependencies = [
"ptr_meta",
]
[[package]]
@@ -2268,6 +2341,15 @@
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rend"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31c1f1959e4db12c985c0283656be0925f1539549db1e47c4bd0b8b599e1ef7"
dependencies = [
"bytecheck",
]
[[package]]
name = "rgit"
version = "0.1.3"
dependencies = [
@@ -2275,7 +2357,6 @@
"arc-swap",
"askama",
"axum",
"bincode",
"bytes",
"clap",
"comrak",
@@ -2287,10 +2368,12 @@
"httparse",
"humantime",
"itertools 0.13.0",
"kanal",
"md5",
"moka",
"path-clean",
"rand",
"rkyv",
"rocksdb",
"rsass",
"rust-ini",
@@ -2316,6 +2399,36 @@
"v_htmlescape",
"xxhash-rust",
"yoke",
]
[[package]]
name = "rkyv"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "395027076c569819ea6035ee62e664f5e03d74e281744f55261dd1afd939212b"
dependencies = [
"bytecheck",
"bytes",
"hashbrown",
"indexmap",
"munge",
"ptr_meta",
"rancor",
"rend",
"rkyv_derive",
"tinyvec",
"uuid",
]
[[package]]
name = "rkyv_derive"
version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cb82b74b4810f07e460852c32f522e979787691b0b7b7439fe473e49d49b2f"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
@@ -20,7 +20,6 @@
"tokio",
"http1",
] }
bincode = "1.3"
bytes = "1.5"
clap = { version = "4.4.10", default-features = false, features = [
"std",
@@ -45,10 +44,12 @@
httparse = "1.7"
humantime = "2.1"
itertools = "0.13.0"
kanal = "0.1.0-pre8"
md5 = "0.7"
moka = { version = "0.12.0", features = ["future"] }
path-clean = "1.0.1"
rand = "0.8.5"
rkyv = "0.8"
rocksdb = { version = "0.22", default-features = false, features = ["snappy"] }
rust-ini = "0.21.1"
serde = { version = "1.0", features = ["derive", "rc"] }
@@ -669,7 +669,7 @@
pub struct CommitUser {
name: String,
email: String,
time: OffsetDateTime,
time: (i64, i32),
}
impl TryFrom<SignatureRef<'_>> for CommitUser {
@@ -679,8 +679,9 @@
Ok(CommitUser {
name: v.name.to_string(),
email: v.email.to_string(),
time: OffsetDateTime::from_unix_timestamp(v.time.seconds)?
.to_offset(UtcOffset::from_whole_seconds(v.time.offset)?),
time: (v.time.seconds, v.time.offset),
})
}
}
@@ -695,7 +696,9 @@
}
pub fn time(&self) -> OffsetDateTime {
self.time
OffsetDateTime::from_unix_timestamp(self.time.0)
.unwrap()
.to_offset(UtcOffset::from_whole_seconds(self.time.1).unwrap())
}
}
@@ -33,7 +33,9 @@
use tower_http::{cors::CorsLayer, timeout::TimeoutLayer};
use tower_layer::layer_fn;
use tracing::{error, info, instrument, warn};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
use tracing_subscriber::{
fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter,
};
use xxhash_rust::const_xxh3;
use crate::{
@@ -122,7 +124,7 @@
std::env::set_var("RUST_LOG", "info");
}
let logger_layer = tracing_subscriber::fmt::layer();
let logger_layer = tracing_subscriber::fmt::layer().with_span_events(FmtSpan::CLOSE);
let env_filter = EnvFilter::from_default_env();
tracing_subscriber::registry()
@@ -28,7 +28,7 @@
</td>
<td>
<a href="/{% if let Some(path) = path %}{{ path }}/{% endif %}{{ repository.name }}">
{%- if let Some(description) = repository.description -%}
{%- if let Some(description) = repository.description.as_ref() -%}
{{- description -}}
{%- else -%}
Unnamed repository; edit this file 'description' to name the repository.
@@ -37,7 +37,7 @@
</td>
<td>
<a href="/{% if let Some(path) = path %}{{ path }}/{% endif %}{{ repository.name }}">
{%- if let Some(owner) = repository.owner -%}
{%- if let Some(owner) = repository.owner.as_ref() -%}
{{- owner -}}
{%- endif -%}
</a>
@@ -45,7 +45,7 @@
<td>
<a href="/{% if let Some(path) = path %}{{ path }}/{% endif %}{{ repository.name }}">
<time datetime="{{ repository.last_modified|format_time }}" title="{{ repository.last_modified|format_time }}">
{{- repository.last_modified.clone()|timeago -}}
{{- repository.last_modified|timeago -}}
</time>
</a>
</td>
@@ -91,7 +91,7 @@
}
}
fs::write(
&out_dir.join("grammar.defs.rs"),
out_dir.join("grammar.defs.rs"),
prettyplease::unparse(
&syn::parse2(quote!(#(#grammar_defs)*)).context("failed to parse grammar defs")?,
),
@@ -100,14 +100,14 @@
let registry = build_grammar_registry(config.grammar.iter().map(|v| v.name.clone()));
fs::write(
&out_dir.join("grammar.registry.rs"),
out_dir.join("grammar.registry.rs"),
prettyplease::unparse(&syn::parse2(registry).context("failed to parse grammar registry")?),
)
.context("failed to write grammar registry")?;
let language = build_language_registry(config.language)?;
fs::write(
&out_dir.join("language.registry.rs"),
out_dir.join("language.registry.rs"),
prettyplease::unparse(&syn::parse2(language)?),
)?;
@@ -214,7 +214,7 @@
thread_local! {
static REGEX: ::std::cell::LazyCell<::regex::RegexSet> = ::std::cell::LazyCell::new(|| {
::regex::RegexSet::new(&[
::regex::RegexSet::new([
#(#injection_regex),*
])
.unwrap()
@@ -433,14 +433,14 @@
fn fetch_git_repository(url: &str, ref_: &str, destination: &Path) -> anyhow::Result<()> {
if !destination.exists() {
let res = Command::new("git").arg("init").arg(&destination).status()?;
let res = Command::new("git").arg("init").arg(destination).status()?;
if !res.success() {
bail!("git init failed with exit code {res}");
}
let res = Command::new("git")
.args(&["remote", "add", "origin", url])
.current_dir(&destination)
.args(["remote", "add", "origin", url])
.current_dir(destination)
.status()?;
if !res.success() {
bail!("git remote failed with exit code {res}");
@@ -448,8 +448,8 @@
}
let res = Command::new("git")
.args(&["rev-parse", "HEAD"])
.current_dir(&destination)
.args(["rev-parse", "HEAD"])
.current_dir(destination)
.output()?
.stdout;
if res == ref_.as_bytes() {
@@ -457,16 +457,16 @@
}
let res = Command::new("git")
.args(&["fetch", "--depth", "1", "origin", ref_])
.current_dir(&destination)
.args(["fetch", "--depth", "1", "origin", ref_])
.current_dir(destination)
.status()?;
if !res.success() {
bail!("git fetch failed with exit code {res}");
}
let res = Command::new("git")
.args(&["reset", "--hard", ref_])
.current_dir(&destination)
.args(["reset", "--hard", ref_])
.current_dir(destination)
.status()?;
if !res.success() {
bail!("git fetch failed with exit code {res}");
@@ -17,7 +17,7 @@
use crate::database::schema::{
commit::Commit,
repository::{Repository, RepositoryId},
repository::{ArchivedRepository, Repository, RepositoryId},
tag::{Tag, TagTree},
};
@@ -51,7 +51,9 @@
};
let id = match Repository::open(db, relative) {
Ok(v) => v.map_or_else(RepositoryId::new, |v| v.get().id),
Ok(v) => v.map_or_else(RepositoryId::new, |v| {
RepositoryId(v.get().id.0.to_native())
}),
Err(error) => {
@@ -61,11 +63,13 @@
}
};
let Some(name) = relative.file_name().map(OsStr::to_string_lossy) else {
let Some(name) = relative.file_name().and_then(OsStr::to_str) else {
continue;
};
let description = std::fs::read(repository.join("description")).unwrap_or_default();
let description = Some(String::from_utf8_lossy(&description)).filter(|v| !v.is_empty());
let description = String::from_utf8(description)
.ok()
.filter(|v| !v.is_empty());
let repository_path = scan_path.join(relative);
@@ -81,15 +85,15 @@
let res = Repository {
id,
name,
name: name.to_string(),
description,
owner: find_gitweb_owner(repository_path.as_path()),
last_modified: find_last_committed_time(&git_repository)
.unwrap_or(OffsetDateTime::UNIX_EPOCH),
default_branch: find_default_branch(&git_repository)
.ok()
.flatten()
.map(Cow::Owned),
last_modified: {
let r =
find_last_committed_time(&git_repository).unwrap_or(OffsetDateTime::UNIX_EPOCH);
(r.unix_timestamp(), r.offset().whole_seconds())
},
default_branch: find_default_branch(&git_repository).ok().flatten(),
}
.insert(db, relative);
@@ -202,7 +206,7 @@
fn branch_index_update(
reference: &mut Reference<'_>,
relative_path: &str,
db_repository: &Repository<'_>,
db_repository: &ArchivedRepository,
db: Arc<rocksdb::DB>,
git_repository: &gix::Repository,
force_reindex: bool,
@@ -218,7 +222,7 @@
let commit = reference.peel_to_commit()?;
let latest_indexed = if let Some(latest_indexed) = commit_tree.fetch_latest_one()? {
if commit.id().as_bytes() == &*latest_indexed.get().hash {
if commit.id().as_bytes() == latest_indexed.get().hash.as_slice() {
info!("No commits since last index");
return Ok(());
}
@@ -246,7 +250,7 @@
let rev = rev?;
if let (false, Some(latest_indexed)) = (seen, &latest_indexed) {
if rev.id.as_bytes() == &*latest_indexed.get().hash {
if rev.id.as_bytes() == latest_indexed.get().hash.as_slice() {
seen = true;
}
@@ -321,7 +325,7 @@
#[instrument(skip(db_repository, db, git_repository))]
fn tag_index_scan(
relative_path: &str,
db_repository: &Repository<'_>,
db_repository: &ArchivedRepository,
db: Arc<rocksdb::DB>,
git_repository: &gix::Repository,
) -> Result<(), anyhow::Error> {
@@ -382,7 +386,7 @@
fn open_repo<P: AsRef<Path> + Debug>(
scan_path: &Path,
relative_path: P,
db_repository: &Repository<'_>,
db_repository: &ArchivedRepository,
db: &rocksdb::DB,
) -> Option<gix::Repository> {
match gix::open(scan_path.join(relative_path.as_ref())) {
@@ -435,13 +439,11 @@
}
}
fn find_gitweb_owner(repository_path: &Path) -> Option<Cow<'_, str>> {
fn find_gitweb_owner(repository_path: &Path) -> Option<String> {
Ini::load_from_file(repository_path.join("config"))
.ok()?
.section(Some("gitweb"))
.and_then(|section| section.get("owner"))
.map(String::from)
.map(Cow::Owned)
.section_mut(Some("gitweb"))
.and_then(|section| section.remove("owner"))
}
@@ -8,18 +8,25 @@
};
use arc_swap::ArcSwap;
use time::format_description::well_known::Rfc3339;
use rkyv::{
rend::{i32_le, i64_le},
tuple::ArchivedTuple2,
};
use time::{format_description::well_known::Rfc3339, OffsetDateTime, UtcOffset};
pub fn format_time(s: impl Into<Timestamp>) -> Result<String, askama::Error> {
let s = s.into().0;
pub fn format_time(s: impl Borrow<time::OffsetDateTime>) -> Result<String, askama::Error> {
(*s.borrow())
.format(&Rfc3339)
.map_err(Box::from)
.map_err(askama::Error::Custom)
}
pub fn timeago(s: impl Borrow<time::OffsetDateTime>) -> Result<String, askama::Error> {
pub fn timeago(s: impl Into<Timestamp>) -> Result<String, askama::Error> {
Ok(timeago::Formatter::new()
.convert((time::OffsetDateTime::now_utc() - *s.borrow()).unsigned_abs()))
.convert((OffsetDateTime::now_utc() - s.into().0).try_into().unwrap()))
}
pub fn file_perms(s: &u16) -> Result<String, askama::Error> {
@@ -52,4 +59,42 @@
});
Ok(url)
}
pub struct Timestamp(OffsetDateTime);
impl From<&ArchivedTuple2<i64_le, i32_le>> for Timestamp {
fn from(value: &ArchivedTuple2<i64_le, i32_le>) -> Self {
Self(
OffsetDateTime::from_unix_timestamp(value.0.to_native())
.unwrap()
.to_offset(UtcOffset::from_whole_seconds(value.1.to_native()).unwrap()),
)
}
}
impl From<(i64, i32)> for Timestamp {
fn from(value: (i64, i32)) -> Self {
Self(
OffsetDateTime::from_unix_timestamp(value.0)
.unwrap()
.to_offset(UtcOffset::from_whole_seconds(value.1).unwrap()),
)
}
}
impl From<&(i64, i32)> for Timestamp {
fn from(value: &(i64, i32)) -> Self {
Self(
OffsetDateTime::from_unix_timestamp(value.0)
.unwrap()
.to_offset(UtcOffset::from_whole_seconds(value.1).unwrap()),
)
}
}
impl From<OffsetDateTime> for Timestamp {
fn from(value: OffsetDateTime) -> Self {
Self(value)
}
}
@@ -24,6 +24,7 @@
let fetched = tokio::task::spawn_blocking(move || Repository::fetch_all(&db))
.await
.context("Failed to join Tokio task")??;
for (k, v) in fetched {
let mut split: Vec<_> = k.split('/').collect();
@@ -1,9 +1,9 @@
use std::{borrow::Cow, ops::Deref, sync::Arc};
use std::sync::Arc;
use anyhow::Context;
use gix::{actor::SignatureRef, bstr::ByteSlice, ObjectId};
use gix::{actor::SignatureRef, ObjectId};
use rkyv::{Archive, Serialize};
use rocksdb::{IteratorMode, ReadOptions, WriteBatch};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use time::{OffsetDateTime, UtcOffset};
use tracing::debug;
use yoke::{Yoke, Yokeable};
@@ -14,33 +14,31 @@
Yoked,
};
#[derive(Serialize, Deserialize, Debug, Yokeable)]
pub struct Commit<'a> {
#[serde(borrow)]
pub summary: Cow<'a, str>,
#[serde(borrow)]
pub message: Cow<'a, str>,
pub author: Author<'a>,
pub committer: Author<'a>,
pub hash: CommitHash<'a>,
#[derive(Serialize, Archive, Debug, Yokeable)]
pub struct Commit {
pub summary: String,
pub message: String,
pub author: Author,
pub committer: Author,
pub hash: [u8; 20],
}
impl<'a> Commit<'a> {
impl Commit {
pub fn new(
commit: &gix::Commit<'_>,
author: SignatureRef<'a>,
committer: SignatureRef<'a>,
author: SignatureRef<'_>,
committer: SignatureRef<'_>,
) -> Result<Self, anyhow::Error> {
let message = commit.message()?;
Ok(Self {
summary: message.summary().to_string().into(),
message: message
.body
.map_or(Cow::Borrowed(""), |v| v.to_string().into()),
summary: message.summary().to_string(),
message: message.body.map(ToString::to_string).unwrap_or_default(),
committer: committer.try_into()?,
author: author.try_into()?,
hash: CommitHash::Oid(commit.id().detach()),
hash: match commit.id().detach() {
ObjectId::Sha1(d) => d,
},
})
}
@@ -49,63 +47,29 @@
}
}
#[derive(Debug)]
pub enum CommitHash<'a> {
Oid(ObjectId),
Bytes(&'a [u8]),
#[derive(Serialize, Archive, Debug)]
pub struct Author {
pub name: String,
pub email: String,
pub time: (i64, i32),
}
impl<'a> Deref for CommitHash<'a> {
type Target = [u8];
fn deref(&self) -> &Self::Target {
match self {
CommitHash::Oid(v) => v.as_bytes(),
CommitHash::Bytes(v) => v,
}
}
}
impl Serialize for CommitHash<'_> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match self {
CommitHash::Oid(v) => serializer.serialize_bytes(v.as_bytes()),
CommitHash::Bytes(v) => serializer.serialize_bytes(v),
}
impl ArchivedAuthor {
pub fn time(&self) -> OffsetDateTime {
OffsetDateTime::from_unix_timestamp(self.time.0.to_native())
.unwrap()
.to_offset(UtcOffset::from_whole_seconds(self.time.1.to_native()).unwrap())
}
}
impl<'a, 'de: 'a> Deserialize<'de> for CommitHash<'a> {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let bytes = <&'a [u8]>::deserialize(deserializer)?;
Ok(Self::Bytes(bytes))
}
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Author<'a> {
#[serde(borrow)]
pub name: Cow<'a, str>,
#[serde(borrow)]
pub email: Cow<'a, str>,
pub time: OffsetDateTime,
}
impl<'a> TryFrom<SignatureRef<'a>> for Author<'a> {
impl TryFrom<SignatureRef<'_>> for Author {
type Error = anyhow::Error;
fn try_from(author: SignatureRef<'a>) -> Result<Self, anyhow::Error> {
fn try_from(author: SignatureRef<'_>) -> Result<Self, anyhow::Error> {
Ok(Self {
name: author.name.to_str_lossy(),
email: author.email.to_str_lossy(),
time: OffsetDateTime::from_unix_timestamp(author.time.seconds)?
.to_offset(UtcOffset::from_whole_seconds(author.time.offset)?),
name: author.name.to_string(),
email: author.email.to_string(),
time: (author.time.seconds, author.time.offset),
})
}
}
@@ -115,7 +79,7 @@
pub prefix: Box<[u8]>,
}
pub type YokedCommit = Yoked<Commit<'static>>;
pub type YokedCommit = Yoked<&'static <Commit as Archive>::Archived>;
impl CommitTree {
pub(super) fn new(db: Arc<rocksdb::DB>, repository: RepositoryId, reference: &str) -> Self {
@@ -170,12 +134,11 @@
return Ok(0);
};
let mut out = [0_u8; std::mem::size_of::<u64>()];
out.copy_from_slice(&res);
let out: [u8; std::mem::size_of::<u64>()] = res.as_ref().try_into()?;
Ok(u64::from_be_bytes(out))
}
fn insert(&self, id: u64, commit: &Commit<'_>, tx: &mut WriteBatch) -> anyhow::Result<()> {
fn insert(&self, id: u64, commit: &Commit, tx: &mut WriteBatch) -> anyhow::Result<()> {
let cf = self
.db
.cf_handle(COMMIT_FAMILY)
@@ -184,7 +147,7 @@
let mut key = self.prefix.to_vec();
key.extend_from_slice(&id.to_be_bytes());
tx.put_cf(cf, key, bincode::serialize(commit)?);
tx.put_cf(cf, key, rkyv::to_bytes::<rkyv::rancor::Error>(commit)?);
Ok(())
}
@@ -202,9 +165,11 @@
return Ok(None);
};
Yoke::try_attach_to_cart(Box::from(value), |data| bincode::deserialize(data))
.map(Some)
.context("Failed to deserialize commit")
Yoke::try_attach_to_cart(Box::from(value), |value| {
rkyv::access::<_, rkyv::rancor::Error>(&value)
})
.context("Failed to deserialize commit")
.map(Some)
}
pub fn fetch_latest(
@@ -240,7 +205,7 @@
.iterator_cf_opt(cf, opts, IteratorMode::End)
.map(|v| {
Yoke::try_attach_to_cart(v.context("failed to read commit")?.1, |data| {
bincode::deserialize(data).context("failed to deserialize")
rkyv::access::<_, rkyv::rancor::Error>(data).context("failed to deserialize")
})
})
.collect::<Result<Vec<_>, anyhow::Error>>()
@@ -9,4 +9,4 @@
pub type Yoked<T> = Yoke<T, Box<[u8]>>;
pub const SCHEMA_VERSION: &str = "1";
pub const SCHEMA_VERSION: &str = "2";
@@ -1,10 +1,9 @@
use std::{borrow::Cow, collections::BTreeMap, ops::Deref, path::Path, sync::Arc};
use std::{collections::BTreeMap, ops::Deref, path::Path, sync::Arc};
use anyhow::{Context, Result};
use rand::random;
use rkyv::{Archive, Serialize};
use rocksdb::IteratorMode;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use yoke::{Yoke, Yokeable};
use crate::database::schema::{
@@ -14,30 +13,26 @@
Yoked,
};
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Yokeable)]
pub struct Repository<'a> {
#[derive(Serialize, Archive, Debug, PartialEq, Eq, Hash, Yokeable)]
pub struct Repository {
pub id: RepositoryId,
#[serde(borrow)]
pub name: Cow<'a, str>,
pub name: String,
#[serde(borrow)]
pub description: Option<Cow<'a, str>>,
pub description: Option<String>,
#[serde(borrow)]
pub owner: Option<Cow<'a, str>>,
pub owner: Option<String>,
pub last_modified: OffsetDateTime,
pub last_modified: (i64, i32),
#[serde(borrow)]
pub default_branch: Option<Cow<'a, str>>,
pub default_branch: Option<String>,
}
pub type YokedRepository = Yoked<Repository<'static>>;
pub type YokedRepository = Yoked<&'static <Repository as Archive>::Archived>;
impl Repository<'_> {
impl Repository {
pub fn exists<P: AsRef<Path>>(database: &rocksdb::DB, path: P) -> Result<bool> {
let cf = database
.cf_handle(REPOSITORY_FAMILY)
@@ -53,11 +48,13 @@
.context("repository column family missing")?;
database
.full_iterator_cf(cf, IteratorMode::Start)
.iterator_cf(cf, IteratorMode::Start)
.filter_map(Result::ok)
.map(|(key, value)| {
let key = String::from_utf8(key.into_vec()).context("invalid repo name")?;
let value = Yoke::try_attach_to_cart(value, |data| bincode::deserialize(data))?;
let value = Yoke::try_attach_to_cart(value, |data| {
rkyv::access::<_, rkyv::rancor::Error>(data)
})?;
Ok((key, value))
})
@@ -70,14 +67,36 @@
.context("repository column family missing")?;
let path = path.as_ref().to_str().context("invalid path")?;
database.put_cf(cf, path, bincode::serialize(self)?)?;
database.put_cf(cf, path, rkyv::to_bytes::<rkyv::rancor::Error>(self)?)?;
Ok(())
}
pub fn open<P: AsRef<Path>>(
database: &rocksdb::DB,
path: P,
) -> Result<Option<YokedRepository>> {
let cf = database
.cf_handle(REPOSITORY_FAMILY)
.context("repository column family missing")?;
let path = path.as_ref().to_str().context("invalid path")?;
let Some(value) = database.get_cf(cf, path)? else {
return Ok(None);
};
Yoke::try_attach_to_cart(value.into_boxed_slice(), |data| {
rkyv::access::<_, rkyv::rancor::Error>(data)
})
.map(Some)
.context("Failed to open repository")
}
}
impl ArchivedRepository {
pub fn delete<P: AsRef<Path>>(&self, database: &rocksdb::DB, path: P) -> Result<()> {
let start_id = self.id.to_be_bytes();
let mut end_id = self.id.to_be_bytes();
let start_id = self.id.0.to_native().to_be_bytes();
let mut end_id = start_id;
*end_id.last_mut().unwrap() += 1;
@@ -100,60 +119,56 @@
database.delete_cf(repo_cf, path)?;
Ok(())
}
pub fn open<P: AsRef<Path>>(
database: &rocksdb::DB,
path: P,
) -> Result<Option<YokedRepository>> {
let cf = database
.cf_handle(REPOSITORY_FAMILY)
.context("repository column family missing")?;
let path = path.as_ref().to_str().context("invalid path")?;
let Some(value) = database.get_cf(cf, path)? else {
return Ok(None);
};
Yoke::try_attach_to_cart(value.into_boxed_slice(), |data| bincode::deserialize(data))
.map(Some)
.context("Failed to open repository")
}
pub fn commit_tree(&self, database: Arc<rocksdb::DB>, reference: &str) -> CommitTree {
CommitTree::new(database, self.id, reference)
CommitTree::new(database, RepositoryId(self.id.0.to_native()), reference)
}
pub fn tag_tree(&self, database: Arc<rocksdb::DB>) -> TagTree {
TagTree::new(database, self.id)
TagTree::new(database, RepositoryId(self.id.0.to_native()))
}
pub fn replace_heads(&self, database: &rocksdb::DB, new_heads: &[String]) -> Result<()> {
pub fn replace_heads(&self, database: &rocksdb::DB, new_heads: &Vec<String>) -> Result<()> {
let cf = database
.cf_handle(REFERENCE_FAMILY)
.context("missing reference column family")?;
database.put_cf(cf, self.id.to_be_bytes(), bincode::serialize(new_heads)?)?;
database.put_cf(
cf,
self.id.0.to_native().to_be_bytes(),
rkyv::to_bytes::<rkyv::rancor::Error>(new_heads)?,
)?;
Ok(())
}
pub fn heads(&self, database: &rocksdb::DB) -> Result<Yoke<Vec<String>, Box<[u8]>>> {
#[allow(clippy::type_complexity)]
pub fn heads(
&self,
database: &rocksdb::DB,
) -> Result<Option<Yoke<&'static ArchivedHeads, Box<[u8]>>>> {
let cf = database
.cf_handle(REFERENCE_FAMILY)
.context("missing reference column family")?;
let Some(bytes) = database.get_cf(cf, self.id.to_be_bytes())? else {
return Ok(Yoke::attach_to_cart(Box::default(), |_| vec![]));
let Some(bytes) = database.get_cf(cf, self.id.0.to_native().to_be_bytes())? else {
return Ok(None);
};
Yoke::try_attach_to_cart(Box::from(bytes), |bytes| bincode::deserialize(bytes))
.context("failed to deserialize heads")
Yoke::try_attach_to_cart(Box::from(bytes), |bytes| {
rkyv::access::<_, rkyv::rancor::Error>(bytes)
})
.context("failed to deserialize heads")
.map(Some)
}
}
#[derive(Serialize, Archive, Debug, Clone, PartialEq, Eq, Hash)]
pub struct Heads(pub Vec<String>);
#[derive(Serialize, Deserialize, Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct RepositoryId(pub(super) u64);
#[derive(Serialize, Archive, Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct RepositoryId(pub u64);
impl RepositoryId {
pub fn new() -> Self {
@@ -1,22 +1,24 @@
use std::{collections::HashSet, sync::Arc};
use anyhow::Context;
use gix::actor::SignatureRef;
use serde::{Deserialize, Serialize};
use rkyv::{Archive, Serialize};
use yoke::{Yoke, Yokeable};
use crate::database::schema::{
commit::Author, prefixes::TAG_FAMILY, repository::RepositoryId, Yoked,
commit::{ArchivedAuthor, Author},
prefixes::TAG_FAMILY,
repository::RepositoryId,
Yoked,
};
#[derive(Serialize, Deserialize, Debug, Yokeable)]
pub struct Tag<'a> {
#[serde(borrow)]
pub tagger: Option<Author<'a>>,
#[derive(Serialize, Archive, Debug, Yokeable)]
pub struct Tag {
pub tagger: Option<Author>,
}
impl<'a> Tag<'a> {
pub fn new(tagger: Option<SignatureRef<'a>>) -> Result<Self, anyhow::Error> {
impl Tag {
pub fn new(tagger: Option<SignatureRef<'_>>) -> Result<Self, anyhow::Error> {
Ok(Self {
tagger: tagger.map(TryFrom::try_from).transpose()?,
})
@@ -32,14 +34,14 @@
prefix: RepositoryId,
}
pub type YokedTag = Yoked<Tag<'static>>;
pub type YokedTag = Yoked<&'static <Tag as Archive>::Archived>;
impl TagTree {
pub(super) fn new(db: Arc<rocksdb::DB>, prefix: RepositoryId) -> Self {
Self { db, prefix }
}
pub fn insert(&self, name: &str, value: &Tag<'_>) -> anyhow::Result<()> {
pub fn insert(&self, name: &str, value: &Tag) -> anyhow::Result<()> {
let cf = self
.db
.cf_handle(TAG_FAMILY)
@@ -48,7 +50,8 @@
let mut db_name = self.prefix.to_be_bytes().to_vec();
db_name.extend_from_slice(name.as_ref());
self.db.put_cf(cf, db_name, bincode::serialize(value)?)?;
self.db
.put_cf(cf, db_name, rkyv::to_bytes::<rkyv::rancor::Error>(value)?)?;
Ok(())
}
@@ -103,14 +106,16 @@
Some((name, value))
})
.map(|(name, value)| {
let value = Yoke::try_attach_to_cart(value, |data| bincode::deserialize(data))?;
let value = Yoke::try_attach_to_cart(value, |data| {
rkyv::access::<_, rkyv::rancor::Error>(data)
})?;
Ok((name, value))
})
.collect::<anyhow::Result<Vec<(String, YokedTag)>>>()?;
res.sort_unstable_by(|a, b| {
let a_tagger = a.1.get().tagger.as_ref().map(|v| v.time);
let b_tagger = b.1.get().tagger.as_ref().map(|v| v.time);
let a_tagger = a.1.get().tagger.as_ref().map(ArchivedAuthor::time);
let b_tagger = b.1.get().tagger.as_ref().map(ArchivedAuthor::time);
b_tagger.cmp(&a_tagger)
});
@@ -1,8 +1,9 @@
use std::{collections::BTreeMap, sync::Arc};
use anyhow::Context;
use askama::Template;
use axum::{response::IntoResponse, Extension};
use rkyv::string::ArchivedString;
use crate::{
into_response,
@@ -29,12 +30,20 @@
.context("Repository does not exist")?;
let mut heads = BTreeMap::new();
for head in repository.get().heads(&db)?.get() {
let commit_tree = repository.get().commit_tree(db.clone(), head);
let name = head.strip_prefix("refs/heads/");
if let Some(heads_db) = repository.get().heads(&db)? {
for head in heads_db
.get()
.0
.as_slice()
.iter()
.map(ArchivedString::as_str)
{
let commit_tree = repository.get().commit_tree(db.clone(), head);
let name = head.strip_prefix("refs/heads/");
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()?) {
heads.insert(name.to_string(), commit);
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()?) {
heads.insert(name.to_string(), commit);
}
}
}
@@ -1,8 +1,9 @@
use std::{collections::BTreeMap, sync::Arc};
use anyhow::Context;
use askama::Template;
use axum::{response::IntoResponse, Extension};
use rkyv::string::ArchivedString;
use crate::{
database::schema::{commit::YokedCommit, repository::YokedRepository},
@@ -32,12 +33,20 @@
let commits = get_default_branch_commits(&repository, &db)?;
let mut heads = BTreeMap::new();
for head in repository.get().heads(&db)?.get() {
let commit_tree = repository.get().commit_tree(db.clone(), head);
let name = head.strip_prefix("refs/heads/");
if let Some(heads_db) = repository.get().heads(&db)? {
for head in heads_db
.get()
.0
.as_slice()
.iter()
.map(ArchivedString::as_str)
{
let commit_tree = repository.get().commit_tree(db.clone(), head);
let name = head.strip_prefix("refs/heads/");
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()?) {
heads.insert(name.to_string(), commit);
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()?) {
heads.insert(name.to_string(), commit);
}
}
}
@@ -43,13 +43,13 @@
<td><a href="/{{ repo.display() }}/tag/?h={{ name }}">{{- name -}}</a></td>
<td><a href="/{{ repo.display() }}/snapshot?h={{ name }}">{{- name -}}.tar.gz</a></td>
<td>
{% if let Some(tagger) = tag.get().tagger -%}
{% if let Some(tagger) = tag.get().tagger.as_ref() -%}
<img src="{{ tagger.email|gravatar }}" width="13" height="13">
{{ tagger.name }}
{%- endif %}
</td>
<td>
{% if let Some(tagger) = tag.get().tagger -%}
{% if let Some(tagger) = tag.get().tagger.as_ref() -%}
<time datetime="{{ tagger.time|format_time }}" title="{{ tagger.time|format_time }}">
{{- tagger.time|timeago -}}
</time>
@@ -75,7 +75,7 @@
<tr>
<td>
<time datetime="{{ commit.committer.time|format_time }}" title="{{ commit.committer.time|format_time }}">
{{- commit.committer.time.clone()|timeago -}}
{{- commit.committer.time|timeago -}}
</time>
</td>
<td><a href="/{{ repo.display() }}/commit/?id={{ commit.hash|hex }}">{{ commit.summary }}</a></td>