From c2201124c0ecd4dacc4ecef79a12e6287839b618 Mon Sep 17 00:00:00 2001 From: Jordan Doyle Date: Fri, 22 Jul 2022 01:52:16 +0100 Subject: [PATCH] Index tags to sled and read them from views --- src/git.rs | 73 ------------------------------------------------------------------------- src/database/indexer.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/methods/filters.rs | 7 +++++-- src/methods/repo.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ templates/repo/refs.html | 4 ++-- templates/repo/summary.html | 8 ++++---- src/database/schema/commit.rs | 2 ++ src/database/schema/mod.rs | 1 + src/database/schema/prefixes.rs | 12 +++++++++++- src/database/schema/repository.rs | 23 +++++++++++++++++++++++ src/database/schema/tag.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ templates/repo/macros/refs.html | 30 ++++++++++++++++-------------- 12 files changed, 271 insertions(+), 112 deletions(-) diff --git a/src/git.rs b/src/git.rs index 631cfaa..c95cde7 100644 --- a/src/git.rs +++ a/src/git.rs @@ -20,7 +20,6 @@ pub struct Git { commits: Cache>, readme_cache: Cache)>>, - refs: Cache>, syntax_set: SyntaxSet, } @@ -36,10 +35,6 @@ .time_to_live(Duration::from_secs(10)) .max_capacity(100) .build(), - refs: Cache::builder() - .time_to_live(Duration::from_secs(10)) - .max_capacity(100) - .build(), syntax_set, } } @@ -200,56 +195,6 @@ }) .await .context("Failed to join Tokio task")? - } - - #[instrument(skip(self))] - pub async fn refs(self: Arc) -> Result, Arc> { - let git = self.git.clone(); - - git.refs - .try_get_with(self.cache_key.clone(), async move { - tokio::task::spawn_blocking(move || { - let repo = self.repo.lock(); - - let ref_iter = repo.references().context("Couldn't get list of references for repository")?; - - let mut built_refs = Refs::default(); - - for ref_ in ref_iter { - let ref_ = ref_?; - - if ref_.is_branch() { - let commit = ref_.peel_to_commit().context("Reference is apparently a branch but I couldn't get to the HEAD of it")?; - - built_refs.branch.push(Branch { - name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(), - commit: commit.try_into()?, - }); - } else if ref_.is_tag() { - if let Ok(tag) = ref_.peel_to_tag() { - built_refs.tag.push(Tag { - name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(), - tagger: tag.tagger().map(TryInto::try_into).transpose()?, - }); - } - } - } - - built_refs.branch.sort_unstable_by(|one, two| { - two.commit.committer.time.cmp(&one.commit.committer.time) - }); - built_refs.tag.sort_unstable_by(|one, two| { - let one_tagger = one.tagger.as_ref().map(|v| v.time); - let two_tagger = two.tagger.as_ref().map(|v| v.time); - two_tagger.cmp(&one_tagger) - }); - - Ok(Arc::new(built_refs)) - }) - .await - .context("Failed to join Tokio task")? - }) - .await } #[instrument(skip(self))] @@ -409,12 +354,6 @@ pub struct FileWithContent { pub metadata: File, pub content: String, -} - -#[derive(Debug, Default)] -pub struct Refs { - pub branch: Vec, - pub tag: Vec, } #[derive(Debug)] @@ -440,19 +379,12 @@ pub tagger: Option, pub message: String, pub tagged_object: Option, -} - -#[derive(Debug)] -pub struct Tag { - pub name: String, - pub tagger: Option, } #[derive(Debug)] pub struct CommitUser { name: String, email: String, - email_md5: String, time: OffsetDateTime, } @@ -463,7 +395,6 @@ Ok(CommitUser { name: String::from_utf8_lossy(v.name_bytes()).into_owned(), email: String::from_utf8_lossy(v.email_bytes()).into_owned(), - email_md5: format!("{:x}", md5::compute(v.email_bytes())), time: OffsetDateTime::from_unix_timestamp(v.when().seconds())?, }) } @@ -476,10 +407,6 @@ pub fn email(&self) -> &str { &self.email - } - - pub fn email_md5(&self) -> &str { - &self.email_md5 } pub fn time(&self) -> OffsetDateTime { diff --git a/src/database/indexer.rs b/src/database/indexer.rs index bb25b4c..0259c49 100644 --- a/src/database/indexer.rs +++ a/src/database/indexer.rs @@ -1,4 +1,5 @@ use git2::Sort; +use std::collections::HashSet; use std::path::{Path, PathBuf}; use time::OffsetDateTime; use tracing::{info, info_span}; @@ -6,12 +7,25 @@ use crate::database::schema::{ commit::Commit, repository::{Repository, RepositoryId}, + tag::Tag, }; pub fn run(db: &sled::Db) { + let span = info_span!("index_update"); + let _entered = span.enter(); + + info!("Starting index update"); + let scan_path = Path::new("/Users/jordan/Code/test-git"); update_repository_metadata(scan_path, db); update_repository_reflog(scan_path, db); + update_repository_tags(scan_path, db); + + info!("Flushing to disk"); + + db.flush().unwrap(); + + info!("Finished index update"); } fn update_repository_metadata(scan_path: &Path, db: &sled::Db) { @@ -52,7 +66,7 @@ } let span = info_span!( - "index_update", + "branch_index_update", reference = reference_name.as_ref(), repository = relative_path ); @@ -93,7 +107,57 @@ // we'll need to add `clear()` to sled's tx api to remove this for to_remove in (i + 1)..(i + 100) { commit_tree.remove(&to_remove.to_be_bytes()).unwrap(); + } + } + } +} + +fn update_repository_tags(scan_path: &Path, db: &sled::Db) { + for (relative_path, db_repository) in Repository::fetch_all(db).unwrap() { + let git_repository = git2::Repository::open(scan_path.join(&relative_path)).unwrap(); + + let tag_tree = db_repository.get().tag_tree(db).unwrap(); + + let git_tags: HashSet<_> = git_repository + .references() + .unwrap() + .filter_map(Result::ok) + .filter(|v| v.name_bytes().starts_with(b"refs/tags/")) + .map(|v| String::from_utf8_lossy(v.name_bytes()).into_owned()) + .collect(); + let indexed_tags: HashSet = tag_tree.list().into_iter().collect(); + + // insert any git tags that are missing from the index + for tag_name in git_tags.difference(&indexed_tags) { + let span = info_span!( + "tag_index_update", + reference = tag_name, + repository = relative_path + ); + let _entered = span.enter(); + + let reference = git_repository.find_reference(tag_name).unwrap(); + + if let Ok(tag) = reference.peel_to_tag() { + info!("Inserting newly discovered tag to index"); + + Tag::new(tag.tagger().as_ref()).insert(&tag_tree, tag_name); } + } + + // remove any extra tags that the index has + // TODO: this also needs to check peel_to_tag + for tag_name in indexed_tags.difference(&git_tags) { + let span = info_span!( + "tag_index_update", + reference = tag_name, + repository = relative_path + ); + let _entered = span.enter(); + + info!("Removing stale tag from index"); + + tag_tree.remove(tag_name); } } } diff --git a/src/methods/filters.rs b/src/methods/filters.rs index f524809..fd79efe 100644 --- a/src/methods/filters.rs +++ a/src/methods/filters.rs @@ -1,8 +1,11 @@ // sorry clippy, we don't have a choice. askama forces this on us #![allow(clippy::unnecessary_wraps, clippy::trivially_copy_pass_by_ref)] -pub fn timeago(s: time::OffsetDateTime) -> Result { - Ok(timeago::Formatter::new().convert((time::OffsetDateTime::now_utc() - s).unsigned_abs())) +use std::borrow::Borrow; + +pub fn timeago(s: impl Borrow) -> Result { + Ok(timeago::Formatter::new() + .convert((time::OffsetDateTime::now_utc() - *s.borrow()).unsigned_abs())) } pub fn file_perms(s: &i32) -> Result { diff --git a/src/methods/repo.rs b/src/methods/repo.rs index dd7cb13..47a6f06 100644 --- a/src/methods/repo.rs +++ a/src/methods/repo.rs @@ -1,4 +1,5 @@ use anyhow::Context; +use std::collections::BTreeMap; use std::{ fmt::{Debug, Display, Formatter}, io::Write, @@ -27,7 +28,9 @@ use yoke::Yoke; use super::filters; -use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, Refs, TreeItem}; +use crate::database::schema::commit::YokedCommit; +use crate::database::schema::tag::YokedTag; +use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, TreeItem}; use crate::{git::Commit, into_response, layers::UnwrapInfallible, Git}; #[derive(Clone)] @@ -159,32 +162,48 @@ .into_response() } +pub struct Refs { + heads: BTreeMap, + tags: Vec<(String, YokedTag)>, +} + #[derive(Template)] #[template(path = "repo/summary.html")] pub struct SummaryView<'a> { repo: Repository, - refs: Arc, + refs: Refs, commit_list: Vec<&'a crate::database::schema::commit::Commit<'a>>, } pub async fn handle_summary( Extension(repo): Extension, - Extension(RepositoryPath(repository_path)): Extension, - Extension(git): Extension>, Extension(db): Extension, ) -> Result { - let open_repo = git.repo(repository_path).await?; - let refs = open_repo.refs().await?; - let repository = crate::database::schema::repository::Repository::open(&db, &*repo)? .context("Repository does not exist")?; let commit_tree = repository.get().commit_tree(&db, "refs/heads/master")?; let commits = commit_tree.fetch_latest(11, 0).await; let commit_list = commits.iter().map(Yoke::get).collect(); + let mut heads = BTreeMap::new(); + for head in repository.get().heads(&db) { + let commit_tree = repository.get().commit_tree(&db, &head)?; + let name = head.strip_prefix("refs/heads/"); + + if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) { + heads.insert(name.to_string(), commit); + } + } + + let tags = repository + .get() + .tag_tree(&db) + .context("Failed to fetch indexed tags")? + .fetch_all(); + Ok(into_response(&SummaryView { repo, - refs, + refs: Refs { heads, tags }, commit_list, })) } @@ -309,18 +328,36 @@ #[template(path = "repo/refs.html")] pub struct RefsView { repo: Repository, - refs: Arc, + refs: Refs, } pub async fn handle_refs( Extension(repo): Extension, - Extension(RepositoryPath(repository_path)): Extension, - Extension(git): Extension>, + Extension(db): Extension, ) -> Result { - let open_repo = git.repo(repository_path).await?; - let refs = open_repo.refs().await?; + let repository = crate::database::schema::repository::Repository::open(&db, &*repo)? + .context("Repository does not exist")?; + + let mut heads = BTreeMap::new(); + for head in repository.get().heads(&db) { + let commit_tree = repository.get().commit_tree(&db, &head)?; + let name = head.strip_prefix("refs/heads/"); - Ok(into_response(&RefsView { repo, refs })) + if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) { + heads.insert(name.to_string(), commit); + } + } + + let tags = repository + .get() + .tag_tree(&db) + .context("Failed to fetch indexed tags")? + .fetch_all(); + + Ok(into_response(&RefsView { + repo, + refs: Refs { heads, tags }, + })) } #[derive(Template)] diff --git a/templates/repo/refs.html b/templates/repo/refs.html index 40f4d06..6f97657 100644 --- a/templates/repo/refs.html +++ a/templates/repo/refs.html @@ -5,7 +5,7 @@ {% block content %} - {% call refs::branch_table(refs.branch) %} + {% call refs::branch_table(refs.heads) %} @@ -16,6 +16,6 @@ - {% call refs::tag_table(refs.tag) %} + {% call refs::tag_table(refs.tags) %}
{% endblock %}diff --git a/templates/repo/summary.html b/templates/repo/summary.html index 09a2897..a9c4b50 100644 --- a/templates/repo/summary.html +++ a/templates/repo/summary.html @@ -5,8 +5,8 @@ {% block content %} - {% call refs::branch_table(refs.branch.iter().take(10)) %} - {% if refs.branch.len() > 10 %} + {% call refs::branch_table(refs.heads.iter().take(10)) %} + {% if refs.heads.len() > 10 %} @@ -26,8 +26,8 @@ - {% call refs::tag_table(refs.tag.iter().take(10)) %} - {% if refs.tag.len() > 10 %} + {% call refs::tag_table(refs.tags.iter().take(10)) %} + {% if refs.tags.len() > 10 %} diff --git a/src/database/schema/commit.rs b/src/database/schema/commit.rs index a2ad226..faa80fe 100644 --- a/src/database/schema/commit.rs +++ a/src/database/schema/commit.rs @@ -85,7 +85,9 @@ #[derive(Serialize, Deserialize, Debug)] pub struct Author<'a> { + #[serde(borrow)] pub name: Cow<'a, str>, + #[serde(borrow)] pub email: Cow<'a, str>, pub time: OffsetDateTime, } diff --git a/src/database/schema/mod.rs b/src/database/schema/mod.rs index 2f51a8b..9609e4f 100644 --- a/src/database/schema/mod.rs +++ a/src/database/schema/mod.rs @@ -6,5 +6,6 @@ pub mod commit; pub mod prefixes; pub mod repository; +pub mod tag; pub type Yoked = Yoke>; diff --git a/src/database/schema/prefixes.rs b/src/database/schema/prefixes.rs index a1e2e90..cad0eec 100644 --- a/src/database/schema/prefixes.rs +++ a/src/database/schema/prefixes.rs @@ -5,7 +5,7 @@ pub enum TreePrefix { Repository = 0, Commit = 100, - _Tag = 101, + Tag = 101, } impl TreePrefix { @@ -31,6 +31,16 @@ prefixed.push(TreePrefix::Commit as u8); prefixed.extend_from_slice(&repository.to_ne_bytes()); prefixed.extend_from_slice(reference); + + prefixed + } + + pub fn tag_id(repository: RepositoryId) -> Vec { + let mut prefixed = Vec::with_capacity( + std::mem::size_of::() + std::mem::size_of::(), + ); + prefixed.push(TreePrefix::Tag as u8); + prefixed.extend_from_slice(&repository.to_ne_bytes()); prefixed } diff --git a/src/database/schema/repository.rs b/src/database/schema/repository.rs index 2b61783..0769fca 100644 --- a/src/database/schema/repository.rs +++ a/src/database/schema/repository.rs @@ -1,7 +1,9 @@ use crate::database::schema::commit::CommitTree; use crate::database::schema::prefixes::TreePrefix; +use crate::database::schema::tag::TagTree; use crate::database::schema::Yoked; use anyhow::{Context, Result}; +use nom::AsBytes; use serde::{Deserialize, Serialize}; use sled::IVec; use std::borrow::Cow; @@ -84,6 +86,27 @@ .context("Failed to open commit tree")?; Ok(CommitTree::new(tree)) + } + + pub fn tag_tree(&self, database: &sled::Db) -> Result { + let tree = database + .open_tree(TreePrefix::tag_id(self.id)) + .context("Failed to open tag tree")?; + + Ok(TagTree::new(tree)) + } + + pub fn heads(&self, database: &sled::Db) -> Vec { + let prefix = TreePrefix::commit_id(self.id, ""); + + database + .tree_names() + .into_iter() + .filter_map(|v| { + v.strip_prefix(prefix.as_bytes()) + .map(|v| String::from_utf8_lossy(v).into_owned()) + }) + .collect() } } diff --git a/src/database/schema/tag.rs b/src/database/schema/tag.rs new file mode 100644 index 0000000..9958ee4 100644 --- /dev/null +++ a/src/database/schema/tag.rs @@ -1,0 +1,92 @@ +use crate::database::schema::commit::Author; +use crate::database::schema::Yoked; +use git2::Signature; +use serde::{Deserialize, Serialize}; +use sled::IVec; +use std::collections::HashSet; +use std::ops::Deref; +use yoke::{Yoke, Yokeable}; + +#[derive(Serialize, Deserialize, Debug, Yokeable)] +pub struct Tag<'a> { + #[serde(borrow)] + pub tagger: Option>, +} + +impl<'a> Tag<'a> { + pub fn new(tagger: Option<&'a Signature<'_>>) -> Self { + Self { + tagger: tagger.map(Into::into), + } + } + + pub fn insert(&self, batch: &TagTree, name: &str) { + batch + .insert(&name.as_bytes(), bincode::serialize(self).unwrap()) + .unwrap(); + } +} + +pub struct TagTree(sled::Tree); + +impl Deref for TagTree { + type Target = sled::Tree; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +pub type YokedTag = Yoked>; + +impl TagTree { + pub(super) fn new(tree: sled::Tree) -> Self { + Self(tree) + } + + pub fn remove(&self, name: &str) -> bool { + self.0.remove(name).unwrap().is_some() + } + + pub fn list(&self) -> HashSet { + self.iter() + .keys() + .filter_map(Result::ok) + .map(|v| String::from_utf8_lossy(&v).into_owned()) + .collect() + } + + pub fn fetch_all(&self) -> Vec<(String, YokedTag)> { + let mut res = self + .iter() + .map(|res| { + let (name, value) = res?; + + let name = String::from_utf8_lossy(&name) + .strip_prefix("refs/tags/") + .unwrap() + .to_string(); + + // internally value is an Arc so it should already be stablederef but because + // of reasons unbeknownst to me, sled has its own Arc implementation so we need + // to box the value as well to get a stablederef... + let value = Box::new(value); + + Ok(( + name, + Yoke::try_attach_to_cart(value, |data: &IVec| bincode::deserialize(data)) + .unwrap(), + )) + }) + .collect::, sled::Error>>() + .unwrap(); + + res.sort_unstable_by(|a, b| { + let a_tagger = a.1.get().tagger.as_ref().map(|v| v.time); + let b_tagger = b.1.get().tagger.as_ref().map(|v| v.time); + b_tagger.cmp(&a_tagger) + }); + + res + } +} diff --git a/templates/repo/macros/refs.html b/templates/repo/macros/refs.html index a3bfecd..316ddcd 100644 --- a/templates/repo/macros/refs.html +++ a/templates/repo/macros/refs.html @@ -9,17 +9,17 @@ -{% for branch in branches %} +{% for (name, commit) in branches %} - - + + @@ -38,20 +38,20 @@ -{% for tag in tags %} +{% for (name, tag) in tags %} - + -- rgit 0.1.3
[...]
[...]
{{ branch.name }}{{ branch.commit.summary() }}{{ name }}{{ commit.get().summary }} - - {{ branch.commit.author().name() }} + + {{ commit.get().author.name }} -
{{ tag.name }}{{ name }} - {% if let Some(tagger) = tag.tagger %} - - {{ tagger.name() }} + {% if let Some(tagger) = tag.get().tagger %} + + {{ tagger.name }} {% endif %} - {% if let Some(tagger) = tag.tagger %} -