Index tags to sled and read them from views
Diff
src/git.rs | 73 -------------------------------------------------------------------------
src/database/indexer.rs | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/methods/filters.rs | 7 +++++--
src/methods/repo.rs | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
templates/repo/refs.html | 4 ++--
templates/repo/summary.html | 8 ++++----
src/database/schema/commit.rs | 2 ++
src/database/schema/mod.rs | 1 +
src/database/schema/prefixes.rs | 12 +++++++++++-
src/database/schema/repository.rs | 23 +++++++++++++++++++++++
src/database/schema/tag.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
templates/repo/macros/refs.html | 30 ++++++++++++++++--------------
12 files changed, 271 insertions(+), 112 deletions(-)
@@ -20,7 +20,6 @@
pub struct Git {
commits: Cache<Oid, Arc<Commit>>,
readme_cache: Cache<PathBuf, Option<(ReadmeFormat, Arc<str>)>>,
refs: Cache<PathBuf, Arc<Refs>>,
syntax_set: SyntaxSet,
}
@@ -36,10 +35,6 @@
.time_to_live(Duration::from_secs(10))
.max_capacity(100)
.build(),
refs: Cache::builder()
.time_to_live(Duration::from_secs(10))
.max_capacity(100)
.build(),
syntax_set,
}
}
@@ -200,56 +195,6 @@
})
.await
.context("Failed to join Tokio task")?
}
#[instrument(skip(self))]
pub async fn refs(self: Arc<Self>) -> Result<Arc<Refs>, Arc<anyhow::Error>> {
let git = self.git.clone();
git.refs
.try_get_with(self.cache_key.clone(), async move {
tokio::task::spawn_blocking(move || {
let repo = self.repo.lock();
let ref_iter = repo.references().context("Couldn't get list of references for repository")?;
let mut built_refs = Refs::default();
for ref_ in ref_iter {
let ref_ = ref_?;
if ref_.is_branch() {
let commit = ref_.peel_to_commit().context("Reference is apparently a branch but I couldn't get to the HEAD of it")?;
built_refs.branch.push(Branch {
name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(),
commit: commit.try_into()?,
});
} else if ref_.is_tag() {
if let Ok(tag) = ref_.peel_to_tag() {
built_refs.tag.push(Tag {
name: String::from_utf8_lossy(ref_.shorthand_bytes()).into_owned(),
tagger: tag.tagger().map(TryInto::try_into).transpose()?,
});
}
}
}
built_refs.branch.sort_unstable_by(|one, two| {
two.commit.committer.time.cmp(&one.commit.committer.time)
});
built_refs.tag.sort_unstable_by(|one, two| {
let one_tagger = one.tagger.as_ref().map(|v| v.time);
let two_tagger = two.tagger.as_ref().map(|v| v.time);
two_tagger.cmp(&one_tagger)
});
Ok(Arc::new(built_refs))
})
.await
.context("Failed to join Tokio task")?
})
.await
}
#[instrument(skip(self))]
@@ -409,12 +354,6 @@
pub struct FileWithContent {
pub metadata: File,
pub content: String,
}
#[derive(Debug, Default)]
pub struct Refs {
pub branch: Vec<Branch>,
pub tag: Vec<Tag>,
}
#[derive(Debug)]
@@ -440,19 +379,12 @@
pub tagger: Option<CommitUser>,
pub message: String,
pub tagged_object: Option<TaggedObject>,
}
#[derive(Debug)]
pub struct Tag {
pub name: String,
pub tagger: Option<CommitUser>,
}
#[derive(Debug)]
pub struct CommitUser {
name: String,
email: String,
email_md5: String,
time: OffsetDateTime,
}
@@ -463,7 +395,6 @@
Ok(CommitUser {
name: String::from_utf8_lossy(v.name_bytes()).into_owned(),
email: String::from_utf8_lossy(v.email_bytes()).into_owned(),
email_md5: format!("{:x}", md5::compute(v.email_bytes())),
time: OffsetDateTime::from_unix_timestamp(v.when().seconds())?,
})
}
@@ -476,10 +407,6 @@
pub fn email(&self) -> &str {
&self.email
}
pub fn email_md5(&self) -> &str {
&self.email_md5
}
pub fn time(&self) -> OffsetDateTime {
@@ -1,4 +1,5 @@
use git2::Sort;
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use time::OffsetDateTime;
use tracing::{info, info_span};
@@ -6,12 +7,25 @@
use crate::database::schema::{
commit::Commit,
repository::{Repository, RepositoryId},
tag::Tag,
};
pub fn run(db: &sled::Db) {
let span = info_span!("index_update");
let _entered = span.enter();
info!("Starting index update");
let scan_path = Path::new("/Users/jordan/Code/test-git");
update_repository_metadata(scan_path, db);
update_repository_reflog(scan_path, db);
update_repository_tags(scan_path, db);
info!("Flushing to disk");
db.flush().unwrap();
info!("Finished index update");
}
fn update_repository_metadata(scan_path: &Path, db: &sled::Db) {
@@ -52,7 +66,7 @@
}
let span = info_span!(
"index_update",
"branch_index_update",
reference = reference_name.as_ref(),
repository = relative_path
);
@@ -93,7 +107,57 @@
for to_remove in (i + 1)..(i + 100) {
commit_tree.remove(&to_remove.to_be_bytes()).unwrap();
}
}
}
}
fn update_repository_tags(scan_path: &Path, db: &sled::Db) {
for (relative_path, db_repository) in Repository::fetch_all(db).unwrap() {
let git_repository = git2::Repository::open(scan_path.join(&relative_path)).unwrap();
let tag_tree = db_repository.get().tag_tree(db).unwrap();
let git_tags: HashSet<_> = git_repository
.references()
.unwrap()
.filter_map(Result::ok)
.filter(|v| v.name_bytes().starts_with(b"refs/tags/"))
.map(|v| String::from_utf8_lossy(v.name_bytes()).into_owned())
.collect();
let indexed_tags: HashSet<String> = tag_tree.list().into_iter().collect();
for tag_name in git_tags.difference(&indexed_tags) {
let span = info_span!(
"tag_index_update",
reference = tag_name,
repository = relative_path
);
let _entered = span.enter();
let reference = git_repository.find_reference(tag_name).unwrap();
if let Ok(tag) = reference.peel_to_tag() {
info!("Inserting newly discovered tag to index");
Tag::new(tag.tagger().as_ref()).insert(&tag_tree, tag_name);
}
}
for tag_name in indexed_tags.difference(&git_tags) {
let span = info_span!(
"tag_index_update",
reference = tag_name,
repository = relative_path
);
let _entered = span.enter();
info!("Removing stale tag from index");
tag_tree.remove(tag_name);
}
}
}
@@ -1,8 +1,11 @@
#![allow(clippy::unnecessary_wraps, clippy::trivially_copy_pass_by_ref)]
pub fn timeago(s: time::OffsetDateTime) -> Result<String, askama::Error> {
Ok(timeago::Formatter::new().convert((time::OffsetDateTime::now_utc() - s).unsigned_abs()))
use std::borrow::Borrow;
pub fn timeago(s: impl Borrow<time::OffsetDateTime>) -> Result<String, askama::Error> {
Ok(timeago::Formatter::new()
.convert((time::OffsetDateTime::now_utc() - *s.borrow()).unsigned_abs()))
}
pub fn file_perms(s: &i32) -> Result<String, askama::Error> {
@@ -1,4 +1,5 @@
use anyhow::Context;
use std::collections::BTreeMap;
use std::{
fmt::{Debug, Display, Formatter},
io::Write,
@@ -27,7 +28,9 @@
use yoke::Yoke;
use super::filters;
use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, Refs, TreeItem};
use crate::database::schema::commit::YokedCommit;
use crate::database::schema::tag::YokedTag;
use crate::git::{DetailedTag, FileWithContent, PathDestination, ReadmeFormat, TreeItem};
use crate::{git::Commit, into_response, layers::UnwrapInfallible, Git};
#[derive(Clone)]
@@ -159,32 +162,48 @@
.into_response()
}
pub struct Refs {
heads: BTreeMap<String, YokedCommit>,
tags: Vec<(String, YokedTag)>,
}
#[derive(Template)]
#[template(path = "repo/summary.html")]
pub struct SummaryView<'a> {
repo: Repository,
refs: Arc<Refs>,
refs: Refs,
commit_list: Vec<&'a crate::database::schema::commit::Commit<'a>>,
}
pub async fn handle_summary(
Extension(repo): Extension<Repository>,
Extension(RepositoryPath(repository_path)): Extension<RepositoryPath>,
Extension(git): Extension<Arc<Git>>,
Extension(db): Extension<sled::Db>,
) -> Result<Response> {
let open_repo = git.repo(repository_path).await?;
let refs = open_repo.refs().await?;
let repository = crate::database::schema::repository::Repository::open(&db, &*repo)?
.context("Repository does not exist")?;
let commit_tree = repository.get().commit_tree(&db, "refs/heads/master")?;
let commits = commit_tree.fetch_latest(11, 0).await;
let commit_list = commits.iter().map(Yoke::get).collect();
let mut heads = BTreeMap::new();
for head in repository.get().heads(&db) {
let commit_tree = repository.get().commit_tree(&db, &head)?;
let name = head.strip_prefix("refs/heads/");
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) {
heads.insert(name.to_string(), commit);
}
}
let tags = repository
.get()
.tag_tree(&db)
.context("Failed to fetch indexed tags")?
.fetch_all();
Ok(into_response(&SummaryView {
repo,
refs,
refs: Refs { heads, tags },
commit_list,
}))
}
@@ -309,18 +328,36 @@
#[template(path = "repo/refs.html")]
pub struct RefsView {
repo: Repository,
refs: Arc<Refs>,
refs: Refs,
}
pub async fn handle_refs(
Extension(repo): Extension<Repository>,
Extension(RepositoryPath(repository_path)): Extension<RepositoryPath>,
Extension(git): Extension<Arc<Git>>,
Extension(db): Extension<sled::Db>,
) -> Result<Response> {
let open_repo = git.repo(repository_path).await?;
let refs = open_repo.refs().await?;
let repository = crate::database::schema::repository::Repository::open(&db, &*repo)?
.context("Repository does not exist")?;
let mut heads = BTreeMap::new();
for head in repository.get().heads(&db) {
let commit_tree = repository.get().commit_tree(&db, &head)?;
let name = head.strip_prefix("refs/heads/");
Ok(into_response(&RefsView { repo, refs }))
if let (Some(name), Some(commit)) = (name, commit_tree.fetch_latest_one()) {
heads.insert(name.to_string(), commit);
}
}
let tags = repository
.get()
.tag_tree(&db)
.context("Failed to fetch indexed tags")?
.fetch_all();
Ok(into_response(&RefsView {
repo,
refs: Refs { heads, tags },
}))
}
#[derive(Template)]
@@ -5,7 +5,7 @@
{% block content %}
<table class="repositories">
{% call refs::branch_table(refs.branch) %}
{% call refs::branch_table(refs.heads) %}
<tbody>
<tr class="separator">
@@ -16,6 +16,6 @@
</tr>
</tbody>
{% call refs::tag_table(refs.tag) %}
{% call refs::tag_table(refs.tags) %}
</table>
{% endblock %}
@@ -5,8 +5,8 @@
{% block content %}
<table class="repositories">
{% call refs::branch_table(refs.branch.iter().take(10)) %}
{% if refs.branch.len() > 10 %}
{% call refs::branch_table(refs.heads.iter().take(10)) %}
{% if refs.heads.len() > 10 %}
<tbody>
<tr class="no-background">
<td><a href="/{{ repo.display() }}/refs" class="no-style">[...]</a></td>
@@ -26,8 +26,8 @@
</tr>
</tbody>
{% call refs::tag_table(refs.tag.iter().take(10)) %}
{% if refs.tag.len() > 10 %}
{% call refs::tag_table(refs.tags.iter().take(10)) %}
{% if refs.tags.len() > 10 %}
<tbody>
<tr class="no-background">
<td><a href="/{{ repo.display() }}/refs" class="no-style">[...]</a></td>
@@ -85,7 +85,9 @@
#[derive(Serialize, Deserialize, Debug)]
pub struct Author<'a> {
#[serde(borrow)]
pub name: Cow<'a, str>,
#[serde(borrow)]
pub email: Cow<'a, str>,
pub time: OffsetDateTime,
}
@@ -6,5 +6,6 @@
pub mod commit;
pub mod prefixes;
pub mod repository;
pub mod tag;
pub type Yoked<T> = Yoke<T, Box<IVec>>;
@@ -5,7 +5,7 @@
pub enum TreePrefix {
Repository = 0,
Commit = 100,
_Tag = 101,
Tag = 101,
}
impl TreePrefix {
@@ -31,6 +31,16 @@
prefixed.push(TreePrefix::Commit as u8);
prefixed.extend_from_slice(&repository.to_ne_bytes());
prefixed.extend_from_slice(reference);
prefixed
}
pub fn tag_id(repository: RepositoryId) -> Vec<u8> {
let mut prefixed = Vec::with_capacity(
std::mem::size_of::<TreePrefix>() + std::mem::size_of::<RepositoryId>(),
);
prefixed.push(TreePrefix::Tag as u8);
prefixed.extend_from_slice(&repository.to_ne_bytes());
prefixed
}
@@ -1,7 +1,9 @@
use crate::database::schema::commit::CommitTree;
use crate::database::schema::prefixes::TreePrefix;
use crate::database::schema::tag::TagTree;
use crate::database::schema::Yoked;
use anyhow::{Context, Result};
use nom::AsBytes;
use serde::{Deserialize, Serialize};
use sled::IVec;
use std::borrow::Cow;
@@ -84,6 +86,27 @@
.context("Failed to open commit tree")?;
Ok(CommitTree::new(tree))
}
pub fn tag_tree(&self, database: &sled::Db) -> Result<TagTree> {
let tree = database
.open_tree(TreePrefix::tag_id(self.id))
.context("Failed to open tag tree")?;
Ok(TagTree::new(tree))
}
pub fn heads(&self, database: &sled::Db) -> Vec<String> {
let prefix = TreePrefix::commit_id(self.id, "");
database
.tree_names()
.into_iter()
.filter_map(|v| {
v.strip_prefix(prefix.as_bytes())
.map(|v| String::from_utf8_lossy(v).into_owned())
})
.collect()
}
}
@@ -1,0 +1,92 @@
use crate::database::schema::commit::Author;
use crate::database::schema::Yoked;
use git2::Signature;
use serde::{Deserialize, Serialize};
use sled::IVec;
use std::collections::HashSet;
use std::ops::Deref;
use yoke::{Yoke, Yokeable};
#[derive(Serialize, Deserialize, Debug, Yokeable)]
pub struct Tag<'a> {
#[serde(borrow)]
pub tagger: Option<Author<'a>>,
}
impl<'a> Tag<'a> {
pub fn new(tagger: Option<&'a Signature<'_>>) -> Self {
Self {
tagger: tagger.map(Into::into),
}
}
pub fn insert(&self, batch: &TagTree, name: &str) {
batch
.insert(&name.as_bytes(), bincode::serialize(self).unwrap())
.unwrap();
}
}
pub struct TagTree(sled::Tree);
impl Deref for TagTree {
type Target = sled::Tree;
fn deref(&self) -> &Self::Target {
&self.0
}
}
pub type YokedTag = Yoked<Tag<'static>>;
impl TagTree {
pub(super) fn new(tree: sled::Tree) -> Self {
Self(tree)
}
pub fn remove(&self, name: &str) -> bool {
self.0.remove(name).unwrap().is_some()
}
pub fn list(&self) -> HashSet<String> {
self.iter()
.keys()
.filter_map(Result::ok)
.map(|v| String::from_utf8_lossy(&v).into_owned())
.collect()
}
pub fn fetch_all(&self) -> Vec<(String, YokedTag)> {
let mut res = self
.iter()
.map(|res| {
let (name, value) = res?;
let name = String::from_utf8_lossy(&name)
.strip_prefix("refs/tags/")
.unwrap()
.to_string();
let value = Box::new(value);
Ok((
name,
Yoke::try_attach_to_cart(value, |data: &IVec| bincode::deserialize(data))
.unwrap(),
))
})
.collect::<Result<Vec<(String, YokedTag)>, sled::Error>>()
.unwrap();
res.sort_unstable_by(|a, b| {
let a_tagger = a.1.get().tagger.as_ref().map(|v| v.time);
let b_tagger = b.1.get().tagger.as_ref().map(|v| v.time);
b_tagger.cmp(&a_tagger)
});
res
}
}
@@ -9,17 +9,17 @@
</thead>
<tbody>
{% for branch in branches %}
{% for (name, commit) in branches %}
<tr>
<td><a href="/{{ repo.display() }}/log/?h={{ branch.name }}">{{ branch.name }}</a></td>
<td><a href="/{{ repo.display() }}/commit/?id={{ branch.commit.oid() }}">{{ branch.commit.summary() }}</a></td>
<td><a href="/{{ repo.display() }}/log/?h={{ name }}">{{ name }}</a></td>
<td><a href="/{{ repo.display() }}/commit/?id={{ commit.get().hash|hex }}">{{ commit.get().summary }}</a></td>
<td>
<img src="https://www.gravatar.com/avatar/{{ branch.commit.author().email_md5() }}?s=13&d=retro" width="13" height="13">
{{ branch.commit.author().name() }}
<img src="https://www.gravatar.com/avatar/{{ commit.get().author.email|md5 }}?s=13&d=retro" width="13" height="13">
{{ commit.get().author.name }}
</td>
<td>
<time datetime="{{ branch.commit.author().time() }}" title="{{ branch.commit.author().time() }}">
{{ branch.commit.author().time()|timeago }}
<time datetime="{{ commit.get().author.time }}" title="{{ commit.get().author.time }}">
{{ commit.get().author.time|timeago }}
</time>
</td>
</tr>
@@ -38,20 +38,20 @@
</thead>
<tbody>
{% for tag in tags %}
{% for (name, tag) in tags %}
<tr>
<td><a href="/{{ repo.display() }}/tag/?h={{ tag.name }}">{{ tag.name }}</a></td>
<td><a href="/{{ repo.display() }}/tag/?h={{ name }}">{{ name }}</a></td>
<td></td>
<td>
{% if let Some(tagger) = tag.tagger %}
<img src="https://www.gravatar.com/avatar/{{ tagger.email_md5() }}?s=13&d=retro" width="13" height="13">
{{ tagger.name() }}
{% if let Some(tagger) = tag.get().tagger %}
<img src="https://www.gravatar.com/avatar/{{ tagger.email|md5 }}?s=13&d=retro" width="13" height="13">
{{ tagger.name }}
{% endif %}
</td>
<td>
{% if let Some(tagger) = tag.tagger %}
<time datetime="{{ tagger.time() }}" title="{{ tagger.time() }}">
{{ tagger.time()|timeago }}
{% if let Some(tagger) = tag.get().tagger %}
<time datetime="{{ tagger.time }}" title="{{ tagger.time }}">
{{ tagger.time|timeago }}
</time>
{% endif %}
</td>