From a3ec9bcddbdbadf70588697f3ee9f585a3263413 Mon Sep 17 00:00:00 2001 From: Alex Butler Date: Tue, 27 Feb 2024 14:40:31 +0000 Subject: [PATCH] Fix cache_releases_older_than --- CHANGELOG.md | 2 +- src/config.rs | 6 ++++-- src/main.rs | 2 +- src/providers/gitlab.rs | 44 +++++++++++++++++++++++++------------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8860798..0870345 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Add info logs for release & metadata fetch latency. - When fetching all releases handle 429 by backing off. - Improve fetch error logging. -- Added crate eligibility cache. +- Added crate eligibility cache. May be controlled with config `cache-releases-older-than`. - Introduce configurable cache backend with a RocksDB implementation (set `cache.type = "rocksdb"` and `cache.path = "cache"` to use it), defaults to `cache.type = "in-memory"`. - Support crate yanking by creating a `yanked` file on the release. - Add `bust-cache` command, invoked via `ssh [registry] -- bust-cache [project] [crate-name] [crate-version]` to remove eligibility cache (ie. after a crate has been yanked) diff --git a/src/config.rs b/src/config.rs index 3a9db39..6660b79 100644 --- a/src/config.rs +++ b/src/config.rs @@ -55,8 +55,10 @@ pub struct GitlabConfig { #[serde(default)] pub metadata_format: MetadataFormat, /// Cache file checksum fetches for all release older than this value. + /// + /// Default zero (cache all releases). #[serde(default, with = "humantime_serde")] - pub cache_releases_older_than: Option, + pub cache_releases_older_than: Duration, } impl GitlabConfig { @@ -135,6 +137,6 @@ fn deser_config() { assert_eq!(gitlab.metadata_format, MetadataFormat::JsonZst); assert_eq!( gitlab.cache_releases_older_than, - Some(Duration::from_secs(2 * 24 * 60 * 60)) + Duration::from_secs(2 * 24 * 60 * 60) ); } diff --git a/src/main.rs b/src/main.rs index d380ec0..2897836 100644 --- a/src/main.rs +++ b/src/main.rs @@ -251,7 +251,7 @@ impl Handler { return Ok(cache); } - info!("Fetching metadata from GitLab"); + debug!("Fetching metadata from GitLab"); // fetch metadata from the provider let metadata = gitlab diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs index f5ccc7b..c164636 100644 --- a/src/providers/gitlab.rs +++ b/src/providers/gitlab.rs @@ -1,10 +1,9 @@ // blocks_in_conditions: didn't work with `#[instrument...`` usage #![allow(clippy::module_name_repetitions, clippy::blocks_in_conditions)] -use crate::cache::{Cache, ConcreteCache, Yoked}; -use crate::providers::EligibilityCacheKey; use crate::{ + cache::{Cache, ConcreteCache, Yoked}, config::{GitlabConfig, MetadataFormat}, - providers::{Release, User}, + providers::{EligibilityCacheKey, Release, User}, }; use anyhow::Context; use async_trait::async_trait; @@ -14,11 +13,10 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use reqwest::{header, Certificate}; use serde::{Deserialize, Serialize}; use smol_str::SmolStr; -use std::borrow::Cow; -use std::sync::Arc; +use std::{borrow::Cow, sync::Arc, time::Duration}; use time::OffsetDateTime; use tokio::sync::Semaphore; -use tracing::{debug, info, info_span, instrument, Instrument}; +use tracing::{debug, info_span, instrument, Instrument}; use url::Url; use yoke::Yoke; @@ -32,6 +30,7 @@ pub struct Gitlab { metadata_format: MetadataFormat, admin_token: Option, cache: ConcreteCache, + cache_checksums_older_than: Duration, } impl Gitlab { @@ -51,6 +50,7 @@ impl Gitlab { metadata_format: config.metadata_format, admin_token: config.admin_token.clone(), cache, + cache_checksums_older_than: config.cache_releases_older_than, }) } @@ -84,7 +84,7 @@ impl Gitlab { return Ok(cached); } - info!("Fetching eligibility for release"); + debug!("Fetching eligibility for release"); let project = utf8_percent_encode(raw_project, NON_ALPHANUMERIC); let package_id = utf8_percent_encode(package_id, NON_ALPHANUMERIC); @@ -119,21 +119,27 @@ impl Gitlab { let expected_file_name = format!("{}-{}.crate", release.name, release.version); // grab the sha256 checksum of the .crate file itself - let release = package_files + let Some(package_file) = package_files .into_iter() .find(|package_file| package_file.file_name == expected_file_name) - .map(|package_file| Release { - name: Cow::Owned(release.name.to_string()), - version: Cow::Owned(release.version.clone()), - checksum: Cow::Owned(package_file.file_sha256), - project: Cow::Owned(raw_project.to_string()), - yanked, - }); + else { + return Ok(Yoke::attach_to_cart(Vec::new(), |_| None)); + }; - self.cache - .put(cache_key, &release) - .await - .context("failed to write to cache")?; + let release = Some(Release { + name: Cow::Owned(release.name.to_string()), + version: Cow::Owned(release.version.clone()), + checksum: Cow::Owned(package_file.file_sha256), + project: Cow::Owned(raw_project.to_string()), + yanked, + }); + + if package_file.created_at + self.cache_checksums_older_than < OffsetDateTime::now_utc() { + self.cache + .put(cache_key, &release) + .await + .context("failed to write to cache")?; + } Ok(Yoke::attach_to_cart(Vec::new(), |_| release)) } -- libgit2 1.7.2