🏡 index : ~doyle/gitlab-cargo-shim.git

e0ead53e576758e2f2fc84e70dbf4685e6132a70 [patch]

ed6c1022b2345a4438ab51c56258862212d3b570

671b14c0edb3eec807f6efa091f0476b4277d7b0

1acdf711ebda5b9a14e959c2f1d8c36cb6006299

e0ead53e576758e2f2fc84e70dbf4685e6132a70.tar.gz

Merge pull request #82 from alexheretic/cache-releases-older-than

Fix cache-releases-older-than support

Diff

 CHANGELOG.md            |  2 +-
 config.toml             |  2 +-
 src/config.rs           |  6 ++++--
 src/main.rs             |  2 +-
 src/providers/gitlab.rs | 44 +++++++++++++++++++++++++-------------------
 5 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8860798..0870345 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@
- Add info logs for release & metadata fetch latency.
- When fetching all releases handle 429 by backing off.
- Improve fetch error logging.
- Added crate eligibility cache.
- Added crate eligibility cache. May be controlled with config `cache-releases-older-than`.
- Introduce configurable cache backend with a RocksDB implementation (set `cache.type = "rocksdb"` and `cache.path = "cache"` to use it), defaults to `cache.type = "in-memory"`.
- Support crate yanking by creating a `yanked` file on the release.
- Add `bust-cache` command, invoked via `ssh [registry] -- bust-cache [project] [crate-name] [crate-version]` to remove eligibility cache (ie. after a crate has been yanked)
diff --git a/config.toml b/config.toml
index e4d15fe..1a7374d 100644
--- a/config.toml
+++ b/config.toml
@@ -26,7 +26,7 @@ uri = "http://127.0.0.1:3000"

## Cache file checksum fetches for all release older than this value.
##
## If omitted no caching will occur.
## If omitted will cache all releases of all ages.
##
## Note: Caching shouldn't be used if published releases are expected to be mutated.
## However, a grace period can allow the majority of crates to benefit from caching
diff --git a/src/config.rs b/src/config.rs
index 3a9db39..6660b79 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -55,8 +55,10 @@ pub struct GitlabConfig {
    #[serde(default)]
    pub metadata_format: MetadataFormat,
    /// Cache file checksum fetches for all release older than this value.
    ///
    /// Default zero (cache all releases).
    #[serde(default, with = "humantime_serde")]
    pub cache_releases_older_than: Option<Duration>,
    pub cache_releases_older_than: Duration,
}

impl GitlabConfig {
@@ -135,6 +137,6 @@ fn deser_config() {
    assert_eq!(gitlab.metadata_format, MetadataFormat::JsonZst);
    assert_eq!(
        gitlab.cache_releases_older_than,
        Some(Duration::from_secs(2 * 24 * 60 * 60))
        Duration::from_secs(2 * 24 * 60 * 60)
    );
}
diff --git a/src/main.rs b/src/main.rs
index d380ec0..2897836 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -251,7 +251,7 @@ impl<U: UserProvider + PackageProvider + Send + Sync + 'static> Handler<U> {
            return Ok(cache);
        }

        info!("Fetching metadata from GitLab");
        debug!("Fetching metadata from GitLab");

        // fetch metadata from the provider
        let metadata = gitlab
diff --git a/src/providers/gitlab.rs b/src/providers/gitlab.rs
index f5ccc7b..c164636 100644
--- a/src/providers/gitlab.rs
+++ b/src/providers/gitlab.rs
@@ -1,10 +1,9 @@
// blocks_in_conditions: didn't work with `#[instrument...`` usage
#![allow(clippy::module_name_repetitions, clippy::blocks_in_conditions)]
use crate::cache::{Cache, ConcreteCache, Yoked};
use crate::providers::EligibilityCacheKey;
use crate::{
    cache::{Cache, ConcreteCache, Yoked},
    config::{GitlabConfig, MetadataFormat},
    providers::{Release, User},
    providers::{EligibilityCacheKey, Release, User},
};
use anyhow::Context;
use async_trait::async_trait;
@@ -14,11 +13,10 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
use reqwest::{header, Certificate};
use serde::{Deserialize, Serialize};
use smol_str::SmolStr;
use std::borrow::Cow;
use std::sync::Arc;
use std::{borrow::Cow, sync::Arc, time::Duration};
use time::OffsetDateTime;
use tokio::sync::Semaphore;
use tracing::{debug, info, info_span, instrument, Instrument};
use tracing::{debug, info_span, instrument, Instrument};
use url::Url;
use yoke::Yoke;

@@ -32,6 +30,7 @@ pub struct Gitlab {
    metadata_format: MetadataFormat,
    admin_token: Option<String>,
    cache: ConcreteCache,
    cache_checksums_older_than: Duration,
}

impl Gitlab {
@@ -51,6 +50,7 @@ impl Gitlab {
            metadata_format: config.metadata_format,
            admin_token: config.admin_token.clone(),
            cache,
            cache_checksums_older_than: config.cache_releases_older_than,
        })
    }

@@ -84,7 +84,7 @@ impl Gitlab {
            return Ok(cached);
        }

        info!("Fetching eligibility for release");
        debug!("Fetching eligibility for release");

        let project = utf8_percent_encode(raw_project, NON_ALPHANUMERIC);
        let package_id = utf8_percent_encode(package_id, NON_ALPHANUMERIC);
@@ -119,21 +119,27 @@ impl Gitlab {
        let expected_file_name = format!("{}-{}.crate", release.name, release.version);

        // grab the sha256 checksum of the .crate file itself
        let release = package_files
        let Some(package_file) = package_files
            .into_iter()
            .find(|package_file| package_file.file_name == expected_file_name)
            .map(|package_file| Release {
                name: Cow::Owned(release.name.to_string()),
                version: Cow::Owned(release.version.clone()),
                checksum: Cow::Owned(package_file.file_sha256),
                project: Cow::Owned(raw_project.to_string()),
                yanked,
            });
        else {
            return Ok(Yoke::attach_to_cart(Vec::new(), |_| None));
        };

        self.cache
            .put(cache_key, &release)
            .await
            .context("failed to write to cache")?;
        let release = Some(Release {
            name: Cow::Owned(release.name.to_string()),
            version: Cow::Owned(release.version.clone()),
            checksum: Cow::Owned(package_file.file_sha256),
            project: Cow::Owned(raw_project.to_string()),
            yanked,
        });

        if package_file.created_at + self.cache_checksums_older_than < OffsetDateTime::now_utc() {
            self.cache
                .put(cache_key, &release)
                .await
                .context("failed to write to cache")?;
        }

        Ok(Yoke::attach_to_cart(Vec::new(), |_| release))
    }