From 15b16763753c3febd6116468ea226e6ca9bf7a12 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 21:06:04 +0200 Subject: [PATCH] feat(cache): purge stale cargo cache entries in retention run TODO 2.9 recorded a retention policy that warned about low disk while freeing 0.0 GB twice in a row, because it governed artifacts, logs and IP leases -- 52 KB in total on this host -- while 195 G of backups sat outside its reach. Adding an unbounded directory of ~645 MB entries, one per Cargo.lock change, without extending retention in the same change would repeat that at a larger scale. The window is the tighter of the global retention and the cache's own, so the aggressive low-disk mode reaches the cache rather than stopping at the light directories. Sizes are read before deleting, not after. Summing st_size over files that have already been unlinked yields 0 -- which is exactly how a policy ends up reporting that it freed nothing while doing real work. A test asserts the reported figure. --- src/ci_orchestrator/commands/retention.py | 45 +++++++- tests/python/test_commands_retention.py | 132 ++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/src/ci_orchestrator/commands/retention.py b/src/ci_orchestrator/commands/retention.py index ffcffe7..cb4748a 100644 --- a/src/ci_orchestrator/commands/retention.py +++ b/src/ci_orchestrator/commands/retention.py @@ -14,7 +14,8 @@ from pathlib import Path import click -from ci_orchestrator.config import load_config +from ci_orchestrator.cargo_cache import CargoCacheStore +from ci_orchestrator.config import Config, load_config @click.group() @@ -93,6 +94,7 @@ def retention_run( _purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if) _purge_old_dirs(log_path, "log", cutoff, what_if=what_if) + _purge_cargo_cache(config, effective_days, what_if=what_if) usage_after = shutil.disk_usage(str(check_path)) free_gb_after = usage_after.free / (1024**3) @@ -132,3 +134,44 @@ def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) else: shutil.rmtree(d, ignore_errors=True) click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}") + + +def _purge_cargo_cache(config: Config, effective_days: int, *, what_if: bool) -> None: + """Purge stale cargo cache entries. + + Entries are the heaviest thing this policy governs -- hundreds of megabytes + each, one per Cargo.lock change -- so leaving them out would repeat TODO + 2.9, where retention warned about low disk while freeing 0.0 GB because + everything that actually weighed was outside its reach. + + The window is the tighter of the global retention and the cache's own, so + the aggressive low-disk mode reaches the cache too. + """ + if not config.cargo_cache.enabled: + click.echo("[RetentionPolicy] cargo cache: disabled — skipping.") + return + + days = min(effective_days, config.cargo_cache.retention_days) + cutoff = datetime.now(tz=UTC) - timedelta(days=days) + store = CargoCacheStore(config.cargo_cache.root) + + # Size the victims first: after a real prune the files are gone and their + # bytes are unreadable, which is how a policy ends up reporting 0.0 GB + # freed while doing real work (TODO 2.9). + doomed = store.prune_older_than(cutoff, what_if=True) + if not doomed: + click.echo( + f"[RetentionPolicy] cargo cache: nothing to purge " + f"(cutoff: {cutoff.strftime('%Y-%m-%d')})." + ) + return + freed_mb = sum(p.stat().st_size for p in doomed if p.exists()) / 1_048_576 + + if not what_if: + store.prune_older_than(cutoff) + + verb = "WhatIf: would purge" if what_if else "Purged" + click.echo( + f"[RetentionPolicy] {verb} {len(doomed)} cargo cache " + f"{'entry' if len(doomed) == 1 else 'entries'} ({freed_mb:.0f} MB)" + ) diff --git a/tests/python/test_commands_retention.py b/tests/python/test_commands_retention.py index 57f22e5..6e1957e 100644 --- a/tests/python/test_commands_retention.py +++ b/tests/python/test_commands_retention.py @@ -11,6 +11,7 @@ from click.testing import CliRunner import ci_orchestrator.commands.retention as ret from ci_orchestrator.__main__ import cli +from ci_orchestrator.config import Config # ── helpers ─────────────────────────────────────────────────────────────────── @@ -191,3 +192,134 @@ def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> assert result.exit_code == 0, result.output assert not old.exists() assert "Purged log" in result.output + + +# ── cargo cache entries ─────────────────────────────────────────────────────── + + +def _cache_config( + tmp_path: Path, *, enabled: bool = True, days: int = 30 +) -> tuple[Path, Config]: + """Point the loaded config at a throwaway cache store. + + Returns ``(store root, config)`` — the root so tests can seed entries, the + config so they can substitute it for ``load_config``. + """ + from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths + + root = tmp_path / "cache" / "cargo" + cfg = Config( + paths=Paths( + root=tmp_path, + templates=tmp_path / "t", + build_vms=tmp_path / "b", + artifacts=tmp_path / "artifacts", + keys=tmp_path / "k", + ), + backend=BackendConfig(), + cargo_cache=CargoCacheConfig(enabled=enabled, root=root, retention_days=days), + ) + return root, cfg + + +def _seed_entry(root: Path, digest: str, *, age_days: int, size: int = 1024) -> Path: + import os + + from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key + + store = CargoCacheStore(root) + src = root.parent / f"seed-{digest}.tar.zst" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_bytes(b"x" * size) + entry = store.publish(cache_key(lock_digest=digest, rustc_version="r", host_triple="t"), src) + stamp = time.time() - age_days * 86400 + os.utime(entry, (stamp, stamp)) + return entry + + +def test_retention_purges_stale_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + fresh = _seed_entry(root, "new", age_days=1) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not old.exists(), "stale entry should be purged" + assert fresh.exists(), "fresh entry should survive" + + +def test_retention_reports_the_bytes_it_freed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """TODO 2.9: a policy that frees space without saying how much is the bug.""" + root, cfg = _cache_config(tmp_path) + _seed_entry(root, "old", age_days=90, size=3 * 1024 * 1024) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "Purged 1 cargo cache entry (3 MB)" in result.output + + +def test_retention_what_if_keeps_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run", "--what-if"]) + assert result.exit_code == 0, result.output + assert old.exists() + assert "would purge 1 cargo cache entry" in result.output + + +def test_retention_skips_the_cache_when_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path, enabled=False) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert old.exists(), "a disabled cache must not be touched" + assert "disabled" in result.output + + +def test_low_disk_tightens_the_cache_window_too( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Aggressive mode must reach the cache — it is the heaviest thing here.""" + root, cfg = _cache_config(tmp_path, days=30) + entry = _seed_entry(root, "old", age_days=10) # young for 30d, old for 7d + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not entry.exists(), "aggressive retention should have reached the cache" + + +def test_retention_missing_cache_store_is_not_an_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _root, cfg = _cache_config(tmp_path) + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "nothing to purge" in result.output