Simone
15b1676375
TODO 2.9 recorded a retention policy that warned about low disk while freeing 0.0 GB twice in a row, because it governed artifacts, logs and IP leases -- 52 KB in total on this host -- while 195 G of backups sat outside its reach. Adding an unbounded directory of ~645 MB entries, one per Cargo.lock change, without extending retention in the same change would repeat that at a larger scale. The window is the tighter of the global retention and the cache's own, so the aggressive low-disk mode reaches the cache rather than stopping at the light directories. Sizes are read before deleting, not after. Summing st_size over files that have already been unlinked yields 0 -- which is exactly how a policy ends up reporting that it freed nothing while doing real work. A test asserts the reported figure.
326 lines
11 KiB
Python
326 lines
11 KiB
Python
"""Tests for ``ci_orchestrator retention run``."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import shutil
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from click.testing import CliRunner
|
|
|
|
import ci_orchestrator.commands.retention as ret
|
|
from ci_orchestrator.__main__ import cli
|
|
from ci_orchestrator.config import Config
|
|
|
|
# ── helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _fake_usage(free_gb: float, total_gb: float = 500.0) -> shutil._ntuple_diskusage:
|
|
gb = 1024**3
|
|
return shutil._ntuple_diskusage(
|
|
int(total_gb * gb),
|
|
int((total_gb - free_gb) * gb),
|
|
int(free_gb * gb),
|
|
)
|
|
|
|
|
|
def _make_old_dir(base: Path, name: str, age_days: int = 40) -> Path:
|
|
d = base / name
|
|
d.mkdir(parents=True)
|
|
old_mtime = time.time() - age_days * 86400
|
|
import os
|
|
|
|
os.utime(d, (old_mtime, old_mtime))
|
|
return d
|
|
|
|
|
|
def _make_recent_dir(base: Path, name: str) -> Path:
|
|
d = base / name
|
|
d.mkdir(parents=True)
|
|
return d
|
|
|
|
|
|
# ── retention run — basic purge ───────────────────────────────────────────────
|
|
|
|
|
|
def test_purge_old_artifact_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
art = tmp_path / "artifacts"
|
|
art.mkdir()
|
|
old = _make_old_dir(art, "job-001")
|
|
recent = _make_recent_dir(art, "job-002")
|
|
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert not old.exists(), "old dir should have been deleted"
|
|
assert recent.exists(), "recent dir should survive"
|
|
assert "Purged artifact" in result.output
|
|
|
|
|
|
def test_nothing_to_purge_message(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
art = tmp_path / "artifacts"
|
|
art.mkdir()
|
|
_make_recent_dir(art, "job-001")
|
|
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert "nothing to purge" in result.output
|
|
|
|
|
|
def test_missing_artifact_dir_skipped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"retention",
|
|
"run",
|
|
"--artifact-dir",
|
|
str(tmp_path / "nonexistent"),
|
|
"--log-dir",
|
|
str(tmp_path / "logs"),
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert "not found" in result.output
|
|
|
|
|
|
# ── aggressive mode ───────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_aggressive_mode_triggered(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
art = tmp_path / "artifacts"
|
|
art.mkdir()
|
|
_make_old_dir(art, "job-001", age_days=10) # 10 days old, > aggressive threshold (7d)
|
|
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=5))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"retention",
|
|
"run",
|
|
"--artifact-dir",
|
|
str(art),
|
|
"--log-dir",
|
|
str(tmp_path / "logs"),
|
|
"--min-free-gb",
|
|
"50",
|
|
"--aggressive-retention-days",
|
|
"7",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert "aggressive retention" in result.output
|
|
assert "WARNING" in result.output
|
|
|
|
|
|
def test_aggressive_mode_not_triggered_when_space_ok(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
art = tmp_path / "artifacts"
|
|
art.mkdir()
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert "Normal retention" in result.output
|
|
assert "WARNING" not in result.output
|
|
|
|
|
|
# ── --what-if ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_what_if_does_not_delete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
art = tmp_path / "artifacts"
|
|
art.mkdir()
|
|
old = _make_old_dir(art, "job-001")
|
|
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"retention",
|
|
"run",
|
|
"--artifact-dir",
|
|
str(art),
|
|
"--log-dir",
|
|
str(tmp_path / "logs"),
|
|
"--what-if",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert old.exists(), "--what-if must not delete anything"
|
|
assert "WhatIf" in result.output
|
|
|
|
|
|
# ── log dir purge ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
log = tmp_path / "logs"
|
|
log.mkdir()
|
|
old = _make_old_dir(log, "run-001")
|
|
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(
|
|
cli,
|
|
[
|
|
"retention",
|
|
"run",
|
|
"--artifact-dir",
|
|
str(tmp_path / "artifacts"),
|
|
"--log-dir",
|
|
str(log),
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert not old.exists()
|
|
assert "Purged log" in result.output
|
|
|
|
|
|
# ── cargo cache entries ───────────────────────────────────────────────────────
|
|
|
|
|
|
def _cache_config(
|
|
tmp_path: Path, *, enabled: bool = True, days: int = 30
|
|
) -> tuple[Path, Config]:
|
|
"""Point the loaded config at a throwaway cache store.
|
|
|
|
Returns ``(store root, config)`` — the root so tests can seed entries, the
|
|
config so they can substitute it for ``load_config``.
|
|
"""
|
|
from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths
|
|
|
|
root = tmp_path / "cache" / "cargo"
|
|
cfg = Config(
|
|
paths=Paths(
|
|
root=tmp_path,
|
|
templates=tmp_path / "t",
|
|
build_vms=tmp_path / "b",
|
|
artifacts=tmp_path / "artifacts",
|
|
keys=tmp_path / "k",
|
|
),
|
|
backend=BackendConfig(),
|
|
cargo_cache=CargoCacheConfig(enabled=enabled, root=root, retention_days=days),
|
|
)
|
|
return root, cfg
|
|
|
|
|
|
def _seed_entry(root: Path, digest: str, *, age_days: int, size: int = 1024) -> Path:
|
|
import os
|
|
|
|
from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key
|
|
|
|
store = CargoCacheStore(root)
|
|
src = root.parent / f"seed-{digest}.tar.zst"
|
|
src.parent.mkdir(parents=True, exist_ok=True)
|
|
src.write_bytes(b"x" * size)
|
|
entry = store.publish(cache_key(lock_digest=digest, rustc_version="r", host_triple="t"), src)
|
|
stamp = time.time() - age_days * 86400
|
|
os.utime(entry, (stamp, stamp))
|
|
return entry
|
|
|
|
|
|
def test_retention_purges_stale_cargo_cache_entries(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
root, cfg = _cache_config(tmp_path)
|
|
old = _seed_entry(root, "old", age_days=90)
|
|
fresh = _seed_entry(root, "new", age_days=1)
|
|
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run"])
|
|
assert result.exit_code == 0, result.output
|
|
assert not old.exists(), "stale entry should be purged"
|
|
assert fresh.exists(), "fresh entry should survive"
|
|
|
|
|
|
def test_retention_reports_the_bytes_it_freed(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""TODO 2.9: a policy that frees space without saying how much is the bug."""
|
|
root, cfg = _cache_config(tmp_path)
|
|
_seed_entry(root, "old", age_days=90, size=3 * 1024 * 1024)
|
|
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "Purged 1 cargo cache entry (3 MB)" in result.output
|
|
|
|
|
|
def test_retention_what_if_keeps_cargo_cache_entries(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
root, cfg = _cache_config(tmp_path)
|
|
old = _seed_entry(root, "old", age_days=90)
|
|
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run", "--what-if"])
|
|
assert result.exit_code == 0, result.output
|
|
assert old.exists()
|
|
assert "would purge 1 cargo cache entry" in result.output
|
|
|
|
|
|
def test_retention_skips_the_cache_when_disabled(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
root, cfg = _cache_config(tmp_path, enabled=False)
|
|
old = _seed_entry(root, "old", age_days=90)
|
|
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run"])
|
|
assert result.exit_code == 0, result.output
|
|
assert old.exists(), "a disabled cache must not be touched"
|
|
assert "disabled" in result.output
|
|
|
|
|
|
def test_low_disk_tightens_the_cache_window_too(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""Aggressive mode must reach the cache — it is the heaviest thing here."""
|
|
root, cfg = _cache_config(tmp_path, days=30)
|
|
entry = _seed_entry(root, "old", age_days=10) # young for 30d, old for 7d
|
|
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run"])
|
|
assert result.exit_code == 0, result.output
|
|
assert not entry.exists(), "aggressive retention should have reached the cache"
|
|
|
|
|
|
def test_retention_missing_cache_store_is_not_an_error(
|
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
_root, cfg = _cache_config(tmp_path)
|
|
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
|
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
|
|
|
result = CliRunner().invoke(cli, ["retention", "run"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "nothing to purge" in result.output
|