diff --git a/src/ci_orchestrator/cargo_cache.py b/src/ci_orchestrator/cargo_cache.py new file mode 100644 index 0000000..f4c347e --- /dev/null +++ b/src/ci_orchestrator/cargo_cache.py @@ -0,0 +1,121 @@ +"""Content-addressed store for compiled Rust dependency artifacts. + +A cache entry is one immutable file per key. Publishing is a ``rename(2)`` +within the store directory, so a reader either sees the previous entry or the +new one and never a half-written file — no advisory locking, which is exactly +what a VMware shared folder cannot give us with several jobs in flight. + +Nothing here knows about SSH or guests: it takes a finished tarball and files +it, or hands back the path of an existing one. That keeps the interesting +logic testable without a VM. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +from datetime import datetime +from pathlib import Path + +# Bump to invalidate every entry at once — e.g. when the prune script changes +# what it strips, making older entries untrustworthy. +CACHE_FORMAT = 1 + +_SAFE_KEY = re.compile(r"^v\d+-[0-9a-f]{32,64}$") +_INCOMING = ".incoming" + + +def cache_key(*, lock_digest: str, rustc_version: str, host_triple: str) -> str: + """Derive the entry key from the inputs that make artifacts incompatible. + + ``lock_digest`` is the sha256 of ``Cargo.lock``: it changes exactly when + the dependency set changes, which is the property that makes this cache + sound. The feature set is deliberately *not* part of the key — cargo + encodes it in each artifact's own hash suffix, so several feature graphs + coexist inside one entry instead of fragmenting it. + + The parts are joined with NUL, which cannot occur in any of them, so + ``("ab", "c")`` and ``("a", "bc")`` cannot collide. + """ + digest = hashlib.sha256( + "\0".join((lock_digest, rustc_version, host_triple)).encode("utf-8") + ).hexdigest() + return f"v{CACHE_FORMAT}-{digest}" + + +class CargoCacheStore: + """A directory of immutable cache entries, one file per key.""" + + def __init__(self, root: Path, *, suffix: str = ".tar.zst") -> None: + self.root = Path(root) + self.suffix = suffix + + # ── paths ──────────────────────────────────────────────────────────── + + def _checked(self, key: str) -> str: + if not _SAFE_KEY.match(key): + raise ValueError(f"unsafe cache key: {key!r}") + return key + + def path_for(self, key: str) -> Path: + """Absolute path of the entry for ``key``. Raises on a malformed key.""" + return self.root / f"{self._checked(key)}{self.suffix}" + + # ── read ───────────────────────────────────────────────────────────── + + def lookup(self, key: str) -> Path | None: + """Return the entry path for ``key``, or ``None`` on a miss.""" + candidate = self.path_for(key) + return candidate if candidate.is_file() else None + + # ── write ──────────────────────────────────────────────────────────── + + def publish(self, key: str, source: Path) -> Path: + """Move ``source`` into the store under ``key``. Returns the entry path. + + The move goes through a staging directory on the same filesystem so the + final step is a rename: concurrent publishers of the same key both + produce a valid entry and the last one wins, with no window in which a + reader can observe a partial file. + """ + source = Path(source) + if not source.is_file(): + raise FileNotFoundError(f"nothing to publish: {source}") + target = self.path_for(key) + staging = self.root / _INCOMING + staging.mkdir(parents=True, exist_ok=True) + tmp = staging / f"{key}.{os.getpid()}{self.suffix}" + try: + os.replace(source, tmp) + except OSError: + # Different filesystem — copy into staging instead, which still + # leaves the publish itself an atomic rename. + shutil.copy2(source, tmp) + source.unlink(missing_ok=True) + os.replace(tmp, target) + return target + + # ── maintenance ────────────────────────────────────────────────────── + + def prune_older_than( + self, cutoff: datetime, *, what_if: bool = False + ) -> list[Path]: + """Delete entries last modified before ``cutoff``. Returns what went.""" + if not self.root.is_dir(): + return [] + removed: list[Path] = [] + for entry in sorted(self.root.glob(f"*{self.suffix}")): + if not entry.is_file(): + continue + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=cutoff.tzinfo) + if mtime >= cutoff: + continue + removed.append(entry) + if not what_if: + entry.unlink(missing_ok=True) + return removed + + +__all__ = ["CACHE_FORMAT", "CargoCacheStore", "cache_key"] diff --git a/tests/python/test_cargo_cache.py b/tests/python/test_cargo_cache.py new file mode 100644 index 0000000..a05ab38 --- /dev/null +++ b/tests/python/test_cargo_cache.py @@ -0,0 +1,206 @@ +"""Tests for ci_orchestrator.cargo_cache — key derivation and the entry store.""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ci_orchestrator.cargo_cache import CACHE_FORMAT, CargoCacheStore, cache_key + +# ── key derivation ─────────────────────────────────────────────────────────── + + +def test_key_is_stable_for_identical_inputs() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a == b + + +def test_key_changes_with_lock_digest() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="def", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_toolchain() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.98.0", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_host_triple() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="aarch64") + assert a != b + + +def test_key_is_filesystem_safe() -> None: + key = cache_key( + lock_digest="a/b+c=", rustc_version="rustc 1.97.1 (x /y)", host_triple="x86_64" + ) + assert all(c.isalnum() or c == "-" for c in key) + assert "/" not in key + + +def test_key_carries_the_format_version() -> None: + key = cache_key(lock_digest="abc", rustc_version="r", host_triple="t") + assert key.startswith(f"v{CACHE_FORMAT}-") + + +def test_key_fields_cannot_be_confused_by_concatenation() -> None: + """``ab|c`` and ``a|bc`` must not collide — the separator has to be real.""" + a = cache_key(lock_digest="ab", rustc_version="c", host_triple="t") + b = cache_key(lock_digest="a", rustc_version="bc", host_triple="t") + assert a != b + + +# ── store ──────────────────────────────────────────────────────────────────── + + +def test_lookup_returns_none_when_absent(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + assert store.lookup(cache_key(lock_digest="a", rustc_version="b", host_triple="c")) is None + + +def test_publish_then_lookup_finds_the_entry(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert store.lookup(key) == published + assert published.read_bytes() == b"payload" + + +def test_publish_consumes_the_source_file(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + assert not src.exists(), "publish must move, not copy — a copy doubles peak disk" + + +def test_publish_overwrites_an_existing_key(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + first = tmp_path / "a.tar.zst" + first.write_bytes(b"first") + store.publish(key, first) + second = tmp_path / "b.tar.zst" + second.write_bytes(b"second") + published = store.publish(key, second) + assert published.read_bytes() == b"second" + + +def test_publish_leaves_no_temp_files_behind(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + staging = tmp_path / ".incoming" + leftovers = list(staging.glob("*")) if staging.is_dir() else [] + assert leftovers == [] + + +def test_publish_creates_the_store_directory(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "not" / "yet") + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert published.is_file() + + +def test_publish_missing_source_raises(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + with pytest.raises(FileNotFoundError): + store.publish(key, tmp_path / "nope.tar.zst") + + +def test_lookup_rejects_a_key_with_a_path_separator(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.lookup("../../etc/passwd") + + +def test_path_for_rejects_an_arbitrary_string(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.path_for("not-a-real-key") + + +def test_lookup_ignores_a_directory_at_the_entry_path(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + store.path_for(key).mkdir(parents=True) + assert store.lookup(key) is None + + +# ── prune ──────────────────────────────────────────────────────────────────── + + +def _age(path: Path, days: int) -> None: + old = (datetime.now(tz=UTC) - timedelta(days=days)).timestamp() + os.utime(path, (old, old)) + + +def _seed(store: CargoCacheStore, tmp_path: Path, digest: str) -> Path: + key = cache_key(lock_digest=digest, rustc_version="b", host_triple="c") + src = tmp_path / f"seed-{digest}.tar.zst" + src.write_bytes(b"x") + return store.publish(key, src) + + +def test_prune_removes_entries_older_than_the_cutoff(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert not entry.exists() + + +def test_prune_keeps_fresh_entries(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "new") + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [] + assert entry.exists() + + +def test_prune_what_if_reports_without_deleting(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than( + datetime.now(tz=UTC) - timedelta(days=30), what_if=True + ) + assert removed == [entry] + assert entry.exists() + + +def test_prune_on_a_missing_store_is_a_noop(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "never-created") + assert store.prune_older_than(datetime.now(tz=UTC)) == [] + + +def test_prune_leaves_unrelated_files_alone(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + stray = (tmp_path / "store") / "README.txt" + stray.write_text("not an entry", encoding="utf-8") + _age(stray, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert stray.exists(), "prune must only touch files with the entry suffix"