feat(cache): content-addressed store for cargo dependency artifacts
One immutable file per key, published by rename(2) inside the store. Two jobs
publishing the same key both produce a valid entry and the last wins; a reader
holding the previous one keeps reading it. No advisory locking, which is the
property a VMware shared folder cannot offer with several jobs in flight --
the objection raised in the furyrs thread.
The key joins its parts with NUL so ("ab","c") and ("a","bc") cannot collide,
and carries a format version so the whole store can be invalidated at once
when the prune rules change and older entries stop being trustworthy.
The feature set is deliberately absent from the key: cargo already encodes it
in each artifact's own hash suffix, so several feature graphs coexist in one
entry rather than fragmenting the store into one entry per variant.
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
"""Tests for ci_orchestrator.cargo_cache — key derivation and the entry store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from ci_orchestrator.cargo_cache import CACHE_FORMAT, CargoCacheStore, cache_key
|
||||
|
||||
# ── key derivation ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_key_is_stable_for_identical_inputs() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_key_changes_with_lock_digest() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="def", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_changes_with_toolchain() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.98.0", host_triple="x86_64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_changes_with_host_triple() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="aarch64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_is_filesystem_safe() -> None:
|
||||
key = cache_key(
|
||||
lock_digest="a/b+c=", rustc_version="rustc 1.97.1 (x /y)", host_triple="x86_64"
|
||||
)
|
||||
assert all(c.isalnum() or c == "-" for c in key)
|
||||
assert "/" not in key
|
||||
|
||||
|
||||
def test_key_carries_the_format_version() -> None:
|
||||
key = cache_key(lock_digest="abc", rustc_version="r", host_triple="t")
|
||||
assert key.startswith(f"v{CACHE_FORMAT}-")
|
||||
|
||||
|
||||
def test_key_fields_cannot_be_confused_by_concatenation() -> None:
|
||||
"""``ab|c`` and ``a|bc`` must not collide — the separator has to be real."""
|
||||
a = cache_key(lock_digest="ab", rustc_version="c", host_triple="t")
|
||||
b = cache_key(lock_digest="a", rustc_version="bc", host_triple="t")
|
||||
assert a != b
|
||||
|
||||
|
||||
# ── store ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_lookup_returns_none_when_absent(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
assert store.lookup(cache_key(lock_digest="a", rustc_version="b", host_triple="c")) is None
|
||||
|
||||
|
||||
def test_publish_then_lookup_finds_the_entry(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
published = store.publish(key, src)
|
||||
assert store.lookup(key) == published
|
||||
assert published.read_bytes() == b"payload"
|
||||
|
||||
|
||||
def test_publish_consumes_the_source_file(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
store.publish(key, src)
|
||||
assert not src.exists(), "publish must move, not copy — a copy doubles peak disk"
|
||||
|
||||
|
||||
def test_publish_overwrites_an_existing_key(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
first = tmp_path / "a.tar.zst"
|
||||
first.write_bytes(b"first")
|
||||
store.publish(key, first)
|
||||
second = tmp_path / "b.tar.zst"
|
||||
second.write_bytes(b"second")
|
||||
published = store.publish(key, second)
|
||||
assert published.read_bytes() == b"second"
|
||||
|
||||
|
||||
def test_publish_leaves_no_temp_files_behind(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
store.publish(key, src)
|
||||
staging = tmp_path / ".incoming"
|
||||
leftovers = list(staging.glob("*")) if staging.is_dir() else []
|
||||
assert leftovers == []
|
||||
|
||||
|
||||
def test_publish_creates_the_store_directory(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "not" / "yet")
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
published = store.publish(key, src)
|
||||
assert published.is_file()
|
||||
|
||||
|
||||
def test_publish_missing_source_raises(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.publish(key, tmp_path / "nope.tar.zst")
|
||||
|
||||
|
||||
def test_lookup_rejects_a_key_with_a_path_separator(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="unsafe cache key"):
|
||||
store.lookup("../../etc/passwd")
|
||||
|
||||
|
||||
def test_path_for_rejects_an_arbitrary_string(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="unsafe cache key"):
|
||||
store.path_for("not-a-real-key")
|
||||
|
||||
|
||||
def test_lookup_ignores_a_directory_at_the_entry_path(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
store.path_for(key).mkdir(parents=True)
|
||||
assert store.lookup(key) is None
|
||||
|
||||
|
||||
# ── prune ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _age(path: Path, days: int) -> None:
|
||||
old = (datetime.now(tz=UTC) - timedelta(days=days)).timestamp()
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def _seed(store: CargoCacheStore, tmp_path: Path, digest: str) -> Path:
|
||||
key = cache_key(lock_digest=digest, rustc_version="b", host_triple="c")
|
||||
src = tmp_path / f"seed-{digest}.tar.zst"
|
||||
src.write_bytes(b"x")
|
||||
return store.publish(key, src)
|
||||
|
||||
|
||||
def test_prune_removes_entries_older_than_the_cutoff(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == [entry]
|
||||
assert not entry.exists()
|
||||
|
||||
|
||||
def test_prune_keeps_fresh_entries(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "new")
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == []
|
||||
assert entry.exists()
|
||||
|
||||
|
||||
def test_prune_what_if_reports_without_deleting(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
|
||||
removed = store.prune_older_than(
|
||||
datetime.now(tz=UTC) - timedelta(days=30), what_if=True
|
||||
)
|
||||
assert removed == [entry]
|
||||
assert entry.exists()
|
||||
|
||||
|
||||
def test_prune_on_a_missing_store_is_a_noop(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "never-created")
|
||||
assert store.prune_older_than(datetime.now(tz=UTC)) == []
|
||||
|
||||
|
||||
def test_prune_leaves_unrelated_files_alone(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
stray = (tmp_path / "store") / "README.txt"
|
||||
stray.write_text("not an entry", encoding="utf-8")
|
||||
_age(stray, days=40)
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == [entry]
|
||||
assert stray.exists(), "prune must only touch files with the entry suffix"
|
||||
Reference in New Issue
Block a user