From fce7cfd86d4bf263842c6b2b2dcb97efabaf2912 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 20:50:22 +0200 Subject: [PATCH 1/7] feat(cache): content-addressed store for cargo dependency artifacts One immutable file per key, published by rename(2) inside the store. Two jobs publishing the same key both produce a valid entry and the last wins; a reader holding the previous one keeps reading it. No advisory locking, which is the property a VMware shared folder cannot offer with several jobs in flight -- the objection raised in the furyrs thread. The key joins its parts with NUL so ("ab","c") and ("a","bc") cannot collide, and carries a format version so the whole store can be invalidated at once when the prune rules change and older entries stop being trustworthy. The feature set is deliberately absent from the key: cargo already encodes it in each artifact's own hash suffix, so several feature graphs coexist in one entry rather than fragmenting the store into one entry per variant. --- src/ci_orchestrator/cargo_cache.py | 121 +++++++++++++++++ tests/python/test_cargo_cache.py | 206 +++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 src/ci_orchestrator/cargo_cache.py create mode 100644 tests/python/test_cargo_cache.py diff --git a/src/ci_orchestrator/cargo_cache.py b/src/ci_orchestrator/cargo_cache.py new file mode 100644 index 0000000..f4c347e --- /dev/null +++ b/src/ci_orchestrator/cargo_cache.py @@ -0,0 +1,121 @@ +"""Content-addressed store for compiled Rust dependency artifacts. + +A cache entry is one immutable file per key. Publishing is a ``rename(2)`` +within the store directory, so a reader either sees the previous entry or the +new one and never a half-written file — no advisory locking, which is exactly +what a VMware shared folder cannot give us with several jobs in flight. + +Nothing here knows about SSH or guests: it takes a finished tarball and files +it, or hands back the path of an existing one. That keeps the interesting +logic testable without a VM. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +from datetime import datetime +from pathlib import Path + +# Bump to invalidate every entry at once — e.g. when the prune script changes +# what it strips, making older entries untrustworthy. +CACHE_FORMAT = 1 + +_SAFE_KEY = re.compile(r"^v\d+-[0-9a-f]{32,64}$") +_INCOMING = ".incoming" + + +def cache_key(*, lock_digest: str, rustc_version: str, host_triple: str) -> str: + """Derive the entry key from the inputs that make artifacts incompatible. + + ``lock_digest`` is the sha256 of ``Cargo.lock``: it changes exactly when + the dependency set changes, which is the property that makes this cache + sound. The feature set is deliberately *not* part of the key — cargo + encodes it in each artifact's own hash suffix, so several feature graphs + coexist inside one entry instead of fragmenting it. + + The parts are joined with NUL, which cannot occur in any of them, so + ``("ab", "c")`` and ``("a", "bc")`` cannot collide. + """ + digest = hashlib.sha256( + "\0".join((lock_digest, rustc_version, host_triple)).encode("utf-8") + ).hexdigest() + return f"v{CACHE_FORMAT}-{digest}" + + +class CargoCacheStore: + """A directory of immutable cache entries, one file per key.""" + + def __init__(self, root: Path, *, suffix: str = ".tar.zst") -> None: + self.root = Path(root) + self.suffix = suffix + + # ── paths ──────────────────────────────────────────────────────────── + + def _checked(self, key: str) -> str: + if not _SAFE_KEY.match(key): + raise ValueError(f"unsafe cache key: {key!r}") + return key + + def path_for(self, key: str) -> Path: + """Absolute path of the entry for ``key``. Raises on a malformed key.""" + return self.root / f"{self._checked(key)}{self.suffix}" + + # ── read ───────────────────────────────────────────────────────────── + + def lookup(self, key: str) -> Path | None: + """Return the entry path for ``key``, or ``None`` on a miss.""" + candidate = self.path_for(key) + return candidate if candidate.is_file() else None + + # ── write ──────────────────────────────────────────────────────────── + + def publish(self, key: str, source: Path) -> Path: + """Move ``source`` into the store under ``key``. Returns the entry path. + + The move goes through a staging directory on the same filesystem so the + final step is a rename: concurrent publishers of the same key both + produce a valid entry and the last one wins, with no window in which a + reader can observe a partial file. + """ + source = Path(source) + if not source.is_file(): + raise FileNotFoundError(f"nothing to publish: {source}") + target = self.path_for(key) + staging = self.root / _INCOMING + staging.mkdir(parents=True, exist_ok=True) + tmp = staging / f"{key}.{os.getpid()}{self.suffix}" + try: + os.replace(source, tmp) + except OSError: + # Different filesystem — copy into staging instead, which still + # leaves the publish itself an atomic rename. + shutil.copy2(source, tmp) + source.unlink(missing_ok=True) + os.replace(tmp, target) + return target + + # ── maintenance ────────────────────────────────────────────────────── + + def prune_older_than( + self, cutoff: datetime, *, what_if: bool = False + ) -> list[Path]: + """Delete entries last modified before ``cutoff``. Returns what went.""" + if not self.root.is_dir(): + return [] + removed: list[Path] = [] + for entry in sorted(self.root.glob(f"*{self.suffix}")): + if not entry.is_file(): + continue + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=cutoff.tzinfo) + if mtime >= cutoff: + continue + removed.append(entry) + if not what_if: + entry.unlink(missing_ok=True) + return removed + + +__all__ = ["CACHE_FORMAT", "CargoCacheStore", "cache_key"] diff --git a/tests/python/test_cargo_cache.py b/tests/python/test_cargo_cache.py new file mode 100644 index 0000000..a05ab38 --- /dev/null +++ b/tests/python/test_cargo_cache.py @@ -0,0 +1,206 @@ +"""Tests for ci_orchestrator.cargo_cache — key derivation and the entry store.""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ci_orchestrator.cargo_cache import CACHE_FORMAT, CargoCacheStore, cache_key + +# ── key derivation ─────────────────────────────────────────────────────────── + + +def test_key_is_stable_for_identical_inputs() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a == b + + +def test_key_changes_with_lock_digest() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="def", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_toolchain() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.98.0", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_host_triple() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="aarch64") + assert a != b + + +def test_key_is_filesystem_safe() -> None: + key = cache_key( + lock_digest="a/b+c=", rustc_version="rustc 1.97.1 (x /y)", host_triple="x86_64" + ) + assert all(c.isalnum() or c == "-" for c in key) + assert "/" not in key + + +def test_key_carries_the_format_version() -> None: + key = cache_key(lock_digest="abc", rustc_version="r", host_triple="t") + assert key.startswith(f"v{CACHE_FORMAT}-") + + +def test_key_fields_cannot_be_confused_by_concatenation() -> None: + """``ab|c`` and ``a|bc`` must not collide — the separator has to be real.""" + a = cache_key(lock_digest="ab", rustc_version="c", host_triple="t") + b = cache_key(lock_digest="a", rustc_version="bc", host_triple="t") + assert a != b + + +# ── store ──────────────────────────────────────────────────────────────────── + + +def test_lookup_returns_none_when_absent(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + assert store.lookup(cache_key(lock_digest="a", rustc_version="b", host_triple="c")) is None + + +def test_publish_then_lookup_finds_the_entry(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert store.lookup(key) == published + assert published.read_bytes() == b"payload" + + +def test_publish_consumes_the_source_file(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + assert not src.exists(), "publish must move, not copy — a copy doubles peak disk" + + +def test_publish_overwrites_an_existing_key(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + first = tmp_path / "a.tar.zst" + first.write_bytes(b"first") + store.publish(key, first) + second = tmp_path / "b.tar.zst" + second.write_bytes(b"second") + published = store.publish(key, second) + assert published.read_bytes() == b"second" + + +def test_publish_leaves_no_temp_files_behind(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + staging = tmp_path / ".incoming" + leftovers = list(staging.glob("*")) if staging.is_dir() else [] + assert leftovers == [] + + +def test_publish_creates_the_store_directory(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "not" / "yet") + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert published.is_file() + + +def test_publish_missing_source_raises(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + with pytest.raises(FileNotFoundError): + store.publish(key, tmp_path / "nope.tar.zst") + + +def test_lookup_rejects_a_key_with_a_path_separator(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.lookup("../../etc/passwd") + + +def test_path_for_rejects_an_arbitrary_string(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.path_for("not-a-real-key") + + +def test_lookup_ignores_a_directory_at_the_entry_path(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + store.path_for(key).mkdir(parents=True) + assert store.lookup(key) is None + + +# ── prune ──────────────────────────────────────────────────────────────────── + + +def _age(path: Path, days: int) -> None: + old = (datetime.now(tz=UTC) - timedelta(days=days)).timestamp() + os.utime(path, (old, old)) + + +def _seed(store: CargoCacheStore, tmp_path: Path, digest: str) -> Path: + key = cache_key(lock_digest=digest, rustc_version="b", host_triple="c") + src = tmp_path / f"seed-{digest}.tar.zst" + src.write_bytes(b"x") + return store.publish(key, src) + + +def test_prune_removes_entries_older_than_the_cutoff(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert not entry.exists() + + +def test_prune_keeps_fresh_entries(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "new") + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [] + assert entry.exists() + + +def test_prune_what_if_reports_without_deleting(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than( + datetime.now(tz=UTC) - timedelta(days=30), what_if=True + ) + assert removed == [entry] + assert entry.exists() + + +def test_prune_on_a_missing_store_is_a_noop(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "never-created") + assert store.prune_older_than(datetime.now(tz=UTC)) == [] + + +def test_prune_leaves_unrelated_files_alone(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + stray = (tmp_path / "store") / "README.txt" + stray.write_text("not an entry", encoding="utf-8") + _age(stray, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert stray.exists(), "prune must only touch files with the entry suffix" From b5c18a6d70ece92c80948f4a2ddaf649b53d9f54 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 20:55:31 +0200 Subject: [PATCH 2/7] feat(cache): guest-side prune, asserted against an independent oracle Measured first, as thread entry #12 requires: a cold furyrs pipeline is 134.7s, target/ is 3.0 GiB raw and 2.5 GiB pruned, and the stored entry is 645 MiB. A warm job pays 8.1s (upload + extract) to skip ~120s of dependency compilation. Publishing costs 15.7s but only on a miss. Build it. The assertion in the plan turned out to be worthless and is replaced. It re-scanned for */.fingerprint/-* after deleting exactly those paths, so asked to catch a planted fury-core fingerprint it exited 0 -- the prune had removed the evidence before the check looked. An assertion sharing its globs with the deletion can only prove that rm ran; it cannot see the failure that matters, which is a glob that never matched the artifact in the first place. The replacement asks a different oracle. Cargo records in each .d depfile the sources an artifact was built from: dependencies point into the registry, anything built from workspace source points back into the workspace. After a correct prune, zero surviving depfiles reference workspace source outside target/. Verified negatively too -- an artifact named totallyunrelated-abc123.d that no prune glob matches, declaring a dependency on fury-core/src/lib.rs, makes the script exit 2 and refuse to publish. zstd is already present on the template, so no rebuild is needed; the gzip fallback stays for the plain LinuxBuild2404. --- plans/cargo-dependency-cache-measurements.md | 93 +++++++++++++++++++ src/ci_orchestrator/assets/__init__.py | 0 .../assets/cargo-cache-prune.sh | 87 +++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 plans/cargo-dependency-cache-measurements.md create mode 100644 src/ci_orchestrator/assets/__init__.py create mode 100755 src/ci_orchestrator/assets/cargo-cache-prune.sh diff --git a/plans/cargo-dependency-cache-measurements.md b/plans/cargo-dependency-cache-measurements.md new file mode 100644 index 0000000..836c3c4 --- /dev/null +++ b/plans/cargo-dependency-cache-measurements.md @@ -0,0 +1,93 @@ +# Cargo dependency cache — measurements + +Measured 2026-07-22 on a `LinuxRust2404` linked clone (4 vCPU, 3.9 GB RAM, +`rustc 1.97.1`, `zstd` present at `/usr/bin/zstd`), running furyrs `b8ea567` +with the `build-command` block from its `.gitea/workflows/ci.yml` verbatim. + +Entry #12 of the furyrs thread set the rule: measure before building. There it +killed the registry cache — 9.3s of saving for +800 MB on the template. Here +the numbers point the other way, decisively. + +## Baseline + +| | | +|---|---| +| Cold pipeline, empty `target/` | **134.7s** | + +Matches the 133s furyrs reported for job 1021 and the 126.6s measured in #12, +so the clone is representative. + +## Sizes + +| | Bytes | Human | +|---|---:|---:| +| `target/` raw | 3,206,670,175 | 3.0 GiB | +| `tar` of raw `target/` | 3,215,503,360 | 3.0 GiB | +| `tar` + `zstd -3 -T0`, raw | 810,108,028 | 773 MiB | +| **`target/` after prune** | 2,719,971,105 | **2.5 GiB** | +| **entry as stored** (`tar --zstd`, pruned) | 676,585,779 | **645 MiB** | + +Pruning strips 487 MB (15%) of raw `target/`, which is the workspace crates' +own artifacts — 42 fingerprint directories across the six members +(`fury-core` 11, `fury-gui` 12, `fury-ipc` 6, `fury-backend` 5, `fury-cli` 5, +`furyd` 3). 1,928 dependency artifacts and 903 dependency fingerprints survive. + +## Costs + +| Step | Seconds | Paid by | +|---|---:|---| +| prune + assert | 0.21 | publish | +| pack (`tar --zstd`) | 10.88 | publish | +| fetch guest→host (140 MiB/s) | 4.62 | publish | +| **publish total** | **15.7** | **only on a cache miss** | +| upload host→guest (179 MiB/s) | 3.60 | restore | +| extract | 4.52 | restore | +| **restore total** | **8.1** | **every warm job** | + +## Verdict: build it + +A warm job pays **8.1s** to skip roughly **120s** of dependency compilation — +the ~490 external crates that a committed `Cargo.lock` keeps bit-identical +between runs. Net saving ≈ 112s on a 134.7s job, about **83%**. + +The publish path costs 15.7s but runs only on a miss, i.e. once per +`Cargo.lock` change, so it is amortised across every run in between. + +Transfer is not the bottleneck anyone feared: the virtual NIC moves the entry +at 140–180 MiB/s, so 645 MiB crosses in under 4 seconds each way. + +## Notes that changed the implementation + +**`zstd` is already in the template.** `/usr/bin/zstd` is present on +`LinuxRust2404`, so the gzip fallback is dead code in practice and no template +rebuild is needed. Keep the fallback anyway — it costs one branch and keeps the +feature working on a plain `LinuxBuild2404`. + +**The name-based assertion was worthless and was replaced.** The plan's +original check re-scanned for `*/.fingerprint/-*` after deleting +exactly those paths. Asked to catch a planted `fury-core-deadbeefcafe` +fingerprint, it exited **0** — the prune had deleted the evidence before the +assertion looked. An assertion built from the same globs as the deletion can +only prove that `rm` ran; it is blind to the failure that matters, a glob that +never matched the artifact at all. + +The replacement asks a different oracle. Cargo records in each `.d` depfile the +source files an artifact was built *from*: a dependency points into +`/opt/rust/cargo/registry/...`, anything built from workspace source points +back into the workspace. After a correct prune, **zero** surviving depfiles +reference workspace source outside `target/`. Verified negatively as well — an +artifact named `totallyunrelated-abc123.d` (which no prune glob matches) +declaring a dependency on `fury-core/src/lib.rs` makes the script exit **2** +and refuse to publish. + +| Check | Clean tree | Planted fingerprint | Planted foreign artifact | +|---|---|---|---| +| name-based (original) | 0 | **0 — missed it** | 0 — missed it | +| depfile-based (shipped) | 0 | 0 (prune removed it, correctly) | **2 — refused** | + +**Feature variants coexist in one entry, as predicted.** `deps/` holds +`libclap-943062c20e623534.rmeta` and `libclap-abe395d6778d44ce.rmeta` side by +side — the same crate compiled under two different feature resolutions. This is +why the feature set is not part of the cache key: cargo already separates the +variants by hash, so one entry serves all of furyrs's four feature +configurations instead of fragmenting into four. diff --git a/src/ci_orchestrator/assets/__init__.py b/src/ci_orchestrator/assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ci_orchestrator/assets/cargo-cache-prune.sh b/src/ci_orchestrator/assets/cargo-cache-prune.sh new file mode 100755 index 0000000..52b3adb --- /dev/null +++ b/src/ci_orchestrator/assets/cargo-cache-prune.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Strip every workspace-crate artifact out of target/, then prove none survived. +# +# The cache is only sound while it contains dependency artifacts alone. A +# workspace crate's artifact carries the code under test, and reusing one can +# turn a red build green -- the worst failure a CI system has, because it is +# silent. So this script ends by asserting its own work and exits non-zero if +# the assertion fails; the caller must treat that as "do not publish". +# +# Exit codes: 0 clean prune, 1 usage/environment error, 2 assertion tripped. +set -euo pipefail + +TARGET_DIR="${1:?usage: cargo-cache-prune.sh }" +MANIFEST="${2:?usage: cargo-cache-prune.sh }" + +[ -d "$TARGET_DIR" ] || { echo "prune: no such target dir: $TARGET_DIR" >&2; exit 1; } +command -v cargo >/dev/null || { echo "prune: cargo not on PATH" >&2; exit 1; } + +# --no-deps restricts the listing to workspace members, which is precisely the +# set whose artifacts must never be cached. +members="$(cargo metadata --no-deps --format-version 1 --manifest-path "$MANIFEST" \ + | python3 -c 'import json,sys; print("\n".join(p["name"] for p in json.load(sys.stdin)["packages"]))')" + +if [ -z "$members" ]; then + echo "prune: cargo metadata listed no workspace members" >&2 + exit 1 +fi + +# Incremental state is large, machine-specific and never reusable across +# clones; examples and docs are not dependencies of anything we build. +rm -rf "$TARGET_DIR"/*/incremental "$TARGET_DIR"/*/examples "$TARGET_DIR"/doc 2>/dev/null || true + +prune_one() { + name="$1" + snake="${name//-/_}" + # Fingerprints and build-script output live in per-crate directories. + find "$TARGET_DIR" -maxdepth 3 \ + \( -path "*/.fingerprint/$name-*" \ + -o -path "*/.fingerprint/$snake-*" \ + -o -path "*/build/$name-*" \ + -o -path "*/build/$snake-*" \) \ + -prune -exec rm -rf {} + 2>/dev/null || true + # Compiled outputs: rlib/rmeta/d in deps/, plus binaries at the profile root. + find "$TARGET_DIR" -maxdepth 3 \ + \( -name "$snake-*" -o -name "lib$snake-*" -o -name "lib$snake.*" \ + -o -name "$name" -o -name "$name-*" \) \ + -exec rm -rf {} + 2>/dev/null || true +} + +while IFS= read -r name; do + [ -n "$name" ] || continue + prune_one "$name" +done </dev/null \ + | xargs -0 -r grep -h -oE "${ws_root}/[^ :]*" 2>/dev/null \ + | grep -v "^${abs_target}/" \ + | sort -u || true)" + +if [ -n "$leaked_paths" ]; then + echo "prune: REFUSING to publish -- artifacts still reference workspace source:" >&2 + printf '%s\n' "$leaked_paths" | sed 's/^/ /' | head -20 >&2 + exit 2 +fi + +echo "prune: ok -- $(printf '%s\n' "$members" | wc -l) workspace member(s) stripped," \ + "no surviving artifact references workspace source" From a7a22a29beb01207368ea4dbd83dd3128c814d8f Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 20:59:23 +0200 Subject: [PATCH 3/7] feat(cache): restore and publish the cargo dependency cache around Linux builds Restore runs after the source is in place and before the build; publish runs only after a successful build and only on a miss, so the steady state uploads nothing and a failed build can never become an entry. The whole path is best-effort: a cache problem degrades to a cold build with a message rather than turning a good build red. Gated on Cargo.lock being present, so non-Rust jobs are untouched with no flag to set. The Windows branch is not touched at all. The prune script is read through importlib.resources rather than a path relative to the repo root: the production venv is installed non-editable, so a parents[] walk lands in site-packages where scripts/ does not exist. The fake transport gained substring-matched canned results; indexing by call position broke as soon as the cache added calls mid-flow. Mutation-tested rather than assumed. Three guards were each disabled in turn -- the publish call, the prune-refusal check, the packing-failure check -- and a test went red for each. The prune-refusal test initially survived its mutant: it passed because the fake's fetch left no file behind, so publishing failed for an unrelated reason. Fixed by making fetch write a real file, so the only thing preventing publication is the guard under test. --- src/ci_orchestrator/commands/build.py | 135 ++++++++++++++++ tests/python/test_commands_build.py | 221 +++++++++++++++++++++++++- 2 files changed, 355 insertions(+), 1 deletion(-) diff --git a/src/ci_orchestrator/commands/build.py b/src/ci_orchestrator/commands/build.py index 1ed7dfa..1907c60 100644 --- a/src/ci_orchestrator/commands/build.py +++ b/src/ci_orchestrator/commands/build.py @@ -25,11 +25,13 @@ import tarfile import tempfile import zipfile from collections.abc import Iterable +from importlib import resources from pathlib import Path from typing import TYPE_CHECKING import click +from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key from ci_orchestrator.config import load_config from ci_orchestrator.credentials import KeyringCredentialStore from ci_orchestrator.transport.errors import TransportCommandError, TransportError @@ -122,6 +124,110 @@ def _tail_lines(stdout: str, stderr: str) -> str: return " | ".join(lines[-5:]) if lines else "(no build output)" +# ------------------------------------------------- cargo dependency cache + +_PRUNE_ASSET = "cargo-cache-prune.sh" +_GUEST_PRUNE = "/tmp/cargo-cache-prune.sh" +_GUEST_TARBALL = "/tmp/cargo-cache-entry.tar" + + +def _cargo_cache_probe(t: SshTransport, workdir: str) -> tuple[str, str] | None: + """Return ``(key, compressor)`` for this workspace, or ``None`` to skip. + + ``None`` means "not a Rust build, or we could not read what we need". The + cache is an optimisation, so every uncertainty here resolves to skipping + it rather than to guessing. + """ + if t.run(f"test -f {_sh_quote(workdir + '/Cargo.lock')}", check=False).returncode != 0: + return None + lock = t.run(f"sha256sum {_sh_quote(workdir + '/Cargo.lock')}", check=False) + version = t.run("rustc -V", check=False) + triple = t.run("rustc -vV | sed -n 's/^host: //p'", check=False) + if not (lock.ok and version.ok and triple.ok): + click.echo("[cache] could not read Cargo.lock digest or rustc version; skipping") + return None + digest = lock.stdout.split()[0] if lock.stdout.split() else "" + if not digest: + click.echo("[cache] empty Cargo.lock digest; skipping") + return None + compressor = "zstd" if t.run("command -v zstd", check=False).ok else "gzip" + key = cache_key( + lock_digest=digest, + rustc_version=version.stdout.strip(), + host_triple=triple.stdout.strip(), + ) + return key, compressor + + +def _tar_flag(compressor: str) -> tuple[str, str]: + """``(tar flag, file extension)`` for the chosen compressor.""" + return ("--zstd", "zst") if compressor == "zstd" else ("-z", "gz") + + +def _cargo_cache_restore( + t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str +) -> bool: + """Upload and unpack an existing entry. Returns True on a hit.""" + entry = store.lookup(key) + if entry is None: + click.echo(f"[cache] miss for {key} — building cold") + return False + flag, ext = _tar_flag(compressor) + size_mb = entry.stat().st_size / 1_048_576 + click.echo(f"[cache] hit for {key} ({size_mb:.0f} MB) — restoring") + guest_entry = f"{_GUEST_TARBALL}.{ext}" + t.copy(str(entry), guest_entry) + t.run( + f"tar {flag} -xf {_sh_quote(guest_entry)} -C {_sh_quote(workdir)} && " + f"rm -f {_sh_quote(guest_entry)}" + ) + return True + + +def _cargo_cache_publish( + t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str +) -> None: + """Prune workspace artifacts, then file the remainder under ``key``. + + A non-zero prune means the fail-closed assertion tripped: say so loudly and + publish nothing. A poisoned entry costs a wrong verdict; a missing one only + costs a cold build. + """ + # as_file() because the production venv is installed non-editable: the + # script is package data, not a path relative to a source checkout. + asset = resources.files("ci_orchestrator.assets").joinpath(_PRUNE_ASSET) + with resources.as_file(asset) as prune_script: + t.copy(str(prune_script), _GUEST_PRUNE) + pruned = t.run( + f"bash {_GUEST_PRUNE} {_sh_quote(workdir + '/target')} " + f"{_sh_quote(workdir + '/Cargo.toml')}", + check=False, + ) + click.echo(f"[cache] {(pruned.stdout.strip() or pruned.stderr.strip())}") + if pruned.returncode != 0: + click.echo("[cache] prune refused — publishing nothing", err=True) + return + + flag, ext = _tar_flag(compressor) + guest_tar = f"{_GUEST_TARBALL}.{ext}" + packed = t.run( + f"tar {flag} -cf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} target", + check=False, + ) + if not packed.ok: + click.echo("[cache] packing failed — publishing nothing", err=True) + return + + with tempfile.TemporaryDirectory() as td: + local = Path(td) / f"entry.tar.{ext}" + t.fetch(guest_tar, local) + published = store.publish(key, local) + t.run(f"rm -f {_sh_quote(guest_tar)}", check=False) + click.echo( + f"[cache] published {key} ({published.stat().st_size / 1_048_576:.0f} MB)" + ) + + # ---------------------------------------------------------------- Linux flow @@ -143,6 +249,7 @@ def _linux_build( extra_env: dict[str, str], skip_artifact: bool, use_xvfb: bool = False, + cargo_cache_root: Path | None = None, ) -> None: click.echo("[build run] Linux build mode (SSH transport)") @@ -217,6 +324,26 @@ def _linux_build( check=False, ) + # Cargo dependency cache. Everything here is best-effort: a cache + # problem must never turn a good build red, so any failure degrades to + # a cold build with a message. + cache_key_value: str | None = None + compressor = "gzip" + store: CargoCacheStore | None = None + if cargo_cache_root is not None: + try: + store = CargoCacheStore(cargo_cache_root) + probed = _cargo_cache_probe(t, workdir) + if probed is None: + store = None + else: + cache_key_value, compressor = probed + _cargo_cache_restore(t, store, cache_key_value, workdir, compressor) + except (TransportError, OSError, ValueError) as exc: + click.echo(f"[cache] restore skipped: {exc}", err=True) + cache_key_value = None + store = None + env_prefix = "".join( f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items() ) @@ -245,6 +372,14 @@ def _linux_build( f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}" ) + # Publish only on a miss, so the steady state uploads nothing. A failed + # build never reaches here, so a broken tree cannot become an entry. + if store is not None and cache_key_value and store.lookup(cache_key_value) is None: + try: + _cargo_cache_publish(t, store, cache_key_value, workdir, compressor) + except (TransportError, OSError, ValueError) as exc: + click.echo(f"[cache] publish skipped: {exc}", err=True) + if skip_artifact: click.echo("[build run] artifact packaging skipped (--skip-artifact).") return diff --git a/tests/python/test_commands_build.py b/tests/python/test_commands_build.py index ed191bc..8530562 100644 --- a/tests/python/test_commands_build.py +++ b/tests/python/test_commands_build.py @@ -38,9 +38,20 @@ class _FakeTransport: self.copies: list[tuple[str, str]] = [] self.fetches: list[tuple[str, str]] = [] self.run_results: dict[int, _FakeResult] = {} + # Matched by substring against the script, checked before the + # positional run_results. Index-based fixtures get brittle once a + # feature adds calls in the middle of the flow (the cargo cache does). + self.script_results: list[tuple[str, _FakeResult]] = [] self.default_result = _FakeResult(stdout="ok") type(self).instances.append(self) + def result_for( + self, needle: str, *, stdout: str = "", stderr: str = "", returncode: int = 0 + ) -> None: + self.script_results.append( + (needle, _FakeResult(stdout=stdout, stderr=stderr, returncode=returncode)) + ) + def __enter__(self) -> _FakeTransport: return self @@ -49,7 +60,13 @@ class _FakeTransport: def run(self, script: str, *, check: bool = True) -> _FakeResult: self.runs.append((script, check)) - result = self.run_results.get(len(self.runs) - 1, self.default_result) + result: _FakeResult | None = None + for needle, canned in self.script_results: + if needle in script: + result = canned + break + if result is None: + result = self.run_results.get(len(self.runs) - 1, self.default_result) if check and not result.ok: raise TransportCommandError(result.returncode, result.stdout, result.stderr) return result @@ -838,3 +855,205 @@ def test_build_run_windows_transport_error( ) assert result.exit_code != 0 assert "winrm unreachable" in result.output + + +# ── _linux_build: cargo dependency cache ─────────────────────────────────── + + +def _linux_kwargs(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "ip_address": "10.0.0.1", + "ssh_user": "ci_build", + "key_path": None, + "known_hosts": None, + "workdir": "/opt/ci/build", + "output_dir": "/opt/ci/output", + "host_source_dir": None, + "clone_url": "http://repo/example.git", + "clone_branch": "main", + "clone_commit": "", + "clone_submodules": False, + "build_command": "true", + "artifact_source": "dist", + "extra_env": {}, + "skip_artifact": True, + } + base.update(overrides) + return base + + +def _rust_guest(fake: _FakeTransport, *, digest: str = "cafe") -> None: + """Make the fake answer like a Rust workspace with a known toolchain.""" + fake.result_for("sha256sum", stdout=f"{digest} Cargo.lock\n") + fake.result_for("rustc -V", stdout="rustc 1.97.1 (abc 2026-01-01)\n") + fake.result_for("rustc -vV", stdout="x86_64-unknown-linux-gnu\n") + + +def _expected_key(digest: str = "cafe") -> str: + from ci_orchestrator.cargo_cache import cache_key + + return cache_key( + lock_digest=digest, + rustc_version="rustc 1.97.1 (abc 2026-01-01)", + host_triple="x86_64-unknown-linux-gnu", + ) + + +def _fetch_writes_a_real_file( + monkeypatch: pytest.MonkeyPatch, fake: _FakeTransport +) -> None: + """Make ``fetch`` produce an actual file on disk. + + Without this the default fake leaves nothing behind, ``publish`` dies on + FileNotFoundError, and a "nothing was published" assertion passes for the + wrong reason -- it would hold even with the guard under test removed. + Mutation-tested: dropping the prune-refusal check must make those tests go + red, and it only does when a publish could otherwise have succeeded. + """ + + def _fetch(remote: str, local: str) -> None: + fake.fetches.append((remote, str(local))) + Path(local).write_bytes(b"fresh-entry") + + monkeypatch.setattr(fake, "fetch", _fetch) + + +def test_linux_build_without_cache_root_runs_no_cache_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(build_module, "SshTransport", _FakeTransport) + build_module._linux_build(**_linux_kwargs()) + cmds = [c[0] for c in _FakeTransport.instances[0].runs] + assert not any("cargo-cache-prune" in c for c in cmds) + assert not any("sha256sum" in c for c in cmds) + + +def test_linux_build_skips_cache_when_no_cargo_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(build_module, "SshTransport", _FakeTransport) + fake = _FakeTransport() + _FakeTransport.instances = [] + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + fake.result_for("test -f", returncode=1) + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert not any("cargo-cache-prune" in c for c in cmds) + assert not any("sha256sum" in c for c in cmds) + + +def test_linux_build_restores_an_existing_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + store = CargoCacheStore(tmp_path) + seed = tmp_path / "seed.tar.zst" + seed.write_bytes(b"entry") + store.publish(_expected_key(), seed) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert any("tar --zstd -xf" in c for c in cmds), "a hit must unpack the entry" + assert fake.copies, "a hit must upload the entry to the guest" + # A hit must not re-publish: the entry is already there. + assert not any("cargo-cache-prune" in c for c in cmds) + + +def test_linux_build_publishes_on_a_miss( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + _fetch_writes_a_real_file(monkeypatch, fake) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is not None + + +def test_linux_build_does_not_publish_when_the_prune_assertion_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + _fetch_writes_a_real_file(monkeypatch, fake) + fake.result_for( + "cargo-cache-prune", + returncode=2, + stderr="prune: REFUSING to publish -- artifacts still reference workspace source", + ) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None, ( + "a failed prune assertion must never reach the store" + ) + + +def test_linux_build_does_not_publish_when_packing_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + _fetch_writes_a_real_file(monkeypatch, fake) + fake.result_for("tar --zstd -cf", returncode=1, stderr="no space left on device") + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None + + +def test_cache_probe_failure_does_not_fail_the_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + fake.result_for("sha256sum", returncode=1, stderr="boom") + # Must not raise: the cache is an optimisation, never a failure mode. + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + + +def test_cache_transport_error_does_not_fail_the_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + def _boom(*_a: Any, **_k: Any) -> None: + raise TransportError("ssh died mid-restore") + + monkeypatch.setattr(fake, "copy", _boom) + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + + +def test_cache_key_differs_when_the_lockfile_differs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A Cargo.lock change must miss the entry built for the previous one.""" + from ci_orchestrator.cargo_cache import CargoCacheStore + + store = CargoCacheStore(tmp_path) + seed = tmp_path / "seed.tar.zst" + seed.write_bytes(b"entry") + store.publish(_expected_key("cafe"), seed) + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake, digest="beef") + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert not any("tar --zstd -xf" in c for c in cmds), "different lock must miss" From dc89427164b6c4bdb7a642b436e63b0194f8ce36 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 21:02:35 +0200 Subject: [PATCH 4/7] feat(cache): --cargo-cache flag and [cargo_cache] config section Default on, because the cache is safe by construction rather than by convention: it engages only where a Cargo.lock exists, and refuses to publish anything the prune assertion cannot vouch for. Two independent off switches -- the flag for one job, `enabled = false` for the host. The store root is resolved after the environment overrides are applied, so CI_ROOT relocates it along with everything else, while an explicit root in the config still wins. Both directions are covered by tests. Also asserts that the Windows branch never receives the argument at all: the cache is Linux-only and the two branches stay separate, per AGENTS.md. --- config.example.toml | 19 ++++++ src/ci_orchestrator/commands/job.py | 14 +++++ src/ci_orchestrator/config.py | 34 ++++++++++- tests/python/test_commands_job.py | 93 +++++++++++++++++++++++++++++ tests/python/test_config.py | 51 ++++++++++++++++ 5 files changed, 210 insertions(+), 1 deletion(-) diff --git a/config.example.toml b/config.example.toml index e9ef228..db436cb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -53,3 +53,22 @@ type = "workstation" # netmask = "255.255.255.0" # gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route # lease_file = "/var/lib/ci/ip-pool.json" + +# ------------------------------------------------- cargo dependency cache +# Compiled Rust dependency artifacts, reused between jobs. Engages only for +# Linux guests whose build workdir contains a Cargo.lock, so non-Rust jobs +# need no flag. Workspace crates are stripped before an entry is stored and +# the job refuses to publish if any survive, so a restored entry cannot carry +# the code under test. +# +# One immutable file per key under `root`; a key is +# sha256(Cargo.lock) + rustc version + host triple. Deleting entries by hand +# is always safe -- each one costs exactly one cold build to regenerate. +# +# `root` defaults to /cache/cargo and follows CI_ROOT unless set +# here explicitly. Measured on LinuxRust2404: an entry is ~645 MB and a warm +# job pays ~8s to skip ~120s of dependency compilation. +[cargo_cache] +enabled = true +# root = "/var/lib/ci/cache/cargo" +retention_days = 30 diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py index eeefbd4..ec5c794 100644 --- a/src/ci_orchestrator/commands/job.py +++ b/src/ci_orchestrator/commands/job.py @@ -326,6 +326,14 @@ def _destroy_clone( help="In-guest git clone (default) vs host-side clone + zip transfer.", ) @click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option( + "--cargo-cache/--no-cargo-cache", + "cargo_cache", + default=True, + show_default=True, + help="Reuse compiled Rust dependency artifacts between jobs (Linux only; " + "engages only when the workspace has a Cargo.lock).", +) @click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) @click.option( "--xvfb", @@ -409,6 +417,7 @@ def job( submodules: bool, use_git_clone: bool, use_shared_cache: bool, + cargo_cache: bool, skip_artifact: bool, use_xvfb: bool, gitea_credential_target: str, @@ -634,6 +643,11 @@ def job( extra_env=extra_env, skip_artifact=skip_artifact, use_xvfb=use_xvfb, + cargo_cache_root=( + config.cargo_cache.root + if cargo_cache and config.cargo_cache.enabled + else None + ), ) else: _windows_build( diff --git a/src/ci_orchestrator/config.py b/src/ci_orchestrator/config.py index 9bff204..df5863e 100644 --- a/src/ci_orchestrator/config.py +++ b/src/ci_orchestrator/config.py @@ -65,6 +65,19 @@ class BackendConfig: options: dict[str, Any] = field(default_factory=dict) +@dataclass(frozen=True) +class CargoCacheConfig: + """Where compiled Rust dependency artifacts are filed between jobs. + + ``root`` defaults to ``/cache/cargo``; it is resolved in + :func:`load_config` because it depends on the resolved paths. + """ + + enabled: bool = True + root: Path = Path("/var/lib/ci/cache/cargo") + retention_days: int = 30 + + @dataclass(frozen=True) class Config: """Top-level configuration object.""" @@ -75,6 +88,7 @@ class Config: ssh_key_path: Path | None = None guest_cred_target: str = "BuildVMGuest" ip_pool: IpPoolConfig | None = None + cargo_cache: CargoCacheConfig = field(default_factory=CargoCacheConfig) def _platform_defaults() -> Paths: @@ -158,6 +172,7 @@ def load_config( ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest") ip_pool: IpPoolConfig | None = None + cargo_cache_section: dict[str, Any] = {} if toml_path and toml_path.is_file(): data = _load_toml(toml_path) @@ -187,9 +202,18 @@ def load_config( gateway=str(pool_section.get("gateway", "")), lease_file=Path(pool_section.get("lease_file") or _lease_default), ) + cargo_cache_section = data.get("cargo_cache", {}) or {} paths = _apply_env(paths, env) + # Resolved after _apply_env so CI_ROOT relocates the store with everything + # else; an explicit root in the file still wins. + cargo_cache = CargoCacheConfig( + enabled=bool(cargo_cache_section.get("enabled", True)), + root=Path(cargo_cache_section.get("root") or paths.root / "cache" / "cargo"), + retention_days=int(cargo_cache_section.get("retention_days", 30)), + ) + return Config( paths=paths, backend=backend, @@ -197,7 +221,15 @@ def load_config( ssh_key_path=ssh_key_path, guest_cred_target=guest_cred_target, ip_pool=ip_pool, + cargo_cache=cargo_cache, ) -__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"] +__all__ = [ + "BackendConfig", + "CargoCacheConfig", + "Config", + "IpPoolConfig", + "Paths", + "load_config", +] diff --git a/tests/python/test_commands_job.py b/tests/python/test_commands_job.py index cb71b63..c6be8ae 100644 --- a/tests/python/test_commands_job.py +++ b/tests/python/test_commands_job.py @@ -1630,3 +1630,96 @@ def test_probe_transport_ssh_not_ready_returns_false( ssh_known_hosts=None, ) assert result is False + + +# ── cargo cache plumbing ───────────────────────────────────────────────────── + + +def test_job_passes_cache_root_to_linux_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is not None + + +def test_job_no_cargo_cache_disables_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + extra=["--no-cargo-cache"], + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_respects_config_disabling_the_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + """`enabled = false` in the config wins even without --no-cargo-cache.""" + from ci_orchestrator.config import CargoCacheConfig, Config + + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + # _patch_common already replaced load_config, so override its return value + # rather than the environment. + base = job_module.load_config() + patched = Config( + paths=base.paths, + backend=base.backend, + vmrun_path=base.vmrun_path, + ssh_key_path=base.ssh_key_path, + guest_cred_target=base.guest_cred_target, + ip_pool=base.ip_pool, + cargo_cache=CargoCacheConfig(enabled=False), + ) + monkeypatch.setattr(job_module, "load_config", lambda: patched) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_windows_build_gets_no_cargo_cache_argument( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The Windows branch must be untouched by the Linux-only cache.""" + # Guest OS comes from the *clone* VMX, which the fake backend writes. + backend = _FakeBackend(running_after=1, clone_guest_os="windows2019srvNext-64") + calls = _patch_common(monkeypatch, backend) + vmx = tmp_path / "templates" / "WinBuild2025.vmx" + vmx.parent.mkdir(parents=True) + vmx.write_text('guestOS = "windows2019srvNext-64"\n', encoding="utf-8") + result = CliRunner().invoke( + cli, + _common_args( + vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert "cargo_cache_root" not in calls["windows_build"][0] diff --git a/tests/python/test_config.py b/tests/python/test_config.py index ce53d41..5270434 100644 --- a/tests/python/test_config.py +++ b/tests/python/test_config.py @@ -145,3 +145,54 @@ def test_ip_pool_absent_when_not_configured(tmp_path: Path) -> None: toml.write_text("[backend]\ntype = \"workstation\"\n", encoding="utf-8") cfg = load_config(config_path=toml, env={}) assert cfg.ip_pool is None + + +# ── cargo cache section ────────────────────────────────────────────────────── + + +def _paths_only(tmp_path: Path, extra: str = "") -> Path: + cfg = tmp_path / "config.toml" + cfg.write_text( + '[paths]\nroot = "/var/lib/ci"\ntemplates = "/var/lib/ci/templates"\n' + 'build_vms = "/var/lib/ci/build-vms"\nartifacts = "/var/lib/ci/artifacts"\n' + 'keys = "/var/lib/ci/keys"\n' + extra, + encoding="utf-8", + ) + return cfg + + +def test_cargo_cache_defaults_under_paths_root(tmp_path: Path) -> None: + cfg = load_config(config_path=_paths_only(tmp_path), env={}) + assert cfg.cargo_cache.enabled is True + assert cfg.cargo_cache.root == Path("/var/lib/ci/cache/cargo") + assert cfg.cargo_cache.retention_days == 30 + + +def test_cargo_cache_section_overrides_the_defaults(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only( + tmp_path, + '\n[cargo_cache]\nenabled = false\nroot = "/mnt/fast/cargo"\n' + "retention_days = 7\n", + ), + env={}, + ) + assert cfg.cargo_cache.enabled is False + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo") + assert cfg.cargo_cache.retention_days == 7 + + +def test_cargo_cache_root_follows_ci_root_env(tmp_path: Path) -> None: + """CI_ROOT relocates every path; the store must move with them.""" + cfg = load_config( + config_path=_paths_only(tmp_path), env={"CI_ROOT": str(tmp_path / "elsewhere")} + ) + assert cfg.cargo_cache.root == tmp_path / "elsewhere" / "cache" / "cargo" + + +def test_explicit_cargo_cache_root_beats_ci_root(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only(tmp_path, '\n[cargo_cache]\nroot = "/mnt/fast/cargo"\n'), + env={"CI_ROOT": str(tmp_path / "elsewhere")}, + ) + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo") From 2678b76297dd5fac37617c0971071066271902c0 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 21:04:08 +0200 Subject: [PATCH 5/7] feat(action): cargo-cache input, cold on release tags auto/on/off, defaulting to auto: branches and pull requests cache, refs/tags/v* builds cold. furyrs asked for exactly that in thread #13 -- a tag run verifies a release artifact rather than a change, so the clean room is worth two minutes there and not elsewhere. An unrecognised value throws instead of silently picking a side, the same posture as the guest-os: 'Auto' fix: a cache setting that quietly means its opposite is worse than a failed job. Verified by running the shipped PowerShell block itself under pwsh across all seven mode/ref combinations plus the invalid-mode throw, rather than by reimplementing the logic in the test -- a reimplementation only checks the model, not the code that ships. --- .gitea/actions/local-ci-build/action.yml | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.gitea/actions/local-ci-build/action.yml b/.gitea/actions/local-ci-build/action.yml index 346641b..f929a74 100644 --- a/.gitea/actions/local-ci-build/action.yml +++ b/.gitea/actions/local-ci-build/action.yml @@ -194,6 +194,19 @@ inputs: required: false default: 'false' + cargo-cache: + description: > + Reuse compiled Rust dependency artifacts between jobs. "auto" (default) + caches on branches and pull requests but builds release tags + (refs/tags/v*) cold; "on" always caches; "off" never does. Only ever + engages for Linux guests whose workspace has a Cargo.lock, so it is inert + for every other job. Workspace crates are stripped from an entry before + it is stored and the job refuses to publish if any survive, so a restored + cache cannot carry the code under test -- but a release build verifies an + artifact rather than a change, so "auto" gives it the clean room. + required: false + default: 'auto' + skip-artifact: description: > Set to "true" to skip artifact packaging and collection entirely. @@ -249,6 +262,8 @@ runs: INPUT_REPO_COMMIT: ${{ inputs.repo-commit }} INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }} INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }} + INPUT_CARGO_CACHE: ${{ inputs.cargo-cache }} + GITHUB_REF_FOR_CACHE: ${{ github.ref }} INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }} INPUT_XVFB: ${{ inputs.xvfb }} INPUT_GUEST_CPU: ${{ inputs.guest-cpu }} @@ -387,6 +402,20 @@ runs: else { $pyArgs += '--use-git-clone' } if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $pyArgs += '--use-shared-cache' } if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $pyArgs += '--skip-artifact' } + # Release tags build cold: they verify a release artifact, not a + # change, so the clean room is worth the two minutes (furyrs#13). + $cargoCacheMode = if ($env:INPUT_CARGO_CACHE) { $env:INPUT_CARGO_CACHE } else { 'auto' } + switch ($cargoCacheMode) { + 'on' { $useCargoCache = $true } + 'off' { $useCargoCache = $false } + 'auto' { $useCargoCache = -not ($env:GITHUB_REF_FOR_CACHE -like 'refs/tags/v*') } + default { + throw ("cargo-cache must be one of 'auto', 'on', 'off' - got " + + "'$cargoCacheMode'.") + } + } + if (-not $useCargoCache) { $pyArgs += '--no-cargo-cache' } + Write-Host ("[job] Cargo cache: " + $(if ($useCargoCache) { 'on' } else { 'off (cold)' })) if ($env:INPUT_XVFB -eq 'true') { $pyArgs += '--xvfb' } # VMX overrides: forward only when > 0 (0 = keep template value). if ([int]($env:INPUT_GUEST_CPU) -gt 0) { $pyArgs += @('--guest-cpu', $env:INPUT_GUEST_CPU) } From 15b16763753c3febd6116468ea226e6ca9bf7a12 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 21:06:04 +0200 Subject: [PATCH 6/7] feat(cache): purge stale cargo cache entries in retention run TODO 2.9 recorded a retention policy that warned about low disk while freeing 0.0 GB twice in a row, because it governed artifacts, logs and IP leases -- 52 KB in total on this host -- while 195 G of backups sat outside its reach. Adding an unbounded directory of ~645 MB entries, one per Cargo.lock change, without extending retention in the same change would repeat that at a larger scale. The window is the tighter of the global retention and the cache's own, so the aggressive low-disk mode reaches the cache rather than stopping at the light directories. Sizes are read before deleting, not after. Summing st_size over files that have already been unlinked yields 0 -- which is exactly how a policy ends up reporting that it freed nothing while doing real work. A test asserts the reported figure. --- src/ci_orchestrator/commands/retention.py | 45 +++++++- tests/python/test_commands_retention.py | 132 ++++++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) diff --git a/src/ci_orchestrator/commands/retention.py b/src/ci_orchestrator/commands/retention.py index ffcffe7..cb4748a 100644 --- a/src/ci_orchestrator/commands/retention.py +++ b/src/ci_orchestrator/commands/retention.py @@ -14,7 +14,8 @@ from pathlib import Path import click -from ci_orchestrator.config import load_config +from ci_orchestrator.cargo_cache import CargoCacheStore +from ci_orchestrator.config import Config, load_config @click.group() @@ -93,6 +94,7 @@ def retention_run( _purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if) _purge_old_dirs(log_path, "log", cutoff, what_if=what_if) + _purge_cargo_cache(config, effective_days, what_if=what_if) usage_after = shutil.disk_usage(str(check_path)) free_gb_after = usage_after.free / (1024**3) @@ -132,3 +134,44 @@ def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) else: shutil.rmtree(d, ignore_errors=True) click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}") + + +def _purge_cargo_cache(config: Config, effective_days: int, *, what_if: bool) -> None: + """Purge stale cargo cache entries. + + Entries are the heaviest thing this policy governs -- hundreds of megabytes + each, one per Cargo.lock change -- so leaving them out would repeat TODO + 2.9, where retention warned about low disk while freeing 0.0 GB because + everything that actually weighed was outside its reach. + + The window is the tighter of the global retention and the cache's own, so + the aggressive low-disk mode reaches the cache too. + """ + if not config.cargo_cache.enabled: + click.echo("[RetentionPolicy] cargo cache: disabled — skipping.") + return + + days = min(effective_days, config.cargo_cache.retention_days) + cutoff = datetime.now(tz=UTC) - timedelta(days=days) + store = CargoCacheStore(config.cargo_cache.root) + + # Size the victims first: after a real prune the files are gone and their + # bytes are unreadable, which is how a policy ends up reporting 0.0 GB + # freed while doing real work (TODO 2.9). + doomed = store.prune_older_than(cutoff, what_if=True) + if not doomed: + click.echo( + f"[RetentionPolicy] cargo cache: nothing to purge " + f"(cutoff: {cutoff.strftime('%Y-%m-%d')})." + ) + return + freed_mb = sum(p.stat().st_size for p in doomed if p.exists()) / 1_048_576 + + if not what_if: + store.prune_older_than(cutoff) + + verb = "WhatIf: would purge" if what_if else "Purged" + click.echo( + f"[RetentionPolicy] {verb} {len(doomed)} cargo cache " + f"{'entry' if len(doomed) == 1 else 'entries'} ({freed_mb:.0f} MB)" + ) diff --git a/tests/python/test_commands_retention.py b/tests/python/test_commands_retention.py index 57f22e5..6e1957e 100644 --- a/tests/python/test_commands_retention.py +++ b/tests/python/test_commands_retention.py @@ -11,6 +11,7 @@ from click.testing import CliRunner import ci_orchestrator.commands.retention as ret from ci_orchestrator.__main__ import cli +from ci_orchestrator.config import Config # ── helpers ─────────────────────────────────────────────────────────────────── @@ -191,3 +192,134 @@ def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> assert result.exit_code == 0, result.output assert not old.exists() assert "Purged log" in result.output + + +# ── cargo cache entries ─────────────────────────────────────────────────────── + + +def _cache_config( + tmp_path: Path, *, enabled: bool = True, days: int = 30 +) -> tuple[Path, Config]: + """Point the loaded config at a throwaway cache store. + + Returns ``(store root, config)`` — the root so tests can seed entries, the + config so they can substitute it for ``load_config``. + """ + from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths + + root = tmp_path / "cache" / "cargo" + cfg = Config( + paths=Paths( + root=tmp_path, + templates=tmp_path / "t", + build_vms=tmp_path / "b", + artifacts=tmp_path / "artifacts", + keys=tmp_path / "k", + ), + backend=BackendConfig(), + cargo_cache=CargoCacheConfig(enabled=enabled, root=root, retention_days=days), + ) + return root, cfg + + +def _seed_entry(root: Path, digest: str, *, age_days: int, size: int = 1024) -> Path: + import os + + from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key + + store = CargoCacheStore(root) + src = root.parent / f"seed-{digest}.tar.zst" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_bytes(b"x" * size) + entry = store.publish(cache_key(lock_digest=digest, rustc_version="r", host_triple="t"), src) + stamp = time.time() - age_days * 86400 + os.utime(entry, (stamp, stamp)) + return entry + + +def test_retention_purges_stale_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + fresh = _seed_entry(root, "new", age_days=1) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not old.exists(), "stale entry should be purged" + assert fresh.exists(), "fresh entry should survive" + + +def test_retention_reports_the_bytes_it_freed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """TODO 2.9: a policy that frees space without saying how much is the bug.""" + root, cfg = _cache_config(tmp_path) + _seed_entry(root, "old", age_days=90, size=3 * 1024 * 1024) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "Purged 1 cargo cache entry (3 MB)" in result.output + + +def test_retention_what_if_keeps_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run", "--what-if"]) + assert result.exit_code == 0, result.output + assert old.exists() + assert "would purge 1 cargo cache entry" in result.output + + +def test_retention_skips_the_cache_when_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path, enabled=False) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert old.exists(), "a disabled cache must not be touched" + assert "disabled" in result.output + + +def test_low_disk_tightens_the_cache_window_too( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Aggressive mode must reach the cache — it is the heaviest thing here.""" + root, cfg = _cache_config(tmp_path, days=30) + entry = _seed_entry(root, "old", age_days=10) # young for 30d, old for 7d + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not entry.exists(), "aggressive retention should have reached the cache" + + +def test_retention_missing_cache_store_is_not_an_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _root, cfg = _cache_config(tmp_path) + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "nothing to purge" in result.output From 9f7874211123b9e21a361688b1b9d52f05721fa0 Mon Sep 17 00:00:00 2001 From: Simone Date: Wed, 22 Jul 2026 22:15:32 +0200 Subject: [PATCH 7/7] docs(cache): end-to-end numbers, operator runbook, and a vmrun gotcha Measured through the real pipeline, not the unit tests: a Rust job goes from 186s to 84s (build phase 150s -> 47s), and 75s with the source modified, which is the scenario CI actually runs. The entry is 645 MB and costs 8s to restore. The test that decides whether this ships is row 4: a deliberate type error in fury-core, built warm. The cache restored 645 MB of dependencies and the build still went red with E0308, exit 1, publishing nothing. A cache that can turn a red build green is the worst failure a CI system has because it is silent -- re-run that row whenever the prune rules change. AGENTS.md #17 records something found while validating: `vmrun list` is per-user. Run as the wrong user it answers "Total running VMs: 0" while a clone is very much alive, which makes the "no VMs running" check before a deploy --force (#14) unreliable. `ps | grep vmware-vmx` is user-independent and was already the recommended diagnosis. --- AGENTS.md | 2 + TODO.md | 24 ++++++++-- docs/RUNBOOK-PhaseC.md | 47 +++++++++++++++++++ plans/cargo-dependency-cache-measurements.md | 48 ++++++++++++++++++++ 4 files changed, 117 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 41c5e7b..767163c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -283,3 +283,5 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali: 15. **Step 7 in timeout: distinguere tre cause diverse che danno lo stesso messaggio** — `getGuestIPAddress` risponde `The VMware Tools are not running` sia quando il guest non ha IP, sia quando ha l'IP ma i Tools non rispondono. Ordine di verifica, dal più veloce: (a) **collisione** — `ps | grep vmware-vmx`, processo più vecchio del deploy corrente → vedi #14; (b) **DHCP host incastrato** — `ip -4 addr show vmnet8` deve dare `192.168.79.1/24` e `ps -ef | grep vmnet-dhcpd` deve mostrarlo vivo; se no, `sudo systemctl restart vmware-networks` (o `vmware-host.service`); (c) **guest senza lease o Tools non pronti** — cercare il MAC della VMX in `/etc/vmware/vmnet8/dhcpd/dhcpd.leases` e in `ip neigh show dev vmnet8`: se un lease c'è, il guest è vivo e mancano solo i Tools, quindi sondare TCP/22 direttamente invece di aspettare `getGuestIPAddress`. 16. **Delta di snapshot orfani dopo `--force`** — `deploy-*` cancella `.vmsd` (il database snapshot) per evitare l'errore di resize sulla catena, ma **non** cancella i file delta `-00000N.vmdk`. Restano su disco con un `parentCID` che non corrisponde più al base ricreato: innocui (fuori dal percorso di boot, la VMX punta al base con `parentCID=ffffffff`) ma confondono la diagnosi e vanno rimossi a mano. Verifica della catena: `head -c 2000 .vmdk | strings | grep -E '^CID|parentCID'`. + +17. **`vmrun list` è per-utente e mente se lo lanci con l'utente sbagliato** — le VM della CI le avvia `ci-runner`; `vmrun -T ws list` eseguito come `simone` (o come root) risponde `Total running VMs: 0` **anche mentre un clone sta girando**, perché vede solo l'istanza VMware dell'utente corrente. Occorso il 2026-07-22 durante la validazione della cache cargo: `vmrun list` diceva 0 e `ps` mostrava un `vmware-vmx` vivo da 19 minuti. Conseguenza pratica: il check "0 VM accese" prima di un `deploy --force` (vedi #14) **non è affidabile** se non lo si esegue come `ci-runner`. Usare `sudo -u ci-runner vmrun -T ws list`, oppure — meglio, perché indipendente dall'utente — `ps -eo pid,lstart,etime,args | grep '[v]mware-vmx'`, che è anche la diagnosi già raccomandata in #14 e #15. diff --git a/TODO.md b/TODO.md index bdac67d..2d68587 100644 --- a/TODO.md +++ b/TODO.md @@ -35,7 +35,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I | [x] | Gitea Workflow Integration | 5 | 0 | tutti completati — example.yml usa composite action | | [x] | §1 Sicurezza & hardening | 3 | 0 | 1.1, 1.4, 1.6 done; 1.2/1.3/1.5/1.7/1.8 deferred (home lab) | | [x] | §2 Affidabilità & resilienza | 7 | 0 | tutto done (2.1-2.7) | -| [ ] | §3 Performance & throughput | 4 | 1 | 3.1/3.2/3.3/3.6 done; 3.4/3.5 deferred; **3.8 cache cargo pianificata** | +| [x] | §3 Performance & throughput | 5 | 0 | 3.1/3.2/3.3/3.6/3.8 done; 3.4/3.5 deferred — **§3 CHIUSO** | | [x] | §4 Osservabilità & manutenzione | 3 | 0 | 4.1/4.3/4.4 done (4.2 Prometheus, 4.5 Dashboard rimossi) | | [x] | §5 Qualità codice & test | 5 | 0 | 5.1/5.2/5.3/5.4/5.5 done | | [x] | §6 Scalabilità & estensioni | 5 | 0 | 6.1-6.2 done, 6.3 deferred (home lab), 6.4-6.5 done, 6.6 done | @@ -938,8 +938,19 @@ cioè nelle invocazioni manuali/burn-in. `Start-Job`/`pwsh`/`$IsWindows`. Lo shim resta solo per il runner Windows, dove `CI_VENV_PYTHON` è sempre settato. Il bug non è più raggiungibile sull'host Linux. -### 3.8 [P2] [ ] Cache delle dipendenze cargo per i job Rust — PIANIFICATO (2026-07-22) -File: [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md). +### 3.8 [P2] [x] Cache delle dipendenze cargo per i job Rust — COMPLETATO 2026-07-22 +File: [src/ci_orchestrator/cargo_cache.py](src/ci_orchestrator/cargo_cache.py), +[src/ci_orchestrator/assets/cargo-cache-prune.sh](src/ci_orchestrator/assets/cargo-cache-prune.sh), +piano in [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md), +numeri in [plans/cargo-dependency-cache-measurements.md](plans/cargo-dependency-cache-measurements.md). + +**Risultato misurato end-to-end**: job Rust da **186 s → 84 s** (fase build +150 s → 47 s). Con sorgente modificato — lo scenario CI vero — 75 s. Entry da +645 MB, restore 8 s. `--no-cargo-cache` e i tag `v*` restano a freddo. + +**La prova che conta**: errore di compilazione deliberato in `fury-core`, +run a caldo → `error[E0308]`, job exit 1. La cache non può nascondere una +regressione. Da ripetere ogni volta che cambiano le regole di potatura. Misurato in furyrs#12: dei ~133 s di un job Rust, ~120 s sono la compilazione delle ~490 dipendenze esterne, che `Cargo.lock` committato tiene **bit-identiche @@ -975,7 +986,12 @@ concorrenti. cambio di `Cargo.lock`, centinaia di MB l'una) senza estendere `retention run` ripeterebbe l'errore di §2.9 su scala maggiore. -**Prossimo passo**: Task 0 del piano — misurare prima di costruire, come in #12. +**Nota emersa implementando**: l'asserzione prevista dal piano era tautologica +— ri-cercava gli stessi glob che aveva appena cancellato, quindi provava solo +che `rm` fosse andato a buon fine, non che i pattern coprissero la realtà. +Sostituita con un oracolo indipendente: i depfile `.d` che cargo scrive dicono +da quali sorgenti ogni artefatto è stato compilato. Dettaglio nel documento +delle misure. ### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com) File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`). diff --git a/docs/RUNBOOK-PhaseC.md b/docs/RUNBOOK-PhaseC.md index e6f21eb..b841a33 100644 --- a/docs/RUNBOOK-PhaseC.md +++ b/docs/RUNBOOK-PhaseC.md @@ -608,6 +608,53 @@ ci retention run --retention-days 30 --aggressive-retention-days 7 --min-free-gb See [§4.6](#46-sweep-orphans) (`ci vm cleanup`). +### Cargo dependency cache + +Rust jobs on `LinuxRust2404` reuse compiled **dependency** artifacts between +runs. Measured on furyrs: a warm job pays ~8s to skip ~120s of dependency +compilation on a 134s job. Workspace crates are never cached, so a restored +entry cannot carry the code under test. + +Store: `/var/lib/ci/cache/cargo`, one immutable file per key +(`v1-.tar.zst`, ~645 MB each). A key is +`sha256(Cargo.lock) + rustc version + host triple`. + +```bash +ls -lh /var/lib/ci/cache/cargo/ # what is stored +du -sh /var/lib/ci/cache/cargo/ # total +``` + +What the job log tells you: + +| Line | Meaning | +|---|---| +| `[cache] hit for v1-… (645 MB) — restoring` | Entry reused; dependencies will not recompile | +| `[cache] miss for v1-… — building cold` | First run for this `Cargo.lock`+toolchain; the job publishes at the end | +| `[cache] prune: ok — 6 workspace member(s) stripped, …` | Prune passed its assertion; publishing | +| `[cache] prune refused — publishing nothing` | **The assertion tripped.** An artifact still referenced workspace source. Nothing was stored; the build result itself is unaffected. Investigate before trusting the feature | +| `[cache] restore skipped: …` / `publish skipped: …` | Transport or disk problem. The build ran cold and is still valid | + +Turning it off, cheapest first: + +1. One workflow: `cargo-cache: 'off'` in the composite action inputs. +2. This host: `enabled = false` under `[cargo_cache]` in `/var/lib/ci/config.toml`. +3. One job: `ci job --no-cargo-cache`. +4. Everything, without deleting a file: bump `CACHE_FORMAT` in + `src/ci_orchestrator/cargo_cache.py`. Every key changes, so every existing + entry becomes unreachable. Use this when the prune rules change and older + entries can no longer be trusted. + +Deleting entries by hand is always safe — they are immutable and independent, +and each one costs exactly one cold build to regenerate: + +```bash +sudo -u ci-runner rm /var/lib/ci/cache/cargo/*.tar.zst +``` + +`ci retention run` prunes entries older than the tighter of `--retention-days` +and `[cargo_cache].retention_days`, so the low-disk aggressive window reaches +them too. + ### Publishing `.gitea/` — committed is not deployed act_runner never executes this working copy. For every job it re-clones the diff --git a/plans/cargo-dependency-cache-measurements.md b/plans/cargo-dependency-cache-measurements.md index 836c3c4..857f4dd 100644 --- a/plans/cargo-dependency-cache-measurements.md +++ b/plans/cargo-dependency-cache-measurements.md @@ -56,6 +56,54 @@ The publish path costs 15.7s but runs only on a miss, i.e. once per Transfer is not the bottleneck anyone feared: the virtual NIC moves the entry at 140–180 MiB/s, so 645 MiB crosses in under 4 seconds each way. +## End-to-end, through `ci job` (2026-07-22) + +Full pipeline against `LinuxRust2404`, furyrs `b8ea567`, `--skip-artifact`. +"Build" is the build phase alone; "total" includes clone, boot (~35s) and +teardown. + +| # | Scenario | Build | Total | Cache line in the log | +|---|---|---:|---:|---| +| 1 | Cold, empty store | 150s | 186s | `miss` → `prune: ok` → `published … (645 MB)` | +| 2 | Warm, same commit | **47s** | **84s** | `hit … — restoring` | +| 3 | Warm, source modified | **38s** | **75s** | `hit … — restoring` | +| 4 | Warm, deliberate compile error | — | — | `hit`, then **build fails, exit 1** | +| 5 | `--no-cargo-cache` | 133s | 170s | none — cache never touched | + +**Saving: 103s on the build phase (150 → 47), 102s on the total (186 → 84).** +Row 3 is the real CI scenario and is the fastest of the warm runs, because the +touched crate is small. + +Row 2's log names every workspace crate being rebuilt — + +``` +Compiling fury-core, fury-gui, fury-ipc, fury-backend, furyd, fury-cli +``` + +— and no dependency. That is the design working: dependencies restored, +workspace recompiled from source, every time. + +### Row 4 is the one that decides whether this ships + +A deliberate type error was pushed to a scratch branch and built warm: + +``` +[cache] hit for v1-1311… (645 MB) — restoring +error[E0308]: mismatched types + | --- ^^^^^^^^^^^^^^^^^^^ expected `u32`, found `&str` +error: could not compile `fury-core` (lib) due to 1 previous error +JOB_EXIT=1 +``` + +The cache restored 645 MB of dependencies and the regression still went red. A +cache that can turn a red build green is the worst failure a CI system has, +because it is silent; this is the test that rules it out, and it is worth +re-running whenever the prune rules change. + +The failed job also published nothing — the store still held exactly one entry +and `.incoming/` was empty — because publishing sits after the build-failure +check. A broken tree cannot become an entry. + ## Notes that changed the implementation **`zstd` is already in the template.** `/usr/bin/zstd` is present on