diff --git a/.gitea/actions/local-ci-build/action.yml b/.gitea/actions/local-ci-build/action.yml index 346641b..f929a74 100644 --- a/.gitea/actions/local-ci-build/action.yml +++ b/.gitea/actions/local-ci-build/action.yml @@ -194,6 +194,19 @@ inputs: required: false default: 'false' + cargo-cache: + description: > + Reuse compiled Rust dependency artifacts between jobs. "auto" (default) + caches on branches and pull requests but builds release tags + (refs/tags/v*) cold; "on" always caches; "off" never does. Only ever + engages for Linux guests whose workspace has a Cargo.lock, so it is inert + for every other job. Workspace crates are stripped from an entry before + it is stored and the job refuses to publish if any survive, so a restored + cache cannot carry the code under test -- but a release build verifies an + artifact rather than a change, so "auto" gives it the clean room. + required: false + default: 'auto' + skip-artifact: description: > Set to "true" to skip artifact packaging and collection entirely. @@ -249,6 +262,8 @@ runs: INPUT_REPO_COMMIT: ${{ inputs.repo-commit }} INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }} INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }} + INPUT_CARGO_CACHE: ${{ inputs.cargo-cache }} + GITHUB_REF_FOR_CACHE: ${{ github.ref }} INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }} INPUT_XVFB: ${{ inputs.xvfb }} INPUT_GUEST_CPU: ${{ inputs.guest-cpu }} @@ -387,6 +402,20 @@ runs: else { $pyArgs += '--use-git-clone' } if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $pyArgs += '--use-shared-cache' } if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $pyArgs += '--skip-artifact' } + # Release tags build cold: they verify a release artifact, not a + # change, so the clean room is worth the two minutes (furyrs#13). + $cargoCacheMode = if ($env:INPUT_CARGO_CACHE) { $env:INPUT_CARGO_CACHE } else { 'auto' } + switch ($cargoCacheMode) { + 'on' { $useCargoCache = $true } + 'off' { $useCargoCache = $false } + 'auto' { $useCargoCache = -not ($env:GITHUB_REF_FOR_CACHE -like 'refs/tags/v*') } + default { + throw ("cargo-cache must be one of 'auto', 'on', 'off' - got " + + "'$cargoCacheMode'.") + } + } + if (-not $useCargoCache) { $pyArgs += '--no-cargo-cache' } + Write-Host ("[job] Cargo cache: " + $(if ($useCargoCache) { 'on' } else { 'off (cold)' })) if ($env:INPUT_XVFB -eq 'true') { $pyArgs += '--xvfb' } # VMX overrides: forward only when > 0 (0 = keep template value). if ([int]($env:INPUT_GUEST_CPU) -gt 0) { $pyArgs += @('--guest-cpu', $env:INPUT_GUEST_CPU) } diff --git a/AGENTS.md b/AGENTS.md index 41c5e7b..767163c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -283,3 +283,5 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali: 15. **Step 7 in timeout: distinguere tre cause diverse che danno lo stesso messaggio** — `getGuestIPAddress` risponde `The VMware Tools are not running` sia quando il guest non ha IP, sia quando ha l'IP ma i Tools non rispondono. Ordine di verifica, dal più veloce: (a) **collisione** — `ps | grep vmware-vmx`, processo più vecchio del deploy corrente → vedi #14; (b) **DHCP host incastrato** — `ip -4 addr show vmnet8` deve dare `192.168.79.1/24` e `ps -ef | grep vmnet-dhcpd` deve mostrarlo vivo; se no, `sudo systemctl restart vmware-networks` (o `vmware-host.service`); (c) **guest senza lease o Tools non pronti** — cercare il MAC della VMX in `/etc/vmware/vmnet8/dhcpd/dhcpd.leases` e in `ip neigh show dev vmnet8`: se un lease c'è, il guest è vivo e mancano solo i Tools, quindi sondare TCP/22 direttamente invece di aspettare `getGuestIPAddress`. 16. **Delta di snapshot orfani dopo `--force`** — `deploy-*` cancella `.vmsd` (il database snapshot) per evitare l'errore di resize sulla catena, ma **non** cancella i file delta `-00000N.vmdk`. Restano su disco con un `parentCID` che non corrisponde più al base ricreato: innocui (fuori dal percorso di boot, la VMX punta al base con `parentCID=ffffffff`) ma confondono la diagnosi e vanno rimossi a mano. Verifica della catena: `head -c 2000 .vmdk | strings | grep -E '^CID|parentCID'`. + +17. **`vmrun list` è per-utente e mente se lo lanci con l'utente sbagliato** — le VM della CI le avvia `ci-runner`; `vmrun -T ws list` eseguito come `simone` (o come root) risponde `Total running VMs: 0` **anche mentre un clone sta girando**, perché vede solo l'istanza VMware dell'utente corrente. Occorso il 2026-07-22 durante la validazione della cache cargo: `vmrun list` diceva 0 e `ps` mostrava un `vmware-vmx` vivo da 19 minuti. Conseguenza pratica: il check "0 VM accese" prima di un `deploy --force` (vedi #14) **non è affidabile** se non lo si esegue come `ci-runner`. Usare `sudo -u ci-runner vmrun -T ws list`, oppure — meglio, perché indipendente dall'utente — `ps -eo pid,lstart,etime,args | grep '[v]mware-vmx'`, che è anche la diagnosi già raccomandata in #14 e #15. diff --git a/TODO.md b/TODO.md index bdac67d..2d68587 100644 --- a/TODO.md +++ b/TODO.md @@ -35,7 +35,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I | [x] | Gitea Workflow Integration | 5 | 0 | tutti completati — example.yml usa composite action | | [x] | §1 Sicurezza & hardening | 3 | 0 | 1.1, 1.4, 1.6 done; 1.2/1.3/1.5/1.7/1.8 deferred (home lab) | | [x] | §2 Affidabilità & resilienza | 7 | 0 | tutto done (2.1-2.7) | -| [ ] | §3 Performance & throughput | 4 | 1 | 3.1/3.2/3.3/3.6 done; 3.4/3.5 deferred; **3.8 cache cargo pianificata** | +| [x] | §3 Performance & throughput | 5 | 0 | 3.1/3.2/3.3/3.6/3.8 done; 3.4/3.5 deferred — **§3 CHIUSO** | | [x] | §4 Osservabilità & manutenzione | 3 | 0 | 4.1/4.3/4.4 done (4.2 Prometheus, 4.5 Dashboard rimossi) | | [x] | §5 Qualità codice & test | 5 | 0 | 5.1/5.2/5.3/5.4/5.5 done | | [x] | §6 Scalabilità & estensioni | 5 | 0 | 6.1-6.2 done, 6.3 deferred (home lab), 6.4-6.5 done, 6.6 done | @@ -938,8 +938,19 @@ cioè nelle invocazioni manuali/burn-in. `Start-Job`/`pwsh`/`$IsWindows`. Lo shim resta solo per il runner Windows, dove `CI_VENV_PYTHON` è sempre settato. Il bug non è più raggiungibile sull'host Linux. -### 3.8 [P2] [ ] Cache delle dipendenze cargo per i job Rust — PIANIFICATO (2026-07-22) -File: [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md). +### 3.8 [P2] [x] Cache delle dipendenze cargo per i job Rust — COMPLETATO 2026-07-22 +File: [src/ci_orchestrator/cargo_cache.py](src/ci_orchestrator/cargo_cache.py), +[src/ci_orchestrator/assets/cargo-cache-prune.sh](src/ci_orchestrator/assets/cargo-cache-prune.sh), +piano in [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md), +numeri in [plans/cargo-dependency-cache-measurements.md](plans/cargo-dependency-cache-measurements.md). + +**Risultato misurato end-to-end**: job Rust da **186 s → 84 s** (fase build +150 s → 47 s). Con sorgente modificato — lo scenario CI vero — 75 s. Entry da +645 MB, restore 8 s. `--no-cargo-cache` e i tag `v*` restano a freddo. + +**La prova che conta**: errore di compilazione deliberato in `fury-core`, +run a caldo → `error[E0308]`, job exit 1. La cache non può nascondere una +regressione. Da ripetere ogni volta che cambiano le regole di potatura. Misurato in furyrs#12: dei ~133 s di un job Rust, ~120 s sono la compilazione delle ~490 dipendenze esterne, che `Cargo.lock` committato tiene **bit-identiche @@ -975,7 +986,12 @@ concorrenti. cambio di `Cargo.lock`, centinaia di MB l'una) senza estendere `retention run` ripeterebbe l'errore di §2.9 su scala maggiore. -**Prossimo passo**: Task 0 del piano — misurare prima di costruire, come in #12. +**Nota emersa implementando**: l'asserzione prevista dal piano era tautologica +— ri-cercava gli stessi glob che aveva appena cancellato, quindi provava solo +che `rm` fosse andato a buon fine, non che i pattern coprissero la realtà. +Sostituita con un oracolo indipendente: i depfile `.d` che cargo scrive dicono +da quali sorgenti ogni artefatto è stato compilato. Dettaglio nel documento +delle misure. ### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com) File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`). diff --git a/config.example.toml b/config.example.toml index e9ef228..db436cb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -53,3 +53,22 @@ type = "workstation" # netmask = "255.255.255.0" # gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route # lease_file = "/var/lib/ci/ip-pool.json" + +# ------------------------------------------------- cargo dependency cache +# Compiled Rust dependency artifacts, reused between jobs. Engages only for +# Linux guests whose build workdir contains a Cargo.lock, so non-Rust jobs +# need no flag. Workspace crates are stripped before an entry is stored and +# the job refuses to publish if any survive, so a restored entry cannot carry +# the code under test. +# +# One immutable file per key under `root`; a key is +# sha256(Cargo.lock) + rustc version + host triple. Deleting entries by hand +# is always safe -- each one costs exactly one cold build to regenerate. +# +# `root` defaults to /cache/cargo and follows CI_ROOT unless set +# here explicitly. Measured on LinuxRust2404: an entry is ~645 MB and a warm +# job pays ~8s to skip ~120s of dependency compilation. +[cargo_cache] +enabled = true +# root = "/var/lib/ci/cache/cargo" +retention_days = 30 diff --git a/docs/RUNBOOK-PhaseC.md b/docs/RUNBOOK-PhaseC.md index e6f21eb..b841a33 100644 --- a/docs/RUNBOOK-PhaseC.md +++ b/docs/RUNBOOK-PhaseC.md @@ -608,6 +608,53 @@ ci retention run --retention-days 30 --aggressive-retention-days 7 --min-free-gb See [§4.6](#46-sweep-orphans) (`ci vm cleanup`). +### Cargo dependency cache + +Rust jobs on `LinuxRust2404` reuse compiled **dependency** artifacts between +runs. Measured on furyrs: a warm job pays ~8s to skip ~120s of dependency +compilation on a 134s job. Workspace crates are never cached, so a restored +entry cannot carry the code under test. + +Store: `/var/lib/ci/cache/cargo`, one immutable file per key +(`v1-.tar.zst`, ~645 MB each). A key is +`sha256(Cargo.lock) + rustc version + host triple`. + +```bash +ls -lh /var/lib/ci/cache/cargo/ # what is stored +du -sh /var/lib/ci/cache/cargo/ # total +``` + +What the job log tells you: + +| Line | Meaning | +|---|---| +| `[cache] hit for v1-… (645 MB) — restoring` | Entry reused; dependencies will not recompile | +| `[cache] miss for v1-… — building cold` | First run for this `Cargo.lock`+toolchain; the job publishes at the end | +| `[cache] prune: ok — 6 workspace member(s) stripped, …` | Prune passed its assertion; publishing | +| `[cache] prune refused — publishing nothing` | **The assertion tripped.** An artifact still referenced workspace source. Nothing was stored; the build result itself is unaffected. Investigate before trusting the feature | +| `[cache] restore skipped: …` / `publish skipped: …` | Transport or disk problem. The build ran cold and is still valid | + +Turning it off, cheapest first: + +1. One workflow: `cargo-cache: 'off'` in the composite action inputs. +2. This host: `enabled = false` under `[cargo_cache]` in `/var/lib/ci/config.toml`. +3. One job: `ci job --no-cargo-cache`. +4. Everything, without deleting a file: bump `CACHE_FORMAT` in + `src/ci_orchestrator/cargo_cache.py`. Every key changes, so every existing + entry becomes unreachable. Use this when the prune rules change and older + entries can no longer be trusted. + +Deleting entries by hand is always safe — they are immutable and independent, +and each one costs exactly one cold build to regenerate: + +```bash +sudo -u ci-runner rm /var/lib/ci/cache/cargo/*.tar.zst +``` + +`ci retention run` prunes entries older than the tighter of `--retention-days` +and `[cargo_cache].retention_days`, so the low-disk aggressive window reaches +them too. + ### Publishing `.gitea/` — committed is not deployed act_runner never executes this working copy. For every job it re-clones the diff --git a/plans/cargo-dependency-cache-measurements.md b/plans/cargo-dependency-cache-measurements.md new file mode 100644 index 0000000..857f4dd --- /dev/null +++ b/plans/cargo-dependency-cache-measurements.md @@ -0,0 +1,141 @@ +# Cargo dependency cache — measurements + +Measured 2026-07-22 on a `LinuxRust2404` linked clone (4 vCPU, 3.9 GB RAM, +`rustc 1.97.1`, `zstd` present at `/usr/bin/zstd`), running furyrs `b8ea567` +with the `build-command` block from its `.gitea/workflows/ci.yml` verbatim. + +Entry #12 of the furyrs thread set the rule: measure before building. There it +killed the registry cache — 9.3s of saving for +800 MB on the template. Here +the numbers point the other way, decisively. + +## Baseline + +| | | +|---|---| +| Cold pipeline, empty `target/` | **134.7s** | + +Matches the 133s furyrs reported for job 1021 and the 126.6s measured in #12, +so the clone is representative. + +## Sizes + +| | Bytes | Human | +|---|---:|---:| +| `target/` raw | 3,206,670,175 | 3.0 GiB | +| `tar` of raw `target/` | 3,215,503,360 | 3.0 GiB | +| `tar` + `zstd -3 -T0`, raw | 810,108,028 | 773 MiB | +| **`target/` after prune** | 2,719,971,105 | **2.5 GiB** | +| **entry as stored** (`tar --zstd`, pruned) | 676,585,779 | **645 MiB** | + +Pruning strips 487 MB (15%) of raw `target/`, which is the workspace crates' +own artifacts — 42 fingerprint directories across the six members +(`fury-core` 11, `fury-gui` 12, `fury-ipc` 6, `fury-backend` 5, `fury-cli` 5, +`furyd` 3). 1,928 dependency artifacts and 903 dependency fingerprints survive. + +## Costs + +| Step | Seconds | Paid by | +|---|---:|---| +| prune + assert | 0.21 | publish | +| pack (`tar --zstd`) | 10.88 | publish | +| fetch guest→host (140 MiB/s) | 4.62 | publish | +| **publish total** | **15.7** | **only on a cache miss** | +| upload host→guest (179 MiB/s) | 3.60 | restore | +| extract | 4.52 | restore | +| **restore total** | **8.1** | **every warm job** | + +## Verdict: build it + +A warm job pays **8.1s** to skip roughly **120s** of dependency compilation — +the ~490 external crates that a committed `Cargo.lock` keeps bit-identical +between runs. Net saving ≈ 112s on a 134.7s job, about **83%**. + +The publish path costs 15.7s but runs only on a miss, i.e. once per +`Cargo.lock` change, so it is amortised across every run in between. + +Transfer is not the bottleneck anyone feared: the virtual NIC moves the entry +at 140–180 MiB/s, so 645 MiB crosses in under 4 seconds each way. + +## End-to-end, through `ci job` (2026-07-22) + +Full pipeline against `LinuxRust2404`, furyrs `b8ea567`, `--skip-artifact`. +"Build" is the build phase alone; "total" includes clone, boot (~35s) and +teardown. + +| # | Scenario | Build | Total | Cache line in the log | +|---|---|---:|---:|---| +| 1 | Cold, empty store | 150s | 186s | `miss` → `prune: ok` → `published … (645 MB)` | +| 2 | Warm, same commit | **47s** | **84s** | `hit … — restoring` | +| 3 | Warm, source modified | **38s** | **75s** | `hit … — restoring` | +| 4 | Warm, deliberate compile error | — | — | `hit`, then **build fails, exit 1** | +| 5 | `--no-cargo-cache` | 133s | 170s | none — cache never touched | + +**Saving: 103s on the build phase (150 → 47), 102s on the total (186 → 84).** +Row 3 is the real CI scenario and is the fastest of the warm runs, because the +touched crate is small. + +Row 2's log names every workspace crate being rebuilt — + +``` +Compiling fury-core, fury-gui, fury-ipc, fury-backend, furyd, fury-cli +``` + +— and no dependency. That is the design working: dependencies restored, +workspace recompiled from source, every time. + +### Row 4 is the one that decides whether this ships + +A deliberate type error was pushed to a scratch branch and built warm: + +``` +[cache] hit for v1-1311… (645 MB) — restoring +error[E0308]: mismatched types + | --- ^^^^^^^^^^^^^^^^^^^ expected `u32`, found `&str` +error: could not compile `fury-core` (lib) due to 1 previous error +JOB_EXIT=1 +``` + +The cache restored 645 MB of dependencies and the regression still went red. A +cache that can turn a red build green is the worst failure a CI system has, +because it is silent; this is the test that rules it out, and it is worth +re-running whenever the prune rules change. + +The failed job also published nothing — the store still held exactly one entry +and `.incoming/` was empty — because publishing sits after the build-failure +check. A broken tree cannot become an entry. + +## Notes that changed the implementation + +**`zstd` is already in the template.** `/usr/bin/zstd` is present on +`LinuxRust2404`, so the gzip fallback is dead code in practice and no template +rebuild is needed. Keep the fallback anyway — it costs one branch and keeps the +feature working on a plain `LinuxBuild2404`. + +**The name-based assertion was worthless and was replaced.** The plan's +original check re-scanned for `*/.fingerprint/-*` after deleting +exactly those paths. Asked to catch a planted `fury-core-deadbeefcafe` +fingerprint, it exited **0** — the prune had deleted the evidence before the +assertion looked. An assertion built from the same globs as the deletion can +only prove that `rm` ran; it is blind to the failure that matters, a glob that +never matched the artifact at all. + +The replacement asks a different oracle. Cargo records in each `.d` depfile the +source files an artifact was built *from*: a dependency points into +`/opt/rust/cargo/registry/...`, anything built from workspace source points +back into the workspace. After a correct prune, **zero** surviving depfiles +reference workspace source outside `target/`. Verified negatively as well — an +artifact named `totallyunrelated-abc123.d` (which no prune glob matches) +declaring a dependency on `fury-core/src/lib.rs` makes the script exit **2** +and refuse to publish. + +| Check | Clean tree | Planted fingerprint | Planted foreign artifact | +|---|---|---|---| +| name-based (original) | 0 | **0 — missed it** | 0 — missed it | +| depfile-based (shipped) | 0 | 0 (prune removed it, correctly) | **2 — refused** | + +**Feature variants coexist in one entry, as predicted.** `deps/` holds +`libclap-943062c20e623534.rmeta` and `libclap-abe395d6778d44ce.rmeta` side by +side — the same crate compiled under two different feature resolutions. This is +why the feature set is not part of the cache key: cargo already separates the +variants by hash, so one entry serves all of furyrs's four feature +configurations instead of fragmenting into four. diff --git a/src/ci_orchestrator/assets/__init__.py b/src/ci_orchestrator/assets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ci_orchestrator/assets/cargo-cache-prune.sh b/src/ci_orchestrator/assets/cargo-cache-prune.sh new file mode 100755 index 0000000..52b3adb --- /dev/null +++ b/src/ci_orchestrator/assets/cargo-cache-prune.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Strip every workspace-crate artifact out of target/, then prove none survived. +# +# The cache is only sound while it contains dependency artifacts alone. A +# workspace crate's artifact carries the code under test, and reusing one can +# turn a red build green -- the worst failure a CI system has, because it is +# silent. So this script ends by asserting its own work and exits non-zero if +# the assertion fails; the caller must treat that as "do not publish". +# +# Exit codes: 0 clean prune, 1 usage/environment error, 2 assertion tripped. +set -euo pipefail + +TARGET_DIR="${1:?usage: cargo-cache-prune.sh }" +MANIFEST="${2:?usage: cargo-cache-prune.sh }" + +[ -d "$TARGET_DIR" ] || { echo "prune: no such target dir: $TARGET_DIR" >&2; exit 1; } +command -v cargo >/dev/null || { echo "prune: cargo not on PATH" >&2; exit 1; } + +# --no-deps restricts the listing to workspace members, which is precisely the +# set whose artifacts must never be cached. +members="$(cargo metadata --no-deps --format-version 1 --manifest-path "$MANIFEST" \ + | python3 -c 'import json,sys; print("\n".join(p["name"] for p in json.load(sys.stdin)["packages"]))')" + +if [ -z "$members" ]; then + echo "prune: cargo metadata listed no workspace members" >&2 + exit 1 +fi + +# Incremental state is large, machine-specific and never reusable across +# clones; examples and docs are not dependencies of anything we build. +rm -rf "$TARGET_DIR"/*/incremental "$TARGET_DIR"/*/examples "$TARGET_DIR"/doc 2>/dev/null || true + +prune_one() { + name="$1" + snake="${name//-/_}" + # Fingerprints and build-script output live in per-crate directories. + find "$TARGET_DIR" -maxdepth 3 \ + \( -path "*/.fingerprint/$name-*" \ + -o -path "*/.fingerprint/$snake-*" \ + -o -path "*/build/$name-*" \ + -o -path "*/build/$snake-*" \) \ + -prune -exec rm -rf {} + 2>/dev/null || true + # Compiled outputs: rlib/rmeta/d in deps/, plus binaries at the profile root. + find "$TARGET_DIR" -maxdepth 3 \ + \( -name "$snake-*" -o -name "lib$snake-*" -o -name "lib$snake.*" \ + -o -name "$name" -o -name "$name-*" \) \ + -exec rm -rf {} + 2>/dev/null || true +} + +while IFS= read -r name; do + [ -n "$name" ] || continue + prune_one "$name" +done </dev/null \ + | xargs -0 -r grep -h -oE "${ws_root}/[^ :]*" 2>/dev/null \ + | grep -v "^${abs_target}/" \ + | sort -u || true)" + +if [ -n "$leaked_paths" ]; then + echo "prune: REFUSING to publish -- artifacts still reference workspace source:" >&2 + printf '%s\n' "$leaked_paths" | sed 's/^/ /' | head -20 >&2 + exit 2 +fi + +echo "prune: ok -- $(printf '%s\n' "$members" | wc -l) workspace member(s) stripped," \ + "no surviving artifact references workspace source" diff --git a/src/ci_orchestrator/cargo_cache.py b/src/ci_orchestrator/cargo_cache.py new file mode 100644 index 0000000..f4c347e --- /dev/null +++ b/src/ci_orchestrator/cargo_cache.py @@ -0,0 +1,121 @@ +"""Content-addressed store for compiled Rust dependency artifacts. + +A cache entry is one immutable file per key. Publishing is a ``rename(2)`` +within the store directory, so a reader either sees the previous entry or the +new one and never a half-written file — no advisory locking, which is exactly +what a VMware shared folder cannot give us with several jobs in flight. + +Nothing here knows about SSH or guests: it takes a finished tarball and files +it, or hands back the path of an existing one. That keeps the interesting +logic testable without a VM. +""" + +from __future__ import annotations + +import hashlib +import os +import re +import shutil +from datetime import datetime +from pathlib import Path + +# Bump to invalidate every entry at once — e.g. when the prune script changes +# what it strips, making older entries untrustworthy. +CACHE_FORMAT = 1 + +_SAFE_KEY = re.compile(r"^v\d+-[0-9a-f]{32,64}$") +_INCOMING = ".incoming" + + +def cache_key(*, lock_digest: str, rustc_version: str, host_triple: str) -> str: + """Derive the entry key from the inputs that make artifacts incompatible. + + ``lock_digest`` is the sha256 of ``Cargo.lock``: it changes exactly when + the dependency set changes, which is the property that makes this cache + sound. The feature set is deliberately *not* part of the key — cargo + encodes it in each artifact's own hash suffix, so several feature graphs + coexist inside one entry instead of fragmenting it. + + The parts are joined with NUL, which cannot occur in any of them, so + ``("ab", "c")`` and ``("a", "bc")`` cannot collide. + """ + digest = hashlib.sha256( + "\0".join((lock_digest, rustc_version, host_triple)).encode("utf-8") + ).hexdigest() + return f"v{CACHE_FORMAT}-{digest}" + + +class CargoCacheStore: + """A directory of immutable cache entries, one file per key.""" + + def __init__(self, root: Path, *, suffix: str = ".tar.zst") -> None: + self.root = Path(root) + self.suffix = suffix + + # ── paths ──────────────────────────────────────────────────────────── + + def _checked(self, key: str) -> str: + if not _SAFE_KEY.match(key): + raise ValueError(f"unsafe cache key: {key!r}") + return key + + def path_for(self, key: str) -> Path: + """Absolute path of the entry for ``key``. Raises on a malformed key.""" + return self.root / f"{self._checked(key)}{self.suffix}" + + # ── read ───────────────────────────────────────────────────────────── + + def lookup(self, key: str) -> Path | None: + """Return the entry path for ``key``, or ``None`` on a miss.""" + candidate = self.path_for(key) + return candidate if candidate.is_file() else None + + # ── write ──────────────────────────────────────────────────────────── + + def publish(self, key: str, source: Path) -> Path: + """Move ``source`` into the store under ``key``. Returns the entry path. + + The move goes through a staging directory on the same filesystem so the + final step is a rename: concurrent publishers of the same key both + produce a valid entry and the last one wins, with no window in which a + reader can observe a partial file. + """ + source = Path(source) + if not source.is_file(): + raise FileNotFoundError(f"nothing to publish: {source}") + target = self.path_for(key) + staging = self.root / _INCOMING + staging.mkdir(parents=True, exist_ok=True) + tmp = staging / f"{key}.{os.getpid()}{self.suffix}" + try: + os.replace(source, tmp) + except OSError: + # Different filesystem — copy into staging instead, which still + # leaves the publish itself an atomic rename. + shutil.copy2(source, tmp) + source.unlink(missing_ok=True) + os.replace(tmp, target) + return target + + # ── maintenance ────────────────────────────────────────────────────── + + def prune_older_than( + self, cutoff: datetime, *, what_if: bool = False + ) -> list[Path]: + """Delete entries last modified before ``cutoff``. Returns what went.""" + if not self.root.is_dir(): + return [] + removed: list[Path] = [] + for entry in sorted(self.root.glob(f"*{self.suffix}")): + if not entry.is_file(): + continue + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=cutoff.tzinfo) + if mtime >= cutoff: + continue + removed.append(entry) + if not what_if: + entry.unlink(missing_ok=True) + return removed + + +__all__ = ["CACHE_FORMAT", "CargoCacheStore", "cache_key"] diff --git a/src/ci_orchestrator/commands/build.py b/src/ci_orchestrator/commands/build.py index 1ed7dfa..1907c60 100644 --- a/src/ci_orchestrator/commands/build.py +++ b/src/ci_orchestrator/commands/build.py @@ -25,11 +25,13 @@ import tarfile import tempfile import zipfile from collections.abc import Iterable +from importlib import resources from pathlib import Path from typing import TYPE_CHECKING import click +from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key from ci_orchestrator.config import load_config from ci_orchestrator.credentials import KeyringCredentialStore from ci_orchestrator.transport.errors import TransportCommandError, TransportError @@ -122,6 +124,110 @@ def _tail_lines(stdout: str, stderr: str) -> str: return " | ".join(lines[-5:]) if lines else "(no build output)" +# ------------------------------------------------- cargo dependency cache + +_PRUNE_ASSET = "cargo-cache-prune.sh" +_GUEST_PRUNE = "/tmp/cargo-cache-prune.sh" +_GUEST_TARBALL = "/tmp/cargo-cache-entry.tar" + + +def _cargo_cache_probe(t: SshTransport, workdir: str) -> tuple[str, str] | None: + """Return ``(key, compressor)`` for this workspace, or ``None`` to skip. + + ``None`` means "not a Rust build, or we could not read what we need". The + cache is an optimisation, so every uncertainty here resolves to skipping + it rather than to guessing. + """ + if t.run(f"test -f {_sh_quote(workdir + '/Cargo.lock')}", check=False).returncode != 0: + return None + lock = t.run(f"sha256sum {_sh_quote(workdir + '/Cargo.lock')}", check=False) + version = t.run("rustc -V", check=False) + triple = t.run("rustc -vV | sed -n 's/^host: //p'", check=False) + if not (lock.ok and version.ok and triple.ok): + click.echo("[cache] could not read Cargo.lock digest or rustc version; skipping") + return None + digest = lock.stdout.split()[0] if lock.stdout.split() else "" + if not digest: + click.echo("[cache] empty Cargo.lock digest; skipping") + return None + compressor = "zstd" if t.run("command -v zstd", check=False).ok else "gzip" + key = cache_key( + lock_digest=digest, + rustc_version=version.stdout.strip(), + host_triple=triple.stdout.strip(), + ) + return key, compressor + + +def _tar_flag(compressor: str) -> tuple[str, str]: + """``(tar flag, file extension)`` for the chosen compressor.""" + return ("--zstd", "zst") if compressor == "zstd" else ("-z", "gz") + + +def _cargo_cache_restore( + t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str +) -> bool: + """Upload and unpack an existing entry. Returns True on a hit.""" + entry = store.lookup(key) + if entry is None: + click.echo(f"[cache] miss for {key} — building cold") + return False + flag, ext = _tar_flag(compressor) + size_mb = entry.stat().st_size / 1_048_576 + click.echo(f"[cache] hit for {key} ({size_mb:.0f} MB) — restoring") + guest_entry = f"{_GUEST_TARBALL}.{ext}" + t.copy(str(entry), guest_entry) + t.run( + f"tar {flag} -xf {_sh_quote(guest_entry)} -C {_sh_quote(workdir)} && " + f"rm -f {_sh_quote(guest_entry)}" + ) + return True + + +def _cargo_cache_publish( + t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str +) -> None: + """Prune workspace artifacts, then file the remainder under ``key``. + + A non-zero prune means the fail-closed assertion tripped: say so loudly and + publish nothing. A poisoned entry costs a wrong verdict; a missing one only + costs a cold build. + """ + # as_file() because the production venv is installed non-editable: the + # script is package data, not a path relative to a source checkout. + asset = resources.files("ci_orchestrator.assets").joinpath(_PRUNE_ASSET) + with resources.as_file(asset) as prune_script: + t.copy(str(prune_script), _GUEST_PRUNE) + pruned = t.run( + f"bash {_GUEST_PRUNE} {_sh_quote(workdir + '/target')} " + f"{_sh_quote(workdir + '/Cargo.toml')}", + check=False, + ) + click.echo(f"[cache] {(pruned.stdout.strip() or pruned.stderr.strip())}") + if pruned.returncode != 0: + click.echo("[cache] prune refused — publishing nothing", err=True) + return + + flag, ext = _tar_flag(compressor) + guest_tar = f"{_GUEST_TARBALL}.{ext}" + packed = t.run( + f"tar {flag} -cf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} target", + check=False, + ) + if not packed.ok: + click.echo("[cache] packing failed — publishing nothing", err=True) + return + + with tempfile.TemporaryDirectory() as td: + local = Path(td) / f"entry.tar.{ext}" + t.fetch(guest_tar, local) + published = store.publish(key, local) + t.run(f"rm -f {_sh_quote(guest_tar)}", check=False) + click.echo( + f"[cache] published {key} ({published.stat().st_size / 1_048_576:.0f} MB)" + ) + + # ---------------------------------------------------------------- Linux flow @@ -143,6 +249,7 @@ def _linux_build( extra_env: dict[str, str], skip_artifact: bool, use_xvfb: bool = False, + cargo_cache_root: Path | None = None, ) -> None: click.echo("[build run] Linux build mode (SSH transport)") @@ -217,6 +324,26 @@ def _linux_build( check=False, ) + # Cargo dependency cache. Everything here is best-effort: a cache + # problem must never turn a good build red, so any failure degrades to + # a cold build with a message. + cache_key_value: str | None = None + compressor = "gzip" + store: CargoCacheStore | None = None + if cargo_cache_root is not None: + try: + store = CargoCacheStore(cargo_cache_root) + probed = _cargo_cache_probe(t, workdir) + if probed is None: + store = None + else: + cache_key_value, compressor = probed + _cargo_cache_restore(t, store, cache_key_value, workdir, compressor) + except (TransportError, OSError, ValueError) as exc: + click.echo(f"[cache] restore skipped: {exc}", err=True) + cache_key_value = None + store = None + env_prefix = "".join( f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items() ) @@ -245,6 +372,14 @@ def _linux_build( f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}" ) + # Publish only on a miss, so the steady state uploads nothing. A failed + # build never reaches here, so a broken tree cannot become an entry. + if store is not None and cache_key_value and store.lookup(cache_key_value) is None: + try: + _cargo_cache_publish(t, store, cache_key_value, workdir, compressor) + except (TransportError, OSError, ValueError) as exc: + click.echo(f"[cache] publish skipped: {exc}", err=True) + if skip_artifact: click.echo("[build run] artifact packaging skipped (--skip-artifact).") return diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py index eeefbd4..ec5c794 100644 --- a/src/ci_orchestrator/commands/job.py +++ b/src/ci_orchestrator/commands/job.py @@ -326,6 +326,14 @@ def _destroy_clone( help="In-guest git clone (default) vs host-side clone + zip transfer.", ) @click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option( + "--cargo-cache/--no-cargo-cache", + "cargo_cache", + default=True, + show_default=True, + help="Reuse compiled Rust dependency artifacts between jobs (Linux only; " + "engages only when the workspace has a Cargo.lock).", +) @click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) @click.option( "--xvfb", @@ -409,6 +417,7 @@ def job( submodules: bool, use_git_clone: bool, use_shared_cache: bool, + cargo_cache: bool, skip_artifact: bool, use_xvfb: bool, gitea_credential_target: str, @@ -634,6 +643,11 @@ def job( extra_env=extra_env, skip_artifact=skip_artifact, use_xvfb=use_xvfb, + cargo_cache_root=( + config.cargo_cache.root + if cargo_cache and config.cargo_cache.enabled + else None + ), ) else: _windows_build( diff --git a/src/ci_orchestrator/commands/retention.py b/src/ci_orchestrator/commands/retention.py index ffcffe7..cb4748a 100644 --- a/src/ci_orchestrator/commands/retention.py +++ b/src/ci_orchestrator/commands/retention.py @@ -14,7 +14,8 @@ from pathlib import Path import click -from ci_orchestrator.config import load_config +from ci_orchestrator.cargo_cache import CargoCacheStore +from ci_orchestrator.config import Config, load_config @click.group() @@ -93,6 +94,7 @@ def retention_run( _purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if) _purge_old_dirs(log_path, "log", cutoff, what_if=what_if) + _purge_cargo_cache(config, effective_days, what_if=what_if) usage_after = shutil.disk_usage(str(check_path)) free_gb_after = usage_after.free / (1024**3) @@ -132,3 +134,44 @@ def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) else: shutil.rmtree(d, ignore_errors=True) click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}") + + +def _purge_cargo_cache(config: Config, effective_days: int, *, what_if: bool) -> None: + """Purge stale cargo cache entries. + + Entries are the heaviest thing this policy governs -- hundreds of megabytes + each, one per Cargo.lock change -- so leaving them out would repeat TODO + 2.9, where retention warned about low disk while freeing 0.0 GB because + everything that actually weighed was outside its reach. + + The window is the tighter of the global retention and the cache's own, so + the aggressive low-disk mode reaches the cache too. + """ + if not config.cargo_cache.enabled: + click.echo("[RetentionPolicy] cargo cache: disabled — skipping.") + return + + days = min(effective_days, config.cargo_cache.retention_days) + cutoff = datetime.now(tz=UTC) - timedelta(days=days) + store = CargoCacheStore(config.cargo_cache.root) + + # Size the victims first: after a real prune the files are gone and their + # bytes are unreadable, which is how a policy ends up reporting 0.0 GB + # freed while doing real work (TODO 2.9). + doomed = store.prune_older_than(cutoff, what_if=True) + if not doomed: + click.echo( + f"[RetentionPolicy] cargo cache: nothing to purge " + f"(cutoff: {cutoff.strftime('%Y-%m-%d')})." + ) + return + freed_mb = sum(p.stat().st_size for p in doomed if p.exists()) / 1_048_576 + + if not what_if: + store.prune_older_than(cutoff) + + verb = "WhatIf: would purge" if what_if else "Purged" + click.echo( + f"[RetentionPolicy] {verb} {len(doomed)} cargo cache " + f"{'entry' if len(doomed) == 1 else 'entries'} ({freed_mb:.0f} MB)" + ) diff --git a/src/ci_orchestrator/config.py b/src/ci_orchestrator/config.py index 9bff204..df5863e 100644 --- a/src/ci_orchestrator/config.py +++ b/src/ci_orchestrator/config.py @@ -65,6 +65,19 @@ class BackendConfig: options: dict[str, Any] = field(default_factory=dict) +@dataclass(frozen=True) +class CargoCacheConfig: + """Where compiled Rust dependency artifacts are filed between jobs. + + ``root`` defaults to ``/cache/cargo``; it is resolved in + :func:`load_config` because it depends on the resolved paths. + """ + + enabled: bool = True + root: Path = Path("/var/lib/ci/cache/cargo") + retention_days: int = 30 + + @dataclass(frozen=True) class Config: """Top-level configuration object.""" @@ -75,6 +88,7 @@ class Config: ssh_key_path: Path | None = None guest_cred_target: str = "BuildVMGuest" ip_pool: IpPoolConfig | None = None + cargo_cache: CargoCacheConfig = field(default_factory=CargoCacheConfig) def _platform_defaults() -> Paths: @@ -158,6 +172,7 @@ def load_config( ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest") ip_pool: IpPoolConfig | None = None + cargo_cache_section: dict[str, Any] = {} if toml_path and toml_path.is_file(): data = _load_toml(toml_path) @@ -187,9 +202,18 @@ def load_config( gateway=str(pool_section.get("gateway", "")), lease_file=Path(pool_section.get("lease_file") or _lease_default), ) + cargo_cache_section = data.get("cargo_cache", {}) or {} paths = _apply_env(paths, env) + # Resolved after _apply_env so CI_ROOT relocates the store with everything + # else; an explicit root in the file still wins. + cargo_cache = CargoCacheConfig( + enabled=bool(cargo_cache_section.get("enabled", True)), + root=Path(cargo_cache_section.get("root") or paths.root / "cache" / "cargo"), + retention_days=int(cargo_cache_section.get("retention_days", 30)), + ) + return Config( paths=paths, backend=backend, @@ -197,7 +221,15 @@ def load_config( ssh_key_path=ssh_key_path, guest_cred_target=guest_cred_target, ip_pool=ip_pool, + cargo_cache=cargo_cache, ) -__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"] +__all__ = [ + "BackendConfig", + "CargoCacheConfig", + "Config", + "IpPoolConfig", + "Paths", + "load_config", +] diff --git a/tests/python/test_cargo_cache.py b/tests/python/test_cargo_cache.py new file mode 100644 index 0000000..a05ab38 --- /dev/null +++ b/tests/python/test_cargo_cache.py @@ -0,0 +1,206 @@ +"""Tests for ci_orchestrator.cargo_cache — key derivation and the entry store.""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from ci_orchestrator.cargo_cache import CACHE_FORMAT, CargoCacheStore, cache_key + +# ── key derivation ─────────────────────────────────────────────────────────── + + +def test_key_is_stable_for_identical_inputs() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a == b + + +def test_key_changes_with_lock_digest() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="def", rustc_version="rustc 1.97.1", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_toolchain() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.98.0", host_triple="x86_64") + assert a != b + + +def test_key_changes_with_host_triple() -> None: + a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64") + b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="aarch64") + assert a != b + + +def test_key_is_filesystem_safe() -> None: + key = cache_key( + lock_digest="a/b+c=", rustc_version="rustc 1.97.1 (x /y)", host_triple="x86_64" + ) + assert all(c.isalnum() or c == "-" for c in key) + assert "/" not in key + + +def test_key_carries_the_format_version() -> None: + key = cache_key(lock_digest="abc", rustc_version="r", host_triple="t") + assert key.startswith(f"v{CACHE_FORMAT}-") + + +def test_key_fields_cannot_be_confused_by_concatenation() -> None: + """``ab|c`` and ``a|bc`` must not collide — the separator has to be real.""" + a = cache_key(lock_digest="ab", rustc_version="c", host_triple="t") + b = cache_key(lock_digest="a", rustc_version="bc", host_triple="t") + assert a != b + + +# ── store ──────────────────────────────────────────────────────────────────── + + +def test_lookup_returns_none_when_absent(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + assert store.lookup(cache_key(lock_digest="a", rustc_version="b", host_triple="c")) is None + + +def test_publish_then_lookup_finds_the_entry(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert store.lookup(key) == published + assert published.read_bytes() == b"payload" + + +def test_publish_consumes_the_source_file(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + assert not src.exists(), "publish must move, not copy — a copy doubles peak disk" + + +def test_publish_overwrites_an_existing_key(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + first = tmp_path / "a.tar.zst" + first.write_bytes(b"first") + store.publish(key, first) + second = tmp_path / "b.tar.zst" + second.write_bytes(b"second") + published = store.publish(key, second) + assert published.read_bytes() == b"second" + + +def test_publish_leaves_no_temp_files_behind(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + store.publish(key, src) + staging = tmp_path / ".incoming" + leftovers = list(staging.glob("*")) if staging.is_dir() else [] + assert leftovers == [] + + +def test_publish_creates_the_store_directory(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "not" / "yet") + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + src = tmp_path / "incoming.tar.zst" + src.write_bytes(b"payload") + published = store.publish(key, src) + assert published.is_file() + + +def test_publish_missing_source_raises(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + with pytest.raises(FileNotFoundError): + store.publish(key, tmp_path / "nope.tar.zst") + + +def test_lookup_rejects_a_key_with_a_path_separator(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.lookup("../../etc/passwd") + + +def test_path_for_rejects_an_arbitrary_string(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + with pytest.raises(ValueError, match="unsafe cache key"): + store.path_for("not-a-real-key") + + +def test_lookup_ignores_a_directory_at_the_entry_path(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path) + key = cache_key(lock_digest="a", rustc_version="b", host_triple="c") + store.path_for(key).mkdir(parents=True) + assert store.lookup(key) is None + + +# ── prune ──────────────────────────────────────────────────────────────────── + + +def _age(path: Path, days: int) -> None: + old = (datetime.now(tz=UTC) - timedelta(days=days)).timestamp() + os.utime(path, (old, old)) + + +def _seed(store: CargoCacheStore, tmp_path: Path, digest: str) -> Path: + key = cache_key(lock_digest=digest, rustc_version="b", host_triple="c") + src = tmp_path / f"seed-{digest}.tar.zst" + src.write_bytes(b"x") + return store.publish(key, src) + + +def test_prune_removes_entries_older_than_the_cutoff(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert not entry.exists() + + +def test_prune_keeps_fresh_entries(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "new") + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [] + assert entry.exists() + + +def test_prune_what_if_reports_without_deleting(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + + removed = store.prune_older_than( + datetime.now(tz=UTC) - timedelta(days=30), what_if=True + ) + assert removed == [entry] + assert entry.exists() + + +def test_prune_on_a_missing_store_is_a_noop(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "never-created") + assert store.prune_older_than(datetime.now(tz=UTC)) == [] + + +def test_prune_leaves_unrelated_files_alone(tmp_path: Path) -> None: + store = CargoCacheStore(tmp_path / "store") + entry = _seed(store, tmp_path, "old") + _age(entry, days=40) + stray = (tmp_path / "store") / "README.txt" + stray.write_text("not an entry", encoding="utf-8") + _age(stray, days=40) + + removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30)) + assert removed == [entry] + assert stray.exists(), "prune must only touch files with the entry suffix" diff --git a/tests/python/test_commands_build.py b/tests/python/test_commands_build.py index ed191bc..8530562 100644 --- a/tests/python/test_commands_build.py +++ b/tests/python/test_commands_build.py @@ -38,9 +38,20 @@ class _FakeTransport: self.copies: list[tuple[str, str]] = [] self.fetches: list[tuple[str, str]] = [] self.run_results: dict[int, _FakeResult] = {} + # Matched by substring against the script, checked before the + # positional run_results. Index-based fixtures get brittle once a + # feature adds calls in the middle of the flow (the cargo cache does). + self.script_results: list[tuple[str, _FakeResult]] = [] self.default_result = _FakeResult(stdout="ok") type(self).instances.append(self) + def result_for( + self, needle: str, *, stdout: str = "", stderr: str = "", returncode: int = 0 + ) -> None: + self.script_results.append( + (needle, _FakeResult(stdout=stdout, stderr=stderr, returncode=returncode)) + ) + def __enter__(self) -> _FakeTransport: return self @@ -49,7 +60,13 @@ class _FakeTransport: def run(self, script: str, *, check: bool = True) -> _FakeResult: self.runs.append((script, check)) - result = self.run_results.get(len(self.runs) - 1, self.default_result) + result: _FakeResult | None = None + for needle, canned in self.script_results: + if needle in script: + result = canned + break + if result is None: + result = self.run_results.get(len(self.runs) - 1, self.default_result) if check and not result.ok: raise TransportCommandError(result.returncode, result.stdout, result.stderr) return result @@ -838,3 +855,205 @@ def test_build_run_windows_transport_error( ) assert result.exit_code != 0 assert "winrm unreachable" in result.output + + +# ── _linux_build: cargo dependency cache ─────────────────────────────────── + + +def _linux_kwargs(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "ip_address": "10.0.0.1", + "ssh_user": "ci_build", + "key_path": None, + "known_hosts": None, + "workdir": "/opt/ci/build", + "output_dir": "/opt/ci/output", + "host_source_dir": None, + "clone_url": "http://repo/example.git", + "clone_branch": "main", + "clone_commit": "", + "clone_submodules": False, + "build_command": "true", + "artifact_source": "dist", + "extra_env": {}, + "skip_artifact": True, + } + base.update(overrides) + return base + + +def _rust_guest(fake: _FakeTransport, *, digest: str = "cafe") -> None: + """Make the fake answer like a Rust workspace with a known toolchain.""" + fake.result_for("sha256sum", stdout=f"{digest} Cargo.lock\n") + fake.result_for("rustc -V", stdout="rustc 1.97.1 (abc 2026-01-01)\n") + fake.result_for("rustc -vV", stdout="x86_64-unknown-linux-gnu\n") + + +def _expected_key(digest: str = "cafe") -> str: + from ci_orchestrator.cargo_cache import cache_key + + return cache_key( + lock_digest=digest, + rustc_version="rustc 1.97.1 (abc 2026-01-01)", + host_triple="x86_64-unknown-linux-gnu", + ) + + +def _fetch_writes_a_real_file( + monkeypatch: pytest.MonkeyPatch, fake: _FakeTransport +) -> None: + """Make ``fetch`` produce an actual file on disk. + + Without this the default fake leaves nothing behind, ``publish`` dies on + FileNotFoundError, and a "nothing was published" assertion passes for the + wrong reason -- it would hold even with the guard under test removed. + Mutation-tested: dropping the prune-refusal check must make those tests go + red, and it only does when a publish could otherwise have succeeded. + """ + + def _fetch(remote: str, local: str) -> None: + fake.fetches.append((remote, str(local))) + Path(local).write_bytes(b"fresh-entry") + + monkeypatch.setattr(fake, "fetch", _fetch) + + +def test_linux_build_without_cache_root_runs_no_cache_commands( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(build_module, "SshTransport", _FakeTransport) + build_module._linux_build(**_linux_kwargs()) + cmds = [c[0] for c in _FakeTransport.instances[0].runs] + assert not any("cargo-cache-prune" in c for c in cmds) + assert not any("sha256sum" in c for c in cmds) + + +def test_linux_build_skips_cache_when_no_cargo_lock( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(build_module, "SshTransport", _FakeTransport) + fake = _FakeTransport() + _FakeTransport.instances = [] + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + fake.result_for("test -f", returncode=1) + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert not any("cargo-cache-prune" in c for c in cmds) + assert not any("sha256sum" in c for c in cmds) + + +def test_linux_build_restores_an_existing_entry( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + store = CargoCacheStore(tmp_path) + seed = tmp_path / "seed.tar.zst" + seed.write_bytes(b"entry") + store.publish(_expected_key(), seed) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert any("tar --zstd -xf" in c for c in cmds), "a hit must unpack the entry" + assert fake.copies, "a hit must upload the entry to the guest" + # A hit must not re-publish: the entry is already there. + assert not any("cargo-cache-prune" in c for c in cmds) + + +def test_linux_build_publishes_on_a_miss( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + _fetch_writes_a_real_file(monkeypatch, fake) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is not None + + +def test_linux_build_does_not_publish_when_the_prune_assertion_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + _fetch_writes_a_real_file(monkeypatch, fake) + fake.result_for( + "cargo-cache-prune", + returncode=2, + stderr="prune: REFUSING to publish -- artifacts still reference workspace source", + ) + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None, ( + "a failed prune assertion must never reach the store" + ) + + +def test_linux_build_does_not_publish_when_packing_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + from ci_orchestrator.cargo_cache import CargoCacheStore + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + _fetch_writes_a_real_file(monkeypatch, fake) + fake.result_for("tar --zstd -cf", returncode=1, stderr="no space left on device") + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None + + +def test_cache_probe_failure_does_not_fail_the_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + fake.result_for("sha256sum", returncode=1, stderr="boom") + # Must not raise: the cache is an optimisation, never a failure mode. + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + + +def test_cache_transport_error_does_not_fail_the_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake) + + def _boom(*_a: Any, **_k: Any) -> None: + raise TransportError("ssh died mid-restore") + + monkeypatch.setattr(fake, "copy", _boom) + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + + +def test_cache_key_differs_when_the_lockfile_differs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A Cargo.lock change must miss the entry built for the previous one.""" + from ci_orchestrator.cargo_cache import CargoCacheStore + + store = CargoCacheStore(tmp_path) + seed = tmp_path / "seed.tar.zst" + seed.write_bytes(b"entry") + store.publish(_expected_key("cafe"), seed) + + fake = _FakeTransport() + monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake) + _rust_guest(fake, digest="beef") + + build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path)) + cmds = [c[0] for c in fake.runs] + assert not any("tar --zstd -xf" in c for c in cmds), "different lock must miss" diff --git a/tests/python/test_commands_job.py b/tests/python/test_commands_job.py index cb71b63..c6be8ae 100644 --- a/tests/python/test_commands_job.py +++ b/tests/python/test_commands_job.py @@ -1630,3 +1630,96 @@ def test_probe_transport_ssh_not_ready_returns_false( ssh_known_hosts=None, ) assert result is False + + +# ── cargo cache plumbing ───────────────────────────────────────────────────── + + +def test_job_passes_cache_root_to_linux_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is not None + + +def test_job_no_cargo_cache_disables_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + extra=["--no-cargo-cache"], + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_respects_config_disabling_the_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + """`enabled = false` in the config wins even without --no-cargo-cache.""" + from ci_orchestrator.config import CargoCacheConfig, Config + + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + # _patch_common already replaced load_config, so override its return value + # rather than the environment. + base = job_module.load_config() + patched = Config( + paths=base.paths, + backend=base.backend, + vmrun_path=base.vmrun_path, + ssh_key_path=base.ssh_key_path, + guest_cred_target=base.guest_cred_target, + ip_pool=base.ip_pool, + cargo_cache=CargoCacheConfig(enabled=False), + ) + monkeypatch.setattr(job_module, "load_config", lambda: patched) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_windows_build_gets_no_cargo_cache_argument( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The Windows branch must be untouched by the Linux-only cache.""" + # Guest OS comes from the *clone* VMX, which the fake backend writes. + backend = _FakeBackend(running_after=1, clone_guest_os="windows2019srvNext-64") + calls = _patch_common(monkeypatch, backend) + vmx = tmp_path / "templates" / "WinBuild2025.vmx" + vmx.parent.mkdir(parents=True) + vmx.write_text('guestOS = "windows2019srvNext-64"\n', encoding="utf-8") + result = CliRunner().invoke( + cli, + _common_args( + vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert "cargo_cache_root" not in calls["windows_build"][0] diff --git a/tests/python/test_commands_retention.py b/tests/python/test_commands_retention.py index 57f22e5..6e1957e 100644 --- a/tests/python/test_commands_retention.py +++ b/tests/python/test_commands_retention.py @@ -11,6 +11,7 @@ from click.testing import CliRunner import ci_orchestrator.commands.retention as ret from ci_orchestrator.__main__ import cli +from ci_orchestrator.config import Config # ── helpers ─────────────────────────────────────────────────────────────────── @@ -191,3 +192,134 @@ def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> assert result.exit_code == 0, result.output assert not old.exists() assert "Purged log" in result.output + + +# ── cargo cache entries ─────────────────────────────────────────────────────── + + +def _cache_config( + tmp_path: Path, *, enabled: bool = True, days: int = 30 +) -> tuple[Path, Config]: + """Point the loaded config at a throwaway cache store. + + Returns ``(store root, config)`` — the root so tests can seed entries, the + config so they can substitute it for ``load_config``. + """ + from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths + + root = tmp_path / "cache" / "cargo" + cfg = Config( + paths=Paths( + root=tmp_path, + templates=tmp_path / "t", + build_vms=tmp_path / "b", + artifacts=tmp_path / "artifacts", + keys=tmp_path / "k", + ), + backend=BackendConfig(), + cargo_cache=CargoCacheConfig(enabled=enabled, root=root, retention_days=days), + ) + return root, cfg + + +def _seed_entry(root: Path, digest: str, *, age_days: int, size: int = 1024) -> Path: + import os + + from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key + + store = CargoCacheStore(root) + src = root.parent / f"seed-{digest}.tar.zst" + src.parent.mkdir(parents=True, exist_ok=True) + src.write_bytes(b"x" * size) + entry = store.publish(cache_key(lock_digest=digest, rustc_version="r", host_triple="t"), src) + stamp = time.time() - age_days * 86400 + os.utime(entry, (stamp, stamp)) + return entry + + +def test_retention_purges_stale_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + fresh = _seed_entry(root, "new", age_days=1) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not old.exists(), "stale entry should be purged" + assert fresh.exists(), "fresh entry should survive" + + +def test_retention_reports_the_bytes_it_freed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """TODO 2.9: a policy that frees space without saying how much is the bug.""" + root, cfg = _cache_config(tmp_path) + _seed_entry(root, "old", age_days=90, size=3 * 1024 * 1024) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "Purged 1 cargo cache entry (3 MB)" in result.output + + +def test_retention_what_if_keeps_cargo_cache_entries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run", "--what-if"]) + assert result.exit_code == 0, result.output + assert old.exists() + assert "would purge 1 cargo cache entry" in result.output + + +def test_retention_skips_the_cache_when_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root, cfg = _cache_config(tmp_path, enabled=False) + old = _seed_entry(root, "old", age_days=90) + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert old.exists(), "a disabled cache must not be touched" + assert "disabled" in result.output + + +def test_low_disk_tightens_the_cache_window_too( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Aggressive mode must reach the cache — it is the heaviest thing here.""" + root, cfg = _cache_config(tmp_path, days=30) + entry = _seed_entry(root, "old", age_days=10) # young for 30d, old for 7d + + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert not entry.exists(), "aggressive retention should have reached the cache" + + +def test_retention_missing_cache_store_is_not_an_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _root, cfg = _cache_config(tmp_path) + monkeypatch.setattr(ret, "load_config", lambda: cfg) + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke(cli, ["retention", "run"]) + assert result.exit_code == 0, result.output + assert "nothing to purge" in result.output diff --git a/tests/python/test_config.py b/tests/python/test_config.py index ce53d41..5270434 100644 --- a/tests/python/test_config.py +++ b/tests/python/test_config.py @@ -145,3 +145,54 @@ def test_ip_pool_absent_when_not_configured(tmp_path: Path) -> None: toml.write_text("[backend]\ntype = \"workstation\"\n", encoding="utf-8") cfg = load_config(config_path=toml, env={}) assert cfg.ip_pool is None + + +# ── cargo cache section ────────────────────────────────────────────────────── + + +def _paths_only(tmp_path: Path, extra: str = "") -> Path: + cfg = tmp_path / "config.toml" + cfg.write_text( + '[paths]\nroot = "/var/lib/ci"\ntemplates = "/var/lib/ci/templates"\n' + 'build_vms = "/var/lib/ci/build-vms"\nartifacts = "/var/lib/ci/artifacts"\n' + 'keys = "/var/lib/ci/keys"\n' + extra, + encoding="utf-8", + ) + return cfg + + +def test_cargo_cache_defaults_under_paths_root(tmp_path: Path) -> None: + cfg = load_config(config_path=_paths_only(tmp_path), env={}) + assert cfg.cargo_cache.enabled is True + assert cfg.cargo_cache.root == Path("/var/lib/ci/cache/cargo") + assert cfg.cargo_cache.retention_days == 30 + + +def test_cargo_cache_section_overrides_the_defaults(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only( + tmp_path, + '\n[cargo_cache]\nenabled = false\nroot = "/mnt/fast/cargo"\n' + "retention_days = 7\n", + ), + env={}, + ) + assert cfg.cargo_cache.enabled is False + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo") + assert cfg.cargo_cache.retention_days == 7 + + +def test_cargo_cache_root_follows_ci_root_env(tmp_path: Path) -> None: + """CI_ROOT relocates every path; the store must move with them.""" + cfg = load_config( + config_path=_paths_only(tmp_path), env={"CI_ROOT": str(tmp_path / "elsewhere")} + ) + assert cfg.cargo_cache.root == tmp_path / "elsewhere" / "cache" / "cargo" + + +def test_explicit_cargo_cache_root_beats_ci_root(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only(tmp_path, '\n[cargo_cache]\nroot = "/mnt/fast/cargo"\n'), + env={"CI_ROOT": str(tmp_path / "elsewhere")}, + ) + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")