Merge branch 'feat/cargo-dependency-cache'
Cargo dependency cache for Rust jobs on LinuxRust2404: 186s -> 84s measured end to end, 75s with the source modified. Dependency artifacts are reused between runs; workspace crates never are, and the job refuses to publish an entry the prune assertion cannot vouch for. Closes thread entries furyrs#7 and furyrs#4. TODO 3.8.
This commit is contained in:
@@ -194,6 +194,19 @@ inputs:
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
cargo-cache:
|
||||
description: >
|
||||
Reuse compiled Rust dependency artifacts between jobs. "auto" (default)
|
||||
caches on branches and pull requests but builds release tags
|
||||
(refs/tags/v*) cold; "on" always caches; "off" never does. Only ever
|
||||
engages for Linux guests whose workspace has a Cargo.lock, so it is inert
|
||||
for every other job. Workspace crates are stripped from an entry before
|
||||
it is stored and the job refuses to publish if any survive, so a restored
|
||||
cache cannot carry the code under test -- but a release build verifies an
|
||||
artifact rather than a change, so "auto" gives it the clean room.
|
||||
required: false
|
||||
default: 'auto'
|
||||
|
||||
skip-artifact:
|
||||
description: >
|
||||
Set to "true" to skip artifact packaging and collection entirely.
|
||||
@@ -249,6 +262,8 @@ runs:
|
||||
INPUT_REPO_COMMIT: ${{ inputs.repo-commit }}
|
||||
INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }}
|
||||
INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }}
|
||||
INPUT_CARGO_CACHE: ${{ inputs.cargo-cache }}
|
||||
GITHUB_REF_FOR_CACHE: ${{ github.ref }}
|
||||
INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }}
|
||||
INPUT_XVFB: ${{ inputs.xvfb }}
|
||||
INPUT_GUEST_CPU: ${{ inputs.guest-cpu }}
|
||||
@@ -387,6 +402,20 @@ runs:
|
||||
else { $pyArgs += '--use-git-clone' }
|
||||
if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $pyArgs += '--use-shared-cache' }
|
||||
if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $pyArgs += '--skip-artifact' }
|
||||
# Release tags build cold: they verify a release artifact, not a
|
||||
# change, so the clean room is worth the two minutes (furyrs#13).
|
||||
$cargoCacheMode = if ($env:INPUT_CARGO_CACHE) { $env:INPUT_CARGO_CACHE } else { 'auto' }
|
||||
switch ($cargoCacheMode) {
|
||||
'on' { $useCargoCache = $true }
|
||||
'off' { $useCargoCache = $false }
|
||||
'auto' { $useCargoCache = -not ($env:GITHUB_REF_FOR_CACHE -like 'refs/tags/v*') }
|
||||
default {
|
||||
throw ("cargo-cache must be one of 'auto', 'on', 'off' - got " +
|
||||
"'$cargoCacheMode'.")
|
||||
}
|
||||
}
|
||||
if (-not $useCargoCache) { $pyArgs += '--no-cargo-cache' }
|
||||
Write-Host ("[job] Cargo cache: " + $(if ($useCargoCache) { 'on' } else { 'off (cold)' }))
|
||||
if ($env:INPUT_XVFB -eq 'true') { $pyArgs += '--xvfb' }
|
||||
# VMX overrides: forward only when > 0 (0 = keep template value).
|
||||
if ([int]($env:INPUT_GUEST_CPU) -gt 0) { $pyArgs += @('--guest-cpu', $env:INPUT_GUEST_CPU) }
|
||||
|
||||
@@ -283,3 +283,5 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali:
|
||||
15. **Step 7 in timeout: distinguere tre cause diverse che danno lo stesso messaggio** — `getGuestIPAddress` risponde `The VMware Tools are not running` sia quando il guest non ha IP, sia quando ha l'IP ma i Tools non rispondono. Ordine di verifica, dal più veloce: (a) **collisione** — `ps | grep vmware-vmx`, processo più vecchio del deploy corrente → vedi #14; (b) **DHCP host incastrato** — `ip -4 addr show vmnet8` deve dare `192.168.79.1/24` e `ps -ef | grep vmnet-dhcpd` deve mostrarlo vivo; se no, `sudo systemctl restart vmware-networks` (o `vmware-host.service`); (c) **guest senza lease o Tools non pronti** — cercare il MAC della VMX in `/etc/vmware/vmnet8/dhcpd/dhcpd.leases` e in `ip neigh show dev vmnet8`: se un lease c'è, il guest è vivo e mancano solo i Tools, quindi sondare TCP/22 direttamente invece di aspettare `getGuestIPAddress`.
|
||||
|
||||
16. **Delta di snapshot orfani dopo `--force`** — `deploy-*` cancella `<vm>.vmsd` (il database snapshot) per evitare l'errore di resize sulla catena, ma **non** cancella i file delta `<disk>-00000N.vmdk`. Restano su disco con un `parentCID` che non corrisponde più al base ricreato: innocui (fuori dal percorso di boot, la VMX punta al base con `parentCID=ffffffff`) ma confondono la diagnosi e vanno rimossi a mano. Verifica della catena: `head -c 2000 <disk>.vmdk | strings | grep -E '^CID|parentCID'`.
|
||||
|
||||
17. **`vmrun list` è per-utente e mente se lo lanci con l'utente sbagliato** — le VM della CI le avvia `ci-runner`; `vmrun -T ws list` eseguito come `simone` (o come root) risponde `Total running VMs: 0` **anche mentre un clone sta girando**, perché vede solo l'istanza VMware dell'utente corrente. Occorso il 2026-07-22 durante la validazione della cache cargo: `vmrun list` diceva 0 e `ps` mostrava un `vmware-vmx` vivo da 19 minuti. Conseguenza pratica: il check "0 VM accese" prima di un `deploy --force` (vedi #14) **non è affidabile** se non lo si esegue come `ci-runner`. Usare `sudo -u ci-runner vmrun -T ws list`, oppure — meglio, perché indipendente dall'utente — `ps -eo pid,lstart,etime,args | grep '[v]mware-vmx'`, che è anche la diagnosi già raccomandata in #14 e #15.
|
||||
|
||||
@@ -35,7 +35,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I
|
||||
| [x] | Gitea Workflow Integration | 5 | 0 | tutti completati — example.yml usa composite action |
|
||||
| [x] | §1 Sicurezza & hardening | 3 | 0 | 1.1, 1.4, 1.6 done; 1.2/1.3/1.5/1.7/1.8 deferred (home lab) |
|
||||
| [x] | §2 Affidabilità & resilienza | 7 | 0 | tutto done (2.1-2.7) |
|
||||
| [ ] | §3 Performance & throughput | 4 | 1 | 3.1/3.2/3.3/3.6 done; 3.4/3.5 deferred; **3.8 cache cargo pianificata** |
|
||||
| [x] | §3 Performance & throughput | 5 | 0 | 3.1/3.2/3.3/3.6/3.8 done; 3.4/3.5 deferred — **§3 CHIUSO** |
|
||||
| [x] | §4 Osservabilità & manutenzione | 3 | 0 | 4.1/4.3/4.4 done (4.2 Prometheus, 4.5 Dashboard rimossi) |
|
||||
| [x] | §5 Qualità codice & test | 5 | 0 | 5.1/5.2/5.3/5.4/5.5 done |
|
||||
| [x] | §6 Scalabilità & estensioni | 5 | 0 | 6.1-6.2 done, 6.3 deferred (home lab), 6.4-6.5 done, 6.6 done |
|
||||
@@ -938,8 +938,19 @@ cioè nelle invocazioni manuali/burn-in.
|
||||
`Start-Job`/`pwsh`/`$IsWindows`. Lo shim resta solo per il runner Windows, dove
|
||||
`CI_VENV_PYTHON` è sempre settato. Il bug non è più raggiungibile sull'host Linux.
|
||||
|
||||
### 3.8 [P2] [ ] Cache delle dipendenze cargo per i job Rust — PIANIFICATO (2026-07-22)
|
||||
File: [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md).
|
||||
### 3.8 [P2] [x] Cache delle dipendenze cargo per i job Rust — COMPLETATO 2026-07-22
|
||||
File: [src/ci_orchestrator/cargo_cache.py](src/ci_orchestrator/cargo_cache.py),
|
||||
[src/ci_orchestrator/assets/cargo-cache-prune.sh](src/ci_orchestrator/assets/cargo-cache-prune.sh),
|
||||
piano in [plans/cargo-dependency-cache.md](plans/cargo-dependency-cache.md),
|
||||
numeri in [plans/cargo-dependency-cache-measurements.md](plans/cargo-dependency-cache-measurements.md).
|
||||
|
||||
**Risultato misurato end-to-end**: job Rust da **186 s → 84 s** (fase build
|
||||
150 s → 47 s). Con sorgente modificato — lo scenario CI vero — 75 s. Entry da
|
||||
645 MB, restore 8 s. `--no-cargo-cache` e i tag `v*` restano a freddo.
|
||||
|
||||
**La prova che conta**: errore di compilazione deliberato in `fury-core`,
|
||||
run a caldo → `error[E0308]`, job exit 1. La cache non può nascondere una
|
||||
regressione. Da ripetere ogni volta che cambiano le regole di potatura.
|
||||
|
||||
Misurato in furyrs#12: dei ~133 s di un job Rust, ~120 s sono la compilazione
|
||||
delle ~490 dipendenze esterne, che `Cargo.lock` committato tiene **bit-identiche
|
||||
@@ -975,7 +986,12 @@ concorrenti.
|
||||
cambio di `Cargo.lock`, centinaia di MB l'una) senza estendere `retention run`
|
||||
ripeterebbe l'errore di §2.9 su scala maggiore.
|
||||
|
||||
**Prossimo passo**: Task 0 del piano — misurare prima di costruire, come in #12.
|
||||
**Nota emersa implementando**: l'asserzione prevista dal piano era tautologica
|
||||
— ri-cercava gli stessi glob che aveva appena cancellato, quindi provava solo
|
||||
che `rm` fosse andato a buon fine, non che i pattern coprissero la realtà.
|
||||
Sostituita con un oracolo indipendente: i depfile `.d` che cargo scrive dicono
|
||||
da quali sorgenti ogni artefatto è stato compilato. Dettaglio nel documento
|
||||
delle misure.
|
||||
|
||||
### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com)
|
||||
File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`).
|
||||
|
||||
@@ -53,3 +53,22 @@ type = "workstation"
|
||||
# netmask = "255.255.255.0"
|
||||
# gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route
|
||||
# lease_file = "/var/lib/ci/ip-pool.json"
|
||||
|
||||
# ------------------------------------------------- cargo dependency cache
|
||||
# Compiled Rust dependency artifacts, reused between jobs. Engages only for
|
||||
# Linux guests whose build workdir contains a Cargo.lock, so non-Rust jobs
|
||||
# need no flag. Workspace crates are stripped before an entry is stored and
|
||||
# the job refuses to publish if any survive, so a restored entry cannot carry
|
||||
# the code under test.
|
||||
#
|
||||
# One immutable file per key under `root`; a key is
|
||||
# sha256(Cargo.lock) + rustc version + host triple. Deleting entries by hand
|
||||
# is always safe -- each one costs exactly one cold build to regenerate.
|
||||
#
|
||||
# `root` defaults to <paths.root>/cache/cargo and follows CI_ROOT unless set
|
||||
# here explicitly. Measured on LinuxRust2404: an entry is ~645 MB and a warm
|
||||
# job pays ~8s to skip ~120s of dependency compilation.
|
||||
[cargo_cache]
|
||||
enabled = true
|
||||
# root = "/var/lib/ci/cache/cargo"
|
||||
retention_days = 30
|
||||
|
||||
@@ -608,6 +608,53 @@ ci retention run --retention-days 30 --aggressive-retention-days 7 --min-free-gb
|
||||
|
||||
See [§4.6](#46-sweep-orphans) (`ci vm cleanup`).
|
||||
|
||||
### Cargo dependency cache
|
||||
|
||||
Rust jobs on `LinuxRust2404` reuse compiled **dependency** artifacts between
|
||||
runs. Measured on furyrs: a warm job pays ~8s to skip ~120s of dependency
|
||||
compilation on a 134s job. Workspace crates are never cached, so a restored
|
||||
entry cannot carry the code under test.
|
||||
|
||||
Store: `/var/lib/ci/cache/cargo`, one immutable file per key
|
||||
(`v1-<sha256>.tar.zst`, ~645 MB each). A key is
|
||||
`sha256(Cargo.lock) + rustc version + host triple`.
|
||||
|
||||
```bash
|
||||
ls -lh /var/lib/ci/cache/cargo/ # what is stored
|
||||
du -sh /var/lib/ci/cache/cargo/ # total
|
||||
```
|
||||
|
||||
What the job log tells you:
|
||||
|
||||
| Line | Meaning |
|
||||
|---|---|
|
||||
| `[cache] hit for v1-… (645 MB) — restoring` | Entry reused; dependencies will not recompile |
|
||||
| `[cache] miss for v1-… — building cold` | First run for this `Cargo.lock`+toolchain; the job publishes at the end |
|
||||
| `[cache] prune: ok — 6 workspace member(s) stripped, …` | Prune passed its assertion; publishing |
|
||||
| `[cache] prune refused — publishing nothing` | **The assertion tripped.** An artifact still referenced workspace source. Nothing was stored; the build result itself is unaffected. Investigate before trusting the feature |
|
||||
| `[cache] restore skipped: …` / `publish skipped: …` | Transport or disk problem. The build ran cold and is still valid |
|
||||
|
||||
Turning it off, cheapest first:
|
||||
|
||||
1. One workflow: `cargo-cache: 'off'` in the composite action inputs.
|
||||
2. This host: `enabled = false` under `[cargo_cache]` in `/var/lib/ci/config.toml`.
|
||||
3. One job: `ci job --no-cargo-cache`.
|
||||
4. Everything, without deleting a file: bump `CACHE_FORMAT` in
|
||||
`src/ci_orchestrator/cargo_cache.py`. Every key changes, so every existing
|
||||
entry becomes unreachable. Use this when the prune rules change and older
|
||||
entries can no longer be trusted.
|
||||
|
||||
Deleting entries by hand is always safe — they are immutable and independent,
|
||||
and each one costs exactly one cold build to regenerate:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner rm /var/lib/ci/cache/cargo/*.tar.zst
|
||||
```
|
||||
|
||||
`ci retention run` prunes entries older than the tighter of `--retention-days`
|
||||
and `[cargo_cache].retention_days`, so the low-disk aggressive window reaches
|
||||
them too.
|
||||
|
||||
### Publishing `.gitea/` — committed is not deployed
|
||||
|
||||
act_runner never executes this working copy. For every job it re-clones the
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
# Cargo dependency cache — measurements
|
||||
|
||||
Measured 2026-07-22 on a `LinuxRust2404` linked clone (4 vCPU, 3.9 GB RAM,
|
||||
`rustc 1.97.1`, `zstd` present at `/usr/bin/zstd`), running furyrs `b8ea567`
|
||||
with the `build-command` block from its `.gitea/workflows/ci.yml` verbatim.
|
||||
|
||||
Entry #12 of the furyrs thread set the rule: measure before building. There it
|
||||
killed the registry cache — 9.3s of saving for +800 MB on the template. Here
|
||||
the numbers point the other way, decisively.
|
||||
|
||||
## Baseline
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Cold pipeline, empty `target/` | **134.7s** |
|
||||
|
||||
Matches the 133s furyrs reported for job 1021 and the 126.6s measured in #12,
|
||||
so the clone is representative.
|
||||
|
||||
## Sizes
|
||||
|
||||
| | Bytes | Human |
|
||||
|---|---:|---:|
|
||||
| `target/` raw | 3,206,670,175 | 3.0 GiB |
|
||||
| `tar` of raw `target/` | 3,215,503,360 | 3.0 GiB |
|
||||
| `tar` + `zstd -3 -T0`, raw | 810,108,028 | 773 MiB |
|
||||
| **`target/` after prune** | 2,719,971,105 | **2.5 GiB** |
|
||||
| **entry as stored** (`tar --zstd`, pruned) | 676,585,779 | **645 MiB** |
|
||||
|
||||
Pruning strips 487 MB (15%) of raw `target/`, which is the workspace crates'
|
||||
own artifacts — 42 fingerprint directories across the six members
|
||||
(`fury-core` 11, `fury-gui` 12, `fury-ipc` 6, `fury-backend` 5, `fury-cli` 5,
|
||||
`furyd` 3). 1,928 dependency artifacts and 903 dependency fingerprints survive.
|
||||
|
||||
## Costs
|
||||
|
||||
| Step | Seconds | Paid by |
|
||||
|---|---:|---|
|
||||
| prune + assert | 0.21 | publish |
|
||||
| pack (`tar --zstd`) | 10.88 | publish |
|
||||
| fetch guest→host (140 MiB/s) | 4.62 | publish |
|
||||
| **publish total** | **15.7** | **only on a cache miss** |
|
||||
| upload host→guest (179 MiB/s) | 3.60 | restore |
|
||||
| extract | 4.52 | restore |
|
||||
| **restore total** | **8.1** | **every warm job** |
|
||||
|
||||
## Verdict: build it
|
||||
|
||||
A warm job pays **8.1s** to skip roughly **120s** of dependency compilation —
|
||||
the ~490 external crates that a committed `Cargo.lock` keeps bit-identical
|
||||
between runs. Net saving ≈ 112s on a 134.7s job, about **83%**.
|
||||
|
||||
The publish path costs 15.7s but runs only on a miss, i.e. once per
|
||||
`Cargo.lock` change, so it is amortised across every run in between.
|
||||
|
||||
Transfer is not the bottleneck anyone feared: the virtual NIC moves the entry
|
||||
at 140–180 MiB/s, so 645 MiB crosses in under 4 seconds each way.
|
||||
|
||||
## End-to-end, through `ci job` (2026-07-22)
|
||||
|
||||
Full pipeline against `LinuxRust2404`, furyrs `b8ea567`, `--skip-artifact`.
|
||||
"Build" is the build phase alone; "total" includes clone, boot (~35s) and
|
||||
teardown.
|
||||
|
||||
| # | Scenario | Build | Total | Cache line in the log |
|
||||
|---|---|---:|---:|---|
|
||||
| 1 | Cold, empty store | 150s | 186s | `miss` → `prune: ok` → `published … (645 MB)` |
|
||||
| 2 | Warm, same commit | **47s** | **84s** | `hit … — restoring` |
|
||||
| 3 | Warm, source modified | **38s** | **75s** | `hit … — restoring` |
|
||||
| 4 | Warm, deliberate compile error | — | — | `hit`, then **build fails, exit 1** |
|
||||
| 5 | `--no-cargo-cache` | 133s | 170s | none — cache never touched |
|
||||
|
||||
**Saving: 103s on the build phase (150 → 47), 102s on the total (186 → 84).**
|
||||
Row 3 is the real CI scenario and is the fastest of the warm runs, because the
|
||||
touched crate is small.
|
||||
|
||||
Row 2's log names every workspace crate being rebuilt —
|
||||
|
||||
```
|
||||
Compiling fury-core, fury-gui, fury-ipc, fury-backend, furyd, fury-cli
|
||||
```
|
||||
|
||||
— and no dependency. That is the design working: dependencies restored,
|
||||
workspace recompiled from source, every time.
|
||||
|
||||
### Row 4 is the one that decides whether this ships
|
||||
|
||||
A deliberate type error was pushed to a scratch branch and built warm:
|
||||
|
||||
```
|
||||
[cache] hit for v1-1311… (645 MB) — restoring
|
||||
error[E0308]: mismatched types
|
||||
| --- ^^^^^^^^^^^^^^^^^^^ expected `u32`, found `&str`
|
||||
error: could not compile `fury-core` (lib) due to 1 previous error
|
||||
JOB_EXIT=1
|
||||
```
|
||||
|
||||
The cache restored 645 MB of dependencies and the regression still went red. A
|
||||
cache that can turn a red build green is the worst failure a CI system has,
|
||||
because it is silent; this is the test that rules it out, and it is worth
|
||||
re-running whenever the prune rules change.
|
||||
|
||||
The failed job also published nothing — the store still held exactly one entry
|
||||
and `.incoming/` was empty — because publishing sits after the build-failure
|
||||
check. A broken tree cannot become an entry.
|
||||
|
||||
## Notes that changed the implementation
|
||||
|
||||
**`zstd` is already in the template.** `/usr/bin/zstd` is present on
|
||||
`LinuxRust2404`, so the gzip fallback is dead code in practice and no template
|
||||
rebuild is needed. Keep the fallback anyway — it costs one branch and keeps the
|
||||
feature working on a plain `LinuxBuild2404`.
|
||||
|
||||
**The name-based assertion was worthless and was replaced.** The plan's
|
||||
original check re-scanned for `*/.fingerprint/<member>-*` after deleting
|
||||
exactly those paths. Asked to catch a planted `fury-core-deadbeefcafe`
|
||||
fingerprint, it exited **0** — the prune had deleted the evidence before the
|
||||
assertion looked. An assertion built from the same globs as the deletion can
|
||||
only prove that `rm` ran; it is blind to the failure that matters, a glob that
|
||||
never matched the artifact at all.
|
||||
|
||||
The replacement asks a different oracle. Cargo records in each `.d` depfile the
|
||||
source files an artifact was built *from*: a dependency points into
|
||||
`/opt/rust/cargo/registry/...`, anything built from workspace source points
|
||||
back into the workspace. After a correct prune, **zero** surviving depfiles
|
||||
reference workspace source outside `target/`. Verified negatively as well — an
|
||||
artifact named `totallyunrelated-abc123.d` (which no prune glob matches)
|
||||
declaring a dependency on `fury-core/src/lib.rs` makes the script exit **2**
|
||||
and refuse to publish.
|
||||
|
||||
| Check | Clean tree | Planted fingerprint | Planted foreign artifact |
|
||||
|---|---|---|---|
|
||||
| name-based (original) | 0 | **0 — missed it** | 0 — missed it |
|
||||
| depfile-based (shipped) | 0 | 0 (prune removed it, correctly) | **2 — refused** |
|
||||
|
||||
**Feature variants coexist in one entry, as predicted.** `deps/` holds
|
||||
`libclap-943062c20e623534.rmeta` and `libclap-abe395d6778d44ce.rmeta` side by
|
||||
side — the same crate compiled under two different feature resolutions. This is
|
||||
why the feature set is not part of the cache key: cargo already separates the
|
||||
variants by hash, so one entry serves all of furyrs's four feature
|
||||
configurations instead of fragmenting into four.
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# Strip every workspace-crate artifact out of target/, then prove none survived.
|
||||
#
|
||||
# The cache is only sound while it contains dependency artifacts alone. A
|
||||
# workspace crate's artifact carries the code under test, and reusing one can
|
||||
# turn a red build green -- the worst failure a CI system has, because it is
|
||||
# silent. So this script ends by asserting its own work and exits non-zero if
|
||||
# the assertion fails; the caller must treat that as "do not publish".
|
||||
#
|
||||
# Exit codes: 0 clean prune, 1 usage/environment error, 2 assertion tripped.
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_DIR="${1:?usage: cargo-cache-prune.sh <target-dir> <manifest-path>}"
|
||||
MANIFEST="${2:?usage: cargo-cache-prune.sh <target-dir> <manifest-path>}"
|
||||
|
||||
[ -d "$TARGET_DIR" ] || { echo "prune: no such target dir: $TARGET_DIR" >&2; exit 1; }
|
||||
command -v cargo >/dev/null || { echo "prune: cargo not on PATH" >&2; exit 1; }
|
||||
|
||||
# --no-deps restricts the listing to workspace members, which is precisely the
|
||||
# set whose artifacts must never be cached.
|
||||
members="$(cargo metadata --no-deps --format-version 1 --manifest-path "$MANIFEST" \
|
||||
| python3 -c 'import json,sys; print("\n".join(p["name"] for p in json.load(sys.stdin)["packages"]))')"
|
||||
|
||||
if [ -z "$members" ]; then
|
||||
echo "prune: cargo metadata listed no workspace members" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Incremental state is large, machine-specific and never reusable across
|
||||
# clones; examples and docs are not dependencies of anything we build.
|
||||
rm -rf "$TARGET_DIR"/*/incremental "$TARGET_DIR"/*/examples "$TARGET_DIR"/doc 2>/dev/null || true
|
||||
|
||||
prune_one() {
|
||||
name="$1"
|
||||
snake="${name//-/_}"
|
||||
# Fingerprints and build-script output live in per-crate directories.
|
||||
find "$TARGET_DIR" -maxdepth 3 \
|
||||
\( -path "*/.fingerprint/$name-*" \
|
||||
-o -path "*/.fingerprint/$snake-*" \
|
||||
-o -path "*/build/$name-*" \
|
||||
-o -path "*/build/$snake-*" \) \
|
||||
-prune -exec rm -rf {} + 2>/dev/null || true
|
||||
# Compiled outputs: rlib/rmeta/d in deps/, plus binaries at the profile root.
|
||||
find "$TARGET_DIR" -maxdepth 3 \
|
||||
\( -name "$snake-*" -o -name "lib$snake-*" -o -name "lib$snake.*" \
|
||||
-o -name "$name" -o -name "$name-*" \) \
|
||||
-exec rm -rf {} + 2>/dev/null || true
|
||||
}
|
||||
|
||||
while IFS= read -r name; do
|
||||
[ -n "$name" ] || continue
|
||||
prune_one "$name"
|
||||
done <<EOF
|
||||
$members
|
||||
EOF
|
||||
|
||||
# ── The assertion ────────────────────────────────────────────────────────────
|
||||
#
|
||||
# This deliberately does NOT re-use the name patterns above. An assertion built
|
||||
# from the same globs as the deletion can only prove that `rm` ran; it is blind
|
||||
# to the failure that actually matters, which is a glob that never matched the
|
||||
# artifact in the first place. Asked to catch a planted fingerprint, such a
|
||||
# check "passes" because the prune deleted the evidence before looking.
|
||||
#
|
||||
# So ask a different oracle: cargo records in each `.d` depfile the source
|
||||
# files an artifact was built FROM. A dependency's depfile points into the
|
||||
# registry; anything built from workspace source points back into the
|
||||
# workspace. That is the property we actually care about, and it holds however
|
||||
# the artifact happens to be named.
|
||||
ws_root="$(cd "$(dirname "$MANIFEST")" && pwd)"
|
||||
abs_target="$(cd "$TARGET_DIR" && pwd)"
|
||||
|
||||
# Every depfile names its own output under target/, so that prefix is excluded;
|
||||
# what remains is genuine source input.
|
||||
leaked_paths="$(find "$abs_target" -name '*.d' -print0 2>/dev/null \
|
||||
| xargs -0 -r grep -h -oE "${ws_root}/[^ :]*" 2>/dev/null \
|
||||
| grep -v "^${abs_target}/" \
|
||||
| sort -u || true)"
|
||||
|
||||
if [ -n "$leaked_paths" ]; then
|
||||
echo "prune: REFUSING to publish -- artifacts still reference workspace source:" >&2
|
||||
printf '%s\n' "$leaked_paths" | sed 's/^/ /' | head -20 >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "prune: ok -- $(printf '%s\n' "$members" | wc -l) workspace member(s) stripped," \
|
||||
"no surviving artifact references workspace source"
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Content-addressed store for compiled Rust dependency artifacts.
|
||||
|
||||
A cache entry is one immutable file per key. Publishing is a ``rename(2)``
|
||||
within the store directory, so a reader either sees the previous entry or the
|
||||
new one and never a half-written file — no advisory locking, which is exactly
|
||||
what a VMware shared folder cannot give us with several jobs in flight.
|
||||
|
||||
Nothing here knows about SSH or guests: it takes a finished tarball and files
|
||||
it, or hands back the path of an existing one. That keeps the interesting
|
||||
logic testable without a VM.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Bump to invalidate every entry at once — e.g. when the prune script changes
|
||||
# what it strips, making older entries untrustworthy.
|
||||
CACHE_FORMAT = 1
|
||||
|
||||
_SAFE_KEY = re.compile(r"^v\d+-[0-9a-f]{32,64}$")
|
||||
_INCOMING = ".incoming"
|
||||
|
||||
|
||||
def cache_key(*, lock_digest: str, rustc_version: str, host_triple: str) -> str:
|
||||
"""Derive the entry key from the inputs that make artifacts incompatible.
|
||||
|
||||
``lock_digest`` is the sha256 of ``Cargo.lock``: it changes exactly when
|
||||
the dependency set changes, which is the property that makes this cache
|
||||
sound. The feature set is deliberately *not* part of the key — cargo
|
||||
encodes it in each artifact's own hash suffix, so several feature graphs
|
||||
coexist inside one entry instead of fragmenting it.
|
||||
|
||||
The parts are joined with NUL, which cannot occur in any of them, so
|
||||
``("ab", "c")`` and ``("a", "bc")`` cannot collide.
|
||||
"""
|
||||
digest = hashlib.sha256(
|
||||
"\0".join((lock_digest, rustc_version, host_triple)).encode("utf-8")
|
||||
).hexdigest()
|
||||
return f"v{CACHE_FORMAT}-{digest}"
|
||||
|
||||
|
||||
class CargoCacheStore:
|
||||
"""A directory of immutable cache entries, one file per key."""
|
||||
|
||||
def __init__(self, root: Path, *, suffix: str = ".tar.zst") -> None:
|
||||
self.root = Path(root)
|
||||
self.suffix = suffix
|
||||
|
||||
# ── paths ────────────────────────────────────────────────────────────
|
||||
|
||||
def _checked(self, key: str) -> str:
|
||||
if not _SAFE_KEY.match(key):
|
||||
raise ValueError(f"unsafe cache key: {key!r}")
|
||||
return key
|
||||
|
||||
def path_for(self, key: str) -> Path:
|
||||
"""Absolute path of the entry for ``key``. Raises on a malformed key."""
|
||||
return self.root / f"{self._checked(key)}{self.suffix}"
|
||||
|
||||
# ── read ─────────────────────────────────────────────────────────────
|
||||
|
||||
def lookup(self, key: str) -> Path | None:
|
||||
"""Return the entry path for ``key``, or ``None`` on a miss."""
|
||||
candidate = self.path_for(key)
|
||||
return candidate if candidate.is_file() else None
|
||||
|
||||
# ── write ────────────────────────────────────────────────────────────
|
||||
|
||||
def publish(self, key: str, source: Path) -> Path:
|
||||
"""Move ``source`` into the store under ``key``. Returns the entry path.
|
||||
|
||||
The move goes through a staging directory on the same filesystem so the
|
||||
final step is a rename: concurrent publishers of the same key both
|
||||
produce a valid entry and the last one wins, with no window in which a
|
||||
reader can observe a partial file.
|
||||
"""
|
||||
source = Path(source)
|
||||
if not source.is_file():
|
||||
raise FileNotFoundError(f"nothing to publish: {source}")
|
||||
target = self.path_for(key)
|
||||
staging = self.root / _INCOMING
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
tmp = staging / f"{key}.{os.getpid()}{self.suffix}"
|
||||
try:
|
||||
os.replace(source, tmp)
|
||||
except OSError:
|
||||
# Different filesystem — copy into staging instead, which still
|
||||
# leaves the publish itself an atomic rename.
|
||||
shutil.copy2(source, tmp)
|
||||
source.unlink(missing_ok=True)
|
||||
os.replace(tmp, target)
|
||||
return target
|
||||
|
||||
# ── maintenance ──────────────────────────────────────────────────────
|
||||
|
||||
def prune_older_than(
|
||||
self, cutoff: datetime, *, what_if: bool = False
|
||||
) -> list[Path]:
|
||||
"""Delete entries last modified before ``cutoff``. Returns what went."""
|
||||
if not self.root.is_dir():
|
||||
return []
|
||||
removed: list[Path] = []
|
||||
for entry in sorted(self.root.glob(f"*{self.suffix}")):
|
||||
if not entry.is_file():
|
||||
continue
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=cutoff.tzinfo)
|
||||
if mtime >= cutoff:
|
||||
continue
|
||||
removed.append(entry)
|
||||
if not what_if:
|
||||
entry.unlink(missing_ok=True)
|
||||
return removed
|
||||
|
||||
|
||||
__all__ = ["CACHE_FORMAT", "CargoCacheStore", "cache_key"]
|
||||
@@ -25,11 +25,13 @@ import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||
@@ -122,6 +124,110 @@ def _tail_lines(stdout: str, stderr: str) -> str:
|
||||
return " | ".join(lines[-5:]) if lines else "(no build output)"
|
||||
|
||||
|
||||
# ------------------------------------------------- cargo dependency cache
|
||||
|
||||
_PRUNE_ASSET = "cargo-cache-prune.sh"
|
||||
_GUEST_PRUNE = "/tmp/cargo-cache-prune.sh"
|
||||
_GUEST_TARBALL = "/tmp/cargo-cache-entry.tar"
|
||||
|
||||
|
||||
def _cargo_cache_probe(t: SshTransport, workdir: str) -> tuple[str, str] | None:
|
||||
"""Return ``(key, compressor)`` for this workspace, or ``None`` to skip.
|
||||
|
||||
``None`` means "not a Rust build, or we could not read what we need". The
|
||||
cache is an optimisation, so every uncertainty here resolves to skipping
|
||||
it rather than to guessing.
|
||||
"""
|
||||
if t.run(f"test -f {_sh_quote(workdir + '/Cargo.lock')}", check=False).returncode != 0:
|
||||
return None
|
||||
lock = t.run(f"sha256sum {_sh_quote(workdir + '/Cargo.lock')}", check=False)
|
||||
version = t.run("rustc -V", check=False)
|
||||
triple = t.run("rustc -vV | sed -n 's/^host: //p'", check=False)
|
||||
if not (lock.ok and version.ok and triple.ok):
|
||||
click.echo("[cache] could not read Cargo.lock digest or rustc version; skipping")
|
||||
return None
|
||||
digest = lock.stdout.split()[0] if lock.stdout.split() else ""
|
||||
if not digest:
|
||||
click.echo("[cache] empty Cargo.lock digest; skipping")
|
||||
return None
|
||||
compressor = "zstd" if t.run("command -v zstd", check=False).ok else "gzip"
|
||||
key = cache_key(
|
||||
lock_digest=digest,
|
||||
rustc_version=version.stdout.strip(),
|
||||
host_triple=triple.stdout.strip(),
|
||||
)
|
||||
return key, compressor
|
||||
|
||||
|
||||
def _tar_flag(compressor: str) -> tuple[str, str]:
|
||||
"""``(tar flag, file extension)`` for the chosen compressor."""
|
||||
return ("--zstd", "zst") if compressor == "zstd" else ("-z", "gz")
|
||||
|
||||
|
||||
def _cargo_cache_restore(
|
||||
t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str
|
||||
) -> bool:
|
||||
"""Upload and unpack an existing entry. Returns True on a hit."""
|
||||
entry = store.lookup(key)
|
||||
if entry is None:
|
||||
click.echo(f"[cache] miss for {key} — building cold")
|
||||
return False
|
||||
flag, ext = _tar_flag(compressor)
|
||||
size_mb = entry.stat().st_size / 1_048_576
|
||||
click.echo(f"[cache] hit for {key} ({size_mb:.0f} MB) — restoring")
|
||||
guest_entry = f"{_GUEST_TARBALL}.{ext}"
|
||||
t.copy(str(entry), guest_entry)
|
||||
t.run(
|
||||
f"tar {flag} -xf {_sh_quote(guest_entry)} -C {_sh_quote(workdir)} && "
|
||||
f"rm -f {_sh_quote(guest_entry)}"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cargo_cache_publish(
|
||||
t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str
|
||||
) -> None:
|
||||
"""Prune workspace artifacts, then file the remainder under ``key``.
|
||||
|
||||
A non-zero prune means the fail-closed assertion tripped: say so loudly and
|
||||
publish nothing. A poisoned entry costs a wrong verdict; a missing one only
|
||||
costs a cold build.
|
||||
"""
|
||||
# as_file() because the production venv is installed non-editable: the
|
||||
# script is package data, not a path relative to a source checkout.
|
||||
asset = resources.files("ci_orchestrator.assets").joinpath(_PRUNE_ASSET)
|
||||
with resources.as_file(asset) as prune_script:
|
||||
t.copy(str(prune_script), _GUEST_PRUNE)
|
||||
pruned = t.run(
|
||||
f"bash {_GUEST_PRUNE} {_sh_quote(workdir + '/target')} "
|
||||
f"{_sh_quote(workdir + '/Cargo.toml')}",
|
||||
check=False,
|
||||
)
|
||||
click.echo(f"[cache] {(pruned.stdout.strip() or pruned.stderr.strip())}")
|
||||
if pruned.returncode != 0:
|
||||
click.echo("[cache] prune refused — publishing nothing", err=True)
|
||||
return
|
||||
|
||||
flag, ext = _tar_flag(compressor)
|
||||
guest_tar = f"{_GUEST_TARBALL}.{ext}"
|
||||
packed = t.run(
|
||||
f"tar {flag} -cf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} target",
|
||||
check=False,
|
||||
)
|
||||
if not packed.ok:
|
||||
click.echo("[cache] packing failed — publishing nothing", err=True)
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
local = Path(td) / f"entry.tar.{ext}"
|
||||
t.fetch(guest_tar, local)
|
||||
published = store.publish(key, local)
|
||||
t.run(f"rm -f {_sh_quote(guest_tar)}", check=False)
|
||||
click.echo(
|
||||
f"[cache] published {key} ({published.stat().st_size / 1_048_576:.0f} MB)"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Linux flow
|
||||
|
||||
|
||||
@@ -143,6 +249,7 @@ def _linux_build(
|
||||
extra_env: dict[str, str],
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool = False,
|
||||
cargo_cache_root: Path | None = None,
|
||||
) -> None:
|
||||
click.echo("[build run] Linux build mode (SSH transport)")
|
||||
|
||||
@@ -217,6 +324,26 @@ def _linux_build(
|
||||
check=False,
|
||||
)
|
||||
|
||||
# Cargo dependency cache. Everything here is best-effort: a cache
|
||||
# problem must never turn a good build red, so any failure degrades to
|
||||
# a cold build with a message.
|
||||
cache_key_value: str | None = None
|
||||
compressor = "gzip"
|
||||
store: CargoCacheStore | None = None
|
||||
if cargo_cache_root is not None:
|
||||
try:
|
||||
store = CargoCacheStore(cargo_cache_root)
|
||||
probed = _cargo_cache_probe(t, workdir)
|
||||
if probed is None:
|
||||
store = None
|
||||
else:
|
||||
cache_key_value, compressor = probed
|
||||
_cargo_cache_restore(t, store, cache_key_value, workdir, compressor)
|
||||
except (TransportError, OSError, ValueError) as exc:
|
||||
click.echo(f"[cache] restore skipped: {exc}", err=True)
|
||||
cache_key_value = None
|
||||
store = None
|
||||
|
||||
env_prefix = "".join(
|
||||
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||
)
|
||||
@@ -245,6 +372,14 @@ def _linux_build(
|
||||
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
|
||||
)
|
||||
|
||||
# Publish only on a miss, so the steady state uploads nothing. A failed
|
||||
# build never reaches here, so a broken tree cannot become an entry.
|
||||
if store is not None and cache_key_value and store.lookup(cache_key_value) is None:
|
||||
try:
|
||||
_cargo_cache_publish(t, store, cache_key_value, workdir, compressor)
|
||||
except (TransportError, OSError, ValueError) as exc:
|
||||
click.echo(f"[cache] publish skipped: {exc}", err=True)
|
||||
|
||||
if skip_artifact:
|
||||
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||
return
|
||||
|
||||
@@ -326,6 +326,14 @@ def _destroy_clone(
|
||||
help="In-guest git clone (default) vs host-side clone + zip transfer.",
|
||||
)
|
||||
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--cargo-cache/--no-cargo-cache",
|
||||
"cargo_cache",
|
||||
default=True,
|
||||
show_default=True,
|
||||
help="Reuse compiled Rust dependency artifacts between jobs (Linux only; "
|
||||
"engages only when the workspace has a Cargo.lock).",
|
||||
)
|
||||
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--xvfb",
|
||||
@@ -409,6 +417,7 @@ def job(
|
||||
submodules: bool,
|
||||
use_git_clone: bool,
|
||||
use_shared_cache: bool,
|
||||
cargo_cache: bool,
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool,
|
||||
gitea_credential_target: str,
|
||||
@@ -634,6 +643,11 @@ def job(
|
||||
extra_env=extra_env,
|
||||
skip_artifact=skip_artifact,
|
||||
use_xvfb=use_xvfb,
|
||||
cargo_cache_root=(
|
||||
config.cargo_cache.root
|
||||
if cargo_cache and config.cargo_cache.enabled
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
_windows_build(
|
||||
|
||||
@@ -14,7 +14,8 @@ from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
from ci_orchestrator.config import Config, load_config
|
||||
|
||||
|
||||
@click.group()
|
||||
@@ -93,6 +94,7 @@ def retention_run(
|
||||
|
||||
_purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if)
|
||||
_purge_old_dirs(log_path, "log", cutoff, what_if=what_if)
|
||||
_purge_cargo_cache(config, effective_days, what_if=what_if)
|
||||
|
||||
usage_after = shutil.disk_usage(str(check_path))
|
||||
free_gb_after = usage_after.free / (1024**3)
|
||||
@@ -132,3 +134,44 @@ def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool)
|
||||
else:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}")
|
||||
|
||||
|
||||
def _purge_cargo_cache(config: Config, effective_days: int, *, what_if: bool) -> None:
|
||||
"""Purge stale cargo cache entries.
|
||||
|
||||
Entries are the heaviest thing this policy governs -- hundreds of megabytes
|
||||
each, one per Cargo.lock change -- so leaving them out would repeat TODO
|
||||
2.9, where retention warned about low disk while freeing 0.0 GB because
|
||||
everything that actually weighed was outside its reach.
|
||||
|
||||
The window is the tighter of the global retention and the cache's own, so
|
||||
the aggressive low-disk mode reaches the cache too.
|
||||
"""
|
||||
if not config.cargo_cache.enabled:
|
||||
click.echo("[RetentionPolicy] cargo cache: disabled — skipping.")
|
||||
return
|
||||
|
||||
days = min(effective_days, config.cargo_cache.retention_days)
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(days=days)
|
||||
store = CargoCacheStore(config.cargo_cache.root)
|
||||
|
||||
# Size the victims first: after a real prune the files are gone and their
|
||||
# bytes are unreadable, which is how a policy ends up reporting 0.0 GB
|
||||
# freed while doing real work (TODO 2.9).
|
||||
doomed = store.prune_older_than(cutoff, what_if=True)
|
||||
if not doomed:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] cargo cache: nothing to purge "
|
||||
f"(cutoff: {cutoff.strftime('%Y-%m-%d')})."
|
||||
)
|
||||
return
|
||||
freed_mb = sum(p.stat().st_size for p in doomed if p.exists()) / 1_048_576
|
||||
|
||||
if not what_if:
|
||||
store.prune_older_than(cutoff)
|
||||
|
||||
verb = "WhatIf: would purge" if what_if else "Purged"
|
||||
click.echo(
|
||||
f"[RetentionPolicy] {verb} {len(doomed)} cargo cache "
|
||||
f"{'entry' if len(doomed) == 1 else 'entries'} ({freed_mb:.0f} MB)"
|
||||
)
|
||||
|
||||
@@ -65,6 +65,19 @@ class BackendConfig:
|
||||
options: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CargoCacheConfig:
|
||||
"""Where compiled Rust dependency artifacts are filed between jobs.
|
||||
|
||||
``root`` defaults to ``<paths.root>/cache/cargo``; it is resolved in
|
||||
:func:`load_config` because it depends on the resolved paths.
|
||||
"""
|
||||
|
||||
enabled: bool = True
|
||||
root: Path = Path("/var/lib/ci/cache/cargo")
|
||||
retention_days: int = 30
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
"""Top-level configuration object."""
|
||||
@@ -75,6 +88,7 @@ class Config:
|
||||
ssh_key_path: Path | None = None
|
||||
guest_cred_target: str = "BuildVMGuest"
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
cargo_cache: CargoCacheConfig = field(default_factory=CargoCacheConfig)
|
||||
|
||||
|
||||
def _platform_defaults() -> Paths:
|
||||
@@ -158,6 +172,7 @@ def load_config(
|
||||
ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None
|
||||
guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest")
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
cargo_cache_section: dict[str, Any] = {}
|
||||
|
||||
if toml_path and toml_path.is_file():
|
||||
data = _load_toml(toml_path)
|
||||
@@ -187,9 +202,18 @@ def load_config(
|
||||
gateway=str(pool_section.get("gateway", "")),
|
||||
lease_file=Path(pool_section.get("lease_file") or _lease_default),
|
||||
)
|
||||
cargo_cache_section = data.get("cargo_cache", {}) or {}
|
||||
|
||||
paths = _apply_env(paths, env)
|
||||
|
||||
# Resolved after _apply_env so CI_ROOT relocates the store with everything
|
||||
# else; an explicit root in the file still wins.
|
||||
cargo_cache = CargoCacheConfig(
|
||||
enabled=bool(cargo_cache_section.get("enabled", True)),
|
||||
root=Path(cargo_cache_section.get("root") or paths.root / "cache" / "cargo"),
|
||||
retention_days=int(cargo_cache_section.get("retention_days", 30)),
|
||||
)
|
||||
|
||||
return Config(
|
||||
paths=paths,
|
||||
backend=backend,
|
||||
@@ -197,7 +221,15 @@ def load_config(
|
||||
ssh_key_path=ssh_key_path,
|
||||
guest_cred_target=guest_cred_target,
|
||||
ip_pool=ip_pool,
|
||||
cargo_cache=cargo_cache,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"]
|
||||
__all__ = [
|
||||
"BackendConfig",
|
||||
"CargoCacheConfig",
|
||||
"Config",
|
||||
"IpPoolConfig",
|
||||
"Paths",
|
||||
"load_config",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
"""Tests for ci_orchestrator.cargo_cache — key derivation and the entry store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from ci_orchestrator.cargo_cache import CACHE_FORMAT, CargoCacheStore, cache_key
|
||||
|
||||
# ── key derivation ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_key_is_stable_for_identical_inputs() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_key_changes_with_lock_digest() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="def", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_changes_with_toolchain() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.98.0", host_triple="x86_64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_changes_with_host_triple() -> None:
|
||||
a = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="x86_64")
|
||||
b = cache_key(lock_digest="abc", rustc_version="rustc 1.97.1", host_triple="aarch64")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_key_is_filesystem_safe() -> None:
|
||||
key = cache_key(
|
||||
lock_digest="a/b+c=", rustc_version="rustc 1.97.1 (x /y)", host_triple="x86_64"
|
||||
)
|
||||
assert all(c.isalnum() or c == "-" for c in key)
|
||||
assert "/" not in key
|
||||
|
||||
|
||||
def test_key_carries_the_format_version() -> None:
|
||||
key = cache_key(lock_digest="abc", rustc_version="r", host_triple="t")
|
||||
assert key.startswith(f"v{CACHE_FORMAT}-")
|
||||
|
||||
|
||||
def test_key_fields_cannot_be_confused_by_concatenation() -> None:
|
||||
"""``ab|c`` and ``a|bc`` must not collide — the separator has to be real."""
|
||||
a = cache_key(lock_digest="ab", rustc_version="c", host_triple="t")
|
||||
b = cache_key(lock_digest="a", rustc_version="bc", host_triple="t")
|
||||
assert a != b
|
||||
|
||||
|
||||
# ── store ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_lookup_returns_none_when_absent(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
assert store.lookup(cache_key(lock_digest="a", rustc_version="b", host_triple="c")) is None
|
||||
|
||||
|
||||
def test_publish_then_lookup_finds_the_entry(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
published = store.publish(key, src)
|
||||
assert store.lookup(key) == published
|
||||
assert published.read_bytes() == b"payload"
|
||||
|
||||
|
||||
def test_publish_consumes_the_source_file(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
store.publish(key, src)
|
||||
assert not src.exists(), "publish must move, not copy — a copy doubles peak disk"
|
||||
|
||||
|
||||
def test_publish_overwrites_an_existing_key(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
first = tmp_path / "a.tar.zst"
|
||||
first.write_bytes(b"first")
|
||||
store.publish(key, first)
|
||||
second = tmp_path / "b.tar.zst"
|
||||
second.write_bytes(b"second")
|
||||
published = store.publish(key, second)
|
||||
assert published.read_bytes() == b"second"
|
||||
|
||||
|
||||
def test_publish_leaves_no_temp_files_behind(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
store.publish(key, src)
|
||||
staging = tmp_path / ".incoming"
|
||||
leftovers = list(staging.glob("*")) if staging.is_dir() else []
|
||||
assert leftovers == []
|
||||
|
||||
|
||||
def test_publish_creates_the_store_directory(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "not" / "yet")
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
src = tmp_path / "incoming.tar.zst"
|
||||
src.write_bytes(b"payload")
|
||||
published = store.publish(key, src)
|
||||
assert published.is_file()
|
||||
|
||||
|
||||
def test_publish_missing_source_raises(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
store.publish(key, tmp_path / "nope.tar.zst")
|
||||
|
||||
|
||||
def test_lookup_rejects_a_key_with_a_path_separator(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="unsafe cache key"):
|
||||
store.lookup("../../etc/passwd")
|
||||
|
||||
|
||||
def test_path_for_rejects_an_arbitrary_string(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
with pytest.raises(ValueError, match="unsafe cache key"):
|
||||
store.path_for("not-a-real-key")
|
||||
|
||||
|
||||
def test_lookup_ignores_a_directory_at_the_entry_path(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path)
|
||||
key = cache_key(lock_digest="a", rustc_version="b", host_triple="c")
|
||||
store.path_for(key).mkdir(parents=True)
|
||||
assert store.lookup(key) is None
|
||||
|
||||
|
||||
# ── prune ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _age(path: Path, days: int) -> None:
|
||||
old = (datetime.now(tz=UTC) - timedelta(days=days)).timestamp()
|
||||
os.utime(path, (old, old))
|
||||
|
||||
|
||||
def _seed(store: CargoCacheStore, tmp_path: Path, digest: str) -> Path:
|
||||
key = cache_key(lock_digest=digest, rustc_version="b", host_triple="c")
|
||||
src = tmp_path / f"seed-{digest}.tar.zst"
|
||||
src.write_bytes(b"x")
|
||||
return store.publish(key, src)
|
||||
|
||||
|
||||
def test_prune_removes_entries_older_than_the_cutoff(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == [entry]
|
||||
assert not entry.exists()
|
||||
|
||||
|
||||
def test_prune_keeps_fresh_entries(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "new")
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == []
|
||||
assert entry.exists()
|
||||
|
||||
|
||||
def test_prune_what_if_reports_without_deleting(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
|
||||
removed = store.prune_older_than(
|
||||
datetime.now(tz=UTC) - timedelta(days=30), what_if=True
|
||||
)
|
||||
assert removed == [entry]
|
||||
assert entry.exists()
|
||||
|
||||
|
||||
def test_prune_on_a_missing_store_is_a_noop(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "never-created")
|
||||
assert store.prune_older_than(datetime.now(tz=UTC)) == []
|
||||
|
||||
|
||||
def test_prune_leaves_unrelated_files_alone(tmp_path: Path) -> None:
|
||||
store = CargoCacheStore(tmp_path / "store")
|
||||
entry = _seed(store, tmp_path, "old")
|
||||
_age(entry, days=40)
|
||||
stray = (tmp_path / "store") / "README.txt"
|
||||
stray.write_text("not an entry", encoding="utf-8")
|
||||
_age(stray, days=40)
|
||||
|
||||
removed = store.prune_older_than(datetime.now(tz=UTC) - timedelta(days=30))
|
||||
assert removed == [entry]
|
||||
assert stray.exists(), "prune must only touch files with the entry suffix"
|
||||
@@ -38,9 +38,20 @@ class _FakeTransport:
|
||||
self.copies: list[tuple[str, str]] = []
|
||||
self.fetches: list[tuple[str, str]] = []
|
||||
self.run_results: dict[int, _FakeResult] = {}
|
||||
# Matched by substring against the script, checked before the
|
||||
# positional run_results. Index-based fixtures get brittle once a
|
||||
# feature adds calls in the middle of the flow (the cargo cache does).
|
||||
self.script_results: list[tuple[str, _FakeResult]] = []
|
||||
self.default_result = _FakeResult(stdout="ok")
|
||||
type(self).instances.append(self)
|
||||
|
||||
def result_for(
|
||||
self, needle: str, *, stdout: str = "", stderr: str = "", returncode: int = 0
|
||||
) -> None:
|
||||
self.script_results.append(
|
||||
(needle, _FakeResult(stdout=stdout, stderr=stderr, returncode=returncode))
|
||||
)
|
||||
|
||||
def __enter__(self) -> _FakeTransport:
|
||||
return self
|
||||
|
||||
@@ -49,7 +60,13 @@ class _FakeTransport:
|
||||
|
||||
def run(self, script: str, *, check: bool = True) -> _FakeResult:
|
||||
self.runs.append((script, check))
|
||||
result = self.run_results.get(len(self.runs) - 1, self.default_result)
|
||||
result: _FakeResult | None = None
|
||||
for needle, canned in self.script_results:
|
||||
if needle in script:
|
||||
result = canned
|
||||
break
|
||||
if result is None:
|
||||
result = self.run_results.get(len(self.runs) - 1, self.default_result)
|
||||
if check and not result.ok:
|
||||
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||
return result
|
||||
@@ -838,3 +855,205 @@ def test_build_run_windows_transport_error(
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "winrm unreachable" in result.output
|
||||
|
||||
|
||||
# ── _linux_build: cargo dependency cache ───────────────────────────────────
|
||||
|
||||
|
||||
def _linux_kwargs(**overrides: Any) -> dict[str, Any]:
|
||||
base: dict[str, Any] = {
|
||||
"ip_address": "10.0.0.1",
|
||||
"ssh_user": "ci_build",
|
||||
"key_path": None,
|
||||
"known_hosts": None,
|
||||
"workdir": "/opt/ci/build",
|
||||
"output_dir": "/opt/ci/output",
|
||||
"host_source_dir": None,
|
||||
"clone_url": "http://repo/example.git",
|
||||
"clone_branch": "main",
|
||||
"clone_commit": "",
|
||||
"clone_submodules": False,
|
||||
"build_command": "true",
|
||||
"artifact_source": "dist",
|
||||
"extra_env": {},
|
||||
"skip_artifact": True,
|
||||
}
|
||||
base.update(overrides)
|
||||
return base
|
||||
|
||||
|
||||
def _rust_guest(fake: _FakeTransport, *, digest: str = "cafe") -> None:
|
||||
"""Make the fake answer like a Rust workspace with a known toolchain."""
|
||||
fake.result_for("sha256sum", stdout=f"{digest} Cargo.lock\n")
|
||||
fake.result_for("rustc -V", stdout="rustc 1.97.1 (abc 2026-01-01)\n")
|
||||
fake.result_for("rustc -vV", stdout="x86_64-unknown-linux-gnu\n")
|
||||
|
||||
|
||||
def _expected_key(digest: str = "cafe") -> str:
|
||||
from ci_orchestrator.cargo_cache import cache_key
|
||||
|
||||
return cache_key(
|
||||
lock_digest=digest,
|
||||
rustc_version="rustc 1.97.1 (abc 2026-01-01)",
|
||||
host_triple="x86_64-unknown-linux-gnu",
|
||||
)
|
||||
|
||||
|
||||
def _fetch_writes_a_real_file(
|
||||
monkeypatch: pytest.MonkeyPatch, fake: _FakeTransport
|
||||
) -> None:
|
||||
"""Make ``fetch`` produce an actual file on disk.
|
||||
|
||||
Without this the default fake leaves nothing behind, ``publish`` dies on
|
||||
FileNotFoundError, and a "nothing was published" assertion passes for the
|
||||
wrong reason -- it would hold even with the guard under test removed.
|
||||
Mutation-tested: dropping the prune-refusal check must make those tests go
|
||||
red, and it only does when a publish could otherwise have succeeded.
|
||||
"""
|
||||
|
||||
def _fetch(remote: str, local: str) -> None:
|
||||
fake.fetches.append((remote, str(local)))
|
||||
Path(local).write_bytes(b"fresh-entry")
|
||||
|
||||
monkeypatch.setattr(fake, "fetch", _fetch)
|
||||
|
||||
|
||||
def test_linux_build_without_cache_root_runs_no_cache_commands(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||
build_module._linux_build(**_linux_kwargs())
|
||||
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||
assert not any("cargo-cache-prune" in c for c in cmds)
|
||||
assert not any("sha256sum" in c for c in cmds)
|
||||
|
||||
|
||||
def test_linux_build_skips_cache_when_no_cargo_lock(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||
fake = _FakeTransport()
|
||||
_FakeTransport.instances = []
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
fake.result_for("test -f", returncode=1)
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
cmds = [c[0] for c in fake.runs]
|
||||
assert not any("cargo-cache-prune" in c for c in cmds)
|
||||
assert not any("sha256sum" in c for c in cmds)
|
||||
|
||||
|
||||
def test_linux_build_restores_an_existing_entry(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake)
|
||||
|
||||
store = CargoCacheStore(tmp_path)
|
||||
seed = tmp_path / "seed.tar.zst"
|
||||
seed.write_bytes(b"entry")
|
||||
store.publish(_expected_key(), seed)
|
||||
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
cmds = [c[0] for c in fake.runs]
|
||||
assert any("tar --zstd -xf" in c for c in cmds), "a hit must unpack the entry"
|
||||
assert fake.copies, "a hit must upload the entry to the guest"
|
||||
# A hit must not re-publish: the entry is already there.
|
||||
assert not any("cargo-cache-prune" in c for c in cmds)
|
||||
|
||||
|
||||
def test_linux_build_publishes_on_a_miss(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake)
|
||||
|
||||
_fetch_writes_a_real_file(monkeypatch, fake)
|
||||
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
assert CargoCacheStore(tmp_path).lookup(_expected_key()) is not None
|
||||
|
||||
|
||||
def test_linux_build_does_not_publish_when_the_prune_assertion_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake)
|
||||
_fetch_writes_a_real_file(monkeypatch, fake)
|
||||
fake.result_for(
|
||||
"cargo-cache-prune",
|
||||
returncode=2,
|
||||
stderr="prune: REFUSING to publish -- artifacts still reference workspace source",
|
||||
)
|
||||
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None, (
|
||||
"a failed prune assertion must never reach the store"
|
||||
)
|
||||
|
||||
|
||||
def test_linux_build_does_not_publish_when_packing_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake)
|
||||
_fetch_writes_a_real_file(monkeypatch, fake)
|
||||
fake.result_for("tar --zstd -cf", returncode=1, stderr="no space left on device")
|
||||
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
assert CargoCacheStore(tmp_path).lookup(_expected_key()) is None
|
||||
|
||||
|
||||
def test_cache_probe_failure_does_not_fail_the_build(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
fake.result_for("sha256sum", returncode=1, stderr="boom")
|
||||
# Must not raise: the cache is an optimisation, never a failure mode.
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
|
||||
|
||||
def test_cache_transport_error_does_not_fail_the_build(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake)
|
||||
|
||||
def _boom(*_a: Any, **_k: Any) -> None:
|
||||
raise TransportError("ssh died mid-restore")
|
||||
|
||||
monkeypatch.setattr(fake, "copy", _boom)
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
|
||||
|
||||
def test_cache_key_differs_when_the_lockfile_differs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A Cargo.lock change must miss the entry built for the previous one."""
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore
|
||||
|
||||
store = CargoCacheStore(tmp_path)
|
||||
seed = tmp_path / "seed.tar.zst"
|
||||
seed.write_bytes(b"entry")
|
||||
store.publish(_expected_key("cafe"), seed)
|
||||
|
||||
fake = _FakeTransport()
|
||||
monkeypatch.setattr(build_module, "SshTransport", lambda *a, **k: fake)
|
||||
_rust_guest(fake, digest="beef")
|
||||
|
||||
build_module._linux_build(**_linux_kwargs(cargo_cache_root=tmp_path))
|
||||
cmds = [c[0] for c in fake.runs]
|
||||
assert not any("tar --zstd -xf" in c for c in cmds), "different lock must miss"
|
||||
|
||||
@@ -1630,3 +1630,96 @@ def test_probe_transport_ssh_not_ready_returns_false(
|
||||
ssh_known_hosts=None,
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
# ── cargo cache plumbing ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_job_passes_cache_root_to_linux_build(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is not None
|
||||
|
||||
|
||||
def test_job_no_cargo_cache_disables_it(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
extra=["--no-cargo-cache"],
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is None
|
||||
|
||||
|
||||
def test_job_respects_config_disabling_the_cache(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
"""`enabled = false` in the config wins even without --no-cargo-cache."""
|
||||
from ci_orchestrator.config import CargoCacheConfig, Config
|
||||
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
# _patch_common already replaced load_config, so override its return value
|
||||
# rather than the environment.
|
||||
base = job_module.load_config()
|
||||
patched = Config(
|
||||
paths=base.paths,
|
||||
backend=base.backend,
|
||||
vmrun_path=base.vmrun_path,
|
||||
ssh_key_path=base.ssh_key_path,
|
||||
guest_cred_target=base.guest_cred_target,
|
||||
ip_pool=base.ip_pool,
|
||||
cargo_cache=CargoCacheConfig(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(job_module, "load_config", lambda: patched)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is None
|
||||
|
||||
|
||||
def test_job_windows_build_gets_no_cargo_cache_argument(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The Windows branch must be untouched by the Linux-only cache."""
|
||||
# Guest OS comes from the *clone* VMX, which the fake backend writes.
|
||||
backend = _FakeBackend(running_after=1, clone_guest_os="windows2019srvNext-64")
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
vmx = tmp_path / "templates" / "WinBuild2025.vmx"
|
||||
vmx.parent.mkdir(parents=True)
|
||||
vmx.write_text('guestOS = "windows2019srvNext-64"\n', encoding="utf-8")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "cargo_cache_root" not in calls["windows_build"][0]
|
||||
|
||||
@@ -11,6 +11,7 @@ from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.retention as ret
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.config import Config
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -191,3 +192,134 @@ def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) ->
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not old.exists()
|
||||
assert "Purged log" in result.output
|
||||
|
||||
|
||||
# ── cargo cache entries ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cache_config(
|
||||
tmp_path: Path, *, enabled: bool = True, days: int = 30
|
||||
) -> tuple[Path, Config]:
|
||||
"""Point the loaded config at a throwaway cache store.
|
||||
|
||||
Returns ``(store root, config)`` — the root so tests can seed entries, the
|
||||
config so they can substitute it for ``load_config``.
|
||||
"""
|
||||
from ci_orchestrator.config import BackendConfig, CargoCacheConfig, Config, Paths
|
||||
|
||||
root = tmp_path / "cache" / "cargo"
|
||||
cfg = Config(
|
||||
paths=Paths(
|
||||
root=tmp_path,
|
||||
templates=tmp_path / "t",
|
||||
build_vms=tmp_path / "b",
|
||||
artifacts=tmp_path / "artifacts",
|
||||
keys=tmp_path / "k",
|
||||
),
|
||||
backend=BackendConfig(),
|
||||
cargo_cache=CargoCacheConfig(enabled=enabled, root=root, retention_days=days),
|
||||
)
|
||||
return root, cfg
|
||||
|
||||
|
||||
def _seed_entry(root: Path, digest: str, *, age_days: int, size: int = 1024) -> Path:
|
||||
import os
|
||||
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key
|
||||
|
||||
store = CargoCacheStore(root)
|
||||
src = root.parent / f"seed-{digest}.tar.zst"
|
||||
src.parent.mkdir(parents=True, exist_ok=True)
|
||||
src.write_bytes(b"x" * size)
|
||||
entry = store.publish(cache_key(lock_digest=digest, rustc_version="r", host_triple="t"), src)
|
||||
stamp = time.time() - age_days * 86400
|
||||
os.utime(entry, (stamp, stamp))
|
||||
return entry
|
||||
|
||||
|
||||
def test_retention_purges_stale_cargo_cache_entries(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
root, cfg = _cache_config(tmp_path)
|
||||
old = _seed_entry(root, "old", age_days=90)
|
||||
fresh = _seed_entry(root, "new", age_days=1)
|
||||
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not old.exists(), "stale entry should be purged"
|
||||
assert fresh.exists(), "fresh entry should survive"
|
||||
|
||||
|
||||
def test_retention_reports_the_bytes_it_freed(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""TODO 2.9: a policy that frees space without saying how much is the bug."""
|
||||
root, cfg = _cache_config(tmp_path)
|
||||
_seed_entry(root, "old", age_days=90, size=3 * 1024 * 1024)
|
||||
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Purged 1 cargo cache entry (3 MB)" in result.output
|
||||
|
||||
|
||||
def test_retention_what_if_keeps_cargo_cache_entries(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
root, cfg = _cache_config(tmp_path)
|
||||
old = _seed_entry(root, "old", age_days=90)
|
||||
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run", "--what-if"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert old.exists()
|
||||
assert "would purge 1 cargo cache entry" in result.output
|
||||
|
||||
|
||||
def test_retention_skips_the_cache_when_disabled(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
root, cfg = _cache_config(tmp_path, enabled=False)
|
||||
old = _seed_entry(root, "old", age_days=90)
|
||||
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert old.exists(), "a disabled cache must not be touched"
|
||||
assert "disabled" in result.output
|
||||
|
||||
|
||||
def test_low_disk_tightens_the_cache_window_too(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Aggressive mode must reach the cache — it is the heaviest thing here."""
|
||||
root, cfg = _cache_config(tmp_path, days=30)
|
||||
entry = _seed_entry(root, "old", age_days=10) # young for 30d, old for 7d
|
||||
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not entry.exists(), "aggressive retention should have reached the cache"
|
||||
|
||||
|
||||
def test_retention_missing_cache_store_is_not_an_error(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_root, cfg = _cache_config(tmp_path)
|
||||
monkeypatch.setattr(ret, "load_config", lambda: cfg)
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(cli, ["retention", "run"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "nothing to purge" in result.output
|
||||
|
||||
@@ -145,3 +145,54 @@ def test_ip_pool_absent_when_not_configured(tmp_path: Path) -> None:
|
||||
toml.write_text("[backend]\ntype = \"workstation\"\n", encoding="utf-8")
|
||||
cfg = load_config(config_path=toml, env={})
|
||||
assert cfg.ip_pool is None
|
||||
|
||||
|
||||
# ── cargo cache section ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _paths_only(tmp_path: Path, extra: str = "") -> Path:
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text(
|
||||
'[paths]\nroot = "/var/lib/ci"\ntemplates = "/var/lib/ci/templates"\n'
|
||||
'build_vms = "/var/lib/ci/build-vms"\nartifacts = "/var/lib/ci/artifacts"\n'
|
||||
'keys = "/var/lib/ci/keys"\n' + extra,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def test_cargo_cache_defaults_under_paths_root(tmp_path: Path) -> None:
|
||||
cfg = load_config(config_path=_paths_only(tmp_path), env={})
|
||||
assert cfg.cargo_cache.enabled is True
|
||||
assert cfg.cargo_cache.root == Path("/var/lib/ci/cache/cargo")
|
||||
assert cfg.cargo_cache.retention_days == 30
|
||||
|
||||
|
||||
def test_cargo_cache_section_overrides_the_defaults(tmp_path: Path) -> None:
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(
|
||||
tmp_path,
|
||||
'\n[cargo_cache]\nenabled = false\nroot = "/mnt/fast/cargo"\n'
|
||||
"retention_days = 7\n",
|
||||
),
|
||||
env={},
|
||||
)
|
||||
assert cfg.cargo_cache.enabled is False
|
||||
assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")
|
||||
assert cfg.cargo_cache.retention_days == 7
|
||||
|
||||
|
||||
def test_cargo_cache_root_follows_ci_root_env(tmp_path: Path) -> None:
|
||||
"""CI_ROOT relocates every path; the store must move with them."""
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(tmp_path), env={"CI_ROOT": str(tmp_path / "elsewhere")}
|
||||
)
|
||||
assert cfg.cargo_cache.root == tmp_path / "elsewhere" / "cache" / "cargo"
|
||||
|
||||
|
||||
def test_explicit_cargo_cache_root_beats_ci_root(tmp_path: Path) -> None:
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(tmp_path, '\n[cargo_cache]\nroot = "/mnt/fast/cargo"\n'),
|
||||
env={"CI_ROOT": str(tmp_path / "elsewhere")},
|
||||
)
|
||||
assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")
|
||||
|
||||
Reference in New Issue
Block a user