feat(cache): restore and publish the cargo dependency cache around Linux builds
Restore runs after the source is in place and before the build; publish runs only after a successful build and only on a miss, so the steady state uploads nothing and a failed build can never become an entry. The whole path is best-effort: a cache problem degrades to a cold build with a message rather than turning a good build red. Gated on Cargo.lock being present, so non-Rust jobs are untouched with no flag to set. The Windows branch is not touched at all. The prune script is read through importlib.resources rather than a path relative to the repo root: the production venv is installed non-editable, so a parents[] walk lands in site-packages where scripts/ does not exist. The fake transport gained substring-matched canned results; indexing by call position broke as soon as the cache added calls mid-flow. Mutation-tested rather than assumed. Three guards were each disabled in turn -- the publish call, the prune-refusal check, the packing-failure check -- and a test went red for each. The prune-refusal test initially survived its mutant: it passed because the fake's fetch left no file behind, so publishing failed for an unrelated reason. Fixed by making fetch write a real file, so the only thing preventing publication is the guard under test.
This commit is contained in:
@@ -25,11 +25,13 @@ import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.cargo_cache import CargoCacheStore, cache_key
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||
@@ -122,6 +124,110 @@ def _tail_lines(stdout: str, stderr: str) -> str:
|
||||
return " | ".join(lines[-5:]) if lines else "(no build output)"
|
||||
|
||||
|
||||
# ------------------------------------------------- cargo dependency cache
|
||||
|
||||
_PRUNE_ASSET = "cargo-cache-prune.sh"
|
||||
_GUEST_PRUNE = "/tmp/cargo-cache-prune.sh"
|
||||
_GUEST_TARBALL = "/tmp/cargo-cache-entry.tar"
|
||||
|
||||
|
||||
def _cargo_cache_probe(t: SshTransport, workdir: str) -> tuple[str, str] | None:
|
||||
"""Return ``(key, compressor)`` for this workspace, or ``None`` to skip.
|
||||
|
||||
``None`` means "not a Rust build, or we could not read what we need". The
|
||||
cache is an optimisation, so every uncertainty here resolves to skipping
|
||||
it rather than to guessing.
|
||||
"""
|
||||
if t.run(f"test -f {_sh_quote(workdir + '/Cargo.lock')}", check=False).returncode != 0:
|
||||
return None
|
||||
lock = t.run(f"sha256sum {_sh_quote(workdir + '/Cargo.lock')}", check=False)
|
||||
version = t.run("rustc -V", check=False)
|
||||
triple = t.run("rustc -vV | sed -n 's/^host: //p'", check=False)
|
||||
if not (lock.ok and version.ok and triple.ok):
|
||||
click.echo("[cache] could not read Cargo.lock digest or rustc version; skipping")
|
||||
return None
|
||||
digest = lock.stdout.split()[0] if lock.stdout.split() else ""
|
||||
if not digest:
|
||||
click.echo("[cache] empty Cargo.lock digest; skipping")
|
||||
return None
|
||||
compressor = "zstd" if t.run("command -v zstd", check=False).ok else "gzip"
|
||||
key = cache_key(
|
||||
lock_digest=digest,
|
||||
rustc_version=version.stdout.strip(),
|
||||
host_triple=triple.stdout.strip(),
|
||||
)
|
||||
return key, compressor
|
||||
|
||||
|
||||
def _tar_flag(compressor: str) -> tuple[str, str]:
|
||||
"""``(tar flag, file extension)`` for the chosen compressor."""
|
||||
return ("--zstd", "zst") if compressor == "zstd" else ("-z", "gz")
|
||||
|
||||
|
||||
def _cargo_cache_restore(
|
||||
t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str
|
||||
) -> bool:
|
||||
"""Upload and unpack an existing entry. Returns True on a hit."""
|
||||
entry = store.lookup(key)
|
||||
if entry is None:
|
||||
click.echo(f"[cache] miss for {key} — building cold")
|
||||
return False
|
||||
flag, ext = _tar_flag(compressor)
|
||||
size_mb = entry.stat().st_size / 1_048_576
|
||||
click.echo(f"[cache] hit for {key} ({size_mb:.0f} MB) — restoring")
|
||||
guest_entry = f"{_GUEST_TARBALL}.{ext}"
|
||||
t.copy(str(entry), guest_entry)
|
||||
t.run(
|
||||
f"tar {flag} -xf {_sh_quote(guest_entry)} -C {_sh_quote(workdir)} && "
|
||||
f"rm -f {_sh_quote(guest_entry)}"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _cargo_cache_publish(
|
||||
t: SshTransport, store: CargoCacheStore, key: str, workdir: str, compressor: str
|
||||
) -> None:
|
||||
"""Prune workspace artifacts, then file the remainder under ``key``.
|
||||
|
||||
A non-zero prune means the fail-closed assertion tripped: say so loudly and
|
||||
publish nothing. A poisoned entry costs a wrong verdict; a missing one only
|
||||
costs a cold build.
|
||||
"""
|
||||
# as_file() because the production venv is installed non-editable: the
|
||||
# script is package data, not a path relative to a source checkout.
|
||||
asset = resources.files("ci_orchestrator.assets").joinpath(_PRUNE_ASSET)
|
||||
with resources.as_file(asset) as prune_script:
|
||||
t.copy(str(prune_script), _GUEST_PRUNE)
|
||||
pruned = t.run(
|
||||
f"bash {_GUEST_PRUNE} {_sh_quote(workdir + '/target')} "
|
||||
f"{_sh_quote(workdir + '/Cargo.toml')}",
|
||||
check=False,
|
||||
)
|
||||
click.echo(f"[cache] {(pruned.stdout.strip() or pruned.stderr.strip())}")
|
||||
if pruned.returncode != 0:
|
||||
click.echo("[cache] prune refused — publishing nothing", err=True)
|
||||
return
|
||||
|
||||
flag, ext = _tar_flag(compressor)
|
||||
guest_tar = f"{_GUEST_TARBALL}.{ext}"
|
||||
packed = t.run(
|
||||
f"tar {flag} -cf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} target",
|
||||
check=False,
|
||||
)
|
||||
if not packed.ok:
|
||||
click.echo("[cache] packing failed — publishing nothing", err=True)
|
||||
return
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
local = Path(td) / f"entry.tar.{ext}"
|
||||
t.fetch(guest_tar, local)
|
||||
published = store.publish(key, local)
|
||||
t.run(f"rm -f {_sh_quote(guest_tar)}", check=False)
|
||||
click.echo(
|
||||
f"[cache] published {key} ({published.stat().st_size / 1_048_576:.0f} MB)"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Linux flow
|
||||
|
||||
|
||||
@@ -143,6 +249,7 @@ def _linux_build(
|
||||
extra_env: dict[str, str],
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool = False,
|
||||
cargo_cache_root: Path | None = None,
|
||||
) -> None:
|
||||
click.echo("[build run] Linux build mode (SSH transport)")
|
||||
|
||||
@@ -217,6 +324,26 @@ def _linux_build(
|
||||
check=False,
|
||||
)
|
||||
|
||||
# Cargo dependency cache. Everything here is best-effort: a cache
|
||||
# problem must never turn a good build red, so any failure degrades to
|
||||
# a cold build with a message.
|
||||
cache_key_value: str | None = None
|
||||
compressor = "gzip"
|
||||
store: CargoCacheStore | None = None
|
||||
if cargo_cache_root is not None:
|
||||
try:
|
||||
store = CargoCacheStore(cargo_cache_root)
|
||||
probed = _cargo_cache_probe(t, workdir)
|
||||
if probed is None:
|
||||
store = None
|
||||
else:
|
||||
cache_key_value, compressor = probed
|
||||
_cargo_cache_restore(t, store, cache_key_value, workdir, compressor)
|
||||
except (TransportError, OSError, ValueError) as exc:
|
||||
click.echo(f"[cache] restore skipped: {exc}", err=True)
|
||||
cache_key_value = None
|
||||
store = None
|
||||
|
||||
env_prefix = "".join(
|
||||
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||
)
|
||||
@@ -245,6 +372,14 @@ def _linux_build(
|
||||
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
|
||||
)
|
||||
|
||||
# Publish only on a miss, so the steady state uploads nothing. A failed
|
||||
# build never reaches here, so a broken tree cannot become an entry.
|
||||
if store is not None and cache_key_value and store.lookup(cache_key_value) is None:
|
||||
try:
|
||||
_cargo_cache_publish(t, store, cache_key_value, workdir, compressor)
|
||||
except (TransportError, OSError, ValueError) as exc:
|
||||
click.echo(f"[cache] publish skipped: {exc}", err=True)
|
||||
|
||||
if skip_artifact:
|
||||
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user