feat(cache): --cargo-cache flag and [cargo_cache] config section
Default on, because the cache is safe by construction rather than by convention: it engages only where a Cargo.lock exists, and refuses to publish anything the prune assertion cannot vouch for. Two independent off switches -- the flag for one job, `enabled = false` for the host. The store root is resolved after the environment overrides are applied, so CI_ROOT relocates it along with everything else, while an explicit root in the config still wins. Both directions are covered by tests. Also asserts that the Windows branch never receives the argument at all: the cache is Linux-only and the two branches stay separate, per AGENTS.md.
This commit is contained in:
@@ -53,3 +53,22 @@ type = "workstation"
|
||||
# netmask = "255.255.255.0"
|
||||
# gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route
|
||||
# lease_file = "/var/lib/ci/ip-pool.json"
|
||||
|
||||
# ------------------------------------------------- cargo dependency cache
|
||||
# Compiled Rust dependency artifacts, reused between jobs. Engages only for
|
||||
# Linux guests whose build workdir contains a Cargo.lock, so non-Rust jobs
|
||||
# need no flag. Workspace crates are stripped before an entry is stored and
|
||||
# the job refuses to publish if any survive, so a restored entry cannot carry
|
||||
# the code under test.
|
||||
#
|
||||
# One immutable file per key under `root`; a key is
|
||||
# sha256(Cargo.lock) + rustc version + host triple. Deleting entries by hand
|
||||
# is always safe -- each one costs exactly one cold build to regenerate.
|
||||
#
|
||||
# `root` defaults to <paths.root>/cache/cargo and follows CI_ROOT unless set
|
||||
# here explicitly. Measured on LinuxRust2404: an entry is ~645 MB and a warm
|
||||
# job pays ~8s to skip ~120s of dependency compilation.
|
||||
[cargo_cache]
|
||||
enabled = true
|
||||
# root = "/var/lib/ci/cache/cargo"
|
||||
retention_days = 30
|
||||
|
||||
@@ -326,6 +326,14 @@ def _destroy_clone(
|
||||
help="In-guest git clone (default) vs host-side clone + zip transfer.",
|
||||
)
|
||||
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--cargo-cache/--no-cargo-cache",
|
||||
"cargo_cache",
|
||||
default=True,
|
||||
show_default=True,
|
||||
help="Reuse compiled Rust dependency artifacts between jobs (Linux only; "
|
||||
"engages only when the workspace has a Cargo.lock).",
|
||||
)
|
||||
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--xvfb",
|
||||
@@ -409,6 +417,7 @@ def job(
|
||||
submodules: bool,
|
||||
use_git_clone: bool,
|
||||
use_shared_cache: bool,
|
||||
cargo_cache: bool,
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool,
|
||||
gitea_credential_target: str,
|
||||
@@ -634,6 +643,11 @@ def job(
|
||||
extra_env=extra_env,
|
||||
skip_artifact=skip_artifact,
|
||||
use_xvfb=use_xvfb,
|
||||
cargo_cache_root=(
|
||||
config.cargo_cache.root
|
||||
if cargo_cache and config.cargo_cache.enabled
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
_windows_build(
|
||||
|
||||
@@ -65,6 +65,19 @@ class BackendConfig:
|
||||
options: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CargoCacheConfig:
|
||||
"""Where compiled Rust dependency artifacts are filed between jobs.
|
||||
|
||||
``root`` defaults to ``<paths.root>/cache/cargo``; it is resolved in
|
||||
:func:`load_config` because it depends on the resolved paths.
|
||||
"""
|
||||
|
||||
enabled: bool = True
|
||||
root: Path = Path("/var/lib/ci/cache/cargo")
|
||||
retention_days: int = 30
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
"""Top-level configuration object."""
|
||||
@@ -75,6 +88,7 @@ class Config:
|
||||
ssh_key_path: Path | None = None
|
||||
guest_cred_target: str = "BuildVMGuest"
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
cargo_cache: CargoCacheConfig = field(default_factory=CargoCacheConfig)
|
||||
|
||||
|
||||
def _platform_defaults() -> Paths:
|
||||
@@ -158,6 +172,7 @@ def load_config(
|
||||
ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None
|
||||
guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest")
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
cargo_cache_section: dict[str, Any] = {}
|
||||
|
||||
if toml_path and toml_path.is_file():
|
||||
data = _load_toml(toml_path)
|
||||
@@ -187,9 +202,18 @@ def load_config(
|
||||
gateway=str(pool_section.get("gateway", "")),
|
||||
lease_file=Path(pool_section.get("lease_file") or _lease_default),
|
||||
)
|
||||
cargo_cache_section = data.get("cargo_cache", {}) or {}
|
||||
|
||||
paths = _apply_env(paths, env)
|
||||
|
||||
# Resolved after _apply_env so CI_ROOT relocates the store with everything
|
||||
# else; an explicit root in the file still wins.
|
||||
cargo_cache = CargoCacheConfig(
|
||||
enabled=bool(cargo_cache_section.get("enabled", True)),
|
||||
root=Path(cargo_cache_section.get("root") or paths.root / "cache" / "cargo"),
|
||||
retention_days=int(cargo_cache_section.get("retention_days", 30)),
|
||||
)
|
||||
|
||||
return Config(
|
||||
paths=paths,
|
||||
backend=backend,
|
||||
@@ -197,7 +221,15 @@ def load_config(
|
||||
ssh_key_path=ssh_key_path,
|
||||
guest_cred_target=guest_cred_target,
|
||||
ip_pool=ip_pool,
|
||||
cargo_cache=cargo_cache,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"]
|
||||
__all__ = [
|
||||
"BackendConfig",
|
||||
"CargoCacheConfig",
|
||||
"Config",
|
||||
"IpPoolConfig",
|
||||
"Paths",
|
||||
"load_config",
|
||||
]
|
||||
|
||||
@@ -1630,3 +1630,96 @@ def test_probe_transport_ssh_not_ready_returns_false(
|
||||
ssh_known_hosts=None,
|
||||
)
|
||||
assert result is False
|
||||
|
||||
|
||||
# ── cargo cache plumbing ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_job_passes_cache_root_to_linux_build(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is not None
|
||||
|
||||
|
||||
def test_job_no_cargo_cache_disables_it(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
extra=["--no-cargo-cache"],
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is None
|
||||
|
||||
|
||||
def test_job_respects_config_disabling_the_cache(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||
) -> None:
|
||||
"""`enabled = false` in the config wins even without --no-cargo-cache."""
|
||||
from ci_orchestrator.config import CargoCacheConfig, Config
|
||||
|
||||
backend = _FakeBackend(running_after=1)
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
# _patch_common already replaced load_config, so override its return value
|
||||
# rather than the environment.
|
||||
base = job_module.load_config()
|
||||
patched = Config(
|
||||
paths=base.paths,
|
||||
backend=base.backend,
|
||||
vmrun_path=base.vmrun_path,
|
||||
ssh_key_path=base.ssh_key_path,
|
||||
guest_cred_target=base.guest_cred_target,
|
||||
ip_pool=base.ip_pool,
|
||||
cargo_cache=CargoCacheConfig(enabled=False),
|
||||
)
|
||||
monkeypatch.setattr(job_module, "load_config", lambda: patched)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
template_vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls["linux_build"][0]["cargo_cache_root"] is None
|
||||
|
||||
|
||||
def test_job_windows_build_gets_no_cargo_cache_argument(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""The Windows branch must be untouched by the Linux-only cache."""
|
||||
# Guest OS comes from the *clone* VMX, which the fake backend writes.
|
||||
backend = _FakeBackend(running_after=1, clone_guest_os="windows2019srvNext-64")
|
||||
calls = _patch_common(monkeypatch, backend)
|
||||
vmx = tmp_path / "templates" / "WinBuild2025.vmx"
|
||||
vmx.parent.mkdir(parents=True)
|
||||
vmx.write_text('guestOS = "windows2019srvNext-64"\n', encoding="utf-8")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
_common_args(
|
||||
vmx,
|
||||
clone_base=tmp_path / "build-vms",
|
||||
artifact_base=tmp_path / "artifacts",
|
||||
),
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "cargo_cache_root" not in calls["windows_build"][0]
|
||||
|
||||
@@ -145,3 +145,54 @@ def test_ip_pool_absent_when_not_configured(tmp_path: Path) -> None:
|
||||
toml.write_text("[backend]\ntype = \"workstation\"\n", encoding="utf-8")
|
||||
cfg = load_config(config_path=toml, env={})
|
||||
assert cfg.ip_pool is None
|
||||
|
||||
|
||||
# ── cargo cache section ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _paths_only(tmp_path: Path, extra: str = "") -> Path:
|
||||
cfg = tmp_path / "config.toml"
|
||||
cfg.write_text(
|
||||
'[paths]\nroot = "/var/lib/ci"\ntemplates = "/var/lib/ci/templates"\n'
|
||||
'build_vms = "/var/lib/ci/build-vms"\nartifacts = "/var/lib/ci/artifacts"\n'
|
||||
'keys = "/var/lib/ci/keys"\n' + extra,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return cfg
|
||||
|
||||
|
||||
def test_cargo_cache_defaults_under_paths_root(tmp_path: Path) -> None:
|
||||
cfg = load_config(config_path=_paths_only(tmp_path), env={})
|
||||
assert cfg.cargo_cache.enabled is True
|
||||
assert cfg.cargo_cache.root == Path("/var/lib/ci/cache/cargo")
|
||||
assert cfg.cargo_cache.retention_days == 30
|
||||
|
||||
|
||||
def test_cargo_cache_section_overrides_the_defaults(tmp_path: Path) -> None:
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(
|
||||
tmp_path,
|
||||
'\n[cargo_cache]\nenabled = false\nroot = "/mnt/fast/cargo"\n'
|
||||
"retention_days = 7\n",
|
||||
),
|
||||
env={},
|
||||
)
|
||||
assert cfg.cargo_cache.enabled is False
|
||||
assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")
|
||||
assert cfg.cargo_cache.retention_days == 7
|
||||
|
||||
|
||||
def test_cargo_cache_root_follows_ci_root_env(tmp_path: Path) -> None:
|
||||
"""CI_ROOT relocates every path; the store must move with them."""
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(tmp_path), env={"CI_ROOT": str(tmp_path / "elsewhere")}
|
||||
)
|
||||
assert cfg.cargo_cache.root == tmp_path / "elsewhere" / "cache" / "cargo"
|
||||
|
||||
|
||||
def test_explicit_cargo_cache_root_beats_ci_root(tmp_path: Path) -> None:
|
||||
cfg = load_config(
|
||||
config_path=_paths_only(tmp_path, '\n[cargo_cache]\nroot = "/mnt/fast/cargo"\n'),
|
||||
env={"CI_ROOT": str(tmp_path / "elsewhere")},
|
||||
)
|
||||
assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")
|
||||
|
||||
Reference in New Issue
Block a user