diff --git a/config.example.toml b/config.example.toml index e9ef228..db436cb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -53,3 +53,22 @@ type = "workstation" # netmask = "255.255.255.0" # gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route # lease_file = "/var/lib/ci/ip-pool.json" + +# ------------------------------------------------- cargo dependency cache +# Compiled Rust dependency artifacts, reused between jobs. Engages only for +# Linux guests whose build workdir contains a Cargo.lock, so non-Rust jobs +# need no flag. Workspace crates are stripped before an entry is stored and +# the job refuses to publish if any survive, so a restored entry cannot carry +# the code under test. +# +# One immutable file per key under `root`; a key is +# sha256(Cargo.lock) + rustc version + host triple. Deleting entries by hand +# is always safe -- each one costs exactly one cold build to regenerate. +# +# `root` defaults to /cache/cargo and follows CI_ROOT unless set +# here explicitly. Measured on LinuxRust2404: an entry is ~645 MB and a warm +# job pays ~8s to skip ~120s of dependency compilation. +[cargo_cache] +enabled = true +# root = "/var/lib/ci/cache/cargo" +retention_days = 30 diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py index eeefbd4..ec5c794 100644 --- a/src/ci_orchestrator/commands/job.py +++ b/src/ci_orchestrator/commands/job.py @@ -326,6 +326,14 @@ def _destroy_clone( help="In-guest git clone (default) vs host-side clone + zip transfer.", ) @click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option( + "--cargo-cache/--no-cargo-cache", + "cargo_cache", + default=True, + show_default=True, + help="Reuse compiled Rust dependency artifacts between jobs (Linux only; " + "engages only when the workspace has a Cargo.lock).", +) @click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) @click.option( "--xvfb", @@ -409,6 +417,7 @@ def job( submodules: bool, use_git_clone: bool, use_shared_cache: bool, + cargo_cache: bool, skip_artifact: bool, use_xvfb: bool, gitea_credential_target: str, @@ -634,6 +643,11 @@ def job( extra_env=extra_env, skip_artifact=skip_artifact, use_xvfb=use_xvfb, + cargo_cache_root=( + config.cargo_cache.root + if cargo_cache and config.cargo_cache.enabled + else None + ), ) else: _windows_build( diff --git a/src/ci_orchestrator/config.py b/src/ci_orchestrator/config.py index 9bff204..df5863e 100644 --- a/src/ci_orchestrator/config.py +++ b/src/ci_orchestrator/config.py @@ -65,6 +65,19 @@ class BackendConfig: options: dict[str, Any] = field(default_factory=dict) +@dataclass(frozen=True) +class CargoCacheConfig: + """Where compiled Rust dependency artifacts are filed between jobs. + + ``root`` defaults to ``/cache/cargo``; it is resolved in + :func:`load_config` because it depends on the resolved paths. + """ + + enabled: bool = True + root: Path = Path("/var/lib/ci/cache/cargo") + retention_days: int = 30 + + @dataclass(frozen=True) class Config: """Top-level configuration object.""" @@ -75,6 +88,7 @@ class Config: ssh_key_path: Path | None = None guest_cred_target: str = "BuildVMGuest" ip_pool: IpPoolConfig | None = None + cargo_cache: CargoCacheConfig = field(default_factory=CargoCacheConfig) def _platform_defaults() -> Paths: @@ -158,6 +172,7 @@ def load_config( ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest") ip_pool: IpPoolConfig | None = None + cargo_cache_section: dict[str, Any] = {} if toml_path and toml_path.is_file(): data = _load_toml(toml_path) @@ -187,9 +202,18 @@ def load_config( gateway=str(pool_section.get("gateway", "")), lease_file=Path(pool_section.get("lease_file") or _lease_default), ) + cargo_cache_section = data.get("cargo_cache", {}) or {} paths = _apply_env(paths, env) + # Resolved after _apply_env so CI_ROOT relocates the store with everything + # else; an explicit root in the file still wins. + cargo_cache = CargoCacheConfig( + enabled=bool(cargo_cache_section.get("enabled", True)), + root=Path(cargo_cache_section.get("root") or paths.root / "cache" / "cargo"), + retention_days=int(cargo_cache_section.get("retention_days", 30)), + ) + return Config( paths=paths, backend=backend, @@ -197,7 +221,15 @@ def load_config( ssh_key_path=ssh_key_path, guest_cred_target=guest_cred_target, ip_pool=ip_pool, + cargo_cache=cargo_cache, ) -__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"] +__all__ = [ + "BackendConfig", + "CargoCacheConfig", + "Config", + "IpPoolConfig", + "Paths", + "load_config", +] diff --git a/tests/python/test_commands_job.py b/tests/python/test_commands_job.py index cb71b63..c6be8ae 100644 --- a/tests/python/test_commands_job.py +++ b/tests/python/test_commands_job.py @@ -1630,3 +1630,96 @@ def test_probe_transport_ssh_not_ready_returns_false( ssh_known_hosts=None, ) assert result is False + + +# ── cargo cache plumbing ───────────────────────────────────────────────────── + + +def test_job_passes_cache_root_to_linux_build( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is not None + + +def test_job_no_cargo_cache_disables_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + extra=["--no-cargo-cache"], + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_respects_config_disabling_the_cache( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + """`enabled = false` in the config wins even without --no-cargo-cache.""" + from ci_orchestrator.config import CargoCacheConfig, Config + + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + # _patch_common already replaced load_config, so override its return value + # rather than the environment. + base = job_module.load_config() + patched = Config( + paths=base.paths, + backend=base.backend, + vmrun_path=base.vmrun_path, + ssh_key_path=base.ssh_key_path, + guest_cred_target=base.guest_cred_target, + ip_pool=base.ip_pool, + cargo_cache=CargoCacheConfig(enabled=False), + ) + monkeypatch.setattr(job_module, "load_config", lambda: patched) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["cargo_cache_root"] is None + + +def test_job_windows_build_gets_no_cargo_cache_argument( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The Windows branch must be untouched by the Linux-only cache.""" + # Guest OS comes from the *clone* VMX, which the fake backend writes. + backend = _FakeBackend(running_after=1, clone_guest_os="windows2019srvNext-64") + calls = _patch_common(monkeypatch, backend) + vmx = tmp_path / "templates" / "WinBuild2025.vmx" + vmx.parent.mkdir(parents=True) + vmx.write_text('guestOS = "windows2019srvNext-64"\n', encoding="utf-8") + result = CliRunner().invoke( + cli, + _common_args( + vmx, + clone_base=tmp_path / "build-vms", + artifact_base=tmp_path / "artifacts", + ), + ) + assert result.exit_code == 0, result.output + assert "cargo_cache_root" not in calls["windows_build"][0] diff --git a/tests/python/test_config.py b/tests/python/test_config.py index ce53d41..5270434 100644 --- a/tests/python/test_config.py +++ b/tests/python/test_config.py @@ -145,3 +145,54 @@ def test_ip_pool_absent_when_not_configured(tmp_path: Path) -> None: toml.write_text("[backend]\ntype = \"workstation\"\n", encoding="utf-8") cfg = load_config(config_path=toml, env={}) assert cfg.ip_pool is None + + +# ── cargo cache section ────────────────────────────────────────────────────── + + +def _paths_only(tmp_path: Path, extra: str = "") -> Path: + cfg = tmp_path / "config.toml" + cfg.write_text( + '[paths]\nroot = "/var/lib/ci"\ntemplates = "/var/lib/ci/templates"\n' + 'build_vms = "/var/lib/ci/build-vms"\nartifacts = "/var/lib/ci/artifacts"\n' + 'keys = "/var/lib/ci/keys"\n' + extra, + encoding="utf-8", + ) + return cfg + + +def test_cargo_cache_defaults_under_paths_root(tmp_path: Path) -> None: + cfg = load_config(config_path=_paths_only(tmp_path), env={}) + assert cfg.cargo_cache.enabled is True + assert cfg.cargo_cache.root == Path("/var/lib/ci/cache/cargo") + assert cfg.cargo_cache.retention_days == 30 + + +def test_cargo_cache_section_overrides_the_defaults(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only( + tmp_path, + '\n[cargo_cache]\nenabled = false\nroot = "/mnt/fast/cargo"\n' + "retention_days = 7\n", + ), + env={}, + ) + assert cfg.cargo_cache.enabled is False + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo") + assert cfg.cargo_cache.retention_days == 7 + + +def test_cargo_cache_root_follows_ci_root_env(tmp_path: Path) -> None: + """CI_ROOT relocates every path; the store must move with them.""" + cfg = load_config( + config_path=_paths_only(tmp_path), env={"CI_ROOT": str(tmp_path / "elsewhere")} + ) + assert cfg.cargo_cache.root == tmp_path / "elsewhere" / "cache" / "cargo" + + +def test_explicit_cargo_cache_root_beats_ci_root(tmp_path: Path) -> None: + cfg = load_config( + config_path=_paths_only(tmp_path, '\n[cargo_cache]\nroot = "/mnt/fast/cargo"\n'), + env={"CI_ROOT": str(tmp_path / "elsewhere")}, + ) + assert cfg.cargo_cache.root == Path("/mnt/fast/cargo")