From 29dfbf1cae32f31e219d0cb502a60c38cf937b21 Mon Sep 17 00:00:00 2001 From: Simone Date: Sun, 17 May 2026 17:00:46 +0200 Subject: [PATCH] feat(job): wire use-git-clone + submodules; default in-guest clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The job command previously discarded --use-git-clone and --submodules (`del`) and always in-guest cloned with submodules off — making both action inputs dead no-ops (submodule repos silently broken). - job.py: --use-git-clone/--host-clone and --submodules/--no-submodules are real booleans defaulting to ON. Default = in-guest clone with submodules. --host-clone clones on the host into a temp dir (_host_clone) and transfers a zip; temp dir cleaned in finally. clone_submodules now honoured (was hardcoded False). - action.yml: use-git-clone / submodules default 'true'; forward the explicit positive/negative flag so the Python default is overridden deterministically. - self-test.yml: add smoke-{windows,linux}-hostclone jobs covering the host-side transport (default jobs already cover in-guest). - tests: default-transport, host-clone, and --no-submodules coverage. Co-Authored-By: Claude Opus 4.7 --- .gitea/actions/local-ci-build/action.yml | 26 ++-- .gitea/workflows/self-test.yml | 31 ++++- src/ci_orchestrator/commands/job.py | 145 +++++++++++++++++------ tests/python/test_commands_job.py | 71 +++++++++++ 4 files changed, 223 insertions(+), 50 deletions(-) diff --git a/.gitea/actions/local-ci-build/action.yml b/.gitea/actions/local-ci-build/action.yml index 6f15bde..19d3b59 100644 --- a/.gitea/actions/local-ci-build/action.yml +++ b/.gitea/actions/local-ci-build/action.yml @@ -48,9 +48,10 @@ inputs: submodules: description: > - Set to "true" to clone the repository with --recurse-submodules. + Recurse submodules when cloning. Default "true". Set to "false" + to skip submodule init/update. required: false - default: 'false' + default: 'true' guest-os: description: > @@ -61,12 +62,14 @@ inputs: use-git-clone: description: > - Set to "true" to clone the repository inside the VM rather than - transferring a zip from the host. Requires Git in the template (see - Install-CIToolchain-WinBuild2025.ps1 Step 12 / Install-CIToolchain-Linux2404.sh). - Preferred for repositories larger than ~200 MB. + Transport for the repo source. Default "true": clone inside the + build VM (requires Git in the template — see + Install-CIToolchain-WinBuild2025.ps1 Step 12 / + Install-CIToolchain-Linux2404.sh). Set to "false" for the + host-side variant: the host clones the repo and transfers a zip + to the guest (useful when the guest cannot reach Gitea). required: false - default: 'false' + default: 'true' configuration: description: > @@ -269,8 +272,13 @@ runs: '--snapshot-name', $resolvedSnapshotName ) - if ($env:INPUT_SUBMODULES -eq 'true') { $pyArgs += '--submodules' } - if ($env:INPUT_USE_GIT_CLONE -eq 'true') { $pyArgs += '--use-git-clone' } + # Submodules and transport default to ON; pass the explicit + # negative flag when the caller opted out so the Python default + # (also ON) is overridden deterministically. + if ($env:INPUT_SUBMODULES -eq 'false') { $pyArgs += '--no-submodules' } + else { $pyArgs += '--submodules' } + if ($env:INPUT_USE_GIT_CLONE -eq 'false') { $pyArgs += '--host-clone' } + else { $pyArgs += '--use-git-clone' } if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $pyArgs += '--use-shared-cache' } if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $pyArgs += '--skip-artifact' } if ($env:INPUT_EXTRA_GUEST_ENV_JSON -and $env:INPUT_EXTRA_GUEST_ENV_JSON -ne '{}') { diff --git a/.gitea/workflows/self-test.yml b/.gitea/workflows/self-test.yml index 6f23c40..70ca413 100644 --- a/.gitea/workflows/self-test.yml +++ b/.gitea/workflows/self-test.yml @@ -4,7 +4,9 @@ # VM clone -> start -> IP detect -> trivial build -> artifact collect -> destroy # # Trigger: workflow_dispatch only (never runs on push/tag). -# Both jobs are independent — a failure in one does not cancel the other. +# Jobs are independent — a failure in one does not cancel the others. +# Covers both source transports: in-guest git clone (default) and the +# host-side clone + zip transfer variant (use-git-clone: 'false'). # # Prerequisites (must be in place before running): # - Runner labels: windows-build:host and linux-build:host active @@ -43,4 +45,29 @@ jobs: artifact-source: '/opt/ci/output' guest-os: 'Linux' job-id-suffix: 'smoke-linux' - artifact-name: 'smoke-linux-${{ github.run_id }}' \ No newline at end of file + artifact-name: 'smoke-linux-${{ github.run_id }}' + + # ── Host-side clone + zip transfer variant (use-git-clone: 'false') ─────── + smoke-windows-hostclone: + runs-on: windows-build + steps: + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main + with: + build-command: 'New-Item -ItemType Directory -Path C:\CI\output -Force | Out-Null; Set-Content C:\CI\output\smoke.txt "smoke-ok"' + artifact-source: 'C:\CI\output' + guest-os: 'Windows' + use-git-clone: 'false' + job-id-suffix: 'smoke-win-hostclone' + artifact-name: 'smoke-windows-hostclone-${{ github.run_id }}' + + smoke-linux-hostclone: + runs-on: linux-build + steps: + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main + with: + build-command: 'mkdir -p /opt/ci/output && echo smoke-ok > /opt/ci/output/smoke.txt' + artifact-source: '/opt/ci/output' + guest-os: 'Linux' + use-git-clone: 'false' + job-id-suffix: 'smoke-linux-hostclone' + artifact-name: 'smoke-linux-hostclone-${{ github.run_id }}' \ No newline at end of file diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py index 5635727..db1aa2a 100644 --- a/src/ci_orchestrator/commands/job.py +++ b/src/ci_orchestrator/commands/job.py @@ -18,7 +18,10 @@ from __future__ import annotations import json import re +import shutil +import subprocess import sys +import tempfile import time import uuid from datetime import UTC, datetime @@ -192,6 +195,43 @@ def _make_clone_name(job_id: str) -> str: return f"Clone_{job_id}_{timestamp}_{suffix}" +def _host_clone( + repo_url: str, branch: str, commit: str, submodules: bool +) -> tuple[Path, Path]: + """Clone the repo on the host into a temp dir (host-side transport). + + Returns ``(tmp_root, src_dir)``. ``src_dir`` is passed as + ``host_source_dir`` to the build (zipped and transferred to the + guest). The caller must remove ``tmp_root`` when done. The repo URL + may embed credentials, so git output is never echoed. + """ + tmp_root = Path(tempfile.mkdtemp(prefix="ci-host-clone-")) + src_dir = tmp_root / "src" + clone = ["git", "clone", "--depth", "1", "--branch", branch] + if submodules: + clone.append("--recurse-submodules") + clone += [repo_url, str(src_dir)] + try: + subprocess.run(clone, check=True, capture_output=True, text=True) + if commit: + subprocess.run( + ["git", "-C", str(src_dir), "fetch", "--depth", "1", + "origin", commit], + check=True, capture_output=True, text=True, + ) + subprocess.run( + ["git", "-C", str(src_dir), "checkout", commit], + check=True, capture_output=True, text=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(tmp_root, ignore_errors=True) + # Do not surface stderr verbatim — it can contain the authed URL. + raise click.ClickException( + f"host-side git clone failed (exit {exc.returncode})" + ) from None + return tmp_root, src_dir + + def _destroy_clone( backend: VmBackend | None, handle: VmHandle | None, @@ -232,8 +272,20 @@ def _destroy_clone( @click.option("--configuration", default="Release", show_default=True) @click.option("--build-command", "build_command", default="") @click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) -@click.option("--submodules", is_flag=True, default=False) -@click.option("--use-git-clone", "use_git_clone", is_flag=True, default=False) +@click.option( + "--submodules/--no-submodules", + "submodules", + default=True, + show_default=True, + help="Recurse submodules when cloning (default: on).", +) +@click.option( + "--use-git-clone/--host-clone", + "use_git_clone", + default=True, + show_default=True, + help="In-guest git clone (default) vs host-side clone + zip transfer.", +) @click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) @click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) @click.option( @@ -326,9 +378,8 @@ def job( Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``. """ - # Reserved for future host-side clone variant; the Python port always - # provisions sources via in-guest ``git clone``. - del submodules, use_git_clone + # Transport: in-guest git clone (default) or host-side clone + zip + # transfer (``--host-clone``). Both honour ``--submodules``. # Inject Gitea credentials into HTTP(S) clone URL so git doesn't prompt. authed_repo_url = repo_url @@ -469,42 +520,58 @@ def job( # ── Phase 4/5: build inside guest ────────────────────────────── click.echo("\n[job] Phase 4/5 — invoking remote build") - if guest_os == "linux": - _linux_build( - ip_address=ip_address, - ssh_user=ssh_user, - key_path=key_path, - known_hosts=ssh_known_hosts_file, - workdir="/opt/ci/build", - output_dir="/opt/ci/output", - host_source_dir=None, - clone_url=authed_repo_url, - clone_branch=branch, - clone_commit=commit, - clone_submodules=False, - build_command=build_command, - artifact_source=guest_artifact_source, - extra_env=extra_env, - skip_artifact=skip_artifact, - ) + host_clone_root: Path | None = None + if use_git_clone: + build_clone_url: str | None = authed_repo_url + build_host_src: str | None = None + click.echo("[job] transport: in-guest git clone") else: - _windows_build( - ip_address=ip_address, - credential_target=target, - workdir="C:\\CI\\build", - output_dir="C:\\CI\\output", - host_source_dir=None, - clone_url=authed_repo_url, - clone_branch=branch, - clone_commit=commit, - clone_submodules=False, - build_command=build_command, - artifact_source=guest_artifact_source, - configuration=configuration, - extra_env=extra_env, - skip_artifact=skip_artifact, - use_shared_cache=use_shared_cache, + click.echo("[job] transport: host-side clone + zip transfer") + host_clone_root, src_dir = _host_clone( + authed_repo_url, branch, commit, submodules ) + build_clone_url = None + build_host_src = str(src_dir) + try: + if guest_os == "linux": + _linux_build( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + workdir="/opt/ci/build", + output_dir="/opt/ci/output", + host_source_dir=build_host_src, + clone_url=build_clone_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + extra_env=extra_env, + skip_artifact=skip_artifact, + ) + else: + _windows_build( + ip_address=ip_address, + credential_target=target, + workdir="C:\\CI\\build", + output_dir="C:\\CI\\output", + host_source_dir=build_host_src, + clone_url=build_clone_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + configuration=configuration, + extra_env=extra_env, + skip_artifact=skip_artifact, + use_shared_cache=use_shared_cache, + ) + finally: + if host_clone_root is not None: + shutil.rmtree(host_clone_root, ignore_errors=True) phase_timings.append(("build", _now() - phase_start)) phase_start = _now() diff --git a/tests/python/test_commands_job.py b/tests/python/test_commands_job.py index 1c9242a..814fcc9 100644 --- a/tests/python/test_commands_job.py +++ b/tests/python/test_commands_job.py @@ -218,6 +218,77 @@ def test_job_happy_path_linux( assert not any((base_clone).iterdir()) or all(not p.exists() for p in base_clone.iterdir()) +def test_job_default_transport_in_guest_with_submodules( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, clone_base=tmp_path / "v", artifact_base=tmp_path / "a" + ), + ) + assert result.exit_code == 0, result.output + kw = calls["linux_build"][0] + # Default: in-guest clone (clone_url set, no host source) + submodules. + assert kw["host_source_dir"] is None + assert kw["clone_url"] and "repo.git" in kw["clone_url"] + assert kw["clone_submodules"] is True + + +def test_job_host_clone_transport( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + root = tmp_path / "hostclone" + src = root / "src" + src.mkdir(parents=True) + + def _fake_host_clone( + repo_url: str, branch: str, commit: str, submodules: bool + ) -> tuple[Path, Path]: + assert submodules is True + return root, src + + monkeypatch.setattr(job_module, "_host_clone", _fake_host_clone) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "v", + artifact_base=tmp_path / "a", + extra=["--host-clone"], + ), + ) + assert result.exit_code == 0, result.output + kw = calls["linux_build"][0] + assert kw["host_source_dir"] == str(src) + assert kw["clone_url"] is None + assert kw["clone_submodules"] is True + # Host-side temp clone removed after the build. + assert not root.exists() + + +def test_job_no_submodules_flag( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path +) -> None: + backend = _FakeBackend(running_after=1) + calls = _patch_common(monkeypatch, backend) + result = CliRunner().invoke( + cli, + _common_args( + template_vmx, + clone_base=tmp_path / "v", + artifact_base=tmp_path / "a", + extra=["--no-submodules"], + ), + ) + assert result.exit_code == 0, result.output + assert calls["linux_build"][0]["clone_submodules"] is False + + def test_job_skip_artifact_branch( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path ) -> None: