"""``job`` sub-command — full CI orchestration. Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a template snapshot, wait for the guest to become reachable, run the build inside the guest, optionally collect artifacts, and **always** destroy the clone (even on failure or ``KeyboardInterrupt``). Composition over re-spawn: this module imports the in-process helpers from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather than re-invoking ``python -m ci_orchestrator`` for each phase. Phase C hook: the backend is selected via :func:`load_backend` so that an ESXi backend can be plugged in by changing ``[backend].type`` in ``config.toml``. ``WorkstationVmrunBackend`` is never imported directly. """ from __future__ import annotations import json import re import shutil import subprocess import sys import tempfile import time import uuid from datetime import UTC, datetime from pathlib import Path from typing import TYPE_CHECKING import click from ci_orchestrator.backends import load_backend from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable from ci_orchestrator.backends.protocol import VmHandle from ci_orchestrator.commands.artifacts import ( _linux_collect, _windows_collect, _write_manifest, ) from ci_orchestrator.commands.build import _linux_build, _windows_build from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir from ci_orchestrator.config import Config, load_config from ci_orchestrator.credentials import KeyringCredentialStore from ci_orchestrator.transport.errors import TransportError from ci_orchestrator.transport.ssh import SshTransport from ci_orchestrator.transport.winrm import WinRmTransport if TYPE_CHECKING: # pragma: no cover from ci_orchestrator.backends.protocol import VmBackend _VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") _GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse") # ────────────────────────────────────────────────────────── helpers def _now() -> float: return time.monotonic() def _read_guest_os_from_vmx(vmx_path: Path) -> str: """Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line.""" try: text = vmx_path.read_text(encoding="utf-8", errors="replace") except OSError: return "windows" m = re.search(r'guestOS\s*=\s*"([^"]+)"', text) if not m: return "windows" detected = m.group(1).lower() if any(h in detected for h in _GUEST_OS_LINUX_HINTS): return "linux" return "windows" def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None: """Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0.""" if cpu <= 0 and memory_mb <= 0: return lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines() out: list[str] = [] seen_cpu = False seen_mem = False for line in lines: low = line.strip().lower() if cpu > 0 and low.startswith("numvcpus"): out.append(f'numvcpus = "{cpu}"') seen_cpu = True elif memory_mb > 0 and low.startswith("memsize"): out.append(f'memsize = "{memory_mb}"') seen_mem = True else: out.append(line) if cpu > 0 and not seen_cpu: out.append(f'numvcpus = "{cpu}"') if memory_mb > 0 and not seen_mem: out.append(f'memsize = "{memory_mb}"') vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8") def _parse_extra_env_json(raw: str) -> dict[str, str]: """Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict.""" raw = (raw or "").strip() if not raw or raw == "{}": return {} try: loaded = json.loads(raw) except json.JSONDecodeError as exc: raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc if not isinstance(loaded, dict): raise click.BadParameter("--extra-env-json must be a JSON object.") out: dict[str, str] = {} for key, value in loaded.items(): if not isinstance(key, str) or not _VALID_ENV_KEY.match(key): raise click.BadParameter( f"--extra-env-json key {key!r} is not a valid env var name." ) out[key] = "" if value is None else str(value) return out def _wait_running( backend: VmBackend, handle: VmHandle, deadline: float, poll: float ) -> bool: while _now() < deadline: try: if backend.is_running(handle): return True except BackendError as exc: click.echo(f"[job] backend probe error: {exc}", err=True) time.sleep(poll) return False def _wait_for_ip( backend: VmBackend, handle: VmHandle, deadline: float, poll: float ) -> str | None: while (now := _now()) < deadline: call_timeout = min(poll, deadline - now) try: ip = backend.get_ip(handle, timeout=call_timeout) except BackendError: ip = None if ip: return ip return None def _probe_transport( *, guest_os: str, ip_address: str, deadline: float, poll: float, credential_target: str, ssh_user: str, ssh_key_path: str | None, ssh_known_hosts: str | None, ) -> bool: """Loop until the guest answers WinRM (Windows) or SSH (Linux).""" if guest_os == "windows": cred = KeyringCredentialStore().get(credential_target) while _now() < deadline: try: with WinRmTransport(ip_address, cred.username, cred.password) as t: if t.is_ready(): return True except TransportError as exc: click.echo(f"[job] WinRM probe failed: {exc}", err=True) time.sleep(poll) return False while _now() < deadline: try: with SshTransport( ip_address, username=ssh_user, key_path=ssh_key_path, known_hosts=ssh_known_hosts, ) as t: if t.is_ready(): return True except TransportError as exc: click.echo(f"[job] SSH probe failed: {exc}", err=True) time.sleep(poll) return False def _make_clone_name(job_id: str) -> str: timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") # uuid4 suffix avoids collisions when several clones share a JobId. suffix = uuid.uuid4().hex[:6] return f"Clone_{job_id}_{timestamp}_{suffix}" def _redact_url_creds(text: str) -> str: """Mask ``scheme://user:pass@host`` credentials in arbitrary text.""" return re.sub(r"(\w+://)[^/\s:@]+:[^/\s@]+@", r"\1***@", text) def _host_clone( repo_url: str, branch: str, commit: str, submodules: bool ) -> tuple[Path, Path]: """Clone the repo on the host into a temp dir (host-side transport). Returns ``(tmp_root, src_dir)``. ``src_dir`` is passed as ``host_source_dir`` to the build (zipped and transferred to the guest). The caller must remove ``tmp_root`` when done. git stderr is surfaced with embedded URL credentials redacted. """ tmp_root = Path(tempfile.mkdtemp(prefix="ci-host-clone-")) src_dir = tmp_root / "src" clone = ["git", "clone", "--depth", "1", "--branch", branch] if submodules: clone.append("--recurse-submodules") clone += [repo_url, str(src_dir)] try: subprocess.run(clone, check=True, capture_output=True, text=True) if commit: subprocess.run( ["git", "-C", str(src_dir), "fetch", "--depth", "1", "origin", commit], check=True, capture_output=True, text=True, ) subprocess.run( ["git", "-C", str(src_dir), "checkout", commit], check=True, capture_output=True, text=True, ) except subprocess.CalledProcessError as exc: shutil.rmtree(tmp_root, ignore_errors=True) detail = _redact_url_creds((exc.stderr or "").strip()) tail = " | ".join(detail.splitlines()[-3:]) if detail else "(no stderr)" raise click.ClickException( f"host-side git clone failed (exit {exc.returncode}): {tail}" ) from None return tmp_root, src_dir def _destroy_clone( backend: VmBackend | None, handle: VmHandle | None, clone_dir: Path | None, ) -> None: """Final cleanup. Tolerates every error — must never raise.""" if backend is not None and handle is not None: try: _stop_with_fallback(backend, handle) except Exception as exc: # pragma: no cover - belt and braces click.echo(f"[job] stop ignored: {exc}", err=True) try: backend.delete(handle) except BackendError as exc: click.echo( f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.", err=True, ) except Exception as exc: # pragma: no cover click.echo(f"[job] delete ignored: {exc}", err=True) if clone_dir is not None and clone_dir.exists(): if _try_remove_dir(clone_dir): click.echo(f"[job] clone directory removed: {clone_dir}") else: click.echo( f"[job] could not fully remove: {clone_dir}", err=True ) # ────────────────────────────────────────────────────────── command @click.command("job") @click.option("--job-id", "job_id", required=True) @click.option("--repo-url", "repo_url", required=True) @click.option("--branch", required=True) @click.option("--commit", default="") @click.option("--configuration", default="Release", show_default=True) @click.option("--build-command", "build_command", default="") @click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) @click.option( "--submodules/--no-submodules", "submodules", default=True, show_default=True, help="Recurse submodules when cloning (default: on).", ) @click.option( "--use-git-clone/--host-clone", "use_git_clone", default=True, show_default=True, help="In-guest git clone (default) vs host-side clone + zip transfer.", ) @click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) @click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) @click.option( "--gitea-credential-target", "gitea_credential_target", default="GiteaPAT", show_default=True, ) @click.option( "--template-path", "template_path", default=None, help="Template VMX path (or backend-specific identifier).", ) @click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True) @click.option("--clone-base-dir", "clone_base_dir", default=None) @click.option("--artifact-base-dir", "artifact_base_dir", default=None) @click.option( "--guest-os", type=click.Choice(["windows", "linux", "auto"], case_sensitive=False), default="auto", show_default=True, ) @click.option( "--guest-credential-target", "guest_credential_target", default=None, help="Override config.guest_cred_target (Windows).", ) @click.option("--ssh-key-path", "ssh_key_path", default=None) @click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) @click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None) @click.option( "--extra-env-json", "extra_env_json", default="", help="JSON object of env vars to inject into the guest before the build.", ) @click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0) @click.option( "--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0 ) @click.option( "--ready-timeout", "ready_timeout", type=click.FloatRange(10.0, 3600.0), default=600.0, show_default=True, help="Total seconds to wait for VM running + IP + transport ready.", ) @click.option( "--poll-interval", "poll_interval", type=click.FloatRange(1.0, 60.0), default=5.0, show_default=True, ) @click.option("--vmrun-path", "vmrun_path", default=None) def job( job_id: str, repo_url: str, branch: str, commit: str, configuration: str, build_command: str, guest_artifact_source: str, submodules: bool, use_git_clone: bool, use_shared_cache: bool, skip_artifact: bool, gitea_credential_target: str, template_path: str | None, snapshot_name: str, clone_base_dir: str | None, artifact_base_dir: str | None, guest_os: str, guest_credential_target: str | None, ssh_key_path: str | None, ssh_user: str, ssh_known_hosts_file: str | None, extra_env_json: str, guest_cpu: int, guest_memory_mb: int, ready_timeout: float, poll_interval: float, vmrun_path: str | None, ) -> None: """Run the full CI pipeline against an ephemeral build VM. Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``. """ # Transport: in-guest git clone (default) or host-side clone + zip # transfer (``--host-clone``). Both honour ``--submodules``. # Inject Gitea credentials into HTTP(S) clone URL so git doesn't prompt. authed_repo_url = repo_url if repo_url and repo_url.startswith(("http://", "https://")): from urllib.parse import urlparse, urlunparse try: gitea_cred = KeyringCredentialStore().get(gitea_credential_target) parsed = urlparse(repo_url) host = parsed.hostname or "" port_part = f":{parsed.port}" if parsed.port else "" netloc = f"{gitea_cred.username}:{gitea_cred.password}@{host}{port_part}" authed_repo_url = urlunparse(parsed._replace(netloc=netloc)) except Exception: pass # no credential stored — try unauthenticated (public repo) config: Config = load_config() if not template_path: raise click.ClickException( "--template-path is required." ) template_p = Path(template_path) if not template_p.is_file(): raise click.ClickException(f"Template VMX not found: {template_path}") base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts base_clone_dir.mkdir(parents=True, exist_ok=True) job_artifact_dir = base_artifact_dir / job_id job_artifact_dir.mkdir(parents=True, exist_ok=True) extra_env = _parse_extra_env_json(extra_env_json) # Build backend from config; allow CLI override for vmrun path. if vmrun_path is not None: from ci_orchestrator.backends.workstation import WorkstationVmrunBackend backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path) else: try: backend = load_backend(config) except BackendNotAvailable as exc: raise click.ClickException(f"backend unavailable: {exc}") from exc clone_name = _make_clone_name(job_id) clone_dir = base_clone_dir / clone_name clone_vmx = clone_dir / f"{clone_name}.vmx" handle: VmHandle | None = None started = datetime.now(tz=UTC) phase_timings: list[tuple[str, float]] = [] phase_start = _now() click.echo("=" * 60) click.echo(f"[job] Job : {job_id}") click.echo(f"[job] Repository : {repo_url}") click.echo(f"[job] Branch : {branch}") if commit: click.echo(f"[job] Commit : {commit}") click.echo(f"[job] Template : {template_path}") click.echo(f"[job] Snapshot : {snapshot_name}") click.echo(f"[job] Artifacts : {job_artifact_dir}") click.echo(f"[job] Started : {started.isoformat()}") click.echo("=" * 60) try: # ── Phase 1/5: clone VM ───────────────────────────────────────── click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}") try: handle = backend.clone_linked( template=str(template_p), snapshot=snapshot_name, name=clone_name, destination=str(clone_vmx), ) except BackendError as exc: raise click.ClickException(f"clone failed: {exc}") from exc if not clone_vmx.is_file(): raise click.ClickException( f"backend reported success but clone VMX is missing: {clone_vmx}" ) if guest_cpu > 0 or guest_memory_mb > 0: click.echo( f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB" ) _apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb) phase_timings.append(("clone", _now() - phase_start)) phase_start = _now() # ── Phase 2/5: start VM ───────────────────────────────────────── click.echo("\n[job] Phase 2/5 — starting VM (headless)") try: backend.start(handle, headless=True) except BackendError as exc: raise click.ClickException(f"vmrun start failed: {exc}") from exc # Resolve guest OS now that the clone exists. if guest_os.lower() == "auto": guest_os = _read_guest_os_from_vmx(clone_vmx) click.echo(f"[job] auto-detected guest OS: {guest_os}") else: guest_os = guest_os.lower() deadline = _now() + ready_timeout if not _wait_running(backend, handle, deadline, poll_interval): raise click.ClickException("timeout waiting for VM to be running.") ip_address = _wait_for_ip(backend, handle, deadline, poll_interval) if not ip_address: raise click.ClickException("timeout waiting for guest IP.") click.echo(f"[job] guest IP: {ip_address}") phase_timings.append(("start", _now() - phase_start)) phase_start = _now() # ── Phase 3/5: wait for transport readiness ──────────────────── click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness") target = guest_credential_target or config.guest_cred_target key_path: str | None = ssh_key_path if key_path is None and config.ssh_key_path is not None: key_path = str(config.ssh_key_path) if not _probe_transport( guest_os=guest_os, ip_address=ip_address, deadline=deadline, poll=poll_interval, credential_target=target, ssh_user=ssh_user, ssh_key_path=key_path, ssh_known_hosts=ssh_known_hosts_file, ): raise click.ClickException( "timeout waiting for guest transport to be ready." ) phase_timings.append(("wait-ready", _now() - phase_start)) phase_start = _now() # ── Phase 4/5: build inside guest ────────────────────────────── click.echo("\n[job] Phase 4/5 — invoking remote build") host_clone_root: Path | None = None if use_git_clone: build_clone_url: str | None = authed_repo_url build_host_src: str | None = None click.echo("[job] transport: in-guest git clone") else: click.echo("[job] transport: host-side clone + zip transfer") host_clone_root, src_dir = _host_clone( authed_repo_url, branch, commit, submodules ) build_clone_url = None build_host_src = str(src_dir) try: if guest_os == "linux": _linux_build( ip_address=ip_address, ssh_user=ssh_user, key_path=key_path, known_hosts=ssh_known_hosts_file, workdir="/opt/ci/build", output_dir="/opt/ci/output", host_source_dir=build_host_src, clone_url=build_clone_url, clone_branch=branch, clone_commit=commit, clone_submodules=submodules, build_command=build_command, artifact_source=guest_artifact_source, extra_env=extra_env, skip_artifact=skip_artifact, ) else: _windows_build( ip_address=ip_address, credential_target=target, workdir="C:\\CI\\build", output_dir="C:\\CI\\output", host_source_dir=build_host_src, clone_url=build_clone_url, clone_branch=branch, clone_commit=commit, clone_submodules=submodules, build_command=build_command, artifact_source=guest_artifact_source, configuration=configuration, extra_env=extra_env, skip_artifact=skip_artifact, use_shared_cache=use_shared_cache, ) finally: if host_clone_root is not None: shutil.rmtree(host_clone_root, ignore_errors=True) phase_timings.append(("build", _now() - phase_start)) phase_start = _now() # ── Phase 5/5: collect artifacts ─────────────────────────────── if skip_artifact: click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)") else: click.echo("\n[job] Phase 5/5 — collecting artifacts") if guest_os == "linux": _linux_collect( ip_address=ip_address, ssh_user=ssh_user, key_path=key_path, known_hosts=ssh_known_hosts_file, guest_path="/opt/ci/output", host_dir=job_artifact_dir, ) else: _windows_collect( ip_address=ip_address, credential_target=target, guest_path="C:\\CI\\output", host_dir=job_artifact_dir, include_logs=True, ) try: count = _write_manifest(job_artifact_dir, job_id, commit) click.echo(f"[job] manifest.json written ({count} file(s))") except OSError as exc: click.echo(f"[job] manifest write failed: {exc}", err=True) phase_timings.append(("artifacts", _now() - phase_start)) elapsed = (datetime.now(tz=UTC) - started).total_seconds() click.echo("\n" + "=" * 60) click.echo("[job] phase durations:") for name, secs in phase_timings: click.echo(f" {name:<14} {int(secs):>5}s") click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s") click.echo("=" * 60) click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s") click.echo(f"[job] Artifacts: {job_artifact_dir}") click.echo("=" * 60) except KeyboardInterrupt: click.echo("\n[job] interrupted by user; cleaning up...", err=True) _destroy_clone(backend, handle, clone_dir) sys.exit(130) except click.ClickException: _destroy_clone(backend, handle, clone_dir) raise except Exception as exc: click.echo(f"\n[job] FAILURE: {exc}", err=True) _destroy_clone(backend, handle, clone_dir) raise click.ClickException(str(exc)) from exc else: _destroy_clone(backend, handle, clone_dir) __all__ = ["job"]