feat(a4): port Invoke-CIJob.ps1 to ci_orchestrator job

Phase A4 of plans/implementation-plan-A-B.md. Implements the full job orchestrator (clone -> start -> wait -> probe -> build -> collect -> guaranteed cleanup) as a new commands/job.py click command, registered under python -m ci_orchestrator job. Backend selection goes through backends.load_backend(config) so Phase C can swap in remote drivers without touching the command. The legacy scripts/Invoke-CIJob.ps1 is replaced by a thin PS 5.1 shim that delegates to the Python CLI; tests/python/test_commands_job.py adds 13 cases covering Linux/Windows happy paths, override application, skip-artifact, and cleanup on every failure mode.
This commit is contained in:
2026-05-14 17:29:56 +02:00
parent 816a15503e
commit f5091d0903
4 changed files with 960 additions and 682 deletions
+550
View File
@@ -0,0 +1,550 @@
"""``job`` sub-command — full CI orchestration.
Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a
template snapshot, wait for the guest to become reachable, run the build
inside the guest, optionally collect artifacts, and **always** destroy the
clone (even on failure or ``KeyboardInterrupt``).
Composition over re-spawn: this module imports the in-process helpers
from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather
than re-invoking ``python -m ci_orchestrator`` for each phase.
Phase C hook: the backend is selected via :func:`load_backend` so that an
ESXi backend can be plugged in by changing ``[backend].type`` in
``config.toml``. ``WorkstationVmrunBackend`` is never imported directly.
"""
from __future__ import annotations
import json
import re
import sys
import time
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING
import click
from ci_orchestrator.backends import load_backend
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
from ci_orchestrator.backends.protocol import VmHandle
from ci_orchestrator.commands.artifacts import (
_linux_collect,
_windows_collect,
_write_manifest,
)
from ci_orchestrator.commands.build import _linux_build, _windows_build
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
from ci_orchestrator.config import Config, load_config
from ci_orchestrator.credentials import KeyringCredentialStore
from ci_orchestrator.transport.errors import TransportError
from ci_orchestrator.transport.ssh import SshTransport
from ci_orchestrator.transport.winrm import WinRmTransport
if TYPE_CHECKING: # pragma: no cover
from ci_orchestrator.backends.protocol import VmBackend
_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse")
# ────────────────────────────────────────────────────────── helpers
def _now() -> float:
return time.monotonic()
def _read_guest_os_from_vmx(vmx_path: Path) -> str:
"""Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line."""
try:
text = vmx_path.read_text(encoding="utf-8", errors="replace")
except OSError:
return "windows"
m = re.search(r'guestOS\s*=\s*"([^"]+)"', text)
if not m:
return "windows"
detected = m.group(1).lower()
if any(h in detected for h in _GUEST_OS_LINUX_HINTS):
return "linux"
return "windows"
def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None:
"""Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0."""
if cpu <= 0 and memory_mb <= 0:
return
lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
out: list[str] = []
seen_cpu = False
seen_mem = False
for line in lines:
low = line.strip().lower()
if cpu > 0 and low.startswith("numvcpus"):
out.append(f'numvcpus = "{cpu}"')
seen_cpu = True
elif memory_mb > 0 and low.startswith("memsize"):
out.append(f'memsize = "{memory_mb}"')
seen_mem = True
else:
out.append(line)
if cpu > 0 and not seen_cpu:
out.append(f'numvcpus = "{cpu}"')
if memory_mb > 0 and not seen_mem:
out.append(f'memsize = "{memory_mb}"')
vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8")
def _parse_extra_env_json(raw: str) -> dict[str, str]:
"""Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict."""
raw = (raw or "").strip()
if not raw or raw == "{}":
return {}
try:
loaded = json.loads(raw)
except json.JSONDecodeError as exc:
raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc
if not isinstance(loaded, dict):
raise click.BadParameter("--extra-env-json must be a JSON object.")
out: dict[str, str] = {}
for key, value in loaded.items():
if not isinstance(key, str) or not _VALID_ENV_KEY.match(key):
raise click.BadParameter(
f"--extra-env-json key {key!r} is not a valid env var name."
)
out[key] = "" if value is None else str(value)
return out
def _wait_running(
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
) -> bool:
while _now() < deadline:
try:
if backend.is_running(handle):
return True
except BackendError as exc:
click.echo(f"[job] backend probe error: {exc}", err=True)
time.sleep(poll)
return False
def _wait_for_ip(
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
) -> str | None:
while _now() < deadline:
try:
ip = backend.get_ip(handle)
except BackendError:
ip = None
if ip:
return ip
time.sleep(poll)
return None
def _probe_transport(
*,
guest_os: str,
ip_address: str,
deadline: float,
poll: float,
credential_target: str,
ssh_user: str,
ssh_key_path: str | None,
ssh_known_hosts: str | None,
) -> bool:
"""Loop until the guest answers WinRM (Windows) or SSH (Linux)."""
if guest_os == "windows":
cred = KeyringCredentialStore().get(credential_target)
while _now() < deadline:
try:
with WinRmTransport(ip_address, cred.username, cred.password) as t:
if t.is_ready():
return True
except TransportError as exc:
click.echo(f"[job] WinRM probe failed: {exc}", err=True)
time.sleep(poll)
return False
while _now() < deadline:
try:
with SshTransport(
ip_address,
username=ssh_user,
key_path=ssh_key_path,
known_hosts=ssh_known_hosts,
) as t:
if t.is_ready():
return True
except TransportError as exc:
click.echo(f"[job] SSH probe failed: {exc}", err=True)
time.sleep(poll)
return False
def _make_clone_name(job_id: str) -> str:
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
# uuid4 suffix avoids collisions when several clones share a JobId.
suffix = uuid.uuid4().hex[:6]
return f"Clone_{job_id}_{timestamp}_{suffix}"
def _destroy_clone(
backend: VmBackend | None,
handle: VmHandle | None,
clone_dir: Path | None,
) -> None:
"""Final cleanup. Tolerates every error — must never raise."""
if backend is not None and handle is not None:
try:
_stop_with_fallback(backend, handle)
except Exception as exc: # pragma: no cover - belt and braces
click.echo(f"[job] stop ignored: {exc}", err=True)
try:
backend.delete(handle)
except BackendError as exc:
click.echo(
f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.",
err=True,
)
except Exception as exc: # pragma: no cover
click.echo(f"[job] delete ignored: {exc}", err=True)
if clone_dir is not None and clone_dir.exists():
if _try_remove_dir(clone_dir):
click.echo(f"[job] clone directory removed: {clone_dir}")
else:
click.echo(
f"[job] could not fully remove: {clone_dir}", err=True
)
# ────────────────────────────────────────────────────────── command
@click.command("job")
@click.option("--job-id", "job_id", required=True)
@click.option("--repo-url", "repo_url", required=True)
@click.option("--branch", required=True)
@click.option("--commit", default="")
@click.option("--configuration", default="Release", show_default=True)
@click.option("--build-command", "build_command", default="")
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
@click.option("--submodules", is_flag=True, default=False)
@click.option("--use-git-clone", "use_git_clone", is_flag=True, default=False)
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
@click.option(
"--gitea-credential-target",
"gitea_credential_target",
default="GiteaPAT",
show_default=True,
)
@click.option(
"--template-path",
"template_path",
default=None,
help="Template VMX path (or backend-specific identifier).",
)
@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True)
@click.option("--clone-base-dir", "clone_base_dir", default=None)
@click.option("--artifact-base-dir", "artifact_base_dir", default=None)
@click.option(
"--guest-os",
type=click.Choice(["windows", "linux", "auto"], case_sensitive=False),
default="auto",
show_default=True,
)
@click.option(
"--guest-credential-target",
"guest_credential_target",
default=None,
help="Override config.guest_cred_target (Windows).",
)
@click.option("--ssh-key-path", "ssh_key_path", default=None)
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
@click.option(
"--extra-env-json",
"extra_env_json",
default="",
help="JSON object of env vars to inject into the guest before the build.",
)
@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0)
@click.option(
"--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0
)
@click.option(
"--ready-timeout",
"ready_timeout",
type=click.FloatRange(10.0, 3600.0),
default=600.0,
show_default=True,
help="Total seconds to wait for VM running + IP + transport ready.",
)
@click.option(
"--poll-interval",
"poll_interval",
type=click.FloatRange(1.0, 60.0),
default=5.0,
show_default=True,
)
@click.option("--vmrun-path", "vmrun_path", default=None)
def job(
job_id: str,
repo_url: str,
branch: str,
commit: str,
configuration: str,
build_command: str,
guest_artifact_source: str,
submodules: bool,
use_git_clone: bool,
use_shared_cache: bool,
skip_artifact: bool,
gitea_credential_target: str,
template_path: str | None,
snapshot_name: str,
clone_base_dir: str | None,
artifact_base_dir: str | None,
guest_os: str,
guest_credential_target: str | None,
ssh_key_path: str | None,
ssh_user: str,
ssh_known_hosts_file: str | None,
extra_env_json: str,
guest_cpu: int,
guest_memory_mb: int,
ready_timeout: float,
poll_interval: float,
vmrun_path: str | None,
) -> None:
"""Run the full CI pipeline against an ephemeral build VM.
Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is
guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``.
"""
# Reserved for future host-side clone variant; the Python port always
# provisions sources via in-guest ``git clone``.
del submodules, use_git_clone, gitea_credential_target
config: Config = load_config()
if not template_path:
raise click.ClickException(
"--template-path is required."
)
template_p = Path(template_path)
if not template_p.is_file():
raise click.ClickException(f"Template VMX not found: {template_path}")
base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts
base_clone_dir.mkdir(parents=True, exist_ok=True)
job_artifact_dir = base_artifact_dir / job_id
job_artifact_dir.mkdir(parents=True, exist_ok=True)
extra_env = _parse_extra_env_json(extra_env_json)
# Build backend from config; allow CLI override for vmrun path.
if vmrun_path is not None:
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
else:
try:
backend = load_backend(config)
except BackendNotAvailable as exc:
raise click.ClickException(f"backend unavailable: {exc}") from exc
clone_name = _make_clone_name(job_id)
clone_dir = base_clone_dir / clone_name
clone_vmx = clone_dir / f"{clone_name}.vmx"
handle: VmHandle | None = None
started = datetime.now(tz=UTC)
phase_timings: list[tuple[str, float]] = []
phase_start = _now()
click.echo("=" * 60)
click.echo(f"[job] Job : {job_id}")
click.echo(f"[job] Repository : {repo_url}")
click.echo(f"[job] Branch : {branch}")
if commit:
click.echo(f"[job] Commit : {commit}")
click.echo(f"[job] Template : {template_path}")
click.echo(f"[job] Snapshot : {snapshot_name}")
click.echo(f"[job] Artifacts : {job_artifact_dir}")
click.echo(f"[job] Started : {started.isoformat()}")
click.echo("=" * 60)
try:
# ── Phase 1/5: clone VM ─────────────────────────────────────────
click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}")
try:
handle = backend.clone_linked(
template=str(template_p),
snapshot=snapshot_name,
name=clone_name,
destination=str(clone_vmx),
)
except BackendError as exc:
raise click.ClickException(f"clone failed: {exc}") from exc
if not clone_vmx.is_file():
raise click.ClickException(
f"backend reported success but clone VMX is missing: {clone_vmx}"
)
if guest_cpu > 0 or guest_memory_mb > 0:
click.echo(
f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB"
)
_apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)
phase_timings.append(("clone", _now() - phase_start))
phase_start = _now()
# ── Phase 2/5: start VM ─────────────────────────────────────────
click.echo("\n[job] Phase 2/5 — starting VM (headless)")
try:
backend.start(handle, headless=True)
except BackendError as exc:
raise click.ClickException(f"vmrun start failed: {exc}") from exc
# Resolve guest OS now that the clone exists.
if guest_os.lower() == "auto":
guest_os = _read_guest_os_from_vmx(clone_vmx)
click.echo(f"[job] auto-detected guest OS: {guest_os}")
else:
guest_os = guest_os.lower()
deadline = _now() + ready_timeout
if not _wait_running(backend, handle, deadline, poll_interval):
raise click.ClickException("timeout waiting for VM to be running.")
ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
if not ip_address:
raise click.ClickException("timeout waiting for guest IP.")
click.echo(f"[job] guest IP: {ip_address}")
phase_timings.append(("start", _now() - phase_start))
phase_start = _now()
# ── Phase 3/5: wait for transport readiness ────────────────────
click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness")
target = guest_credential_target or config.guest_cred_target
key_path: str | None = ssh_key_path
if key_path is None and config.ssh_key_path is not None:
key_path = str(config.ssh_key_path)
if not _probe_transport(
guest_os=guest_os,
ip_address=ip_address,
deadline=deadline,
poll=poll_interval,
credential_target=target,
ssh_user=ssh_user,
ssh_key_path=key_path,
ssh_known_hosts=ssh_known_hosts_file,
):
raise click.ClickException(
"timeout waiting for guest transport to be ready."
)
phase_timings.append(("wait-ready", _now() - phase_start))
phase_start = _now()
# ── Phase 4/5: build inside guest ──────────────────────────────
click.echo("\n[job] Phase 4/5 — invoking remote build")
if guest_os == "linux":
_linux_build(
ip_address=ip_address,
ssh_user=ssh_user,
key_path=key_path,
known_hosts=ssh_known_hosts_file,
workdir="/opt/ci/build",
output_dir="/opt/ci/output",
host_source_dir=None,
clone_url=repo_url,
clone_branch=branch,
clone_commit=commit,
clone_submodules=False,
build_command=build_command,
artifact_source=guest_artifact_source,
extra_env=extra_env,
skip_artifact=skip_artifact,
)
else:
_windows_build(
ip_address=ip_address,
credential_target=target,
workdir="C:\\CI\\build",
artifact_zip="C:\\CI\\output\\artifacts.zip",
host_source_dir=None,
clone_url=repo_url,
clone_branch=branch,
clone_commit=commit,
clone_submodules=False,
build_command=build_command,
artifact_source=guest_artifact_source,
configuration=configuration,
extra_env=extra_env,
skip_artifact=skip_artifact,
use_shared_cache=use_shared_cache,
)
phase_timings.append(("build", _now() - phase_start))
phase_start = _now()
# ── Phase 5/5: collect artifacts ───────────────────────────────
if skip_artifact:
click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)")
else:
click.echo("\n[job] Phase 5/5 — collecting artifacts")
if guest_os == "linux":
_linux_collect(
ip_address=ip_address,
ssh_user=ssh_user,
key_path=key_path,
known_hosts=ssh_known_hosts_file,
guest_path="/opt/ci/output",
host_dir=job_artifact_dir,
)
else:
_windows_collect(
ip_address=ip_address,
credential_target=target,
guest_path="C:\\CI\\output\\artifacts.zip",
host_dir=job_artifact_dir,
include_logs=True,
)
try:
count = _write_manifest(job_artifact_dir, job_id, commit)
click.echo(f"[job] manifest.json written ({count} file(s))")
except OSError as exc:
click.echo(f"[job] manifest write failed: {exc}", err=True)
phase_timings.append(("artifacts", _now() - phase_start))
elapsed = (datetime.now(tz=UTC) - started).total_seconds()
click.echo("\n" + "=" * 60)
click.echo("[job] phase durations:")
for name, secs in phase_timings:
click.echo(f" {name:<14} {int(secs):>5}s")
click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s")
click.echo("=" * 60)
click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s")
click.echo(f"[job] Artifacts: {job_artifact_dir}")
click.echo("=" * 60)
except KeyboardInterrupt:
click.echo("\n[job] interrupted by user; cleaning up...", err=True)
_destroy_clone(backend, handle, clone_dir)
sys.exit(130)
except click.ClickException:
_destroy_clone(backend, handle, clone_dir)
raise
except Exception as exc:
click.echo(f"\n[job] FAILURE: {exc}", err=True)
_destroy_clone(backend, handle, clone_dir)
raise click.ClickException(str(exc)) from exc
else:
_destroy_clone(backend, handle, clone_dir)
__all__ = ["job"]