feat(cli): port burn-in/smoke/validate/creds to Python (Phase C1-C6)

Add four native command groups so the Linux host no longer needs pwsh for manual ops. __main__ registers all four at once, so they ship together. - bench run (C2): N concurrent jobs in-process via concurrent.futures, per-round orphan-clone + stale-lock assertions. Replaces Test-CapacityBurnIn.ps1 + Start-BurnInTest*.ps1. No Start-Job/pwsh/$IsWindows. - bench measure (C3): clone/start/IP/transport/destroy timings appended to benchmark.jsonl (legacy field names kept). Replaces Measure-CIBenchmark.ps1. - smoke run (C4): one E2E job; asserts exit 0 + artifact dir + job/success event. ns7zip/nsinnounp presets. Replaces Test-Smoke.ps1 + Test-*Build-Linux.ps1. - validate host (C5): vmrun/snapshot/keyring/permissions/systemd checks. - validate guest(C6): transport probe with explicit failure cause. - creds set (C6): keyring writer matching credentials.py read scheme; password via stdin/prompt only. Replaces Set-CIGuestCredential.ps1. All groups import backends.load_backend only (Phase D ESXi path stays open). Suite green, coverage 95.10% (gate 90%); new modules at 100%. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-07 20:15:10 +02:00
parent 8f1b3b7eb1
commit d63fca5967
9 changed files with 2934 additions and 0 deletions
@@ -0,0 +1,351 @@
+"""``smoke`` sub-commands.
+
+Phase C4 ports the manual smoke / E2E build tooling to Python so the Linux
+host no longer needs ``pwsh``:
+
+* ``smoke run`` — end-to-end no-op (or real build) job, replacing
+  ``Test-Smoke.ps1`` and ``Test-*Build-Linux.ps1``.
+
+The command runs **one** full end-to-end job through the in-process
+``job`` orchestrator (reusing ``commands/job.py`` — no re-spawn of
+``python -m ci_orchestrator``) and then asserts the three smoke
+invariants of ``Test-Smoke.ps1``:
+
+1. the pipeline exited 0,
+2. the per-job artifact directory was created, and
+3. a ``job``/``success`` event is present in the ``invoke-ci.jsonl``
+   event log (the same JSONL format ``report job`` consumes).
+
+Because the in-process ``job`` command does not itself emit the JSONL
+event log (that was the job of the retired ``Invoke-CIJob.ps1``), this
+command writes the ``job``/``start`` and ``job``/``success`` events
+around the orchestration, exactly as the PowerShell wrapper did.
+
+Design rule: import ``backends.load_backend`` and the existing transports
+only — never ``WorkstationVmrunBackend`` directly (keeps the Phase D ESXi
+path open).
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+
+import click
+
+from ci_orchestrator.commands.job import job as job_command
+from ci_orchestrator.config import Config, load_config
+
+# ─────────────────────────────────────────────────────── OS presets
+
+# Per-guest defaults mirroring ``Test-Smoke.ps1``: a trivial build that
+# writes a marker file into the standard output directory, plus the
+# matching template / snapshot / artifact-source for the OS family.
+_LINUX_TEMPLATE = "/var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx"
+_WINDOWS_TEMPLATE = r"F:\CI\Templates\WinBuild2025\WinBuild2025.vmx"
+
+_NOOP_BUILD_LINUX = "mkdir -p /opt/ci/output && echo smoke-ok > /opt/ci/output/smoke.txt"
+_NOOP_BUILD_WINDOWS = (
+    "New-Item -ItemType Directory -Path C:\\CI\\output -Force | Out-Null; "
+    'Set-Content C:\\CI\\output\\smoke.txt "smoke-ok"'
+)
+
+# Real Linux build E2E presets (ports of ``Test-*Build-Linux.ps1``).
+# Exposed via ``--preset`` so the no-op default is never overridden by a
+# hard-coded ``F:\`` Windows path. Each preset supplies a build command
+# and the in-guest artifact source directory.
+_PRESETS: dict[str, tuple[str, str]] = {
+    "ns7zip": (
+        "python3 build_plugin.py --host linux --7zip-version 26.01 "
+        "--configs x64-unicode --verbose",
+        "plugins",
+    ),
+    "nsinnounp": (
+        "python3 build_plugin.py --final --dist-dir dist",
+        "dist",
+    ),
+}
+
+
+def _default_template(guest_os: str) -> str:
+    return _LINUX_TEMPLATE if guest_os == "linux" else _WINDOWS_TEMPLATE
+
+
+def _default_snapshot(guest_os: str) -> str:
+    return "BaseClean-Linux" if guest_os == "linux" else "BaseClean"
+
+
+def _noop_build_command(guest_os: str) -> tuple[str, str]:
+    """Return ``(build_command, guest_artifact_source)`` for the marker job."""
+    if guest_os == "linux":
+        return _NOOP_BUILD_LINUX, "/opt/ci/output"
+    return _NOOP_BUILD_WINDOWS, "C:\\CI\\output"
+
+
+def _utcnow_iso() -> str:
+    return datetime.now(tz=UTC).isoformat()
+
+
+def _write_event(
+    jsonl_path: Path,
+    *,
+    job_id: str,
+    phase: str,
+    status: str,
+    elapsed_sec: float | None = None,
+    error: str | None = None,
+) -> None:
+    """Append one event to ``invoke-ci.jsonl`` in ``report job`` format."""
+    data: dict[str, object] = {}
+    if elapsed_sec is not None:
+        data["elapsedSec"] = int(elapsed_sec)
+    if error is not None:
+        data["error"] = error
+    record: dict[str, object] = {
+        "jobId": job_id,
+        "phase": phase,
+        "status": status,
+        "ts": _utcnow_iso(),
+        "data": data,
+    }
+    jsonl_path.parent.mkdir(parents=True, exist_ok=True)
+    with jsonl_path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record) + "\n")
+
+
+def _has_job_success(jsonl_path: Path) -> bool:
+    """True iff the JSONL log contains a ``job``/``success`` event."""
+    try:
+        text = jsonl_path.read_text(encoding="utf-8")
+    except OSError:
+        return False
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            obj = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(obj, dict):
+            continue
+        if obj.get("phase") == "job" and obj.get("status") == "success":
+            return True
+    return False
+
+
+# ────────────────────────────────────────────────────────── command
+
+
+@click.group("smoke")
+def smoke() -> None:
+    """End-to-end smoke and build validation."""
+
+
+@smoke.command("run")
+@click.option(
+    "--guest-os",
+    type=click.Choice(["windows", "linux"], case_sensitive=False),
+    default="linux",
+    show_default=True,
+    help="Guest OS family to smoke-test.",
+)
+@click.option(
+    "--build-command",
+    "build_command",
+    default="",
+    help="Build command to run (default: a marker no-op job).",
+)
+@click.option(
+    "--preset",
+    type=click.Choice(sorted(_PRESETS), case_sensitive=False),
+    default=None,
+    help="Real Linux build E2E preset (ns7zip / nsinnounp). Linux only.",
+)
+@click.option(
+    "--repo-url",
+    "repo_url",
+    default="https://example.invalid/smoke-test.git",
+    show_default=True,
+    help="Repository to build (default: a placeholder; the no-op marker "
+    "job never depends on the clone).",
+)
+@click.option("--branch", default="main", show_default=True)
+@click.option(
+    "--job-id",
+    "job_id",
+    default=None,
+    help="Job id (default: smoke-<timestamp>).",
+)
+@click.option(
+    "--template-path",
+    "template_path",
+    default=None,
+    help="Template VMX path (default: per-guest-os standard template).",
+)
+@click.option(
+    "--snapshot-name",
+    "snapshot_name",
+    default=None,
+    help="Snapshot to clone from (default: per-guest-os BaseClean[-Linux]).",
+)
+@click.option(
+    "--ssh-key-path",
+    "ssh_key_path",
+    default=None,
+    help="SSH private key for the guest (Linux).",
+)
+@click.option(
+    "--log-dir",
+    "log_dir",
+    default=None,
+    help="Base directory for invoke-ci.jsonl logs (default: CI_ARTIFACTS).",
+)
+def smoke_run(
+    guest_os: str,
+    build_command: str,
+    preset: str | None,
+    repo_url: str,
+    branch: str,
+    job_id: str | None,
+    template_path: str | None,
+    snapshot_name: str | None,
+    ssh_key_path: str | None,
+    log_dir: str | None,
+) -> None:
+    """Run one end-to-end job and assert success + artifact + event."""
+    guest_os = guest_os.lower()
+
+    if preset is not None:
+        preset = preset.lower()
+        if guest_os != "linux":
+            raise click.UsageError("--preset is only supported with --guest-os linux.")
+        if build_command:
+            raise click.UsageError("--preset and --build-command are mutually exclusive.")
+        preset_cmd, artifact_source = _PRESETS[preset]
+        effective_build_command = preset_cmd
+    elif build_command:
+        effective_build_command = build_command
+        # A custom command writes to the standard output dir for the OS.
+        _, artifact_source = _noop_build_command(guest_os)
+    else:
+        effective_build_command, artifact_source = _noop_build_command(guest_os)
+
+    config: Config = load_config()
+    resolved_job_id = job_id or f"smoke-{datetime.now(tz=UTC).strftime('%Y%m%d-%H%M%S')}"
+    resolved_template = template_path or _default_template(guest_os)
+    resolved_snapshot = snapshot_name or _default_snapshot(guest_os)
+
+    artifact_base: Path = config.paths.artifacts
+    # Logs live alongside (not inside) the artifacts tree so that writing the
+    # event log never accidentally creates the per-job artifact directory and
+    # masks a build that produced no artifacts (mirrors F:\CI\Logs vs
+    # F:\CI\Artifacts in Test-Smoke.ps1).
+    log_base = Path(log_dir) if log_dir else config.paths.root / "logs"
+    job_artifact_dir = artifact_base / resolved_job_id
+    jsonl_path = log_base / resolved_job_id / "invoke-ci.jsonl"
+
+    click.echo("=" * 60)
+    click.echo(f"[smoke] Guest OS   : {guest_os}")
+    click.echo(f"[smoke] Template   : {resolved_template}")
+    click.echo(f"[smoke] Snapshot   : {resolved_snapshot}")
+    click.echo(f"[smoke] Job id     : {resolved_job_id}")
+    if preset is not None:
+        click.echo(f"[smoke] Preset     : {preset}")
+    click.echo(f"[smoke] Artifacts  : {job_artifact_dir}")
+    click.echo("=" * 60)
+
+    _write_event(
+        jsonl_path, job_id=resolved_job_id, phase="job", status="start"
+    )
+
+    start = time.monotonic()
+    failed = False
+    try:
+        job_command.callback(  # type: ignore[misc]
+            job_id=resolved_job_id,
+            repo_url=repo_url,
+            branch=branch,
+            commit="",
+            configuration="Release",
+            build_command=effective_build_command,
+            guest_artifact_source=artifact_source,
+            submodules=True,
+            use_git_clone=True,
+            use_shared_cache=False,
+            skip_artifact=False,
+            use_xvfb=False,
+            gitea_credential_target="GiteaPAT",
+            template_path=resolved_template,
+            snapshot_name=resolved_snapshot,
+            clone_base_dir=None,
+            artifact_base_dir=str(artifact_base),
+            guest_os=guest_os,
+            guest_credential_target=None,
+            ssh_key_path=ssh_key_path,
+            ssh_user="ci_build",
+            ssh_known_hosts_file=None,
+            extra_env_json="",
+            guest_cpu=0,
+            guest_memory_mb=0,
+            ready_timeout=600.0,
+            poll_interval=5.0,
+            vmrun_path=None,
+        )
+    except click.ClickException as exc:
+        failed = True
+        _write_event(
+            jsonl_path,
+            job_id=resolved_job_id,
+            phase="job",
+            status="failure",
+            elapsed_sec=time.monotonic() - start,
+            error=exc.format_message(),
+        )
+        click.echo(f"[smoke] FAIL: pipeline raised: {exc.format_message()}", err=True)
+
+    if not failed:
+        _write_event(
+            jsonl_path,
+            job_id=resolved_job_id,
+            phase="job",
+            status="success",
+            elapsed_sec=time.monotonic() - start,
+        )
+
+    # ── Assert the three smoke invariants ───────────────────────────────
+    click.echo("\n[smoke] Verifying results...")
+    checks_ok = True
+
+    if failed:
+        checks_ok = False
+    else:
+        click.echo("[smoke] OK: pipeline exit 0")
+
+    if job_artifact_dir.is_dir():
+        click.echo(f"[smoke] OK: artifact dir exists: {job_artifact_dir}")
+    else:
+        click.echo(
+            f"[smoke] FAIL: artifact dir not found: {job_artifact_dir}", err=True
+        )
+        checks_ok = False
+
+    if _has_job_success(jsonl_path):
+        click.echo("[smoke] OK: JSONL contains job/success event")
+    else:
+        click.echo(
+            f"[smoke] FAIL: JSONL missing job/success event in: {jsonl_path}",
+            err=True,
+        )
+        checks_ok = False
+
+    click.echo("")
+    if checks_ok:
+        click.echo(f"[smoke] PASSED — {guest_os} smoke test completed successfully.")
+        return
+    raise click.ClickException("smoke test FAILED — one or more checks did not pass.")
+
+
+__all__ = ["smoke"]