local-ci-cd-system/src/ci_orchestrator/commands/job.py

"""``job`` sub-command — full CI orchestration.

Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a
template snapshot, wait for the guest to become reachable, run the build
inside the guest, optionally collect artifacts, and **always** destroy the
clone (even on failure or ``KeyboardInterrupt``).

Composition over re-spawn: this module imports the in-process helpers
from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather
than re-invoking ``python -m ci_orchestrator`` for each phase.

Phase C hook: the backend is selected via :func:`load_backend` so that an
ESXi backend can be plugged in by changing ``[backend].type`` in
``config.toml``. ``WorkstationVmrunBackend`` is never imported directly.
"""

from __future__ import annotations

import json
import re
import shutil
import subprocess
import sys
import tempfile
import time
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import TYPE_CHECKING

import click

from ci_orchestrator.backends import load_backend
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
from ci_orchestrator.backends.protocol import VmHandle
from ci_orchestrator.commands.artifacts import (
    _linux_collect,
    _windows_collect,
    _write_manifest,
)
from ci_orchestrator.commands.build import _linux_build, _windows_build
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
from ci_orchestrator.config import Config, load_config
from ci_orchestrator.credentials import KeyringCredentialStore
from ci_orchestrator.transport.errors import TransportError
from ci_orchestrator.transport.ssh import SshTransport
from ci_orchestrator.transport.winrm import WinRmTransport

if TYPE_CHECKING:  # pragma: no cover
    from ci_orchestrator.backends.protocol import VmBackend


_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse")


# ────────────────────────────────────────────────────────── helpers


def _now() -> float:
    return time.monotonic()


def _read_guest_os_from_vmx(vmx_path: Path) -> str:
    """Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line."""
    try:
        text = vmx_path.read_text(encoding="utf-8", errors="replace")
    except OSError:
        return "windows"
    m = re.search(r'guestOS\s*=\s*"([^"]+)"', text)
    if not m:
        return "windows"
    detected = m.group(1).lower()
    if any(h in detected for h in _GUEST_OS_LINUX_HINTS):
        return "linux"
    return "windows"


def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None:
    """Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0."""
    if cpu <= 0 and memory_mb <= 0:
        return
    lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
    out: list[str] = []
    seen_cpu = False
    seen_mem = False
    for line in lines:
        low = line.strip().lower()
        if cpu > 0 and low.startswith("numvcpus"):
            out.append(f'numvcpus = "{cpu}"')
            seen_cpu = True
        elif memory_mb > 0 and low.startswith("memsize"):
            out.append(f'memsize = "{memory_mb}"')
            seen_mem = True
        else:
            out.append(line)
    if cpu > 0 and not seen_cpu:
        out.append(f'numvcpus = "{cpu}"')
    if memory_mb > 0 and not seen_mem:
        out.append(f'memsize = "{memory_mb}"')
    vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8")


def _parse_extra_env_json(raw: str) -> dict[str, str]:
    """Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict."""
    raw = (raw or "").strip()
    if not raw or raw == "{}":
        return {}
    try:
        loaded = json.loads(raw)
    except json.JSONDecodeError as exc:
        raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc
    if not isinstance(loaded, dict):
        raise click.BadParameter("--extra-env-json must be a JSON object.")
    out: dict[str, str] = {}
    for key, value in loaded.items():
        if not isinstance(key, str) or not _VALID_ENV_KEY.match(key):
            raise click.BadParameter(
                f"--extra-env-json key {key!r} is not a valid env var name."
            )
        out[key] = "" if value is None else str(value)
    return out


def _wait_running(
    backend: VmBackend, handle: VmHandle, deadline: float, poll: float
) -> bool:
    while _now() < deadline:
        try:
            if backend.is_running(handle):
                return True
        except BackendError as exc:
            click.echo(f"[job] backend probe error: {exc}", err=True)
        time.sleep(poll)
    return False


def _wait_for_ip(
    backend: VmBackend, handle: VmHandle, deadline: float, poll: float
) -> str | None:
    while (now := _now()) < deadline:
        call_timeout = min(poll, deadline - now)
        try:
            ip = backend.get_ip(handle, timeout=call_timeout)
        except BackendError:
            ip = None
        if ip:
            return ip
    return None


def _probe_transport(
    *,
    guest_os: str,
    ip_address: str,
    deadline: float,
    poll: float,
    credential_target: str,
    ssh_user: str,
    ssh_key_path: str | None,
    ssh_known_hosts: str | None,
) -> bool:
    """Loop until the guest answers WinRM (Windows) or SSH (Linux)."""
    if guest_os == "windows":
        cred = KeyringCredentialStore().get(credential_target)
        while _now() < deadline:
            try:
                with WinRmTransport(ip_address, cred.username, cred.password) as t:
                    if t.is_ready():
                        return True
            except TransportError as exc:
                click.echo(f"[job] WinRM probe failed: {exc}", err=True)
            time.sleep(poll)
        return False
    while _now() < deadline:
        try:
            with SshTransport(
                ip_address,
                username=ssh_user,
                key_path=ssh_key_path,
                known_hosts=ssh_known_hosts,
            ) as t:
                if t.is_ready():
                    return True
        except TransportError as exc:
            click.echo(f"[job] SSH probe failed: {exc}", err=True)
        time.sleep(poll)
    return False


def _make_clone_name(job_id: str) -> str:
    timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
    # uuid4 suffix avoids collisions when several clones share a JobId.
    suffix = uuid.uuid4().hex[:6]
    return f"Clone_{job_id}_{timestamp}_{suffix}"


def _redact_url_creds(text: str) -> str:
    """Mask ``scheme://user:pass@host`` credentials in arbitrary text."""
    return re.sub(r"(\w+://)[^/\s:@]+:[^/\s@]+@", r"\1***@", text)


def _host_clone(
    repo_url: str, branch: str, commit: str, submodules: bool
) -> tuple[Path, Path]:
    """Clone the repo on the host into a temp dir (host-side transport).

    Returns ``(tmp_root, src_dir)``. ``src_dir`` is passed as
    ``host_source_dir`` to the build (zipped and transferred to the
    guest). The caller must remove ``tmp_root`` when done. git stderr is
    surfaced with embedded URL credentials redacted.
    """
    tmp_root = Path(tempfile.mkdtemp(prefix="ci-host-clone-"))
    src_dir = tmp_root / "src"
    clone = ["git", "clone", "--depth", "1", "--branch", branch]
    if submodules:
        clone.append("--recurse-submodules")
    clone += [repo_url, str(src_dir)]
    try:
        subprocess.run(clone, check=True, capture_output=True, text=True)
        if commit:
            subprocess.run(
                ["git", "-C", str(src_dir), "fetch", "--depth", "1",
                 "origin", commit],
                check=True, capture_output=True, text=True,
            )
            subprocess.run(
                ["git", "-C", str(src_dir), "checkout", commit],
                check=True, capture_output=True, text=True,
            )
    except subprocess.CalledProcessError as exc:
        shutil.rmtree(tmp_root, ignore_errors=True)
        detail = _redact_url_creds((exc.stderr or "").strip())
        tail = " | ".join(detail.splitlines()[-3:]) if detail else "(no stderr)"
        raise click.ClickException(
            f"host-side git clone failed (exit {exc.returncode}): {tail}"
        ) from None
    return tmp_root, src_dir


def _destroy_clone(
    backend: VmBackend | None,
    handle: VmHandle | None,
    clone_dir: Path | None,
) -> None:
    """Final cleanup. Tolerates every error — must never raise."""
    if backend is not None and handle is not None:
        try:
            _stop_with_fallback(backend, handle)
        except Exception as exc:  # pragma: no cover - belt and braces
            click.echo(f"[job] stop ignored: {exc}", err=True)
        try:
            backend.delete(handle)
        except BackendError as exc:
            click.echo(
                f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.",
                err=True,
            )
        except Exception as exc:  # pragma: no cover
            click.echo(f"[job] delete ignored: {exc}", err=True)
    if clone_dir is not None and clone_dir.exists():
        if _try_remove_dir(clone_dir):
            click.echo(f"[job] clone directory removed: {clone_dir}")
        else:
            click.echo(
                f"[job] could not fully remove: {clone_dir}", err=True
            )


# ────────────────────────────────────────────────────────── command


@click.command("job")
@click.option("--job-id", "job_id", required=True)
@click.option("--repo-url", "repo_url", required=True)
@click.option("--branch", required=True)
@click.option("--commit", default="")
@click.option("--configuration", default="Release", show_default=True)
@click.option("--build-command", "build_command", default="")
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
@click.option(
    "--submodules/--no-submodules",
    "submodules",
    default=True,
    show_default=True,
    help="Recurse submodules when cloning (default: on).",
)
@click.option(
    "--use-git-clone/--host-clone",
    "use_git_clone",
    default=True,
    show_default=True,
    help="In-guest git clone (default) vs host-side clone + zip transfer.",
)
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
@click.option(
    "--gitea-credential-target",
    "gitea_credential_target",
    default="GiteaPAT",
    show_default=True,
)
@click.option(
    "--template-path",
    "template_path",
    default=None,
    help="Template VMX path (or backend-specific identifier).",
)
@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True)
@click.option("--clone-base-dir", "clone_base_dir", default=None)
@click.option("--artifact-base-dir", "artifact_base_dir", default=None)
@click.option(
    "--guest-os",
    type=click.Choice(["windows", "linux", "auto"], case_sensitive=False),
    default="auto",
    show_default=True,
)
@click.option(
    "--guest-credential-target",
    "guest_credential_target",
    default=None,
    help="Override config.guest_cred_target (Windows).",
)
@click.option("--ssh-key-path", "ssh_key_path", default=None)
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
@click.option(
    "--extra-env-json",
    "extra_env_json",
    default="",
    help="JSON object of env vars to inject into the guest before the build.",
)
@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0)
@click.option(
    "--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0
)
@click.option(
    "--ready-timeout",
    "ready_timeout",
    type=click.FloatRange(10.0, 3600.0),
    default=600.0,
    show_default=True,
    help="Total seconds to wait for VM running + IP + transport ready.",
)
@click.option(
    "--poll-interval",
    "poll_interval",
    type=click.FloatRange(1.0, 60.0),
    default=5.0,
    show_default=True,
)
@click.option("--vmrun-path", "vmrun_path", default=None)
def job(
    job_id: str,
    repo_url: str,
    branch: str,
    commit: str,
    configuration: str,
    build_command: str,
    guest_artifact_source: str,
    submodules: bool,
    use_git_clone: bool,
    use_shared_cache: bool,
    skip_artifact: bool,
    gitea_credential_target: str,
    template_path: str | None,
    snapshot_name: str,
    clone_base_dir: str | None,
    artifact_base_dir: str | None,
    guest_os: str,
    guest_credential_target: str | None,
    ssh_key_path: str | None,
    ssh_user: str,
    ssh_known_hosts_file: str | None,
    extra_env_json: str,
    guest_cpu: int,
    guest_memory_mb: int,
    ready_timeout: float,
    poll_interval: float,
    vmrun_path: str | None,
) -> None:
    """Run the full CI pipeline against an ephemeral build VM.

    Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is
    guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``.
    """
    # Transport: in-guest git clone (default) or host-side clone + zip
    # transfer (``--host-clone``). Both honour ``--submodules``.

    # Inject Gitea credentials into HTTP(S) clone URL so git doesn't prompt.
    authed_repo_url = repo_url
    if repo_url and repo_url.startswith(("http://", "https://")):
        from urllib.parse import urlparse, urlunparse

        try:
            gitea_cred = KeyringCredentialStore().get(gitea_credential_target)
            parsed = urlparse(repo_url)
            host = parsed.hostname or ""
            port_part = f":{parsed.port}" if parsed.port else ""
            netloc = f"{gitea_cred.username}:{gitea_cred.password}@{host}{port_part}"
            authed_repo_url = urlunparse(parsed._replace(netloc=netloc))
        except Exception:
            pass  # no credential stored — try unauthenticated (public repo)

    config: Config = load_config()
    if not template_path:
        raise click.ClickException(
            "--template-path is required."
        )
    template_p = Path(template_path)
    if not template_p.is_file():
        raise click.ClickException(f"Template VMX not found: {template_path}")

    base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
    base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts
    base_clone_dir.mkdir(parents=True, exist_ok=True)
    job_artifact_dir = base_artifact_dir / job_id
    job_artifact_dir.mkdir(parents=True, exist_ok=True)

    extra_env = _parse_extra_env_json(extra_env_json)

    # Build backend from config; allow CLI override for vmrun path.
    if vmrun_path is not None:
        from ci_orchestrator.backends.workstation import WorkstationVmrunBackend

        backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
    else:
        try:
            backend = load_backend(config)
        except BackendNotAvailable as exc:
            raise click.ClickException(f"backend unavailable: {exc}") from exc

    clone_name = _make_clone_name(job_id)
    clone_dir = base_clone_dir / clone_name
    clone_vmx = clone_dir / f"{clone_name}.vmx"

    handle: VmHandle | None = None
    started = datetime.now(tz=UTC)
    phase_timings: list[tuple[str, float]] = []
    phase_start = _now()

    click.echo("=" * 60)
    click.echo(f"[job] Job          : {job_id}")
    click.echo(f"[job] Repository   : {repo_url}")
    click.echo(f"[job] Branch       : {branch}")
    if commit:
        click.echo(f"[job] Commit       : {commit}")
    click.echo(f"[job] Template     : {template_path}")
    click.echo(f"[job] Snapshot     : {snapshot_name}")
    click.echo(f"[job] Artifacts    : {job_artifact_dir}")
    click.echo(f"[job] Started      : {started.isoformat()}")
    click.echo("=" * 60)

    try:
        # ── Phase 1/5: clone VM ─────────────────────────────────────────
        click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}")
        try:
            handle = backend.clone_linked(
                template=str(template_p),
                snapshot=snapshot_name,
                name=clone_name,
                destination=str(clone_vmx),
            )
        except BackendError as exc:
            raise click.ClickException(f"clone failed: {exc}") from exc
        if not clone_vmx.is_file():
            raise click.ClickException(
                f"backend reported success but clone VMX is missing: {clone_vmx}"
            )

        if guest_cpu > 0 or guest_memory_mb > 0:
            click.echo(
                f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB"
            )
            _apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)

        phase_timings.append(("clone", _now() - phase_start))
        phase_start = _now()

        # ── Phase 2/5: start VM ─────────────────────────────────────────
        click.echo("\n[job] Phase 2/5 — starting VM (headless)")
        try:
            backend.start(handle, headless=True)
        except BackendError as exc:
            raise click.ClickException(f"vmrun start failed: {exc}") from exc

        # Resolve guest OS now that the clone exists.
        if guest_os.lower() == "auto":
            guest_os = _read_guest_os_from_vmx(clone_vmx)
            click.echo(f"[job] auto-detected guest OS: {guest_os}")
        else:
            guest_os = guest_os.lower()

        deadline = _now() + ready_timeout
        if not _wait_running(backend, handle, deadline, poll_interval):
            raise click.ClickException("timeout waiting for VM to be running.")
        ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
        if not ip_address:
            raise click.ClickException("timeout waiting for guest IP.")
        click.echo(f"[job] guest IP: {ip_address}")
        phase_timings.append(("start", _now() - phase_start))
        phase_start = _now()

        # ── Phase 3/5: wait for transport readiness ────────────────────
        click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness")
        target = guest_credential_target or config.guest_cred_target
        key_path: str | None = ssh_key_path
        if key_path is None and config.ssh_key_path is not None:
            key_path = str(config.ssh_key_path)

        if not _probe_transport(
            guest_os=guest_os,
            ip_address=ip_address,
            deadline=deadline,
            poll=poll_interval,
            credential_target=target,
            ssh_user=ssh_user,
            ssh_key_path=key_path,
            ssh_known_hosts=ssh_known_hosts_file,
        ):
            raise click.ClickException(
                "timeout waiting for guest transport to be ready."
            )
        phase_timings.append(("wait-ready", _now() - phase_start))
        phase_start = _now()

        # ── Phase 4/5: build inside guest ──────────────────────────────
        click.echo("\n[job] Phase 4/5 — invoking remote build")
        host_clone_root: Path | None = None
        if use_git_clone:
            build_clone_url: str | None = authed_repo_url
            build_host_src: str | None = None
            click.echo("[job] transport: in-guest git clone")
        else:
            click.echo("[job] transport: host-side clone + zip transfer")
            host_clone_root, src_dir = _host_clone(
                authed_repo_url, branch, commit, submodules
            )
            build_clone_url = None
            build_host_src = str(src_dir)
        try:
            if guest_os == "linux":
                _linux_build(
                    ip_address=ip_address,
                    ssh_user=ssh_user,
                    key_path=key_path,
                    known_hosts=ssh_known_hosts_file,
                    workdir="/opt/ci/build",
                    output_dir="/opt/ci/output",
                    host_source_dir=build_host_src,
                    clone_url=build_clone_url,
                    clone_branch=branch,
                    clone_commit=commit,
                    clone_submodules=submodules,
                    build_command=build_command,
                    artifact_source=guest_artifact_source,
                    extra_env=extra_env,
                    skip_artifact=skip_artifact,
                )
            else:
                _windows_build(
                    ip_address=ip_address,
                    credential_target=target,
                    workdir="C:\\CI\\build",
                    output_dir="C:\\CI\\output",
                    host_source_dir=build_host_src,
                    clone_url=build_clone_url,
                    clone_branch=branch,
                    clone_commit=commit,
                    clone_submodules=submodules,
                    build_command=build_command,
                    artifact_source=guest_artifact_source,
                    configuration=configuration,
                    extra_env=extra_env,
                    skip_artifact=skip_artifact,
                    use_shared_cache=use_shared_cache,
                )
        finally:
            if host_clone_root is not None:
                shutil.rmtree(host_clone_root, ignore_errors=True)
        phase_timings.append(("build", _now() - phase_start))
        phase_start = _now()

        # ── Phase 5/5: collect artifacts ───────────────────────────────
        if skip_artifact:
            click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)")
        else:
            click.echo("\n[job] Phase 5/5 — collecting artifacts")
            if guest_os == "linux":
                _linux_collect(
                    ip_address=ip_address,
                    ssh_user=ssh_user,
                    key_path=key_path,
                    known_hosts=ssh_known_hosts_file,
                    guest_path="/opt/ci/output",
                    host_dir=job_artifact_dir,
                )
            else:
                _windows_collect(
                    ip_address=ip_address,
                    credential_target=target,
                    guest_path="C:\\CI\\output",
                    host_dir=job_artifact_dir,
                    include_logs=True,
                )
            try:
                count = _write_manifest(job_artifact_dir, job_id, commit)
                click.echo(f"[job] manifest.json written ({count} file(s))")
            except OSError as exc:
                click.echo(f"[job] manifest write failed: {exc}", err=True)
        phase_timings.append(("artifacts", _now() - phase_start))

        elapsed = (datetime.now(tz=UTC) - started).total_seconds()
        click.echo("\n" + "=" * 60)
        click.echo("[job] phase durations:")
        for name, secs in phase_timings:
            click.echo(f"  {name:<14} {int(secs):>5}s")
        click.echo(f"  {'TOTAL':<14} {int(elapsed):>5}s")
        click.echo("=" * 60)
        click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s")
        click.echo(f"[job] Artifacts: {job_artifact_dir}")
        click.echo("=" * 60)
    except KeyboardInterrupt:
        click.echo("\n[job] interrupted by user; cleaning up...", err=True)
        _destroy_clone(backend, handle, clone_dir)
        sys.exit(130)
    except click.ClickException:
        _destroy_clone(backend, handle, clone_dir)
        raise
    except Exception as exc:
        click.echo(f"\n[job] FAILURE: {exc}", err=True)
        _destroy_clone(backend, handle, clone_dir)
        raise click.ClickException(str(exc)) from exc
    else:
        _destroy_clone(backend, handle, clone_dir)


__all__ = ["job"]