feat(a3): port build pipeline (vm new, build run, artifacts collect)
Implements Phase A3 of plans/implementation-plan-A-B.md:
- commands/vm.py: vm new (replaces New-BuildVM.ps1)
- commands/build.py: build run (replaces Invoke-RemoteBuild.ps1) with WinRM/SSH dispatch and stdout streaming for act_runner
- commands/artifacts.py: artifacts collect (replaces Get-BuildArtifacts.ps1) using transport.fetch()
- 3 PS scripts reduced to shims preserving \0
- Pester tests/{New-BuildVM,Wait-VMReady,Remove-BuildVM}.Tests.ps1 removed; equivalent cases covered in pytest
- Phase C hook: build/artifacts accept opaque VmHandle/vmx; no Windows path assumptions
Tests: 91 pytest, ruff clean, mypy --strict clean, coverage 78.27% (>=70 gate).
Master checklist + step A3 attivita + 'definizione di fatto' updated; A3-closeout.md added.
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
"""CLI entry point.
|
||||
|
||||
Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor``
|
||||
(disk/runner) and ``report`` (job). Subsequent phases add ``vm new``,
|
||||
``build``, ``artifacts``, ``job``.
|
||||
(disk/runner) and ``report`` (job). Phase A3 adds ``vm new``, ``build``,
|
||||
``artifacts``. Subsequent phases add ``job``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,6 +12,8 @@ import time as time
|
||||
import click
|
||||
|
||||
from ci_orchestrator import __version__
|
||||
from ci_orchestrator.commands.artifacts import artifacts
|
||||
from ci_orchestrator.commands.build import build
|
||||
from ci_orchestrator.commands.monitor import monitor
|
||||
from ci_orchestrator.commands.report import report
|
||||
from ci_orchestrator.commands.vm import vm
|
||||
@@ -34,6 +36,8 @@ def cli() -> None:
|
||||
|
||||
cli.add_command(wait_ready)
|
||||
cli.add_command(vm)
|
||||
cli.add_command(build)
|
||||
cli.add_command(artifacts)
|
||||
cli.add_command(monitor)
|
||||
cli.add_command(report)
|
||||
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""``artifacts`` sub-commands.
|
||||
|
||||
Phase A3 ports the artifact-collection script:
|
||||
|
||||
* ``artifacts collect`` — replaces ``scripts/Get-BuildArtifacts.ps1``
|
||||
|
||||
Uses the abstract :meth:`~ci_orchestrator.transport.winrm.WinRmTransport.fetch`
|
||||
and the SSH transport (via a tar+fetch helper) so no platform-specific
|
||||
``Copy-Item`` / ``scp.exe`` invocations leak into command code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import shlex
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
|
||||
# ---------------------------------------------------------------- group
|
||||
|
||||
|
||||
@click.group("artifacts")
|
||||
def artifacts() -> None:
|
||||
"""Artifact transfer commands."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- helpers
|
||||
|
||||
|
||||
def _sh_quote(value: str) -> str:
|
||||
return shlex.quote(value)
|
||||
|
||||
|
||||
def _ps_quote(value: str) -> str:
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _sha256_hex(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(64 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def _write_manifest(host_dir: Path, job_id: str, commit: str) -> int:
|
||||
"""Write ``manifest.json`` summarising every collected file. Returns count."""
|
||||
files = [
|
||||
p
|
||||
for p in host_dir.rglob("*")
|
||||
if p.is_file() and p.name != "manifest.json"
|
||||
]
|
||||
entries = []
|
||||
for f in files:
|
||||
rel = f.relative_to(host_dir).as_posix()
|
||||
entries.append(
|
||||
{
|
||||
"name": rel,
|
||||
"size": f.stat().st_size,
|
||||
"sha256": _sha256_hex(f),
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"ts": datetime.now(tz=UTC).isoformat(),
|
||||
"jobId": job_id,
|
||||
"commit": commit,
|
||||
"files": entries,
|
||||
}
|
||||
out = host_dir / "manifest.json"
|
||||
out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||
return len(entries)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Linux flow
|
||||
|
||||
|
||||
def _linux_collect(
|
||||
*,
|
||||
ip_address: str,
|
||||
ssh_user: str,
|
||||
key_path: str | None,
|
||||
known_hosts: str | None,
|
||||
guest_path: str,
|
||||
host_dir: Path,
|
||||
) -> None:
|
||||
"""Tar the guest directory, fetch the single archive, extract locally."""
|
||||
click.echo(f"[artifacts collect] tarring guest:{guest_path}")
|
||||
archive_remote = f"/tmp/ci-artifacts-{ip_address.replace('.', '-')}.tar.gz"
|
||||
with SshTransport(
|
||||
ip_address,
|
||||
username=ssh_user,
|
||||
key_path=key_path,
|
||||
known_hosts=known_hosts,
|
||||
) as t:
|
||||
# Build a tarball whose contents are *inside* guest_path. Strip the
|
||||
# parent component so files land directly under host_dir.
|
||||
t.run(
|
||||
f"tar -czf {_sh_quote(archive_remote)} "
|
||||
f"-C {_sh_quote(guest_path)} ."
|
||||
)
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
local_tar = Path(td) / "artifacts.tar.gz"
|
||||
t.fetch(archive_remote, str(local_tar))
|
||||
with tarfile.open(local_tar, "r:gz") as tf:
|
||||
# ``data`` filter (Python 3.12+) protects against path
|
||||
# traversal in malicious tarballs.
|
||||
tf.extractall(host_dir, filter="data")
|
||||
finally:
|
||||
t.run(f"rm -f {_sh_quote(archive_remote)}", check=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Windows flow
|
||||
|
||||
|
||||
def _windows_collect(
|
||||
*,
|
||||
ip_address: str,
|
||||
credential_target: str,
|
||||
guest_path: str,
|
||||
host_dir: Path,
|
||||
include_logs: bool,
|
||||
) -> None:
|
||||
cred = KeyringCredentialStore().get(credential_target)
|
||||
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||
# Verify artifact exists.
|
||||
probe = t.run(
|
||||
f"if (Test-Path {_ps_quote(guest_path)}) {{ 'OK' }} else {{ 'MISSING' }}",
|
||||
check=False,
|
||||
)
|
||||
if "MISSING" in probe.stdout:
|
||||
raise click.ClickException(f"artifact not found in guest: {guest_path}")
|
||||
|
||||
artifact_name = Path(guest_path).name
|
||||
host_dest = host_dir / artifact_name
|
||||
click.echo(f"[artifacts collect] fetching {artifact_name}")
|
||||
t.fetch(guest_path, str(host_dest))
|
||||
|
||||
if include_logs:
|
||||
# List log files in the build dir and fetch each one.
|
||||
list_ps = (
|
||||
"Get-ChildItem -Path 'C:\\CI\\build' -Filter '*.log' -Recurse "
|
||||
"-ErrorAction SilentlyContinue | "
|
||||
"ForEach-Object { $_.FullName }"
|
||||
)
|
||||
listing = t.run(list_ps, check=False)
|
||||
for line in (listing.stdout or "").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
dest = host_dir / Path(line).name
|
||||
click.echo(f"[artifacts collect] fetching log: {Path(line).name}")
|
||||
try:
|
||||
t.fetch(line, str(dest))
|
||||
except TransportError as exc:
|
||||
click.echo(
|
||||
f"[artifacts collect] log fetch failed ({line}): {exc}",
|
||||
err=True,
|
||||
)
|
||||
|
||||
if not host_dest.is_file() or host_dest.stat().st_size == 0:
|
||||
raise click.ClickException(
|
||||
f"artifact missing or empty after fetch: {host_dest}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- CLI
|
||||
|
||||
|
||||
@artifacts.command("collect")
|
||||
@click.option("--ip-address", "ip_address", required=True)
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="windows",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--guest-artifact-path",
|
||||
"guest_artifact_path",
|
||||
required=True,
|
||||
help="Guest path: file (Windows) or directory (Linux).",
|
||||
)
|
||||
@click.option(
|
||||
"--host-artifact-dir",
|
||||
"host_artifact_dir",
|
||||
required=True,
|
||||
help="Local directory to deposit artifacts (created if missing).",
|
||||
)
|
||||
@click.option("--include-logs", "include_logs", is_flag=True, default=False)
|
||||
@click.option("--credential-target", "credential_target", default=None)
|
||||
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||
@click.option("--ssh-key-path", "ssh_key_path", default=None)
|
||||
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
|
||||
@click.option("--job-id", "job_id", default="")
|
||||
@click.option("--commit", "commit", default="")
|
||||
def artifacts_collect(
|
||||
ip_address: str,
|
||||
guest_os: str,
|
||||
guest_artifact_path: str,
|
||||
host_artifact_dir: str,
|
||||
include_logs: bool,
|
||||
credential_target: str | None,
|
||||
ssh_user: str,
|
||||
ssh_key_path: str | None,
|
||||
ssh_known_hosts_file: str | None,
|
||||
job_id: str,
|
||||
commit: str,
|
||||
) -> None:
|
||||
"""Copy build artifacts from a guest VM to a local directory.
|
||||
|
||||
Mirrors ``scripts/Get-BuildArtifacts.ps1``. Writes ``manifest.json``
|
||||
when ``--job-id`` or ``--commit`` is provided.
|
||||
"""
|
||||
guest_os = guest_os.lower()
|
||||
host_dir = Path(host_artifact_dir)
|
||||
host_dir.mkdir(parents=True, exist_ok=True)
|
||||
config = load_config()
|
||||
|
||||
try:
|
||||
if guest_os == "linux":
|
||||
key_path: str | None = ssh_key_path
|
||||
if key_path is None and config.ssh_key_path is not None:
|
||||
key_path = str(config.ssh_key_path)
|
||||
_linux_collect(
|
||||
ip_address=ip_address,
|
||||
ssh_user=ssh_user,
|
||||
key_path=key_path,
|
||||
known_hosts=ssh_known_hosts_file,
|
||||
guest_path=guest_artifact_path,
|
||||
host_dir=host_dir,
|
||||
)
|
||||
else:
|
||||
target = credential_target or config.guest_cred_target
|
||||
_windows_collect(
|
||||
ip_address=ip_address,
|
||||
credential_target=target,
|
||||
guest_path=guest_artifact_path,
|
||||
host_dir=host_dir,
|
||||
include_logs=include_logs,
|
||||
)
|
||||
except (TransportError, TransportCommandError) as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
transferred = [p for p in host_dir.rglob("*") if p.is_file()]
|
||||
if not transferred:
|
||||
raise click.ClickException(
|
||||
f"no files were transferred into {host_dir}"
|
||||
)
|
||||
click.echo(f"[artifacts collect] {len(transferred)} file(s) collected.")
|
||||
|
||||
if job_id or commit:
|
||||
count = _write_manifest(host_dir, job_id, commit)
|
||||
click.echo(f"[artifacts collect] manifest.json written ({count} file(s)).")
|
||||
|
||||
|
||||
__all__ = ["artifacts", "artifacts_collect"]
|
||||
@@ -0,0 +1,509 @@
|
||||
"""``build`` sub-commands.
|
||||
|
||||
Phase A3 ports the remote build pipeline:
|
||||
|
||||
* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1``
|
||||
|
||||
The original PS script supports two source-provisioning modes (host-side
|
||||
clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom
|
||||
build command and optional artifact packaging. This Python port preserves
|
||||
the mechanics while keeping the guest workdir and artifact paths fully
|
||||
parameterised — see Phase C hooks below.
|
||||
|
||||
Phase C hook: the guest workdir / artifact paths are passed as opaque
|
||||
strings; this module makes no assumption about Windows vs POSIX path
|
||||
syntax beyond a sensible default per ``--guest-os``. ``build run`` does
|
||||
not import ``WorkstationVmrunBackend`` directly; it relies on the
|
||||
transport layer to talk to whatever guest the orchestrator points it at.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- group
|
||||
|
||||
|
||||
@click.group("build")
|
||||
def build() -> None:
|
||||
"""Remote build commands."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- helpers
|
||||
|
||||
|
||||
def _parse_extra_env(items: Iterable[str]) -> dict[str, str]:
|
||||
"""Parse ``KEY=VALUE`` pairs into a dict.
|
||||
|
||||
Empty entries are ignored. A pair without ``=`` raises ``BadParameter``.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
for raw in items:
|
||||
if not raw:
|
||||
continue
|
||||
if "=" not in raw:
|
||||
raise click.BadParameter(
|
||||
f"--extra-env value must be KEY=VALUE, got: {raw!r}"
|
||||
)
|
||||
key, value = raw.split("=", 1)
|
||||
if not key:
|
||||
raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}")
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def _zip_dir(source_dir: Path, archive_path: Path) -> None:
|
||||
"""Create a deterministic ZIP archive containing the contents of ``source_dir``."""
|
||||
with zipfile.ZipFile(
|
||||
archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1
|
||||
) as zf:
|
||||
for item in source_dir.rglob("*"):
|
||||
if item.is_file():
|
||||
zf.write(item, arcname=item.relative_to(source_dir))
|
||||
|
||||
|
||||
def _tar_dir(source_dir: Path, archive_path: Path) -> None:
|
||||
"""Create a gzip tar archive containing the contents of ``source_dir``."""
|
||||
with tarfile.open(archive_path, "w:gz") as tf:
|
||||
for item in source_dir.iterdir():
|
||||
tf.add(item, arcname=item.name)
|
||||
|
||||
|
||||
def _ps_quote(value: str) -> str:
|
||||
"""Quote a value for embedding in a PowerShell single-quoted string."""
|
||||
return "'" + value.replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _sh_quote(value: str) -> str:
|
||||
"""POSIX shell single-quote escaping."""
|
||||
return shlex.quote(value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Linux flow
|
||||
|
||||
|
||||
def _linux_build(
|
||||
*,
|
||||
ip_address: str,
|
||||
ssh_user: str,
|
||||
key_path: str | None,
|
||||
known_hosts: str | None,
|
||||
workdir: str,
|
||||
output_dir: str,
|
||||
host_source_dir: str | None,
|
||||
clone_url: str | None,
|
||||
clone_branch: str,
|
||||
clone_commit: str,
|
||||
clone_submodules: bool,
|
||||
build_command: str,
|
||||
artifact_source: str,
|
||||
extra_env: dict[str, str],
|
||||
skip_artifact: bool,
|
||||
) -> None:
|
||||
click.echo("[build run] Linux build mode (SSH transport)")
|
||||
|
||||
with SshTransport(
|
||||
ip_address,
|
||||
username=ssh_user,
|
||||
key_path=key_path,
|
||||
known_hosts=known_hosts,
|
||||
) as t:
|
||||
t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
|
||||
|
||||
if host_source_dir:
|
||||
click.echo(f"[build run] packaging host source: {host_source_dir}")
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
tar_path = Path(td) / "ci-src.tar.gz"
|
||||
_tar_dir(Path(host_source_dir), tar_path)
|
||||
guest_tar = f"/tmp/{tar_path.name}"
|
||||
t.copy(str(tar_path), guest_tar)
|
||||
click.echo(f"[build run] extracting source in guest: {workdir}")
|
||||
t.run(
|
||||
f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} "
|
||||
f"&& rm -f {_sh_quote(guest_tar)}"
|
||||
)
|
||||
elif clone_url:
|
||||
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
|
||||
cmd = (
|
||||
f"git clone --depth 1 --branch {_sh_quote(clone_branch)} "
|
||||
f"{'--recurse-submodules ' if clone_submodules else ''}"
|
||||
f"{_sh_quote(clone_url)} {_sh_quote(workdir)}"
|
||||
)
|
||||
t.run(cmd)
|
||||
if clone_commit:
|
||||
t.run(
|
||||
f"git -C {_sh_quote(workdir)} checkout {_sh_quote(clone_commit)}"
|
||||
)
|
||||
else:
|
||||
raise click.ClickException(
|
||||
"Linux build requires either --host-source-dir or --clone-url."
|
||||
)
|
||||
|
||||
env_prefix = "".join(
|
||||
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||
)
|
||||
cmd = build_command or "make"
|
||||
full_cmd = f"{env_prefix}cd {_sh_quote(workdir)} && {cmd}"
|
||||
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
|
||||
result = t.run(full_cmd, check=False)
|
||||
# Stream guest stdout/stderr after the fact (paramiko buffers the channel).
|
||||
if result.stdout:
|
||||
sys.stdout.write(result.stdout)
|
||||
if result.stderr:
|
||||
sys.stderr.write(result.stderr)
|
||||
if result.returncode != 0:
|
||||
raise click.ClickException(
|
||||
f"build command failed (exit {result.returncode})"
|
||||
)
|
||||
|
||||
if skip_artifact:
|
||||
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||
return
|
||||
|
||||
t.run(f"rm -rf {_sh_quote(output_dir)} && mkdir -p {_sh_quote(output_dir)}")
|
||||
src = artifact_source or "dist"
|
||||
cp_cmd = (
|
||||
f"if [ -d {_sh_quote(workdir + '/' + src)} ]; then "
|
||||
f"cp -r {_sh_quote(workdir + '/' + src + '/.')} {_sh_quote(output_dir + '/')}; "
|
||||
f"elif [ -e {_sh_quote(workdir + '/' + src)} ]; then "
|
||||
f"cp {_sh_quote(workdir + '/' + src)} {_sh_quote(output_dir + '/')}; "
|
||||
f"else echo '[build run] WARNING: artifact source not found: {src}' >&2; fi"
|
||||
)
|
||||
t.run(cp_cmd)
|
||||
click.echo(f"[build run] artifact staged at {output_dir}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- Windows flow
|
||||
|
||||
|
||||
def _windows_build(
|
||||
*,
|
||||
ip_address: str,
|
||||
credential_target: str,
|
||||
workdir: str,
|
||||
artifact_zip: str,
|
||||
host_source_dir: str | None,
|
||||
clone_url: str | None,
|
||||
clone_branch: str,
|
||||
clone_commit: str,
|
||||
clone_submodules: bool,
|
||||
build_command: str,
|
||||
artifact_source: str,
|
||||
configuration: str,
|
||||
extra_env: dict[str, str],
|
||||
skip_artifact: bool,
|
||||
use_shared_cache: bool,
|
||||
) -> None:
|
||||
click.echo("[build run] Windows build mode (WinRM transport)")
|
||||
cred = KeyringCredentialStore().get(credential_target)
|
||||
|
||||
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||
# Prepare workdir + artifact output directory.
|
||||
prep_ps = (
|
||||
f"$work = {_ps_quote(workdir)}; "
|
||||
f"$out = Split-Path {_ps_quote(artifact_zip)} -Parent; "
|
||||
"if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; "
|
||||
"if (Test-Path $work) { Remove-Item $work -Recurse -Force }; "
|
||||
"New-Item -ItemType Directory -Path $work -Force | Out-Null"
|
||||
)
|
||||
t.run(prep_ps)
|
||||
|
||||
if host_source_dir:
|
||||
click.echo(f"[build run] zipping host source: {host_source_dir}")
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
zip_path = Path(td) / "ci-src.zip"
|
||||
_zip_dir(Path(host_source_dir), zip_path)
|
||||
size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1)
|
||||
click.echo(f"[build run] uploading source archive ({size_mb} MB)")
|
||||
guest_zip = "C:\\CI\\src-transfer.zip"
|
||||
t.copy(str(zip_path), guest_zip)
|
||||
t.run(
|
||||
f"Expand-Archive -Path {_ps_quote(guest_zip)} "
|
||||
f"-DestinationPath {_ps_quote(workdir)} -Force; "
|
||||
f"Remove-Item {_ps_quote(guest_zip)} -Force"
|
||||
)
|
||||
elif clone_url:
|
||||
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
|
||||
sub_flag = " --recurse-submodules" if clone_submodules else ""
|
||||
t.run(
|
||||
f"$env:GIT_TERMINAL_PROMPT = '0'; "
|
||||
f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); "
|
||||
f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} "
|
||||
f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)"
|
||||
)
|
||||
if clone_commit:
|
||||
t.run(
|
||||
f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}"
|
||||
)
|
||||
else:
|
||||
raise click.ClickException(
|
||||
"Windows build requires either --host-source-dir or --clone-url."
|
||||
)
|
||||
|
||||
# Inject env vars + run the build.
|
||||
env_setup = "; ".join(
|
||||
f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items()
|
||||
)
|
||||
if env_setup:
|
||||
env_setup += "; "
|
||||
if use_shared_cache:
|
||||
env_setup += (
|
||||
"$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; "
|
||||
"$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; "
|
||||
)
|
||||
if build_command:
|
||||
run_cmd = build_command
|
||||
click.echo(f"[build run] custom build command: {run_cmd}")
|
||||
else:
|
||||
click.echo(
|
||||
f"[build run] default build: dotnet restore + build (configuration={configuration})"
|
||||
)
|
||||
run_cmd = (
|
||||
"dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; "
|
||||
f"dotnet build --configuration {_ps_quote(configuration)} "
|
||||
"--output (Join-Path (Get-Location) 'bin\\CIOutput')"
|
||||
)
|
||||
|
||||
full_ps = (
|
||||
f"{env_setup}Set-Location {_ps_quote(workdir)}; "
|
||||
f"$ErrorActionPreference='Continue'; "
|
||||
f"& cmd /c {_ps_quote(run_cmd + ' 2>&1')}; "
|
||||
"exit $LASTEXITCODE"
|
||||
)
|
||||
result = t.run(full_ps, check=False)
|
||||
if result.stdout:
|
||||
sys.stdout.write(result.stdout)
|
||||
if result.stderr:
|
||||
sys.stderr.write(result.stderr)
|
||||
if result.returncode != 0:
|
||||
raise click.ClickException(
|
||||
f"build command failed (exit {result.returncode})"
|
||||
)
|
||||
|
||||
if skip_artifact:
|
||||
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||
return
|
||||
|
||||
# Package the artifact source into the requested zip on the guest.
|
||||
if build_command:
|
||||
src_rel = artifact_source or "dist"
|
||||
src_expr = f"Join-Path {_ps_quote(workdir)} {_ps_quote(src_rel)}"
|
||||
else:
|
||||
src_expr = f"Join-Path {_ps_quote(workdir)} 'bin\\CIOutput'"
|
||||
|
||||
pkg_ps = (
|
||||
f"$src = {src_expr}; "
|
||||
f"$zip = {_ps_quote(artifact_zip)}; "
|
||||
"if (-not (Test-Path $src)) { "
|
||||
" Write-Host \"[build run] artifact source not found: $src\"; "
|
||||
" exit 2 } ; "
|
||||
"$dir = Split-Path $zip -Parent; "
|
||||
"if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }; "
|
||||
"if (Test-Path $zip) { Remove-Item $zip -Force }; "
|
||||
"Compress-Archive -Path (Join-Path $src '*') -DestinationPath $zip -Force"
|
||||
)
|
||||
t.run(pkg_ps)
|
||||
click.echo(f"[build run] artifact written to {artifact_zip}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- CLI
|
||||
|
||||
|
||||
@build.command("run")
|
||||
@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.")
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="windows",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--credential-target",
|
||||
"credential_target",
|
||||
default=None,
|
||||
help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.",
|
||||
)
|
||||
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||
@click.option(
|
||||
"--ssh-key-path",
|
||||
"ssh_key_path",
|
||||
default=None,
|
||||
help="SSH private key path (Linux). Defaults to config.ssh_key_path.",
|
||||
)
|
||||
@click.option(
|
||||
"--ssh-known-hosts-file",
|
||||
"ssh_known_hosts_file",
|
||||
default=None,
|
||||
help="Persistent known_hosts file (Linux). Default: permissive (None).",
|
||||
)
|
||||
@click.option("--host-source-dir", "host_source_dir", default=None)
|
||||
@click.option("--clone-url", "clone_url", default=None)
|
||||
@click.option("--clone-branch", "clone_branch", default="main", show_default=True)
|
||||
@click.option("--clone-commit", "clone_commit", default="")
|
||||
@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--gitea-credential-target",
|
||||
"gitea_credential_target",
|
||||
default="GiteaPAT",
|
||||
show_default=True,
|
||||
help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.",
|
||||
)
|
||||
@click.option("--build-command", "build_command", default="")
|
||||
@click.option(
|
||||
"--guest-work-dir",
|
||||
"guest_work_dir",
|
||||
default=None,
|
||||
help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-artifact-zip",
|
||||
"guest_artifact_zip",
|
||||
default=None,
|
||||
help="Artifact zip path inside the Windows guest (default: C:\\CI\\output\\artifacts.zip).",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-linux-work-dir",
|
||||
"guest_linux_work_dir",
|
||||
default=None,
|
||||
help="Alias for --guest-work-dir, kept for shim compatibility (Linux).",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-linux-output-dir",
|
||||
"guest_linux_output_dir",
|
||||
default="/opt/ci/output",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
|
||||
@click.option("--configuration", default="Release", show_default=True)
|
||||
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--extra-env",
|
||||
"extra_env",
|
||||
multiple=True,
|
||||
help="Repeatable KEY=VALUE pairs injected into the guest environment.",
|
||||
)
|
||||
def build_run(
|
||||
ip_address: str,
|
||||
guest_os: str,
|
||||
credential_target: str | None,
|
||||
ssh_user: str,
|
||||
ssh_key_path: str | None,
|
||||
ssh_known_hosts_file: str | None,
|
||||
host_source_dir: str | None,
|
||||
clone_url: str | None,
|
||||
clone_branch: str,
|
||||
clone_commit: str,
|
||||
clone_submodules: bool,
|
||||
gitea_credential_target: str,
|
||||
build_command: str,
|
||||
guest_work_dir: str | None,
|
||||
guest_artifact_zip: str | None,
|
||||
guest_linux_work_dir: str | None,
|
||||
guest_linux_output_dir: str,
|
||||
guest_artifact_source: str,
|
||||
configuration: str,
|
||||
skip_artifact: bool,
|
||||
use_shared_cache: bool,
|
||||
extra_env: tuple[str, ...],
|
||||
) -> None:
|
||||
"""Run a build inside a guest VM, optionally packaging the result.
|
||||
|
||||
Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via
|
||||
one of:
|
||||
|
||||
* ``--host-source-dir`` — local directory zipped/tarred and uploaded;
|
||||
* ``--clone-url`` — ``git clone`` invoked inside the guest.
|
||||
|
||||
Phase C hook: ``--guest-work-dir`` and ``--guest-artifact-zip`` are
|
||||
parameterised; defaults are applied here, never inside the transport
|
||||
layer.
|
||||
"""
|
||||
del gitea_credential_target # not yet honoured by Python port
|
||||
guest_os = guest_os.lower()
|
||||
|
||||
if host_source_dir and clone_url:
|
||||
raise click.BadParameter(
|
||||
"Provide either --host-source-dir or --clone-url, not both."
|
||||
)
|
||||
if not host_source_dir and not clone_url:
|
||||
raise click.BadParameter(
|
||||
"Provide either --host-source-dir or --clone-url."
|
||||
)
|
||||
if host_source_dir and not Path(host_source_dir).is_dir():
|
||||
raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}")
|
||||
|
||||
extra = _parse_extra_env(extra_env)
|
||||
config = load_config()
|
||||
|
||||
if guest_os == "linux":
|
||||
workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build"
|
||||
key_path: str | None = ssh_key_path
|
||||
if key_path is None and config.ssh_key_path is not None:
|
||||
key_path = str(config.ssh_key_path)
|
||||
try:
|
||||
_linux_build(
|
||||
ip_address=ip_address,
|
||||
ssh_user=ssh_user,
|
||||
key_path=key_path,
|
||||
known_hosts=ssh_known_hosts_file,
|
||||
workdir=workdir,
|
||||
output_dir=guest_linux_output_dir,
|
||||
host_source_dir=host_source_dir,
|
||||
clone_url=clone_url,
|
||||
clone_branch=clone_branch,
|
||||
clone_commit=clone_commit,
|
||||
clone_submodules=clone_submodules,
|
||||
build_command=build_command,
|
||||
artifact_source=guest_artifact_source,
|
||||
extra_env=extra,
|
||||
skip_artifact=skip_artifact,
|
||||
)
|
||||
except (TransportError, TransportCommandError) as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
else:
|
||||
workdir = guest_work_dir or "C:\\CI\\build"
|
||||
artifact_zip = guest_artifact_zip or "C:\\CI\\output\\artifacts.zip"
|
||||
target = credential_target or config.guest_cred_target
|
||||
try:
|
||||
_windows_build(
|
||||
ip_address=ip_address,
|
||||
credential_target=target,
|
||||
workdir=workdir,
|
||||
artifact_zip=artifact_zip,
|
||||
host_source_dir=host_source_dir,
|
||||
clone_url=clone_url,
|
||||
clone_branch=clone_branch,
|
||||
clone_commit=clone_commit,
|
||||
clone_submodules=clone_submodules,
|
||||
build_command=build_command,
|
||||
artifact_source=guest_artifact_source,
|
||||
configuration=configuration,
|
||||
extra_env=extra,
|
||||
skip_artifact=skip_artifact,
|
||||
use_shared_cache=use_shared_cache,
|
||||
)
|
||||
except (TransportError, TransportCommandError) as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
|
||||
__all__ = ["build", "build_run"]
|
||||
@@ -354,4 +354,116 @@ def vm_cleanup(
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── new
|
||||
|
||||
|
||||
@vm.command("new")
|
||||
@click.option(
|
||||
"--template",
|
||||
"--template-path",
|
||||
"template",
|
||||
required=True,
|
||||
help="Identifier of the template VM (Workstation: VMX path; ESXi: managed-object ref).",
|
||||
)
|
||||
@click.option(
|
||||
"--snapshot",
|
||||
"--snapshot-name",
|
||||
"snapshot",
|
||||
default="BaseClean",
|
||||
show_default=True,
|
||||
help="Snapshot name on the template to clone from.",
|
||||
)
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"clone_base_dir",
|
||||
required=True,
|
||||
help="Directory under which the new clone folder will be created.",
|
||||
)
|
||||
@click.option(
|
||||
"--job-id",
|
||||
"job_id",
|
||||
required=True,
|
||||
help="Unique job identifier (used to name the clone folder).",
|
||||
)
|
||||
@click.option(
|
||||
"--vmrun-path",
|
||||
"vmrun_path",
|
||||
default=None,
|
||||
help="Override vmrun binary path.",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="windows",
|
||||
show_default=True,
|
||||
help="Informational; reserved for backend hints in Phase C.",
|
||||
)
|
||||
def vm_new(
|
||||
template: str,
|
||||
snapshot: str,
|
||||
clone_base_dir: str,
|
||||
job_id: str,
|
||||
vmrun_path: str | None,
|
||||
guest_os: str,
|
||||
) -> None:
|
||||
"""Create a linked clone of the template VM for an ephemeral CI build.
|
||||
|
||||
Mirrors ``scripts/New-BuildVM.ps1``. Prints the clone identifier
|
||||
(Workstation: VMX path) on stdout on success.
|
||||
|
||||
Phase C hook: ``--template`` is treated as an opaque string; the backend
|
||||
is responsible for interpretation. Workstation builds the destination
|
||||
path under ``--clone-base-dir`` from ``--job-id`` + timestamp.
|
||||
"""
|
||||
del guest_os # currently informational only
|
||||
# Verify the template identifier looks plausible: for Workstation it is
|
||||
# a VMX path. For other backends in Phase C this validation will move
|
||||
# into the backend itself.
|
||||
template_path = Path(template)
|
||||
if not template_path.is_file():
|
||||
raise click.ClickException(f"Template VMX not found: {template}")
|
||||
|
||||
base = Path(clone_base_dir)
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||
clone_name = f"Clone_{job_id}_{timestamp}"
|
||||
clone_dir = base / clone_name
|
||||
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||
|
||||
click.echo("[vm new] creating linked clone...")
|
||||
click.echo(f" template : {template}")
|
||||
click.echo(f" snapshot : {snapshot}")
|
||||
click.echo(f" clone vmx: {clone_vmx}")
|
||||
|
||||
try:
|
||||
backend = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
raise click.ClickException(f"vmrun unavailable: {exc}") from exc
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
handle = backend.clone_linked(
|
||||
template=str(template_path),
|
||||
snapshot=snapshot,
|
||||
name=clone_name,
|
||||
destination=str(clone_vmx),
|
||||
)
|
||||
except BackendError as exc:
|
||||
# Clean up partial clone directory if vmrun left one behind.
|
||||
if clone_dir.exists():
|
||||
_try_remove_dir(clone_dir)
|
||||
raise click.ClickException(f"clone failed: {exc}") from exc
|
||||
|
||||
if not clone_vmx.is_file():
|
||||
raise click.ClickException(
|
||||
f"backend reported success but clone VMX is missing: {clone_vmx}"
|
||||
)
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
click.echo(f"[vm new] clone created in {elapsed:.1f}s")
|
||||
# Final stdout line: the identifier itself, so PS callers can capture it.
|
||||
click.echo(handle.identifier)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm"]
|
||||
|
||||
Reference in New Issue
Block a user