Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(a3): port build pipeline (vm new, build run, artifacts collect)

Browse Source 
Implements Phase A3 of plans/implementation-plan-A-B.md:

- commands/vm.py: vm new (replaces New-BuildVM.ps1)

- commands/build.py: build run (replaces Invoke-RemoteBuild.ps1) with WinRM/SSH dispatch and stdout streaming for act_runner

- commands/artifacts.py: artifacts collect (replaces Get-BuildArtifacts.ps1) using transport.fetch()

- 3 PS scripts reduced to shims preserving \0

- Pester tests/{New-BuildVM,Wait-VMReady,Remove-BuildVM}.Tests.ps1 removed; equivalent cases covered in pytest

- Phase C hook: build/artifacts accept opaque VmHandle/vmx; no Windows path assumptions

Tests: 91 pytest, ruff clean, mypy --strict clean, coverage 78.27% (>=70 gate).

Master checklist + step A3 attivita + 'definizione di fatto' updated; A3-closeout.md added.
This commit is contained in:
Simone
2026-05-14 17:20:34 +02:00
parent 096ba7fe16
commit 816a15503e
15 changed files with 1900 additions and 1061 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ci_orchestrator/commands/build.py
+509
View File
@@ -0,0 +1,509 @@
"""``build`` sub-commands.
Phase A3 ports the remote build pipeline:
* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1``
The original PS script supports two source-provisioning modes (host-side
clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom
build command and optional artifact packaging. This Python port preserves
the mechanics while keeping the guest workdir and artifact paths fully
parameterised — see Phase C hooks below.
Phase C hook: the guest workdir / artifact paths are passed as opaque
strings; this module makes no assumption about Windows vs POSIX path
syntax beyond a sensible default per ``--guest-os``. ``build run`` does
not import ``WorkstationVmrunBackend`` directly; it relies on the
transport layer to talk to whatever guest the orchestrator points it at.
"""
from __future__ import annotations
import shlex
import sys
import tarfile
import tempfile
import zipfile
from collections.abc import Iterable
from pathlib import Path
from typing import TYPE_CHECKING
import click
from ci_orchestrator.config import load_config
from ci_orchestrator.credentials import KeyringCredentialStore
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
from ci_orchestrator.transport.ssh import SshTransport
from ci_orchestrator.transport.winrm import WinRmTransport
if TYPE_CHECKING: # pragma: no cover
pass
# ---------------------------------------------------------------- group
@click.group("build")
def build() -> None:
"""Remote build commands."""
# ---------------------------------------------------------------- helpers
def _parse_extra_env(items: Iterable[str]) -> dict[str, str]:
"""Parse ``KEY=VALUE`` pairs into a dict.
Empty entries are ignored. A pair without ``=`` raises ``BadParameter``.
"""
out: dict[str, str] = {}
for raw in items:
if not raw:
continue
if "=" not in raw:
raise click.BadParameter(
f"--extra-env value must be KEY=VALUE, got: {raw!r}"
)
key, value = raw.split("=", 1)
if not key:
raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}")
out[key] = value
return out
def _zip_dir(source_dir: Path, archive_path: Path) -> None:
"""Create a deterministic ZIP archive containing the contents of ``source_dir``."""
with zipfile.ZipFile(
archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1
) as zf:
for item in source_dir.rglob("*"):
if item.is_file():
zf.write(item, arcname=item.relative_to(source_dir))
def _tar_dir(source_dir: Path, archive_path: Path) -> None:
"""Create a gzip tar archive containing the contents of ``source_dir``."""
with tarfile.open(archive_path, "w:gz") as tf:
for item in source_dir.iterdir():
tf.add(item, arcname=item.name)
def _ps_quote(value: str) -> str:
"""Quote a value for embedding in a PowerShell single-quoted string."""
return "'" + value.replace("'", "''") + "'"
def _sh_quote(value: str) -> str:
"""POSIX shell single-quote escaping."""
return shlex.quote(value)
# ---------------------------------------------------------------- Linux flow
def _linux_build(
*,
ip_address: str,
ssh_user: str,
key_path: str | None,
known_hosts: str | None,
workdir: str,
output_dir: str,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
build_command: str,
artifact_source: str,
extra_env: dict[str, str],
skip_artifact: bool,
) -> None:
click.echo("[build run] Linux build mode (SSH transport)")
with SshTransport(
ip_address,
username=ssh_user,
key_path=key_path,
known_hosts=known_hosts,
) as t:
t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
if host_source_dir:
click.echo(f"[build run] packaging host source: {host_source_dir}")
with tempfile.TemporaryDirectory() as td:
tar_path = Path(td) / "ci-src.tar.gz"
_tar_dir(Path(host_source_dir), tar_path)
guest_tar = f"/tmp/{tar_path.name}"
t.copy(str(tar_path), guest_tar)
click.echo(f"[build run] extracting source in guest: {workdir}")
t.run(
f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} "
f"&& rm -f {_sh_quote(guest_tar)}"
)
elif clone_url:
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
cmd = (
f"git clone --depth 1 --branch {_sh_quote(clone_branch)} "
f"{'--recurse-submodules ' if clone_submodules else ''}"
f"{_sh_quote(clone_url)} {_sh_quote(workdir)}"
)
t.run(cmd)
if clone_commit:
t.run(
f"git -C {_sh_quote(workdir)} checkout {_sh_quote(clone_commit)}"
)
else:
raise click.ClickException(
"Linux build requires either --host-source-dir or --clone-url."
)
env_prefix = "".join(
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
)
cmd = build_command or "make"
full_cmd = f"{env_prefix}cd {_sh_quote(workdir)} && {cmd}"
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
result = t.run(full_cmd, check=False)
# Stream guest stdout/stderr after the fact (paramiko buffers the channel).
if result.stdout:
sys.stdout.write(result.stdout)
if result.stderr:
sys.stderr.write(result.stderr)
if result.returncode != 0:
raise click.ClickException(
f"build command failed (exit {result.returncode})"
)
if skip_artifact:
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
return
t.run(f"rm -rf {_sh_quote(output_dir)} && mkdir -p {_sh_quote(output_dir)}")
src = artifact_source or "dist"
cp_cmd = (
f"if [ -d {_sh_quote(workdir + '/' + src)} ]; then "
f"cp -r {_sh_quote(workdir + '/' + src + '/.')} {_sh_quote(output_dir + '/')}; "
f"elif [ -e {_sh_quote(workdir + '/' + src)} ]; then "
f"cp {_sh_quote(workdir + '/' + src)} {_sh_quote(output_dir + '/')}; "
f"else echo '[build run] WARNING: artifact source not found: {src}' >&2; fi"
)
t.run(cp_cmd)
click.echo(f"[build run] artifact staged at {output_dir}")
# ---------------------------------------------------------------- Windows flow
def _windows_build(
*,
ip_address: str,
credential_target: str,
workdir: str,
artifact_zip: str,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
build_command: str,
artifact_source: str,
configuration: str,
extra_env: dict[str, str],
skip_artifact: bool,
use_shared_cache: bool,
) -> None:
click.echo("[build run] Windows build mode (WinRM transport)")
cred = KeyringCredentialStore().get(credential_target)
with WinRmTransport(ip_address, cred.username, cred.password) as t:
# Prepare workdir + artifact output directory.
prep_ps = (
f"$work = {_ps_quote(workdir)}; "
f"$out = Split-Path {_ps_quote(artifact_zip)} -Parent; "
"if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; "
"if (Test-Path $work) { Remove-Item $work -Recurse -Force }; "
"New-Item -ItemType Directory -Path $work -Force | Out-Null"
)
t.run(prep_ps)
if host_source_dir:
click.echo(f"[build run] zipping host source: {host_source_dir}")
with tempfile.TemporaryDirectory() as td:
zip_path = Path(td) / "ci-src.zip"
_zip_dir(Path(host_source_dir), zip_path)
size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1)
click.echo(f"[build run] uploading source archive ({size_mb} MB)")
guest_zip = "C:\\CI\\src-transfer.zip"
t.copy(str(zip_path), guest_zip)
t.run(
f"Expand-Archive -Path {_ps_quote(guest_zip)} "
f"-DestinationPath {_ps_quote(workdir)} -Force; "
f"Remove-Item {_ps_quote(guest_zip)} -Force"
)
elif clone_url:
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
sub_flag = " --recurse-submodules" if clone_submodules else ""
t.run(
f"$env:GIT_TERMINAL_PROMPT = '0'; "
f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); "
f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} "
f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)"
)
if clone_commit:
t.run(
f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}"
)
else:
raise click.ClickException(
"Windows build requires either --host-source-dir or --clone-url."
)
# Inject env vars + run the build.
env_setup = "; ".join(
f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items()
)
if env_setup:
env_setup += "; "
if use_shared_cache:
env_setup += (
"$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; "
"$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; "
)
if build_command:
run_cmd = build_command
click.echo(f"[build run] custom build command: {run_cmd}")
else:
click.echo(
f"[build run] default build: dotnet restore + build (configuration={configuration})"
)
run_cmd = (
"dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; "
f"dotnet build --configuration {_ps_quote(configuration)} "
"--output (Join-Path (Get-Location) 'bin\\CIOutput')"
)
full_ps = (
f"{env_setup}Set-Location {_ps_quote(workdir)}; "
f"$ErrorActionPreference='Continue'; "
f"& cmd /c {_ps_quote(run_cmd + ' 2>&1')}; "
"exit $LASTEXITCODE"
)
result = t.run(full_ps, check=False)
if result.stdout:
sys.stdout.write(result.stdout)
if result.stderr:
sys.stderr.write(result.stderr)
if result.returncode != 0:
raise click.ClickException(
f"build command failed (exit {result.returncode})"
)
if skip_artifact:
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
return
# Package the artifact source into the requested zip on the guest.
if build_command:
src_rel = artifact_source or "dist"
src_expr = f"Join-Path {_ps_quote(workdir)} {_ps_quote(src_rel)}"
else:
src_expr = f"Join-Path {_ps_quote(workdir)} 'bin\\CIOutput'"
pkg_ps = (
f"$src = {src_expr}; "
f"$zip = {_ps_quote(artifact_zip)}; "
"if (-not (Test-Path $src)) { "
" Write-Host \"[build run] artifact source not found: $src\"; "
" exit 2 } ; "
"$dir = Split-Path $zip -Parent; "
"if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null }; "
"if (Test-Path $zip) { Remove-Item $zip -Force }; "
"Compress-Archive -Path (Join-Path $src '*') -DestinationPath $zip -Force"
)
t.run(pkg_ps)
click.echo(f"[build run] artifact written to {artifact_zip}")
# ---------------------------------------------------------------- CLI
@build.command("run")
@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.")
@click.option(
"--guest-os",
type=click.Choice(["windows", "linux"], case_sensitive=False),
default="windows",
show_default=True,
)
@click.option(
"--credential-target",
"credential_target",
default=None,
help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.",
)
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
@click.option(
"--ssh-key-path",
"ssh_key_path",
default=None,
help="SSH private key path (Linux). Defaults to config.ssh_key_path.",
)
@click.option(
"--ssh-known-hosts-file",
"ssh_known_hosts_file",
default=None,
help="Persistent known_hosts file (Linux). Default: permissive (None).",
)
@click.option("--host-source-dir", "host_source_dir", default=None)
@click.option("--clone-url", "clone_url", default=None)
@click.option("--clone-branch", "clone_branch", default="main", show_default=True)
@click.option("--clone-commit", "clone_commit", default="")
@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False)
@click.option(
"--gitea-credential-target",
"gitea_credential_target",
default="GiteaPAT",
show_default=True,
help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.",
)
@click.option("--build-command", "build_command", default="")
@click.option(
"--guest-work-dir",
"guest_work_dir",
default=None,
help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).",
)
@click.option(
"--guest-artifact-zip",
"guest_artifact_zip",
default=None,
help="Artifact zip path inside the Windows guest (default: C:\\CI\\output\\artifacts.zip).",
)
@click.option(
"--guest-linux-work-dir",
"guest_linux_work_dir",
default=None,
help="Alias for --guest-work-dir, kept for shim compatibility (Linux).",
)
@click.option(
"--guest-linux-output-dir",
"guest_linux_output_dir",
default="/opt/ci/output",
show_default=True,
)
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
@click.option("--configuration", default="Release", show_default=True)
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
@click.option(
"--extra-env",
"extra_env",
multiple=True,
help="Repeatable KEY=VALUE pairs injected into the guest environment.",
)
def build_run(
ip_address: str,
guest_os: str,
credential_target: str | None,
ssh_user: str,
ssh_key_path: str | None,
ssh_known_hosts_file: str | None,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
gitea_credential_target: str,
build_command: str,
guest_work_dir: str | None,
guest_artifact_zip: str | None,
guest_linux_work_dir: str | None,
guest_linux_output_dir: str,
guest_artifact_source: str,
configuration: str,
skip_artifact: bool,
use_shared_cache: bool,
extra_env: tuple[str, ...],
) -> None:
"""Run a build inside a guest VM, optionally packaging the result.
Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via
one of:
* ``--host-source-dir`` — local directory zipped/tarred and uploaded;
* ``--clone-url`` — ``git clone`` invoked inside the guest.
Phase C hook: ``--guest-work-dir`` and ``--guest-artifact-zip`` are
parameterised; defaults are applied here, never inside the transport
layer.
"""
del gitea_credential_target # not yet honoured by Python port
guest_os = guest_os.lower()
if host_source_dir and clone_url:
raise click.BadParameter(
"Provide either --host-source-dir or --clone-url, not both."
)
if not host_source_dir and not clone_url:
raise click.BadParameter(
"Provide either --host-source-dir or --clone-url."
)
if host_source_dir and not Path(host_source_dir).is_dir():
raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}")
extra = _parse_extra_env(extra_env)
config = load_config()
if guest_os == "linux":
workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build"
key_path: str | None = ssh_key_path
if key_path is None and config.ssh_key_path is not None:
key_path = str(config.ssh_key_path)
try:
_linux_build(
ip_address=ip_address,
ssh_user=ssh_user,
key_path=key_path,
known_hosts=ssh_known_hosts_file,
workdir=workdir,
output_dir=guest_linux_output_dir,
host_source_dir=host_source_dir,
clone_url=clone_url,
clone_branch=clone_branch,
clone_commit=clone_commit,
clone_submodules=clone_submodules,
build_command=build_command,
artifact_source=guest_artifact_source,
extra_env=extra,
skip_artifact=skip_artifact,
)
except (TransportError, TransportCommandError) as exc:
raise click.ClickException(str(exc)) from exc
else:
workdir = guest_work_dir or "C:\\CI\\build"
artifact_zip = guest_artifact_zip or "C:\\CI\\output\\artifacts.zip"
target = credential_target or config.guest_cred_target
try:
_windows_build(
ip_address=ip_address,
credential_target=target,
workdir=workdir,
artifact_zip=artifact_zip,
host_source_dir=host_source_dir,
clone_url=clone_url,
clone_branch=clone_branch,
clone_commit=clone_commit,
clone_submodules=clone_submodules,
build_command=build_command,
artifact_source=guest_artifact_source,
configuration=configuration,
extra_env=extra,
skip_artifact=skip_artifact,
use_shared_cache=use_shared_cache,
)
except (TransportError, TransportCommandError) as exc:
raise click.ClickException(str(exc)) from exc
__all__ = ["build", "build_run"]