Files
local-ci-cd-system/src/ci_orchestrator/commands/build.py
T
Simone 6f51d44f92 feat(build): optional xvfb-run for Linux GUI builds
Add an --xvfb switch to 'build run' and 'job' that wraps the Linux build
command in 'xvfb-run -a sh -c ...', giving GUI toolkits (GTK4/PyGObject)
a headless $DISPLAY. Exported env vars precede xvfb-run so they
propagate into the virtual-display session. No-op (with a notice) for
Windows guests.

The local-ci-build composite action exposes it as the 'xvfb' input
(default false), forwarded as INPUT_XVFB -> --xvfb, mirroring the
use-shared-cache/skip-artifact wiring.

Docs: WORKFLOW-AUTHORING.md parameter table + Linux example note;
LINUX-TEMPLATE-SETUP.md Step 4a (the template ships python3-gi
gir1.2-gtk-4.0 xvfb). Adds build-run unit tests (wrap, plain, Windows
no-op).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-06 13:38:34 +02:00

642 lines
25 KiB
Python

"""``build`` sub-commands.
Phase A3 ports the remote build pipeline:
* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1``
The original PS script supports two source-provisioning modes (host-side
clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom
build command and optional artifact packaging. This Python port preserves
the mechanics while keeping the guest workdir and artifact paths fully
parameterised — see Phase C hooks below.
Phase C hook: the guest workdir / artifact paths are passed as opaque
strings; this module makes no assumption about Windows vs POSIX path
syntax beyond a sensible default per ``--guest-os``. ``build run`` does
not import ``WorkstationVmrunBackend`` directly; it relies on the
transport layer to talk to whatever guest the orchestrator points it at.
"""
from __future__ import annotations
import shlex
import sys
import tarfile
import tempfile
import zipfile
from collections.abc import Iterable
from pathlib import Path
from typing import TYPE_CHECKING
import click
from ci_orchestrator.config import load_config
from ci_orchestrator.credentials import KeyringCredentialStore
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
from ci_orchestrator.transport.ssh import SshTransport
from ci_orchestrator.transport.winrm import WinRmTransport
if TYPE_CHECKING: # pragma: no cover
pass
# ---------------------------------------------------------------- group
@click.group("build")
def build() -> None:
"""Remote build commands."""
# ---------------------------------------------------------------- helpers
def _parse_extra_env(items: Iterable[str]) -> dict[str, str]:
"""Parse ``KEY=VALUE`` pairs into a dict.
Empty entries are ignored. A pair without ``=`` raises ``BadParameter``.
"""
out: dict[str, str] = {}
for raw in items:
if not raw:
continue
if "=" not in raw:
raise click.BadParameter(
f"--extra-env value must be KEY=VALUE, got: {raw!r}"
)
key, value = raw.split("=", 1)
if not key:
raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}")
out[key] = value
return out
def _zip_dir(source_dir: Path, archive_path: Path) -> None:
"""Create a deterministic ZIP archive containing the contents of ``source_dir``."""
with zipfile.ZipFile(
archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1
) as zf:
for item in source_dir.rglob("*"):
if item.is_file():
zf.write(item, arcname=item.relative_to(source_dir))
def _tar_dir(source_dir: Path, archive_path: Path) -> None:
"""Create a gzip tar archive containing the contents of ``source_dir``."""
with tarfile.open(archive_path, "w:gz") as tf:
for item in source_dir.iterdir():
tf.add(item, arcname=item.name)
def _ps_quote(value: str) -> str:
"""Quote a value for embedding in a PowerShell single-quoted string."""
return "'" + value.replace("'", "''") + "'"
def _sh_quote(value: str) -> str:
"""POSIX shell single-quote escaping."""
return shlex.quote(value)
def _report_build_output(stdout: str, stderr: str) -> str:
"""Emit guest build stdout/stderr to the CI log, flushed.
Returns the last few lines for the failure message so the cause is
visible even if stream buffering hides the body on a non-zero exit.
"""
click.echo("[build run] ----- build output -----")
if stdout:
sys.stdout.write(stdout if stdout.endswith("\n") else stdout + "\n")
sys.stdout.flush()
if stderr:
sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n")
sys.stderr.flush()
click.echo("[build run] ----- end build output -----")
return _tail_lines(stdout, stderr)
def _tail_lines(stdout: str, stderr: str) -> str:
"""Last few output lines for a failure message (no printing)."""
combined = (stderr or "").strip() or (stdout or "").strip()
lines = combined.splitlines()
return " | ".join(lines[-5:]) if lines else "(no build output)"
# ---------------------------------------------------------------- Linux flow
def _linux_build(
*,
ip_address: str,
ssh_user: str,
key_path: str | None,
known_hosts: str | None,
workdir: str,
output_dir: str,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
build_command: str,
artifact_source: str,
extra_env: dict[str, str],
skip_artifact: bool,
use_xvfb: bool = False,
) -> None:
click.echo("[build run] Linux build mode (SSH transport)")
with SshTransport(
ip_address,
username=ssh_user,
key_path=key_path,
known_hosts=known_hosts,
) as t:
# Clock-skew guard. Ephemeral clones can boot with a wrong clock
# (e.g. RTC kept in local time → guest reads it as UTC and runs hours
# ahead). Enable NTP and BLOCK until the clock is actually synced, so
# the (possibly backward) correction step happens HERE — before the
# sources are stamped below. If the step landed *after* the touch,
# make would again see source mtimes in the future and warn about
# clock skew. Do NOT run `hwclock -s`: on a clone whose RTC is in
# local time it pushes the clock further ahead, the very skew we are
# removing. Best-effort (check=False); the touch after checkout is the
# backstop when NTP is unreachable and the clock simply stays put.
t.run(
"sudo timedatectl set-ntp true 2>/dev/null || true; "
'for _ in $(seq 1 60); do '
'[ "$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" '
"= yes ] && break; sleep 1; done",
check=False,
)
t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
if host_source_dir:
click.echo(f"[build run] packaging host source: {host_source_dir}")
with tempfile.TemporaryDirectory() as td:
tar_path = Path(td) / "ci-src.tar.gz"
_tar_dir(Path(host_source_dir), tar_path)
guest_tar = f"/tmp/{tar_path.name}"
t.copy(str(tar_path), guest_tar)
click.echo(f"[build run] extracting source in guest: {workdir}")
t.run(
f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} "
f"&& rm -f {_sh_quote(guest_tar)}"
)
elif clone_url:
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
cmd = (
f"GIT_TERMINAL_PROMPT=0 "
f"git clone --depth 1 --branch {_sh_quote(clone_branch)} "
f"{'--recurse-submodules ' if clone_submodules else ''}"
f"{_sh_quote(clone_url)} {_sh_quote(workdir)}"
)
t.run(cmd)
if clone_commit:
# The shallow --depth 1 --branch clone only has the branch
# tip. If the branch advanced between workflow trigger and
# runner pickup, the pinned commit is absent ("reference is
# not a tree"). Fetch exactly that commit, then check out.
t.run(
f"git -C {_sh_quote(workdir)} fetch --depth 1 origin "
f"{_sh_quote(clone_commit)} && "
f"git -C {_sh_quote(workdir)} checkout "
f"{_sh_quote(clone_commit)}"
)
else:
raise click.ClickException(
"Linux build requires either --host-source-dir or --clone-url."
)
# Normalise source mtimes to the (now-synced) guest clock so make
# never sees a file dated in the future and skips the skew warning
# plus the spurious rebuild it triggers.
t.run(
f"find {_sh_quote(workdir)} -exec touch -d {_sh_quote('now')} {{}} +",
check=False,
)
env_prefix = "".join(
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
)
cmd = build_command or "make"
# Optionally run the build under a headless X server (xvfb-run) so GUI
# toolkits (e.g. GTK4/PyGObject) have a $DISPLAY. The exported env vars
# precede xvfb-run so they propagate into the virtual-display session;
# -a auto-selects a free server number. The whole `cd && build` is
# wrapped via `sh -c` so xvfb-run drives a single command.
inner = f"cd {_sh_quote(workdir)} && {cmd}"
run_part = f"xvfb-run -a sh -c {_sh_quote(inner)}" if use_xvfb else inner
# PYTHONUNBUFFERED so the guest python flushes promptly and the
# streamed output is timely rather than emitted in one block.
full_cmd = f"{env_prefix}export PYTHONUNBUFFERED=1; {run_part}"
xvfb_note = " [xvfb-run]" if use_xvfb else ""
click.echo(
f"[build run] running build{xvfb_note} (env vars redacted): "
f"cd {workdir} && {cmd}"
)
click.echo("[build run] ----- build output (live) -----")
result = t.run_streaming(full_cmd, check=False)
click.echo("[build run] ----- end build output -----")
if result.returncode != 0:
raise click.ClickException(
"build command failed "
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
)
if skip_artifact:
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
return
# Resolve the artifact source: absolute path as-is, else relative
# to the build workdir. (artifact_source may be absolute, e.g.
# /opt/ci/output; the old `workdir + '/' + src` produced
# /opt/ci/build//opt/ci/output and silently found nothing.)
src = artifact_source or "dist"
srcpath = src.rstrip("/") if src.startswith("/") else workdir.rstrip("/") + "/" + src
out = output_dir.rstrip("/")
if srcpath == out or srcpath.startswith(out + "/"):
# The build wrote directly into the collect dir; do NOT wipe it
# (that deleted the build output) and nothing to copy. Fail
# loudly if it is missing or empty rather than collecting an
# empty artifact.
click.echo(
f"[build run] artifact source is the output dir ({out}); collecting in place"
)
t.run(
f"if [ ! -d {_sh_quote(out)} ] || "
f'[ -z "$(ls -A {_sh_quote(out)} 2>/dev/null)" ]; then '
f"echo '[build run] artifact source not found or empty: {out}' >&2; "
f"exit 1; fi"
)
else:
t.run(f"rm -rf {_sh_quote(out)} && mkdir -p {_sh_quote(out)}")
cp_cmd = (
f"if [ -d {_sh_quote(srcpath)} ]; then "
f"cp -r {_sh_quote(srcpath + '/.')} {_sh_quote(out + '/')}; "
f"elif [ -e {_sh_quote(srcpath)} ]; then "
f"cp {_sh_quote(srcpath)} {_sh_quote(out + '/')}; "
f"else echo '[build run] artifact source not found: {srcpath}' >&2; "
f"exit 1; fi"
)
t.run(cp_cmd)
click.echo(f"[build run] artifact staged at {out}")
# ---------------------------------------------------------------- Windows flow
def _windows_build(
*,
ip_address: str,
credential_target: str,
workdir: str,
output_dir: str,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
build_command: str,
artifact_source: str,
configuration: str,
extra_env: dict[str, str],
skip_artifact: bool,
use_shared_cache: bool,
) -> None:
click.echo("[build run] Windows build mode (WinRM transport)")
cred = KeyringCredentialStore().get(credential_target)
with WinRmTransport(ip_address, cred.username, cred.password) as t:
# Prepare workdir + artifact output directory.
prep_ps = (
f"$work = {_ps_quote(workdir)}; "
f"$out = {_ps_quote(output_dir)}; "
"if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; "
"if (Test-Path $work) { Remove-Item $work -Recurse -Force }; "
"New-Item -ItemType Directory -Path $work -Force | Out-Null"
)
t.run(prep_ps)
if host_source_dir:
click.echo(f"[build run] zipping host source: {host_source_dir}")
with tempfile.TemporaryDirectory() as td:
zip_path = Path(td) / "ci-src.zip"
_zip_dir(Path(host_source_dir), zip_path)
size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1)
click.echo(f"[build run] uploading source archive ({size_mb} MB)")
guest_zip = "C:\\CI\\src-transfer.zip"
t.copy(str(zip_path), guest_zip)
t.run(
f"Expand-Archive -Path {_ps_quote(guest_zip)} "
f"-DestinationPath {_ps_quote(workdir)} -Force; "
f"Remove-Item {_ps_quote(guest_zip)} -Force"
)
elif clone_url:
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
sub_flag = " --recurse-submodules" if clone_submodules else ""
t.run(
f"$env:GIT_TERMINAL_PROMPT = '0'; "
f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); "
f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} "
f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)"
)
if clone_commit:
# Shallow clone has only the branch tip; fetch the pinned
# commit explicitly so checkout works even if the branch
# advanced between trigger and runner pickup.
t.run(
f"git -C {_ps_quote(workdir)} fetch --depth 1 origin "
f"{_ps_quote(clone_commit)}; "
f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}"
)
else:
raise click.ClickException(
"Windows build requires either --host-source-dir or --clone-url."
)
# Inject env vars + run the build.
env_setup = "; ".join(
f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items()
)
if env_setup:
env_setup += "; "
if use_shared_cache:
env_setup += (
"$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; "
"$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; "
)
if build_command:
run_cmd = build_command
click.echo(f"[build run] custom build command: {run_cmd}")
else:
click.echo(
f"[build run] default build: dotnet restore + build (configuration={configuration})"
)
run_cmd = (
"dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; "
f"dotnet build --configuration {_ps_quote(configuration)} "
"--output (Join-Path (Get-Location) 'bin\\CIOutput')"
)
# Run the build command as PowerShell on the Windows guest. It is
# authored in PS by the workflow (New-Item, Set-Content, ...) and
# may also invoke native exes (python, dotnet). Wrapping it in
# `cmd /c` broke PS-native commands ("'New-Item' is not
# recognized"). LASTEXITCODE is seeded to 0 so a PS-only command
# (which never sets it) exits cleanly; native tools still set it.
full_ps = (
f"{env_setup}Set-Location {_ps_quote(workdir)}; "
f"$ErrorActionPreference='Continue'; "
f"$global:LASTEXITCODE = 0; "
f"{run_cmd}; "
"if ($null -eq $LASTEXITCODE) { exit 0 } else { exit $LASTEXITCODE }"
)
result = t.run(full_ps, check=False)
tail = _report_build_output(result.stdout, result.stderr)
if result.returncode != 0:
raise click.ClickException(
f"build command failed (exit {result.returncode}): {tail}"
)
if skip_artifact:
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
return
# Stage the artifact source into the collect dir as raw files
# (no intermediate zip — `actions/upload-artifact` produces the
# single final archive; mirrors the Linux flow, avoids zip-in-zip).
# artifact_source may be absolute (e.g. C:\CI\output) or relative
# to the build workdir; PowerShell Join-Path does not treat a
# rooted segment as absolute, so resolve explicitly. When the
# source IS the collect dir (build wrote there directly), collect
# in place — do NOT wipe it (that deleted the build output).
src_rel = (artifact_source or "dist") if build_command else "bin\\CIOutput"
pkg_ps = (
f"$work = {_ps_quote(workdir)}; "
f"$rel = {_ps_quote(src_rel)}; "
"$src = if ([System.IO.Path]::IsPathRooted($rel)) "
"{ $rel } else { Join-Path $work $rel }; "
f"$out = {_ps_quote(output_dir)}; "
"$sf = [System.IO.Path]::GetFullPath($src).TrimEnd('\\'); "
"$of = [System.IO.Path]::GetFullPath($out).TrimEnd('\\'); "
"$inPlace = ($sf -ieq $of) -or "
"$sf.StartsWith($of + '\\', "
"[System.StringComparison]::OrdinalIgnoreCase); "
"if ($inPlace) { "
" if (-not (Test-Path $out) -or "
" -not (Get-ChildItem -Force -- $out)) { "
" throw \"[build run] artifact source not found or empty: $out\" } "
"} else { "
" if (-not (Test-Path $src)) { "
" throw \"[build run] artifact source not found: $src\" }; "
" if (Test-Path $out) { Remove-Item $out -Recurse -Force }; "
" New-Item -ItemType Directory -Path $out -Force | Out-Null; "
" if (Test-Path $src -PathType Leaf) { "
" Copy-Item $src -Destination $out -Force "
" } else { "
" Copy-Item (Join-Path $src '*') -Destination $out -Recurse -Force } "
"}"
)
t.run(pkg_ps)
click.echo(f"[build run] artifact staged at {output_dir}")
# ---------------------------------------------------------------- CLI
@build.command("run")
@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.")
@click.option(
"--guest-os",
type=click.Choice(["windows", "linux"], case_sensitive=False),
default="windows",
show_default=True,
)
@click.option(
"--credential-target",
"credential_target",
default=None,
help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.",
)
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
@click.option(
"--ssh-key-path",
"ssh_key_path",
default=None,
help="SSH private key path (Linux). Defaults to config.ssh_key_path.",
)
@click.option(
"--ssh-known-hosts-file",
"ssh_known_hosts_file",
default=None,
help="Persistent known_hosts file (Linux). Default: permissive (None).",
)
@click.option("--host-source-dir", "host_source_dir", default=None)
@click.option("--clone-url", "clone_url", default=None)
@click.option("--clone-branch", "clone_branch", default="main", show_default=True)
@click.option("--clone-commit", "clone_commit", default="")
@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False)
@click.option(
"--gitea-credential-target",
"gitea_credential_target",
default="GiteaPAT",
show_default=True,
help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.",
)
@click.option("--build-command", "build_command", default="")
@click.option(
"--guest-work-dir",
"guest_work_dir",
default=None,
help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).",
)
@click.option(
"--guest-output-dir",
"guest_output_dir",
default=None,
help="Collect dir inside the Windows guest (default: C:\\CI\\output). "
"Raw build outputs are staged here; upload-artifact zips them.",
)
@click.option(
"--guest-linux-work-dir",
"guest_linux_work_dir",
default=None,
help="Alias for --guest-work-dir, kept for shim compatibility (Linux).",
)
@click.option(
"--guest-linux-output-dir",
"guest_linux_output_dir",
default="/opt/ci/output",
show_default=True,
)
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
@click.option("--configuration", default="Release", show_default=True)
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
@click.option(
"--xvfb",
"use_xvfb",
is_flag=True,
default=False,
help="Linux only: run the build under xvfb-run (headless X for GUI toolkits).",
)
@click.option(
"--extra-env",
"extra_env",
multiple=True,
help="Repeatable KEY=VALUE pairs injected into the guest environment.",
)
def build_run(
ip_address: str,
guest_os: str,
credential_target: str | None,
ssh_user: str,
ssh_key_path: str | None,
ssh_known_hosts_file: str | None,
host_source_dir: str | None,
clone_url: str | None,
clone_branch: str,
clone_commit: str,
clone_submodules: bool,
gitea_credential_target: str,
build_command: str,
guest_work_dir: str | None,
guest_output_dir: str | None,
guest_linux_work_dir: str | None,
guest_linux_output_dir: str,
guest_artifact_source: str,
configuration: str,
skip_artifact: bool,
use_shared_cache: bool,
use_xvfb: bool,
extra_env: tuple[str, ...],
) -> None:
"""Run a build inside a guest VM, optionally packaging the result.
Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via
one of:
* ``--host-source-dir`` — local directory zipped/tarred and uploaded;
* ``--clone-url`` — ``git clone`` invoked inside the guest.
Phase C hook: ``--guest-work-dir`` and ``--guest-output-dir`` are
parameterised; defaults are applied here, never inside the transport
layer.
"""
del gitea_credential_target # not yet honoured by Python port
guest_os = guest_os.lower()
if host_source_dir and clone_url:
raise click.BadParameter(
"Provide either --host-source-dir or --clone-url, not both."
)
if not host_source_dir and not clone_url:
raise click.BadParameter(
"Provide either --host-source-dir or --clone-url."
)
if host_source_dir and not Path(host_source_dir).is_dir():
raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}")
extra = _parse_extra_env(extra_env)
config = load_config()
if guest_os == "linux":
workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build"
key_path: str | None = ssh_key_path
if key_path is None and config.ssh_key_path is not None:
key_path = str(config.ssh_key_path)
try:
_linux_build(
ip_address=ip_address,
ssh_user=ssh_user,
key_path=key_path,
known_hosts=ssh_known_hosts_file,
workdir=workdir,
output_dir=guest_linux_output_dir,
host_source_dir=host_source_dir,
clone_url=clone_url,
clone_branch=clone_branch,
clone_commit=clone_commit,
clone_submodules=clone_submodules,
build_command=build_command,
artifact_source=guest_artifact_source,
extra_env=extra,
skip_artifact=skip_artifact,
use_xvfb=use_xvfb,
)
except (TransportError, TransportCommandError) as exc:
raise click.ClickException(str(exc)) from exc
else:
if use_xvfb:
click.echo("[build run] --xvfb ignored for Windows guest.")
workdir = guest_work_dir or "C:\\CI\\build"
output_dir = guest_output_dir or "C:\\CI\\output"
target = credential_target or config.guest_cred_target
try:
_windows_build(
ip_address=ip_address,
credential_target=target,
workdir=workdir,
output_dir=output_dir,
host_source_dir=host_source_dir,
clone_url=clone_url,
clone_branch=clone_branch,
clone_commit=clone_commit,
clone_submodules=clone_submodules,
build_command=build_command,
artifact_source=guest_artifact_source,
configuration=configuration,
extra_env=extra,
skip_artifact=skip_artifact,
use_shared_cache=use_shared_cache,
)
except (TransportError, TransportCommandError) as exc:
raise click.ClickException(str(exc)) from exc
__all__ = ["build", "build_run"]