From 04ca316f17665474bd824c62be48f26b913631ea Mon Sep 17 00:00:00 2001 From: Simone Date: Mon, 8 Jun 2026 00:11:36 +0200 Subject: [PATCH 1/3] feat(vm): add `vm open` to launch the VMware Workstation GUI Open a VMX in the interactive Workstation GUI (not the headless vmrun path) for hands-on debugging of a clone. Spawns `vmware` detached (own session, stdio detached) and returns the child PID. Flags: --vmx (required), --power-on (-x), --fullscreen (-X), --new-window (-n), --display (sudo -u ci-runner strips $DISPLAY, so allow forcing it), --vmware-path. Warns when no X display is set. 7 tests; vm.py 96%. Co-Authored-By: Claude Opus 4.8 --- src/ci_orchestrator/commands/vm.py | 114 +++++++++++++++++++++++++++++ tests/python/test_commands_vm.py | 91 +++++++++++++++++++++++ 2 files changed, 205 insertions(+) diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py index e1911dc..15761cf 100644 --- a/src/ci_orchestrator/commands/vm.py +++ b/src/ci_orchestrator/commands/vm.py @@ -12,7 +12,9 @@ instead of scanning the local filesystem. from __future__ import annotations +import os import shutil +import subprocess import time from datetime import UTC, datetime, timedelta from pathlib import Path @@ -483,4 +485,116 @@ def vm_new( click.echo(handle.identifier) +# ────────────────────────────────────────────────────────────────────────── open + + +def _launch_gui(argv: list[str], env: dict[str, str]) -> int: + """Spawn the VMware Workstation GUI detached. Returns the child PID. + + Isolated so tests can monkeypatch it without spawning a real process. The + GUI must outlive the CLI, so the child is started in its own session with + its stdio detached. + """ + proc = subprocess.Popen( # argv is a fixed list, never a shell string + argv, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return proc.pid + + +@vm.command("open") +@click.option( + "--vmx", + "--vm-path", + "vmx", + required=True, + help="Path to the VMX file to open in the VMware Workstation GUI.", +) +@click.option( + "--vmware-path", + "vmware_path", + default="vmware", + show_default=True, + help="Path to the VMware Workstation GUI binary.", +) +@click.option( + "--power-on", + "power_on", + is_flag=True, + default=False, + help="Power the VM on when it is opened (vmware -x).", +) +@click.option( + "--fullscreen", + is_flag=True, + default=False, + help="Power on and enter full-screen mode (vmware -X). Implies --power-on.", +) +@click.option( + "--new-window", + "new_window", + is_flag=True, + default=False, + help="Open in a new window instead of a tab (vmware -n).", +) +@click.option( + "--display", + "display", + default=None, + help="X display to use (default: inherit $DISPLAY). e.g. ':0'.", +) +def vm_open( + vmx: str, + vmware_path: str, + power_on: bool, + fullscreen: bool, + new_window: bool, + display: str | None, +) -> None: + """Open a VMX in the VMware Workstation GUI (run as the ci-runner user). + + Launches the interactive GUI — unlike the headless ``vmrun`` path the rest + of the orchestrator uses. The GUI needs an X display: when invoked through + ``sudo -u ci-runner`` the display is usually stripped, so authorise the + service user first, e.g. ``xhost +SI:localuser:ci-runner``, and pass + ``--display :0`` (or export ``DISPLAY``). + """ + vmx_path = Path(vmx) + if not vmx_path.is_file(): + raise click.ClickException(f"VMX file not found: {vmx}") + + gui = shutil.which(vmware_path) or vmware_path + + env = dict(os.environ) + if display: + env["DISPLAY"] = display + if not env.get("DISPLAY"): + click.echo( + "[vm open] WARNING: no X display set ($DISPLAY empty and --display " + "omitted). The GUI will fail to start. Pass --display :0 and run " + "`xhost +SI:localuser:ci-runner` from a desktop session first.", + err=True, + ) + + argv = [gui] + if new_window: + argv.append("-n") + if fullscreen: + argv.append("-X") + elif power_on: + argv.append("-x") + argv.append(str(vmx_path)) + + click.echo(f"[vm open] launching VMware GUI: {' '.join(argv)}") + try: + pid = _launch_gui(argv, env) + except (OSError, ValueError) as exc: + raise click.ClickException(f"failed to launch VMware GUI: {exc}") from exc + click.echo(f"[vm open] VMware GUI started (pid {pid}); display={env.get('DISPLAY', '')}") + + __all__ = ["cleanup_orphans", "vm"] diff --git a/tests/python/test_commands_vm.py b/tests/python/test_commands_vm.py index 172863d..18fb5e7 100644 --- a/tests/python/test_commands_vm.py +++ b/tests/python/test_commands_vm.py @@ -497,3 +497,94 @@ def test_vm_cleanup_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_pat ) assert result.exit_code == 0 assert "vmrun not available" in result.output + + +# ── vm open ─────────────────────────────────────────────────────────────────── + + +def _capture_launch(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """Patch _launch_gui to record argv/env instead of spawning a process.""" + captured: dict[str, Any] = {} + + def _fake(argv: list[str], env: dict[str, str]) -> int: + captured["argv"] = argv + captured["env"] = env + return 4242 + + monkeypatch.setattr(vm_module, "_launch_gui", _fake) + return captured + + +def test_vm_open_missing_vmx_errors(tmp_path: Path) -> None: + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(tmp_path / "nope.vmx")]) + assert result.exit_code != 0 + assert "VMX file not found" in result.output + + +def test_vm_open_happy_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + cap = _capture_launch(monkeypatch) + monkeypatch.setenv("DISPLAY", ":0") + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(vmx)]) + assert result.exit_code == 0, result.output + assert cap["argv"][-1] == str(vmx) + assert "-x" not in cap["argv"] and "-X" not in cap["argv"] + assert "pid 4242" in result.output + + +def test_vm_open_power_on_adds_x(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + cap = _capture_launch(monkeypatch) + monkeypatch.setenv("DISPLAY", ":0") + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(vmx), "--power-on"]) + assert result.exit_code == 0, result.output + assert "-x" in cap["argv"] + + +def test_vm_open_fullscreen_adds_capital_x(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + cap = _capture_launch(monkeypatch) + monkeypatch.setenv("DISPLAY", ":0") + result = CliRunner().invoke( + cli, ["vm", "open", "--vmx", str(vmx), "--fullscreen", "--new-window"] + ) + assert result.exit_code == 0, result.output + assert "-X" in cap["argv"] and "-n" in cap["argv"] + + +def test_vm_open_display_option_overrides_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + cap = _capture_launch(monkeypatch) + monkeypatch.delenv("DISPLAY", raising=False) + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(vmx), "--display", ":1"]) + assert result.exit_code == 0, result.output + assert cap["env"]["DISPLAY"] == ":1" + assert "display=:1" in result.output + + +def test_vm_open_warns_without_display(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + _capture_launch(monkeypatch) + monkeypatch.delenv("DISPLAY", raising=False) + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(vmx)]) + assert result.exit_code == 0, result.output + assert "no X display" in result.output + + +def test_vm_open_launch_failure_is_reported(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + vmx = tmp_path / "clone.vmx" + vmx.write_text("config") + + def _boom(argv: list[str], env: dict[str, str]) -> int: + raise OSError("vmware not found") + + monkeypatch.setattr(vm_module, "_launch_gui", _boom) + monkeypatch.setenv("DISPLAY", ":0") + result = CliRunner().invoke(cli, ["vm", "open", "--vmx", str(vmx)]) + assert result.exit_code != 0 + assert "failed to launch VMware GUI" in result.output From 2241d6af7f43d6d2ebf0bc0347408ed8fe33e0e2 Mon Sep 17 00:00:00 2001 From: Simone Date: Mon, 8 Jun 2026 00:11:36 +0200 Subject: [PATCH 2/3] feat(setup): add `ci` shell alias helper + install it from setup-host scripts/install-ci-alias.sh installs `ci` (runs the orchestrator as ci-runner via the prod venv) and `ci-me` (current user). Idempotent, marker-bounded; supports --system (/etc/profile.d), --uninstall. setup-host-linux.sh gains a step [7/7] that installs it system-wide, passing CI_PROD_PYTHON/CI_RUN_USER. Co-Authored-By: Claude Opus 4.8 --- scripts/install-ci-alias.sh | 105 ++++++++++++++++++++++++++++++++++++ setup-host-linux.sh | 32 +++++++---- 2 files changed, 128 insertions(+), 9 deletions(-) create mode 100755 scripts/install-ci-alias.sh diff --git a/scripts/install-ci-alias.sh b/scripts/install-ci-alias.sh new file mode 100755 index 0000000..dd3aa13 --- /dev/null +++ b/scripts/install-ci-alias.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# install-ci-alias.sh — install a `ci` shell helper that runs the CI +# orchestrator as the ci-runner service user against the production venv. +# +# After install: ci validate host == sudo -u ci-runner \ +# /opt/ci/venv/bin/python -m ci_orchestrator validate host +# +# Usage: +# ./scripts/install-ci-alias.sh # install into the current user's +# # ~/.bashrc and ~/.zshrc (if present) +# ./scripts/install-ci-alias.sh --system # install system-wide via +# # /etc/profile.d/ci-alias.sh (needs sudo) +# ./scripts/install-ci-alias.sh --uninstall # remove the installed block +# +# Idempotent: re-running updates the block in place instead of duplicating it. +set -euo pipefail + +PROD_PY="${CI_PROD_PYTHON:-/opt/ci/venv/bin/python}" +RUN_USER="${CI_RUN_USER:-ci-runner}" +MARK_BEGIN="# >>> ci-orchestrator alias >>>" +MARK_END="# <<< ci-orchestrator alias <<<" + +log() { printf '[install-ci-alias] %s\n' "$*"; } +die() { printf '[install-ci-alias] ERROR: %s\n' "$*" >&2; exit 1; } + +block() { + cat </dev/null || true + fi +} + +install_into() { + local file="$1" + strip_block "$file" + { printf '\n'; block; } >> "$file" + log "updated $file" +} + +uninstall_from() { + local file="$1" + if [ -f "$file" ] && grep -qF "$MARK_BEGIN" "$file"; then + strip_block "$file" + log "removed block from $file" + fi +} + +main() { + local mode="user" action="install" + case "${1:-}" in + --system) mode="system" ;; + --uninstall) action="uninstall" ;; + "" ) ;; + * ) die "unknown argument: $1 (use --system or --uninstall)" ;; + esac + + # Sanity: warn (don't fail) if the production venv is missing. + [ -x "$PROD_PY" ] || log "WARNING: $PROD_PY not found — alias installed anyway; fix the venv before use." + + if [ "$mode" = "system" ]; then + local target="/etc/profile.d/ci-alias.sh" + if [ "$action" = "uninstall" ]; then + sudo rm -f "$target" && log "removed $target" + else + block | sudo tee "$target" >/dev/null + sudo chmod 0644 "$target" + log "installed $target (applies to all login shells)" + fi + else + local files=() + [ -f "$HOME/.bashrc" ] && files+=("$HOME/.bashrc") + [ -f "$HOME/.zshrc" ] && files+=("$HOME/.zshrc") + # If neither exists, create ~/.bashrc so the alias lands somewhere. + [ ${#files[@]} -eq 0 ] && files+=("$HOME/.bashrc") + + for f in "${files[@]}"; do + if [ "$action" = "uninstall" ]; then uninstall_from "$f"; else install_into "$f"; fi + done + fi + + if [ "$action" = "install" ]; then + log "done. Open a new shell, or run: source ~/.bashrc" + log "try: ci validate host" + else + log "done. Open a new shell to drop the alias." + fi +} + +main "$@" diff --git a/setup-host-linux.sh b/setup-host-linux.sh index 31bb6db..ff04e0e 100755 --- a/setup-host-linux.sh +++ b/setup-host-linux.sh @@ -13,6 +13,7 @@ # 4. Installa Python 3.11+ e crea venv produzione in /opt/ci/venv # 5. (Opzionale) Clona il repo e installa il package nel venv # 6. (Opt-in, --with-pwsh) Installa PowerShell Core +# 7. Installa l'alias shell `ci` a livello di sistema (/etc/profile.d/) # L'host Linux NON ha più bisogno di pwsh per il normale funzionamento # (orchestratore Python). Installalo solo se ti serve per script legacy. @@ -93,7 +94,7 @@ echo " Utente : $CI_USER" echo "" # ── Step 1: utente di servizio ──────────────────────────────────────────────── -info "[1/6] Utente di servizio $CI_USER" +info "[1/7] Utente di servizio $CI_USER" if id "$CI_USER" &>/dev/null; then warn "Utente $CI_USER esiste già — skip" else @@ -102,7 +103,7 @@ else fi # ── Step 2: mount partizione CI ─────────────────────────────────────────────── -info "[2/6] Mount $CI_DISK → $CI_ROOT" +info "[2/7] Mount $CI_DISK → $CI_ROOT" if mountpoint -q "$CI_ROOT" 2>/dev/null; then warn "$CI_ROOT già montato — skip riorganizzazione e fstab" @@ -160,7 +161,7 @@ else fi # ── Step 3: layout directory e permessi ─────────────────────────────────────── -info "[3/6] Layout directory e permessi" +info "[3/7] Layout directory e permessi" # Installa acl se mancante if ! command -v setfacl &>/dev/null; then @@ -186,7 +187,7 @@ echo " Layout OK — ACL default su build-vms applicata" # ── Step 4: Python 3.11+ e venv ────────────────────────────────────────────── if [[ "$SKIP_PYTHON" == false ]]; then - info "[4/6] Python 3.11+ e venv produzione" + info "[4/7] Python 3.11+ e venv produzione" # Cerca la prima versione di Python >= 3.11 già installata PYTHON_BIN="" @@ -217,12 +218,12 @@ if [[ "$SKIP_PYTHON" == false ]]; then echo " venv creato in $OPT_CI/venv ($($PYTHON_BIN --version))" fi else - info "[4/6] Python/venv — SKIPPED" + info "[4/7] Python/venv — SKIPPED" fi # ── Step 5: Clone repo e pip install ───────────────────────────────────────── if [[ "$SKIP_CLONE" == false ]]; then - info "[5/6] Clone repo e pip install" + info "[5/7] Clone repo e pip install" REPO_DIR="$OPT_CI/local-ci-cd-system" if [[ -d "$REPO_DIR/.git" ]]; then warn "Repo già presente in $REPO_DIR — skip clone (eseguire git pull manualmente)" @@ -236,12 +237,12 @@ if [[ "$SKIP_CLONE" == false ]]; then || die "ci_orchestrator non importabile" echo " Verifica OK: ${verify_out%%$'\n'*}" else - info "[5/6] Clone repo — SKIPPED" + info "[5/7] Clone repo — SKIPPED" fi # ── Step 6: PowerShell Core (opt-in via --with-pwsh) ────────────────────────── if [[ "$WITH_PWSH" == true ]]; then - info "[6/6] PowerShell Core" + info "[6/7] PowerShell Core" if command -v pwsh &>/dev/null; then warn "pwsh già installato: $(pwsh --version) — skip" else @@ -263,7 +264,20 @@ if [[ "$WITH_PWSH" == true ]]; then echo " Installato: $(pwsh --version)" fi else - info "[6/6] PowerShell Core — SKIPPED (opt-in con --with-pwsh)" + info "[6/7] PowerShell Core — SKIPPED (opt-in con --with-pwsh)" +fi + +# ── Step 7: alias shell `ci` (system-wide) ──────────────────────────────────── +info "[7/7] Alias shell 'ci'" +ALIAS_SCRIPT="$OPT_CI/local-ci-cd-system/scripts/install-ci-alias.sh" +if [[ -x "$ALIAS_SCRIPT" ]]; then + # Installa /etc/profile.d/ci-alias.sh per tutti gli utenti login. + # Passa il venv/utente correnti così l'alias punta a questa installazione. + CI_PROD_PYTHON="$OPT_CI/venv/bin/python" CI_RUN_USER="$CI_USER" \ + bash "$ALIAS_SCRIPT" --system + echo " Alias 'ci' installato — apri una nuova shell e prova: ci validate host" +else + warn "install-ci-alias.sh non trovato in $ALIAS_SCRIPT — skip (clona il repo, Step 5)" fi # ── Riepilogo ───────────────────────────────────────────────────────────────── From f13b24956e217d04a546b02aa88b6ec7b85aae9a Mon Sep 17 00:00:00 2001 From: Simone Date: Mon, 8 Jun 2026 00:11:36 +0200 Subject: [PATCH 3/3] docs: add complete manual CLI runbook (all commands + ci alias + vm open) Rewrite docs/RUNBOOK-PhaseC.md as a full manual-command reference: every ci_orchestrator command with a working example as the ci-runner user, the `ci` alias setup, quick-reference table, manual VM lifecycle (incl. vm open GUI launch with the xhost/--display procedure), templating, maintenance, Phase C testing, cutover checklist and a troubleshooting map. Co-Authored-By: Claude Opus 4.8 --- docs/RUNBOOK-PhaseC.md | 343 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 docs/RUNBOOK-PhaseC.md diff --git a/docs/RUNBOOK-PhaseC.md b/docs/RUNBOOK-PhaseC.md new file mode 100644 index 0000000..4e0c10c --- /dev/null +++ b/docs/RUNBOOK-PhaseC.md @@ -0,0 +1,343 @@ +# RUNBOOK — manual CLI reference (Linux host, pwsh-free) + +Simplified but **complete** reference for every `ci_orchestrator` command you can +run by hand on the Linux host. All commands run as the **`ci-runner`** service +user against the live config (`/var/lib/ci/config.toml`). + +Two notations are used: + +- **Full form:** `sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator ` +- **Short form (alias):** `ci ` — after installing the helper (see §0). + +Examples below use the **short form**. Drop `ci` for the literal command. + +--- + +## 0. Setup (do once) + +### 0.1 Refresh the production venv (mandatory after a code change/merge) + +```bash +cd /opt/ci/local-ci-cd-system +sudo /opt/ci/venv/bin/python -m pip install . # non-editable, see CLAUDE.md +sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help | grep -E 'bench|smoke|validate|creds' +``` + +### 0.2 Install the `ci` alias + +```bash +./scripts/install-ci-alias.sh # into ~/.bashrc + ~/.zshrc (idempotent) +# --system install /etc/profile.d/ci-alias.sh for all users (needs sudo) +# --uninstall remove it +source ~/.bashrc # or open a new shell +ci validate host # smoke test the alias +``` + +The script defines two helpers: + +| Helper | Runs as | Use for | +|---------|---------------|---------| +| `ci` | `ci-runner` (sudo) | everything (writes keyring, starts VMs) | +| `ci-me` | current user | read-only checks where sudo is noise | + +### 0.3 Live environment + +| Thing | Value | +|-------|-------| +| Service user | `ci-runner` · Prod venv `/opt/ci/venv/bin/python` | +| Config | `/var/lib/ci/config.toml` | +| Linux template | `/var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx` (snap `BaseClean-Linux`) | +| Windows templates | `/var/lib/ci/templates/WinBuild2025` · `WinBuild2022` (snap `BaseClean`) | +| Clones / Artifacts | `/var/lib/ci/build-vms` · `/var/lib/ci/artifacts` | +| Linux SSH key | `/var/lib/ci/keys/ci_linux` · cred target `BuildVMGuest` | + +--- + +## 1. Quick reference — every command + +| Command | Does | Section | +|---------|------|---------| +| `ci validate host` | host prerequisites check (no VM) | §2 | +| `ci validate guest --host IP` | probe a running guest's transport | §2 | +| `ci monitor disk` | free-space alert | §2 | +| `ci monitor runner` | watch + auto-restart act_runner | §2 | +| `ci report job` | job summaries from JSONL logs | §2 | +| `ci job ...` | **full pipeline** clone→build→collect→destroy | §3 | +| `ci vm new ...` | linked-clone a template | §4 | +| `ci wait-ready --vmx ...` | poll until SSH/WinRM ready | §4 | +| `ci build run ...` | run a build in a running guest | §4 | +| `ci artifacts collect ...` | copy artifacts out of a guest | §4 | +| `ci vm remove --vmx ...` | stop + delete one clone | §4 | +| `ci vm cleanup` | sweep orphaned clones + stale locks | §4 / §6 | +| `ci vm open --vmx ...` | open a VMX in the VMware Workstation GUI | §4 | +| `ci creds set --user ...` | store guest credentials in keyring | §5 | +| `ci template deploy-linux` | build the Linux template VM | §5 | +| `ci template prepare-linux` | provision Linux template over SSH | §5 | +| `ci template prepare-win` | provision Windows template over WinRM | §5 | +| `ci template backup` | timestamped copy of template dirs | §5 | +| `ci retention run` | purge old artifacts/logs | §6 | +| `ci bench measure ...` | phase-timing benchmark | §7 | +| `ci bench run ...` | concurrency burn-in | §7 | +| `ci smoke run ...` | one E2E job + assertions | §7 | + +--- + +## 2. Health, monitoring & reporting (safe, no VM boot) + +### `validate host` — run this first, every day +```bash +ci validate host ; echo "exit=$?" +``` +Checks vmrun, template snapshots, keyring, `/var/lib/ci` permissions, `act-runner` +unit. Exit `0` + all `[OK ]`; a `[FAIL]` line names the broken check. + +### `validate guest` — diagnose a running guest's transport +```bash +ci validate guest --host 192.168.79.200 --ssh --guest-os linux # SSH +ci validate guest --host 192.168.79.201 --winrm --guest-os windows # WinRM +``` +Connects + runs a trivial command, printing the explicit cause on failure +(connect vs auth vs command). The guest must already be booted. + +### `monitor disk` — free-space alert +```bash +ci monitor disk --min-free-gb 50 +ci monitor disk --min-free-gb 50 --json # machine-readable one-liner +ci monitor disk --min-free-gb 50 --webhook-url "$DISCORD_WEBHOOK" +``` + +### `monitor runner` — keep act_runner alive +```bash +ci monitor runner --service-name act-runner --max-restarts 3 +``` +Restarts the runner up to N times per window; optional `--webhook-url`, +`--gitea-url`, `--gitea-credential-target`. + +### `report job` — read job history +```bash +ci report job --last 10 # 10 most recent, table +ci report job --failed # only failures +ci report job --job-id smoke-20260607-2030 # phase breakdown for one job +ci report job --last 20 --json # JSON out +``` + +--- + +## 3. Full pipeline (`job`) — the normal way to run a build + +One command does everything: clone → wait-ready → build → collect → destroy. +This is what the runner invokes per push; you can run it by hand too. + +```bash +# Linux build, in-guest git clone, no artifact packaging +ci job \ + --job-id manual-$(date +%s) \ + --repo-url https://gitea.emulab.it/Simone/myrepo.git \ + --branch main \ + --guest-os linux \ + --build-command "make all" \ + --use-git-clone +``` + +Key flags: `--commit`, `--configuration Release`, `--guest-artifact-source dist`, +`--submodules/--no-submodules`, `--use-git-clone/--host-clone`, `--use-shared-cache`, +`--skip-artifact`, `--xvfb` (Linux GUI builds), `--guest-cpu N`, `--guest-memory-mb N`, +`--template-path`, `--snapshot-name`, `--ready-timeout`, `--extra-env-json '{"K":"V"}'`, +`--gitea-credential-target` (private repos). Required: `--job-id --repo-url --branch`. + +> For a zero-config end-to-end check, prefer `smoke run` (§7) over hand-rolling `job`. + +--- + +## 4. Manual VM lifecycle (the steps `job` runs internally) + +Use these to drive a build stage-by-stage, e.g. when debugging a single phase. + +### 4.1 Clone a VM +```bash +ci vm new \ + --template /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx \ + --snapshot BaseClean-Linux \ + --clone-base-dir /var/lib/ci/build-vms \ + --job-id dbg-1 \ + --guest-os linux \ + --start # prints the clone VMX path on stdout +``` + +### 4.2 Wait until reachable +```bash +ci wait-ready \ + --vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx \ + --guest-os linux \ + --ssh-user ci_build \ + --ssh-key-path /var/lib/ci/keys/ci_linux \ + --timeout 180 +``` +Prints the guest IP when ready. `--ip-address` skips polling; `--credential-target` +for WinRM guests. + +### 4.3 Run the build (guest already up) +```bash +ci build run \ + --ip-address 192.168.79.200 \ + --guest-os linux \ + --ssh-user ci_build --ssh-key-path /var/lib/ci/keys/ci_linux \ + --clone-url https://gitea.emulab.it/Simone/myrepo.git --clone-branch main \ + --build-command "make all" \ + --guest-artifact-source dist +``` +Windows guests use `--credential-target BuildVMGuest` instead of the SSH flags. +Repeatable `--extra-env KEY=VALUE`; `--xvfb` Linux GUI; `--use-shared-cache`. + +### 4.4 Collect artifacts +```bash +ci artifacts collect \ + --ip-address 192.168.79.200 \ + --guest-os linux \ + --ssh-user ci_build --ssh-key-path /var/lib/ci/keys/ci_linux \ + --guest-artifact-path dist \ + --host-artifact-dir /var/lib/ci/artifacts/dbg-1 \ + --include-logs --job-id dbg-1 +``` + +### 4.5 Destroy the clone +```bash +ci vm remove --vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx +ci vm remove --vmx --force # skip soft stop +``` + +### 4.6 Sweep orphans (also a maintenance task, §6) +```bash +ci vm cleanup --what-if # list only +ci vm cleanup --max-age-hours 4 --lock-file /var/lib/ci/vm-start.lock +``` + +### 4.7 Open a VM in the Workstation GUI (interactive debugging) +Launches the **interactive** VMware Workstation GUI (not the headless vmrun path) +so you can watch/poke a clone by hand. + +```bash +# authorise the service user on your desktop X session first (run as YOUR user): +xhost +SI:localuser:ci-runner + +# then open the VMX as ci-runner, forcing the display: +ci vm open --vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx --display :0 +ci vm open --vmx --display :0 --power-on # also power it on (-x) +ci vm open --vmx --display :0 --fullscreen # power on + fullscreen (-X) +``` +The GUI needs an X display; `sudo -u ci-runner` strips `$DISPLAY`, so pass +`--display :0` (or export `DISPLAY`). Other flags: `--new-window` (`-n`), +`--vmware-path` to override the binary. The command spawns the GUI detached and +returns immediately (prints the child PID). + +--- + +## 5. Credentials & template provisioning + +### `creds set` — store the guest login (replaces Set-CIGuestCredential.ps1) +```bash +ci creds set --target BuildVMGuest --user ci_build # hidden prompt + printf '%s' 'SuperSecret123' | ci creds set --target BuildVMGuest --user ci_build --password-stdin +``` +Password is never on the command line. Verify via `validate host` keyring check. +(Leading space before `printf` keeps the secret out of shell history.) + +### Template management +```bash +# Build the Linux template VM from scratch (Ubuntu 24.04) +ci template deploy-linux + +# Provision an existing Linux template over SSH (installs toolchain, sets snapshot prep) +ci template prepare-linux + +# Provision a Windows template over WinRM (Linux-host driven) +ci template prepare-win + +# Timestamped backup of the template directories +ci template backup +``` +Run `ci template --help` for the (many) per-template options. + +--- + +## 6. Maintenance + +### `retention run` — purge old artifacts & logs +```bash +ci retention run --what-if # dry run first +ci retention run --retention-days 30 --aggressive-retention-days 7 --min-free-gb 50 +``` +Switches to aggressive retention automatically when free space drops below +`--min-free-gb`. + +### Orphan sweep — see §4.6 (`vm cleanup`). + +--- + +## 7. Phase C testing (burn-in / benchmark / smoke) + +### `smoke run` — one E2E job + assertions (replaces Test-Smoke.ps1) +```bash +ci smoke run --guest-os linux # no-op marker job (fastest) +ci smoke run --guest-os windows +ci smoke run --guest-os linux --preset ns7zip # real Linux build E2E +``` +Asserts: exit 0 + artifact dir under `/var/lib/ci/artifacts//` + +`job/success` event in `invoke-ci.jsonl`. + +### `bench measure` — phase timings (replaces Measure-CIBenchmark.ps1) +```bash +ci bench measure --guest-os linux --iterations 1 +ci bench measure --guest-os linux --iterations 4 # match historical baselines +sudo -u ci-runner tail -n1 /var/lib/ci/artifacts/benchmark.jsonl +``` +Times clone/start/IP/transport/destroy; appends to `benchmark.jsonl` +(legacy field names preserved for trend continuity). + +### `bench run` — concurrency burn-in (replaces Test-CapacityBurnIn.ps1) +```bash +ci bench run --guest-os linux --concurrency 2 --rounds 2 # quick confidence +ci bench run --guest-os linux --concurrency 4 --rounds 10 # standard gate +ci bench run --guest-os windows --concurrency 3 --rounds 10 # WinRM unstable at 4x (TODO §3.6) +sudo -u ci-runner cat "$(ls -t /var/lib/ci/artifacts/bench/*.json | head -1)" +``` +Per round asserts: all jobs exit 0, no orphan clone dir, no stale `vm-start.lock`. +JSON report under `/var/lib/ci/artifacts/bench/.json`. + +--- + +## 8. Cutover checklist & uninstalling pwsh + +Repeat for ≥1 week with pwsh **still installed** (A/B safety net): + +- [ ] Normal `git push` CI pipeline green. +- [ ] `ci validate host` green daily. +- [ ] `ci bench run --guest-os linux --concurrency 4 --rounds 10` green (no orphans/locks). +- [ ] `ci smoke run` green on `--guest-os linux` **and** `--guest-os windows`. +- [ ] `ci bench measure` timings comparable to historical baselines. +- [ ] No procedure forced a fallback to a `.ps1` on the Linux host. + +Then — only after a fully green week: + +```bash +pwsh --version +sudo apt-get remove powershell +ci validate host # must still exit 0 without pwsh +``` +Rollback: `sudo ./setup-host-linux.sh --with-pwsh` reinstalls pwsh; the original +`.ps1` scripts remain in `scripts/`. + +--- + +## 9. Troubleshooting quick map + +| Symptom | Cause | Fix | +|---------|-------|-----| +| `No such command 'bench'` | prod venv not refreshed | §0.1 `pip install .` | +| `validate host` keyring `[FAIL]` | credential missing/wrong user | §5 `creds set` | +| `validate guest` connect failure | VM not booted / wrong IP / firewall | confirm VM running + IP | +| `bench run` round FAIL: orphan | clone not destroyed | `ci vm cleanup`; check vmrun + `/var/lib/ci/build-vms` | +| WinRM jobs drop at concurrency 4 | known instability (TODO §3.6) | use `--concurrency 3` | +| IP acquire timeout (Linux) | vmnet8 DHCP wedged | `sudo systemctl restart vmware-networks` | +| `vmrun` "operation was canceled" | vmmon/vmnet unbuilt after kernel bump | `sudo vmware-modconfig --console --install-all` (or reboot; systemd guard rebuilds) | +| `ci: command not found` | alias not loaded | `source ~/.bashrc` or re-run §0.2 |