Simone
a0497ffa4e
The ISO repack (extract + xorriso, ~minutes) ran on every deploy. Cache it per VM dir: store an inputs key (base ISO size+mtime + rendered autounattend.xml + post-install.ps1) in a `.iso.inputs` sidecar next to install-noprompt.iso. Step 4 skips the repack when the patched ISO and a matching sidecar already exist, and re-patches automatically when the base ISO or either answer file changes.
2338 lines
103 KiB
Python
2338 lines
103 KiB
Python
"""``template`` sub-commands — CI template VM management.
|
|
|
|
Commands
|
|
--------
|
|
template backup
|
|
Ports ``scripts/Backup-CITemplate.ps1`` to Python. Creates timestamped
|
|
7z archives of VMware template directories and prunes old backups beyond
|
|
``--keep-count``. Stops ``act-runner.service`` before copying and
|
|
restarts it afterwards; pass ``--skip-runner-stop`` to bypass this.
|
|
|
|
template prepare-win
|
|
Provisions a Windows template VM end-to-end via pypsrp (replaces the
|
|
``Prepare-WinBuild*.ps1`` PowerShell scripts that required PSWSMan /
|
|
New-PSSession — unreliable on Linux hosts).
|
|
|
|
Steps performed by ``template prepare-win``
|
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
1. Resolves ``Install-CIToolchain-<vmx-stem>.ps1`` from
|
|
``<repo-root>/template/`` (or ``--toolchain-script``).
|
|
2. Optionally locates ``template/guest-setup/windows/ci-static-ip.ps1``
|
|
for the ip_pool feature.
|
|
3. Starts the VM via vmrun (skipped if already running).
|
|
4. Auto-detects guest IP via VMware Tools (skipped if ``--ip``
|
|
supplied).
|
|
5. Waits for TCP/5986 then probes WinRM session readiness.
|
|
6. Uploads and executes the toolchain script inside the guest. Handles
|
|
the Windows Update reboot-required exit code (3010) in a loop
|
|
(max 10 iterations).
|
|
7. Installs the ``CI-StaticIp`` scheduled task (``SYSTEM``, At Startup)
|
|
so each linked clone applies a pre-assigned static IP from
|
|
``guestinfo.ip-assignment`` before WinRM starts.
|
|
8. Runs post-setup remote validation (11 guest-state checks via a
|
|
single ``ConvertTo-Json`` call).
|
|
9. Final reboot → re-affirms autologin registry keys → graceful
|
|
``Stop-Computer``.
|
|
10. Waits for the VM to power off (120 s timeout).
|
|
11. Creates the ``--snapshot`` (default ``BaseClean``); optionally
|
|
deletes and recreates if a snapshot with that name already exists.
|
|
12. If ``--store-credential`` is set, writes build-user credentials into
|
|
the system keyring (Windows → Credential Manager; Linux →
|
|
SecretService / file backend) using the two-entry scheme read by
|
|
:class:`~ci_orchestrator.credentials.KeyringCredentialStore`.
|
|
|
|
Run from the repository root so toolchain and guest-setup scripts are
|
|
auto-discovered under ``template/``.
|
|
|
|
Usage example::
|
|
|
|
python -m ci_orchestrator template prepare-win \\
|
|
--vmx /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \\
|
|
--admin-password AdminPass1! \\
|
|
--build-password CIBuildPass1! \\
|
|
--store-credential
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import contextlib
|
|
import fnmatch
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import click
|
|
import paramiko
|
|
|
|
from ci_orchestrator.backends.protocol import VmHandle
|
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
|
from ci_orchestrator.config import load_config
|
|
from ci_orchestrator.transport.errors import TransportError
|
|
from ci_orchestrator.transport.winrm import WinRmResult, WinRmTransport
|
|
from ci_orchestrator.winimage import autounattend as win_autounattend
|
|
from ci_orchestrator.winimage import iso_patch as win_iso_patch
|
|
from ci_orchestrator.winimage import uup as win_uup
|
|
|
|
|
|
@click.group()
|
|
def template() -> None:
|
|
"""VMware template management."""
|
|
|
|
|
|
@template.command("backup")
|
|
@click.option(
|
|
"--template-path",
|
|
default=None,
|
|
help=(
|
|
"Template directory to back up "
|
|
"(default: <CI_TEMPLATES>/WinBuild2025)."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--all-templates",
|
|
is_flag=True,
|
|
help="Back up every subdirectory under CI_TEMPLATES in a single pass.",
|
|
)
|
|
@click.option(
|
|
"--backup-base-dir",
|
|
default="/var/lib/ci/backups",
|
|
show_default=True,
|
|
help="Parent directory for timestamped backup folders.",
|
|
)
|
|
@click.option(
|
|
"--keep-count",
|
|
default=3,
|
|
show_default=True,
|
|
help="Number of most-recent backups to retain per template.",
|
|
)
|
|
@click.option(
|
|
"--skip-runner-stop",
|
|
is_flag=True,
|
|
help="Skip stopping/restarting act-runner.service.",
|
|
)
|
|
@click.option(
|
|
"--what-if",
|
|
is_flag=True,
|
|
help="Dry run — log what would happen without copying or deleting.",
|
|
)
|
|
def template_backup(
|
|
template_path: str | None,
|
|
all_templates: bool,
|
|
backup_base_dir: str,
|
|
keep_count: int,
|
|
skip_runner_stop: bool,
|
|
what_if: bool,
|
|
) -> None:
|
|
"""Create timestamped backup of CI template VM directories."""
|
|
config = load_config()
|
|
backup_base = Path(backup_base_dir)
|
|
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
|
|
|
if all_templates:
|
|
tmpl_root = config.paths.templates
|
|
if not tmpl_root.exists():
|
|
raise click.ClickException(f"Templates directory not found: {tmpl_root}")
|
|
to_backup = sorted(d for d in tmpl_root.iterdir() if d.is_dir())
|
|
if not to_backup:
|
|
raise click.ClickException(
|
|
f"No template subdirectories found under {tmpl_root}"
|
|
)
|
|
else:
|
|
p = Path(template_path) if template_path else config.paths.templates / "WinBuild2025"
|
|
if not p.is_dir():
|
|
raise click.ClickException(f"Template directory not found: {p}")
|
|
to_backup = [p]
|
|
|
|
runner_was_stopped = False
|
|
try:
|
|
if not skip_runner_stop:
|
|
runner_was_stopped = _stop_runner(what_if=what_if)
|
|
|
|
if not what_if:
|
|
backup_base.mkdir(parents=True, exist_ok=True)
|
|
|
|
for tmpl_dir in to_backup:
|
|
name_tag = f"{tmpl_dir.name}_{timestamp}" if all_templates else timestamp
|
|
archive = backup_base / f"Template_{name_tag}.7z"
|
|
|
|
if what_if:
|
|
click.echo(
|
|
f"[Backup-CITemplate] WhatIf: would compress {tmpl_dir} -> {archive}"
|
|
)
|
|
else:
|
|
click.echo(f"[Backup-CITemplate] Compressing {tmpl_dir} -> {archive} ...")
|
|
_compress_7z(tmpl_dir, archive)
|
|
size_mb = archive.stat().st_size // (1024 * 1024)
|
|
click.echo(
|
|
f"[Backup-CITemplate] Backup complete: {archive} ({size_mb} MB)"
|
|
)
|
|
|
|
template_label = tmpl_dir.name if all_templates else None
|
|
_prune_backups(backup_base, template_label, keep_count, what_if=what_if)
|
|
|
|
finally:
|
|
if runner_was_stopped:
|
|
_start_runner(what_if=what_if)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────── helpers
|
|
|
|
|
|
def _stop_runner(*, what_if: bool) -> bool:
|
|
"""Stop act-runner.service. Returns True if it was running."""
|
|
result = subprocess.run(
|
|
["systemctl", "is-active", "act-runner.service"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.stdout.strip() != "active":
|
|
return False
|
|
if what_if:
|
|
click.echo("[Backup-CITemplate] WhatIf: would stop act-runner.service")
|
|
return True
|
|
click.echo("[Backup-CITemplate] Stopping act-runner.service ...")
|
|
subprocess.run(["systemctl", "stop", "act-runner.service"], check=True)
|
|
click.echo("[Backup-CITemplate] act-runner.service stopped.")
|
|
return True
|
|
|
|
|
|
def _start_runner(*, what_if: bool) -> None:
|
|
if what_if:
|
|
click.echo("[Backup-CITemplate] WhatIf: would start act-runner.service")
|
|
return
|
|
click.echo("[Backup-CITemplate] Restarting act-runner.service ...")
|
|
subprocess.run(["systemctl", "start", "act-runner.service"], check=True)
|
|
result = subprocess.run(
|
|
["systemctl", "is-active", "act-runner.service"],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
click.echo(
|
|
f"[Backup-CITemplate] act-runner.service status: {result.stdout.strip()}"
|
|
)
|
|
|
|
|
|
def _compress_7z(source: Path, archive: Path) -> None:
|
|
"""Create a 7z archive of ``source`` at ``archive`` using compression level 1."""
|
|
subprocess.run(
|
|
["7z", "a", "-mx=1", "-mmt=on", str(archive), str(source)],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
def _prune_backups(
|
|
backup_base: Path,
|
|
template_name: str | None,
|
|
keep_count: int,
|
|
*,
|
|
what_if: bool,
|
|
) -> None:
|
|
if template_name:
|
|
pattern = f"Template_{template_name}_????????_??????.7z"
|
|
else:
|
|
pattern = "Template_????????_??????.7z"
|
|
|
|
existing = sorted(
|
|
[
|
|
f
|
|
for f in backup_base.iterdir()
|
|
if f.is_file() and fnmatch.fnmatch(f.name, pattern)
|
|
],
|
|
key=lambda f: f.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
|
|
to_remove = existing[keep_count:]
|
|
for old in to_remove:
|
|
if what_if:
|
|
click.echo(
|
|
f"[Backup-CITemplate] WhatIf: would prune old backup: {old.name}"
|
|
)
|
|
else:
|
|
click.echo(f"[Backup-CITemplate] Pruning old backup: {old.name}")
|
|
old.unlink()
|
|
|
|
if to_remove:
|
|
label = template_name or "template"
|
|
click.echo(f"[Backup-CITemplate] Retained {keep_count} backup(s) for {label}.")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template prepare-win helpers
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
def _ps_escape(s: str) -> str:
|
|
"""Escape *s* for use inside a PowerShell single-quoted string."""
|
|
return s.replace("'", "''")
|
|
|
|
|
|
def _wait_tcp(host: str, port: int, timeout: float) -> bool:
|
|
"""Return True once TCP *port* on *host* accepts a connection within *timeout* s."""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=5.0):
|
|
return True
|
|
except OSError:
|
|
time.sleep(5)
|
|
return False
|
|
|
|
|
|
def _parse_exit_marker(stdout: str) -> int:
|
|
"""Extract ``CI_EXITCODE:N`` written by the invoke wrapper; default 0."""
|
|
for line in reversed(stdout.splitlines()):
|
|
stripped = line.strip()
|
|
if stripped.startswith("CI_EXITCODE:"):
|
|
try:
|
|
return int(stripped.split(":", 1)[1])
|
|
except (IndexError, ValueError):
|
|
pass
|
|
return 0
|
|
|
|
|
|
def _strip_exit_marker(stdout: str) -> str:
|
|
"""Remove ``CI_EXITCODE:N`` lines from output before display."""
|
|
return "\n".join(
|
|
line for line in stdout.splitlines() if not line.strip().startswith("CI_EXITCODE:")
|
|
)
|
|
|
|
|
|
def _wait_winrm(transport: WinRmTransport, timeout: int) -> None:
|
|
"""Poll *transport*.is_ready() until True or *timeout* seconds elapse."""
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if transport.is_ready():
|
|
return
|
|
time.sleep(10)
|
|
click.echo(" ... waiting for WinRM ...")
|
|
raise click.ClickException(f"WinRM not ready within {timeout}s.")
|
|
|
|
|
|
def _run_with_heartbeat(
|
|
transport: WinRmTransport,
|
|
script: str,
|
|
*,
|
|
check: bool = True,
|
|
heartbeat: int = 30,
|
|
) -> WinRmResult:
|
|
"""Run *script* via *transport* and print a heartbeat line every *heartbeat* s.
|
|
|
|
pypsrp's execute_ps is fully blocking — no streaming. The heartbeat
|
|
thread reassures the operator that the command is still running during
|
|
long-running steps (e.g. Windows Update, which can take 20-40 min).
|
|
"""
|
|
result_holder: list[WinRmResult] = []
|
|
exc_holder: list[BaseException] = []
|
|
stop_event = threading.Event()
|
|
|
|
def _worker() -> None:
|
|
try:
|
|
result_holder.append(transport.run(script, check=check))
|
|
except BaseException as exc:
|
|
exc_holder.append(exc)
|
|
finally:
|
|
stop_event.set()
|
|
|
|
worker = threading.Thread(target=_worker, daemon=True)
|
|
worker.start()
|
|
elapsed = 0
|
|
while not stop_event.wait(timeout=float(heartbeat)):
|
|
elapsed += heartbeat
|
|
click.echo(f" ... still running ({elapsed}s elapsed) ...")
|
|
worker.join()
|
|
if exc_holder:
|
|
raise exc_holder[0]
|
|
return result_holder[0]
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template prepare-win command
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@template.command("prepare-win")
|
|
@click.option("--vmx", required=True, metavar="PATH", help="Template VM .vmx file.")
|
|
@click.option("--ip", "guest_ip", default="", help="Guest IP (auto-detect via VMware Tools if omitted).")
|
|
@click.option("--admin-username", default="Administrator", show_default=True)
|
|
@click.option("--admin-password", default="", help="Administrator password (prompted if omitted).")
|
|
@click.option("--build-username", default="ci_build", show_default=True)
|
|
@click.option("--build-password", default="", help="CI build user password (prompted if omitted).")
|
|
@click.option("--snapshot", default="BaseClean", show_default=True, help="Snapshot name.")
|
|
@click.option("--toolchain-script", "toolchain_script", default=None, metavar="PATH",
|
|
help="Path to Install-CIToolchain-*.ps1 (auto-detected from VMX stem if omitted).")
|
|
@click.option("--skip-windows-update", is_flag=True, help="Pass -SkipWindowsUpdate to toolchain script.")
|
|
@click.option("--skip-tier2", is_flag=True, help="Pass -SkipTier2 to toolchain script.")
|
|
@click.option("--skip-cleanup", is_flag=True, help="Pass -SkipCleanup to toolchain script.")
|
|
@click.option("--winrm-timeout", default=300, show_default=True, help="Seconds to wait for WinRM.")
|
|
@click.option("--auth", default="basic", show_default=True,
|
|
type=click.Choice(["basic", "ntlm", "negotiate"]),
|
|
help="WinRM authentication protocol.")
|
|
@click.option("--store-credential", is_flag=True,
|
|
help="Store build-user credentials in the system keyring after provisioning.")
|
|
@click.option("--credential-target", default="BuildVMGuest", show_default=True,
|
|
help="Keyring target name for the stored credential.")
|
|
@click.option("--recreate-snapshot", is_flag=True,
|
|
help="Delete and recreate the snapshot if it already exists (no prompt).")
|
|
def template_prepare_win(
|
|
vmx: str,
|
|
guest_ip: str,
|
|
admin_username: str,
|
|
admin_password: str,
|
|
build_username: str,
|
|
build_password: str,
|
|
snapshot: str,
|
|
toolchain_script: str | None,
|
|
skip_windows_update: bool,
|
|
skip_tier2: bool,
|
|
skip_cleanup: bool,
|
|
winrm_timeout: int,
|
|
auth: str,
|
|
store_credential: bool,
|
|
credential_target: str,
|
|
recreate_snapshot: bool,
|
|
) -> None:
|
|
"""Provision a Windows template VM via pypsrp (Linux-host compatible).
|
|
|
|
Replaces Prepare-WinBuild*.ps1 for hosts where PSWSMan/New-PSSession is
|
|
unavailable or unreliable. Run from the repo root so toolchain and
|
|
guest-setup scripts are auto-discovered under template/.
|
|
"""
|
|
vmx_path = Path(vmx).resolve()
|
|
if not vmx_path.is_file():
|
|
raise click.ClickException(f"VMX not found: {vmx_path}")
|
|
|
|
# --- Resolve toolchain script ---
|
|
if toolchain_script is None:
|
|
candidate = Path.cwd() / "template" / f"Install-CIToolchain-{vmx_path.stem}.ps1"
|
|
if not candidate.is_file():
|
|
raise click.ClickException(
|
|
f"Toolchain script not found at {candidate}. "
|
|
"Run from the repo root or pass --toolchain-script."
|
|
)
|
|
toolchain_path = candidate
|
|
else:
|
|
toolchain_path = Path(toolchain_script)
|
|
if not toolchain_path.is_file():
|
|
raise click.ClickException(f"Toolchain script not found: {toolchain_path}")
|
|
|
|
# --- Locate CI-StaticIp guest script ---
|
|
_sip_candidate = Path.cwd() / "template" / "guest-setup" / "windows" / "ci-static-ip.ps1"
|
|
static_ip_path: Path | None
|
|
if _sip_candidate.is_file():
|
|
static_ip_path = _sip_candidate
|
|
else:
|
|
click.echo(
|
|
f"[prepare-win] WARNING: {_sip_candidate} not found — "
|
|
"ip_pool static-IP feature will not work on this template.",
|
|
err=True,
|
|
)
|
|
static_ip_path = None
|
|
|
|
# --- Prompt for passwords ---
|
|
if not admin_password:
|
|
admin_password = click.prompt(
|
|
f"Template VM {admin_username} password", hide_input=True
|
|
)
|
|
if not build_password:
|
|
build_password = click.prompt(f"New {build_username} password", hide_input=True)
|
|
|
|
# Save tty state now (terminal clean) so we can restore it before any
|
|
# interactive prompt at the end — pypsrp alters tty settings during
|
|
# WinRM sessions, causing readline() to receive \r instead of \n.
|
|
_tty_fd = sys.stdin.fileno() if sys.stdin.isatty() else -1
|
|
_saved_tty_attrs: list[object] = []
|
|
if _tty_fd >= 0: # pragma: no cover
|
|
try:
|
|
import termios
|
|
_saved_tty_attrs.append(termios.tcgetattr(_tty_fd))
|
|
except Exception:
|
|
pass
|
|
|
|
config = load_config()
|
|
backend = WorkstationVmrunBackend(vmrun_path=config.vmrun_path)
|
|
handle = VmHandle(identifier=str(vmx_path), name=vmx_path.stem)
|
|
|
|
if not backend.is_running(handle):
|
|
click.echo(f"[prepare-win] Starting VM: {vmx_path.name} ...")
|
|
backend.start(handle, headless=True)
|
|
else:
|
|
click.echo(f"[prepare-win] VM already running: {vmx_path.name}")
|
|
|
|
if not guest_ip:
|
|
click.echo("[prepare-win] Waiting for guest IP (vmrun getGuestIPAddress -wait) ...")
|
|
detected = backend.get_ip(handle, timeout=300.0)
|
|
if not detected:
|
|
raise click.ClickException(
|
|
"Could not get guest IP. Ensure VMware Tools is installed."
|
|
)
|
|
guest_ip = detected
|
|
click.echo(f"[prepare-win] Guest IP: {guest_ip}")
|
|
|
|
# --- Wait for WinRM TCP then probe session ---
|
|
click.echo(f"[prepare-win] Waiting for TCP/5986 on {guest_ip} ({winrm_timeout}s) ...")
|
|
if not _wait_tcp(guest_ip, 5986, float(winrm_timeout)):
|
|
raise click.ClickException(f"TCP/5986 not reachable on {guest_ip}.")
|
|
|
|
transport = WinRmTransport(
|
|
host=guest_ip,
|
|
username=admin_username,
|
|
password=admin_password,
|
|
port=5986,
|
|
ssl=True,
|
|
cert_validation=False,
|
|
auth=auth,
|
|
connection_timeout=30,
|
|
)
|
|
click.echo("[prepare-win] Probing WinRM session ...")
|
|
_wait_winrm(transport, winrm_timeout)
|
|
click.echo("[prepare-win] WinRM ready.")
|
|
|
|
try:
|
|
# Ensure C:\CI exists
|
|
transport.run(
|
|
"if (-not (Test-Path 'C:\\CI')) "
|
|
"{ New-Item -ItemType Directory 'C:\\CI' -Force | Out-Null }"
|
|
)
|
|
|
|
# Upload toolchain script
|
|
guest_script = f"C:\\CI\\{toolchain_path.name}"
|
|
click.echo(f"[prepare-win] Uploading {toolchain_path.name} -> {guest_script} ...")
|
|
transport.copy(toolchain_path, guest_script)
|
|
click.echo("[prepare-win] Upload complete.")
|
|
|
|
# Build args string
|
|
e = _ps_escape
|
|
args_parts = [
|
|
f"-BuildPassword '{e(build_password)}'",
|
|
f"-BuildUsername '{e(build_username)}'",
|
|
f"-AdminPassword '{e(admin_password)}'",
|
|
]
|
|
if skip_windows_update:
|
|
args_parts.append("-SkipWindowsUpdate:$true")
|
|
if skip_tier2:
|
|
args_parts.append("-SkipTier2:$true")
|
|
args_parts.append("-SkipCleanup:$true") # always skip during WU loop; cleanup runs separately after
|
|
args_str = " ".join(args_parts)
|
|
|
|
# Run toolchain script — loop handles Windows Update reboot (exit 3010)
|
|
_MAX_ITER = 10
|
|
exit_code = 0
|
|
for iteration in range(1, _MAX_ITER + 1):
|
|
click.echo(
|
|
f"[prepare-win] Running {toolchain_path.name} "
|
|
f"(iteration {iteration}/{_MAX_ITER}) ..."
|
|
)
|
|
invoke = (
|
|
"Set-ExecutionPolicy Bypass -Scope Process -Force\n"
|
|
"$ec = 0\n"
|
|
"try {\n"
|
|
f" & '{e(guest_script)}' {args_str}\n"
|
|
" $ec = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE }\n"
|
|
"} catch {\n"
|
|
" Write-Error $_\n"
|
|
" $ec = 1\n"
|
|
"}\n"
|
|
'"CI_EXITCODE:$ec"\n'
|
|
)
|
|
try:
|
|
result = _run_with_heartbeat(transport, invoke, check=False)
|
|
exit_code = _parse_exit_marker(result.stdout)
|
|
display = _strip_exit_marker(result.stdout).strip()
|
|
if display:
|
|
click.echo(display)
|
|
if result.stderr.strip():
|
|
click.echo(result.stderr.strip(), err=True)
|
|
except TransportError:
|
|
click.echo(
|
|
"[prepare-win] Transport error — VM likely rebooted mid-update "
|
|
"(treating as 3010)."
|
|
)
|
|
exit_code = 3010
|
|
|
|
click.echo(f"[prepare-win] Toolchain exit code: {exit_code}")
|
|
|
|
if exit_code == 0:
|
|
break
|
|
if exit_code != 3010:
|
|
raise click.ClickException(
|
|
f"Toolchain script failed (exit {exit_code})."
|
|
)
|
|
if iteration == _MAX_ITER:
|
|
raise click.ClickException(
|
|
f"Windows Update did not converge after {_MAX_ITER} iterations."
|
|
)
|
|
|
|
click.echo("[prepare-win] Windows Update requires reboot — rebooting ...")
|
|
with contextlib.suppress(TransportError):
|
|
transport.run("Restart-Computer -Force", check=False)
|
|
transport.close()
|
|
time.sleep(30)
|
|
click.echo(f"[prepare-win] Waiting for WinRM after reboot ({winrm_timeout}s) ...")
|
|
_wait_winrm(transport, winrm_timeout)
|
|
click.echo("[prepare-win] WinRM ready after reboot.")
|
|
|
|
# Install CI-StaticIp scheduled task
|
|
if static_ip_path is not None:
|
|
click.echo("[prepare-win] Installing CI-StaticIp scheduled task ...")
|
|
transport.copy(static_ip_path, "C:\\CI\\ci-static-ip.ps1")
|
|
transport.run(r"""
|
|
$action = New-ScheduledTaskAction -Execute 'powershell.exe' `
|
|
-Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1'
|
|
$trigger = New-ScheduledTaskTrigger -AtStartup
|
|
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -LogonType ServiceAccount -RunLevel Highest
|
|
Register-ScheduledTask -TaskName 'CI-StaticIp' `
|
|
-Action $action -Trigger $trigger -Principal $principal -Force | Out-Null
|
|
""")
|
|
click.echo("[prepare-win] CI-StaticIp task registered.")
|
|
|
|
# Post-setup validation
|
|
click.echo("[prepare-win] Running post-setup validation ...")
|
|
esc_u = e(build_username)
|
|
val_result = transport.run(
|
|
"$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine')"
|
|
" + ';' + [System.Environment]::GetEnvironmentVariable('PATH','User')\n"
|
|
"$msbuild = 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2026"
|
|
"\\BuildTools\\MSBuild\\Current\\Bin\\MSBuild.exe'\n"
|
|
"@{\n"
|
|
" WinRMRunning = (Get-Service WinRM -EA SilentlyContinue).Status -eq 'Running'\n"
|
|
f" UserExists = [bool](Get-LocalUser -Name '{esc_u}' -EA SilentlyContinue)\n"
|
|
f" UserIsAdmin = [bool](net localgroup Administrators 2>&1 | Select-String -SimpleMatch '{esc_u}')\n"
|
|
" UACDisabled = (Get-ItemProperty 'HKLM:\\SOFTWARE\\Microsoft\\Windows"
|
|
"\\CurrentVersion\\Policies\\System' -Name EnableLUA -EA SilentlyContinue).EnableLUA -eq 0\n"
|
|
" FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)\n"
|
|
" DotNetExists = [bool](Get-Command dotnet -EA SilentlyContinue)\n"
|
|
" PythonExists = Test-Path 'C:\\Python\\python.exe' -PathType Leaf\n"
|
|
" MSBuildExists = Test-Path $msbuild -PathType Leaf\n"
|
|
" CIDirsPresent = (Test-Path 'C:\\CI\\build') -and (Test-Path 'C:\\CI\\output')"
|
|
" -and (Test-Path 'C:\\CI\\scripts')\n"
|
|
" StaticIpTask = [bool](Get-ScheduledTask -TaskName 'CI-StaticIp' -EA SilentlyContinue)\n"
|
|
" StaticIpScript = Test-Path 'C:\\CI\\ci-static-ip.ps1' -PathType Leaf\n"
|
|
"} | ConvertTo-Json -Compress\n",
|
|
check=False,
|
|
)
|
|
if val_result.ok and val_result.stdout.strip():
|
|
try:
|
|
state: dict[str, object] = json.loads(val_result.stdout.strip())
|
|
failed = [k for k, v in state.items() if not v]
|
|
for k, v in state.items():
|
|
mark = "OK " if v else "FAIL"
|
|
click.echo(f" [{mark}] {k}")
|
|
if failed:
|
|
click.echo(
|
|
f"[prepare-win] WARNING: {len(failed)} check(s) failed: "
|
|
f"{', '.join(failed)}",
|
|
err=True,
|
|
)
|
|
else:
|
|
click.echo("[prepare-win] All post-setup checks passed.")
|
|
except (json.JSONDecodeError, AttributeError):
|
|
click.echo(val_result.stdout)
|
|
else:
|
|
click.echo(
|
|
f"[prepare-win] Validation script error:\n{val_result.stderr}", err=True
|
|
)
|
|
|
|
# Final reboot to flush installation state
|
|
click.echo("[prepare-win] Final reboot to flush installation state ...")
|
|
with contextlib.suppress(TransportError):
|
|
transport.run("Restart-Computer -Force", check=False)
|
|
transport.close()
|
|
time.sleep(30)
|
|
click.echo("[prepare-win] Waiting for WinRM after final reboot ...")
|
|
_wait_winrm(transport, winrm_timeout)
|
|
|
|
# Disk cleanup — runs after final reboot on fully-installed system
|
|
if not skip_cleanup:
|
|
click.echo("[prepare-win] Running post-update disk cleanup (-CleanupOnly) ...")
|
|
cleanup_invoke = (
|
|
"Set-ExecutionPolicy Bypass -Scope Process -Force\n"
|
|
"$ec = 0\n"
|
|
"try {\n"
|
|
f" & '{e(guest_script)}' -BuildPassword '{e(build_password)}'"
|
|
f" -BuildUsername '{e(build_username)}'"
|
|
f" -AdminPassword '{e(admin_password)}' -CleanupOnly:$true\n"
|
|
" $ec = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE }\n"
|
|
"} catch {\n"
|
|
" Write-Error $_\n"
|
|
" $ec = 1\n"
|
|
"}\n"
|
|
'"CI_EXITCODE:$ec"\n'
|
|
)
|
|
cleanup_result = _run_with_heartbeat(transport, cleanup_invoke, check=False)
|
|
cleanup_ec = _parse_exit_marker(cleanup_result.stdout)
|
|
display = _strip_exit_marker(cleanup_result.stdout).strip()
|
|
if display:
|
|
click.echo(display)
|
|
if cleanup_result.stderr.strip():
|
|
click.echo(cleanup_result.stderr.strip(), err=True)
|
|
if cleanup_ec != 0:
|
|
raise click.ClickException(f"Disk cleanup failed (exit {cleanup_ec}).")
|
|
click.echo("[prepare-win] Post-update cleanup complete.")
|
|
|
|
# Re-affirm autologin
|
|
click.echo("[prepare-win] Re-affirming autologin registry settings ...")
|
|
transport.run(
|
|
"$wl = 'HKLM:\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon'\n"
|
|
"Set-ItemProperty $wl AutoAdminLogon '1' -Type String -Force\n"
|
|
f"Set-ItemProperty $wl DefaultUserName '{e(admin_username)}' -Type String -Force\n"
|
|
"Set-ItemProperty $wl DefaultDomainName '.' -Type String -Force\n"
|
|
f"if ('{e(admin_password)}' -ne '') {{\n"
|
|
f" Set-ItemProperty $wl DefaultPassword '{e(admin_password)}' -Type String -Force\n"
|
|
"}\n"
|
|
)
|
|
|
|
# Graceful shutdown
|
|
click.echo("[prepare-win] Sending graceful shutdown ...")
|
|
with contextlib.suppress(TransportError):
|
|
transport.run("Stop-Computer -Force", check=False)
|
|
|
|
finally:
|
|
transport.close()
|
|
|
|
# Wait for VM to power off
|
|
click.echo("[prepare-win] Waiting for VM to power off (120s) ...")
|
|
off_deadline = time.monotonic() + 120
|
|
while time.monotonic() < off_deadline:
|
|
if not backend.is_running(handle):
|
|
break
|
|
time.sleep(5)
|
|
click.echo(" ... waiting for shutdown ...")
|
|
else:
|
|
raise click.ClickException("VM did not power off within 120s.")
|
|
click.echo("[prepare-win] VM powered off.")
|
|
|
|
# Restore tty state before interactive prompt — pypsrp alters terminal
|
|
# settings and readline() would receive \r instead of \n on Enter.
|
|
if _saved_tty_attrs and _tty_fd >= 0: # pragma: no cover
|
|
try:
|
|
import termios
|
|
termios.tcsetattr(_tty_fd, termios.TCSADRAIN, _saved_tty_attrs[0]) # type: ignore[arg-type]
|
|
except Exception:
|
|
pass
|
|
|
|
# Create snapshot
|
|
vmrun_exe = backend.vmrun_path
|
|
existing = backend.list_snapshots(handle)
|
|
if snapshot in existing:
|
|
click.echo(f"[prepare-win] Snapshot '{snapshot}' already exists.")
|
|
if recreate_snapshot:
|
|
do_recreate = True
|
|
else:
|
|
sys.stdout.write(f"[prepare-win] Delete and recreate '{snapshot}'? [y/N] ")
|
|
sys.stdout.flush()
|
|
raw = sys.stdin.readline()
|
|
do_recreate = raw.strip().lower() in ("y", "yes")
|
|
if not do_recreate:
|
|
click.echo("[prepare-win] Keeping existing snapshot. Provisioning complete.")
|
|
return
|
|
click.echo(f"[prepare-win] Deleting snapshot '{snapshot}' ...")
|
|
subprocess.run(
|
|
[vmrun_exe, "-T", "ws", "deleteSnapshot", str(vmx_path), snapshot],
|
|
check=True,
|
|
)
|
|
|
|
click.echo(f"[prepare-win] Creating snapshot '{snapshot}' ...")
|
|
subprocess.run(
|
|
[vmrun_exe, "-T", "ws", "snapshot", str(vmx_path), snapshot],
|
|
check=True,
|
|
)
|
|
click.echo(f"[prepare-win] Snapshot '{snapshot}' created.")
|
|
|
|
# Store credentials in system keyring (Windows Credential Manager on Windows,
|
|
# SecretService / file backend on Linux) using the two-entry scheme so
|
|
# KeyringCredentialStore.get() can retrieve them without knowing the username.
|
|
if store_credential:
|
|
try:
|
|
import keyring
|
|
except ImportError as exc:
|
|
raise click.ClickException(
|
|
"keyring is not installed; run `pip install keyring`."
|
|
) from exc
|
|
keyring.set_password(f"{credential_target}:meta", "username", build_username)
|
|
keyring.set_password(credential_target, build_username, build_password)
|
|
click.echo(
|
|
f"[prepare-win] Credentials stored in keyring "
|
|
f"(target='{credential_target}', user='{build_username}')."
|
|
)
|
|
|
|
click.echo("[prepare-win] Provisioning complete.")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template deploy-linux helpers
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
def _deploy_step(msg: str, step: int, start_from: int) -> None:
|
|
if start_from > step:
|
|
click.echo(f"\n=== {msg} === [SKIP]")
|
|
else:
|
|
click.echo(f"\n=== {msg} ===")
|
|
|
|
|
|
def _find_vdisk_manager(explicit: str | None) -> str:
|
|
if explicit:
|
|
p = Path(explicit)
|
|
if not p.is_file():
|
|
raise click.ClickException(f"vmware-vdiskmanager not found: {p}")
|
|
return str(p)
|
|
found = shutil.which("vmware-vdiskmanager")
|
|
if found:
|
|
return found
|
|
raise click.ClickException(
|
|
"vmware-vdiskmanager not in PATH. Pass --vdisk-manager or install VMware Workstation."
|
|
)
|
|
|
|
|
|
def _clean_force_artifacts(vm_dir: Path, disk_stem: str) -> list[str]:
|
|
"""Remove snapshot-chain and lock artifacts left in a reused VM directory.
|
|
|
|
``deploy-linux --force`` reuses the existing VM directory. If a previous
|
|
run (or a prepare-linux snapshot) left a snapshot chain behind, the disk
|
|
in the .vmx is "part of a snapshot chain" and ``vmware-vdiskmanager``
|
|
refuses to resize it at Step 4. We delete the snapshot delta disks, the
|
|
snapshot metadata (.vmsn), the snapshot database (.vmsd) and stale lock
|
|
files (.lck) so Step 4 sees a plain, chain-free disk.
|
|
|
|
The base disk itself is left untouched — Step 4 overwrites it from the
|
|
cached cloud image. Returns the sorted list of removed file names (for
|
|
logging and tests).
|
|
"""
|
|
removed: list[str] = []
|
|
patterns = (
|
|
f"{disk_stem}-[0-9]*.vmdk", # snapshot delta/redo disks
|
|
"*.vmsn", # snapshot memory/state
|
|
"*.vmsd", # snapshot database
|
|
"*.lck", # stale lock files/dirs
|
|
)
|
|
for pattern in patterns:
|
|
for path in vm_dir.glob(pattern):
|
|
if path.is_dir():
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
else:
|
|
path.unlink(missing_ok=True)
|
|
removed.append(path.name)
|
|
return sorted(removed)
|
|
|
|
|
|
def _download_vmdk(url: str, dest: Path) -> None:
|
|
click.echo(f" Downloading {url} ...")
|
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
|
try:
|
|
urllib.request.urlretrieve(url, str(tmp))
|
|
tmp.rename(dest)
|
|
except Exception:
|
|
if tmp.exists():
|
|
tmp.unlink()
|
|
raise
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as fh:
|
|
for chunk in iter(lambda: fh.read(65536), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest().upper()
|
|
|
|
|
|
def _patched_iso_inputs_key(
|
|
base_iso: Path, autounattend_xml: str, post_install_ps1: str
|
|
) -> str:
|
|
"""Cache key for a patched ISO — changes if the base ISO or answer files do."""
|
|
st = base_iso.stat()
|
|
h = hashlib.sha256()
|
|
h.update(f"{base_iso.name}|{st.st_size}|{int(st.st_mtime)}\n".encode())
|
|
h.update(autounattend_xml.encode("utf-8"))
|
|
h.update(b"\x00")
|
|
h.update(post_install_ps1.encode("utf-8"))
|
|
return h.hexdigest()
|
|
|
|
|
|
def _build_seed_iso(staging_dir: Path, iso_path: Path) -> None:
|
|
tool = shutil.which("genisoimage") or shutil.which("mkisofs")
|
|
if not tool:
|
|
raise click.ClickException("genisoimage/mkisofs not found. Install genisoimage.")
|
|
if iso_path.exists():
|
|
iso_path.unlink()
|
|
subprocess.run(
|
|
[tool, "-o", str(iso_path), "-V", "cidata", "-J", "-quiet", str(staging_dir)],
|
|
check=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
|
|
def _read_guestinfo_ip(vmrun: str, vmx: str) -> str | None:
|
|
"""Read guestinfo.ci-ip written by ci-report-ip service inside the guest."""
|
|
try:
|
|
result = subprocess.run(
|
|
[vmrun, "readVariable", vmx, "runtimeConfig", "guestinfo.ci-ip"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
ip = result.stdout.strip()
|
|
if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
|
|
return ip
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
return None
|
|
|
|
|
|
def _poll_guest_ip_deploy(vmrun: str, vmx: str, timeout: float) -> str | None:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
# Prefer guestinfo.ci-ip written by ci-report-ip service (faster, no
|
|
# VMware Tools DHCP heuristic); fall back to getGuestIPAddress for
|
|
# first-boot VMs that don't have ci-report-ip installed yet.
|
|
ip = _read_guestinfo_ip(vmrun, vmx)
|
|
if ip:
|
|
click.echo(f" ... guest IP from guestinfo.ci-ip: {ip}")
|
|
return ip
|
|
try:
|
|
result = subprocess.run(
|
|
[vmrun, "getGuestIPAddress", vmx],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
check=False,
|
|
)
|
|
ip = result.stdout.strip()
|
|
if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
|
|
return ip
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
click.echo(" ... waiting for guest IP (open-vm-tools) ...")
|
|
time.sleep(15)
|
|
return None
|
|
|
|
|
|
def _wait_ssh_port_deploy(
|
|
host: str, port: int, timeout: float, interval: float
|
|
) -> bool:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with socket.create_connection((host, port), timeout=5.0):
|
|
return True
|
|
except OSError:
|
|
click.echo(f" ... waiting for SSH port {port} on {host} ...")
|
|
time.sleep(interval)
|
|
return False
|
|
|
|
|
|
def _verify_cloud_init_done(
|
|
host: str,
|
|
cloud_user: str,
|
|
ssh_key: str,
|
|
timeout: float,
|
|
interval: float = 15.0,
|
|
) -> bool:
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
try:
|
|
client.connect(
|
|
host,
|
|
username=cloud_user,
|
|
key_filename=ssh_key,
|
|
timeout=5.0,
|
|
auth_timeout=5.0,
|
|
banner_timeout=10.0,
|
|
look_for_keys=False,
|
|
allow_agent=False,
|
|
)
|
|
_, stdout_fh, _ = client.exec_command(
|
|
"test -f /var/lib/cloud/instance/boot-finished && echo OK",
|
|
timeout=10.0,
|
|
)
|
|
if stdout_fh.read().decode().strip() == "OK":
|
|
return True
|
|
except Exception as exc:
|
|
click.echo(
|
|
f" ... not ready yet ({str(exc)[:80]}) -- retrying in {interval:.0f}s ..."
|
|
)
|
|
finally:
|
|
client.close()
|
|
time.sleep(interval)
|
|
return False
|
|
|
|
|
|
def _patch_vmx_key(vmx_path: Path, key: str, value: str) -> None:
|
|
text = vmx_path.read_text(encoding="ascii", errors="replace")
|
|
pattern = re.compile(rf"^\s*{re.escape(key)}\s*=.*$", re.MULTILINE)
|
|
line = f'{key} = "{value}"'
|
|
text = pattern.sub(line, text) if pattern.search(text) else text.rstrip("\n") + f"\n{line}\n"
|
|
vmx_path.write_text(text, encoding="ascii")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template deploy-linux command
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@template.command("deploy-linux")
|
|
@click.option(
|
|
"--vmx", "vmx_path_str", default=None, metavar="PATH",
|
|
help="VMX file to create (default: <CI_TEMPLATES>/LinuxBuild2404/LinuxBuild2404.vmx).",
|
|
)
|
|
@click.option("--vm-name", default="LinuxBuild2404", show_default=True, help="VM display name.")
|
|
@click.option("--cloud-user", default="ci_build", show_default=True, help="cloud-init username.")
|
|
@click.option(
|
|
"--hostname", "guest_hostname", default="ci-linux-template", show_default=True,
|
|
help="Guest hostname.",
|
|
)
|
|
@click.option("--timezone", default="Europe/Rome", show_default=True, help="IANA timezone (reserved).")
|
|
@click.option("--disk-size-gb", default=40, show_default=True, help="Disk size after resize (GB).")
|
|
@click.option("--memory-mb", default=4096, show_default=True, help="Guest RAM (MiB).")
|
|
@click.option("--vcpu-count", default=4, show_default=True, help="vCPU count.")
|
|
@click.option("--cores-per-socket", default=2, show_default=True, help="Cores per socket.")
|
|
@click.option(
|
|
"--snapshot-name", default="PostInstall-Linux", show_default=True,
|
|
help="Snapshot name recorded in summary (taken by prepare-linux).",
|
|
)
|
|
@click.option(
|
|
"--ssh-key", "ssh_key_str", default=None, metavar="PATH",
|
|
help="SSH private key (default: from config ssh_key_path or <CI_KEYS>/ci_linux).",
|
|
)
|
|
@click.option(
|
|
"--vmdk-url",
|
|
default="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.vmdk",
|
|
show_default=True,
|
|
help="Ubuntu cloud VMDK download URL.",
|
|
)
|
|
@click.option(
|
|
"--vmdk-cache", "vmdk_cache_str", default=None, metavar="PATH",
|
|
help="Local VMDK cache path (default: <CI_ROOT>/iso/noble-server-cloudimg-amd64.vmdk).",
|
|
)
|
|
@click.option(
|
|
"--vmdk-sha256", "vmdk_sha256", default="",
|
|
help="Expected SHA256 of cached VMDK (empty = skip verification).",
|
|
)
|
|
@click.option(
|
|
"--seed-iso", "seed_iso_str", default=None, metavar="PATH",
|
|
help="Seed ISO path (default: <VM dir>/cloud-init-seed.iso).",
|
|
)
|
|
@click.option(
|
|
"--ssh-timeout", default=300, show_default=True, help="Seconds to wait for SSH port 22.",
|
|
)
|
|
@click.option(
|
|
"--ssh-poll-interval", default=10, show_default=True,
|
|
help="Seconds between SSH port poll attempts.",
|
|
)
|
|
@click.option("--show-gui", is_flag=True, help="Power on with VMware Workstation GUI visible.")
|
|
@click.option("--force", is_flag=True, help="Allow overwriting an existing VM directory.")
|
|
@click.option(
|
|
"--diag-password", "diag_password", default="",
|
|
help="Set plaintext password on ubuntu account for console login (diagnostic only).",
|
|
)
|
|
@click.option(
|
|
"--vdisk-manager", "vdisk_manager_str", default=None, metavar="PATH",
|
|
help="Path to vmware-vdiskmanager (auto-detected if omitted).",
|
|
)
|
|
@click.option(
|
|
"--start-from-step", "start_from_step", default=1,
|
|
type=click.IntRange(1, 10), show_default=True,
|
|
help=(
|
|
"Skip to this step (artifacts from skipped steps must exist). "
|
|
"Steps: 1=prereqs+vmdk 2=cloud-init 3=seed-ISO 4=vmdk-copy+resize "
|
|
"5=VMX 6=power-on 7=guest-IP 8=SSH-port 9=cloud-init-verify 10=detach-ISO"
|
|
),
|
|
)
|
|
def template_deploy_linux(
|
|
vmx_path_str: str | None,
|
|
vm_name: str,
|
|
cloud_user: str,
|
|
guest_hostname: str,
|
|
timezone: str,
|
|
disk_size_gb: int,
|
|
memory_mb: int,
|
|
vcpu_count: int,
|
|
cores_per_socket: int,
|
|
snapshot_name: str,
|
|
ssh_key_str: str | None,
|
|
vmdk_url: str,
|
|
vmdk_cache_str: str | None,
|
|
vmdk_sha256: str,
|
|
seed_iso_str: str | None,
|
|
ssh_timeout: int,
|
|
ssh_poll_interval: int,
|
|
show_gui: bool,
|
|
force: bool,
|
|
diag_password: str,
|
|
vdisk_manager_str: str | None,
|
|
start_from_step: int,
|
|
) -> None:
|
|
"""Deploy Ubuntu 24.04 LTS build VM template (Linux VMware Workstation host).
|
|
|
|
Ports Deploy-LinuxBuild2404.ps1 for Linux hosts. Steps 1-10 mirror
|
|
the PowerShell original; ISO creation uses genisoimage instead of IMAPI2FS,
|
|
cloud-init verification uses paramiko instead of ssh.exe.
|
|
"""
|
|
_ = timezone # accepted for parity with PS script; set timezone via prepare-linux
|
|
|
|
config = load_config()
|
|
|
|
# ── Resolve paths ──────────────────────────────────────────────────────────
|
|
vmx_path = (
|
|
Path(vmx_path_str).resolve()
|
|
if vmx_path_str
|
|
else config.paths.templates / "LinuxBuild2404" / "LinuxBuild2404.vmx"
|
|
)
|
|
vm_dir = vmx_path.parent
|
|
|
|
ssh_key_path: Path
|
|
if ssh_key_str:
|
|
ssh_key_path = Path(ssh_key_str)
|
|
elif config.ssh_key_path:
|
|
ssh_key_path = config.ssh_key_path
|
|
else:
|
|
ssh_key_path = config.paths.keys / "ci_linux"
|
|
|
|
ssh_pub_key_path = ssh_key_path.with_suffix(ssh_key_path.suffix + ".pub")
|
|
|
|
vmdk_cache = (
|
|
Path(vmdk_cache_str)
|
|
if vmdk_cache_str
|
|
else config.paths.root / "iso" / "noble-server-cloudimg-amd64.vmdk"
|
|
)
|
|
seed_iso_path = Path(seed_iso_str) if seed_iso_str else vm_dir / "cloud-init-seed.iso"
|
|
staging_dir = vm_dir / "_cloud_init_staging"
|
|
dest_vmdk = vm_dir / "LinuxBuild2404.vmdk"
|
|
|
|
# ── Resolve tools ──────────────────────────────────────────────────────────
|
|
vmrun = config.vmrun_path or shutil.which("vmrun") or "vmrun"
|
|
vdisk_mgr = _find_vdisk_manager(vdisk_manager_str)
|
|
|
|
# guestinfo base64 payloads — populated in step 2, consumed in step 5;
|
|
# empty when step 2 is skipped (guestinfo then omitted from VMX).
|
|
guest_user_data_b64 = ""
|
|
guest_meta_data_b64 = ""
|
|
guest_ip: str | None = None
|
|
|
|
if start_from_step > 1:
|
|
click.echo(
|
|
f" [SKIP] Steps 1..{start_from_step - 1} -- starting from Step {start_from_step}"
|
|
)
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 1: validate prerequisites + download cloud VMDK
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step("Step 1: validate prerequisites + download cloud VMDK", 1, start_from_step)
|
|
if start_from_step <= 1:
|
|
checks: list[tuple[str, bool]] = [
|
|
(f"vmrun found: {vmrun}", bool(shutil.which(vmrun) or Path(vmrun).is_file())),
|
|
(f"vmware-vdiskmanager found: {vdisk_mgr}", Path(vdisk_mgr).is_file()),
|
|
(
|
|
"genisoimage/mkisofs in PATH",
|
|
bool(shutil.which("genisoimage") or shutil.which("mkisofs")),
|
|
),
|
|
(f"SSH private key: {ssh_key_path}", ssh_key_path.is_file()),
|
|
(f"SSH public key: {ssh_pub_key_path}", ssh_pub_key_path.is_file()),
|
|
]
|
|
for label, ok in checks:
|
|
if not ok:
|
|
raise click.ClickException(f"[PreFlight] {label}")
|
|
click.echo(f" [OK] {label}")
|
|
|
|
if vm_dir.exists():
|
|
if not force:
|
|
raise click.ClickException(
|
|
f"[PreFlight] VM directory already exists: {vm_dir}\n"
|
|
" Use --force to overwrite."
|
|
)
|
|
click.echo(f" [WARN] --force set -- reusing existing VM directory: {vm_dir}")
|
|
cleaned = _clean_force_artifacts(vm_dir, vmx_path.stem)
|
|
if cleaned:
|
|
click.echo(
|
|
" [OK] Cleared stale snapshot/lock artifacts (avoids resize "
|
|
f"snapshot-chain error): {', '.join(cleaned)}"
|
|
)
|
|
else:
|
|
vm_dir.mkdir(parents=True, exist_ok=True)
|
|
click.echo(f" [OK] VM directory ready: {vm_dir}")
|
|
|
|
if not vmdk_cache.is_file():
|
|
vmdk_cache.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
_download_vmdk(vmdk_url, vmdk_cache)
|
|
click.echo(f" [OK] Download complete: {vmdk_cache}")
|
|
except Exception as exc:
|
|
raise click.ClickException(f"[Deploy] VMDK download failed: {exc}") from exc
|
|
else:
|
|
click.echo(f" [OK] VMDK found in cache: {vmdk_cache}")
|
|
|
|
if vmdk_sha256:
|
|
click.echo(" Verifying VMDK SHA256 ...")
|
|
actual = _sha256_file(vmdk_cache)
|
|
if actual != vmdk_sha256.upper():
|
|
raise click.ClickException(
|
|
f"[Deploy] VMDK SHA256 mismatch!\n"
|
|
f" Expected: {vmdk_sha256.upper()}\n"
|
|
f" Actual: {actual}"
|
|
)
|
|
click.echo(f" [OK] VMDK SHA256: {actual}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 2: generate cloud-init user-data + meta-data
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step("Step 2: generate cloud-init user-data + meta-data", 2, start_from_step)
|
|
if start_from_step <= 2:
|
|
pub_key = ssh_pub_key_path.read_text(encoding="utf-8").strip()
|
|
|
|
if staging_dir.exists():
|
|
shutil.rmtree(staging_dir)
|
|
staging_dir.mkdir(parents=True)
|
|
|
|
meta_data = f"instance-id: linuxbuild-001\nlocal-hostname: {guest_hostname}\n"
|
|
user_data = (
|
|
"#cloud-config\n"
|
|
f"hostname: {guest_hostname}\n"
|
|
# Suppress cloud-init's first-boot apt update/upgrade. It would hold
|
|
# the dpkg lock for minutes and delay open-vm-tools startup past the
|
|
# step-7 IP-detection poll. The real toolchain apt work runs later
|
|
# in prepare-linux where it is managed.
|
|
"package_update: false\n"
|
|
"package_upgrade: false\n"
|
|
"users:\n"
|
|
f" - name: {cloud_user}\n"
|
|
" shell: /bin/bash\n"
|
|
" sudo: 'ALL=(ALL) NOPASSWD:ALL'\n"
|
|
" ssh_authorized_keys:\n"
|
|
f" - {pub_key}\n"
|
|
)
|
|
if diag_password:
|
|
click.echo(
|
|
" [WARN] --diag-password set: console login enabled for ubuntu "
|
|
"(diagnostic mode)"
|
|
)
|
|
user_data += (
|
|
f"chpasswd:\n list: |\n ubuntu:{diag_password}\n expire: false\n"
|
|
)
|
|
|
|
(staging_dir / "meta-data").write_text(meta_data, encoding="utf-8")
|
|
(staging_dir / "user-data").write_text(user_data, encoding="utf-8")
|
|
click.echo(f" [OK] meta-data written: {staging_dir / 'meta-data'}")
|
|
click.echo(f" [OK] user-data written: {staging_dir / 'user-data'}")
|
|
|
|
guest_meta_data_b64 = base64.b64encode(meta_data.encode()).decode()
|
|
guest_user_data_b64 = base64.b64encode(user_data.encode()).decode()
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 3: build cloud-init seed ISO (nocloud, label: cidata)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(
|
|
"Step 3: build cloud-init seed ISO (nocloud, label: cidata)", 3, start_from_step
|
|
)
|
|
if start_from_step <= 3:
|
|
try:
|
|
_build_seed_iso(staging_dir, seed_iso_path)
|
|
except subprocess.CalledProcessError as exc:
|
|
raise click.ClickException(
|
|
f"[Deploy Step 3] ISO creation failed:\n{exc.stderr}"
|
|
) from exc
|
|
if not seed_iso_path.is_file() or seed_iso_path.stat().st_size <= 4096:
|
|
raise click.ClickException("[Deploy Step 3] Seed ISO missing or too small.")
|
|
click.echo(
|
|
f" [OK] Seed ISO: {seed_iso_path} "
|
|
f"({seed_iso_path.stat().st_size // 1024} KB)"
|
|
)
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 4: copy + resize cloud VMDK
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(f"Step 4: copy + resize cloud VMDK to {disk_size_gb} GB", 4, start_from_step)
|
|
if start_from_step <= 4:
|
|
if dest_vmdk.exists() and force:
|
|
dest_vmdk.unlink()
|
|
click.echo(
|
|
f" Converting {vmdk_cache} -> {dest_vmdk} "
|
|
"(streamOptimized -> monolithicSparse) ..."
|
|
)
|
|
r = subprocess.run(
|
|
[vdisk_mgr, "-r", str(vmdk_cache), "-t", "0", str(dest_vmdk)],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
if r.returncode != 0:
|
|
raise click.ClickException(
|
|
f"[Deploy Step 4] vmware-vdiskmanager convert failed "
|
|
f"(exit {r.returncode}):\n{r.stderr}"
|
|
)
|
|
click.echo(f" [OK] VMDK converted: {dest_vmdk}")
|
|
|
|
click.echo(f" Resizing VMDK to {disk_size_gb} GB ...")
|
|
r = subprocess.run(
|
|
[vdisk_mgr, "-x", f"{disk_size_gb}GB", str(dest_vmdk)],
|
|
check=False, capture_output=True, text=True,
|
|
)
|
|
if r.returncode != 0:
|
|
raise click.ClickException(
|
|
f"[Deploy Step 4] vmware-vdiskmanager resize failed "
|
|
f"(exit {r.returncode}):\n{r.stderr}"
|
|
)
|
|
click.echo(f" [OK] VMDK resized: {dest_vmdk}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 5: generate VMX
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step("Step 5: generate VMX", 5, start_from_step)
|
|
if start_from_step <= 5:
|
|
vmx_lines = [
|
|
'.encoding = "UTF-8"',
|
|
'config.version = "8"',
|
|
'virtualHW.version = "19"',
|
|
f'displayName = "{vm_name}"',
|
|
'guestOS = "ubuntu-64"',
|
|
'firmware = "efi"',
|
|
f'memsize = "{memory_mb}"',
|
|
f'numvcpus = "{vcpu_count}"',
|
|
f'cpuid.coresPerSocket = "{cores_per_socket}"',
|
|
'scsi0.present = "TRUE"',
|
|
'scsi0.virtualDev = "pvscsi"',
|
|
'scsi0:0.present = "TRUE"',
|
|
'scsi0:0.fileName = "LinuxBuild2404.vmdk"',
|
|
'scsi0:0.deviceType = "scsi-hardDisk"',
|
|
'ethernet0.present = "TRUE"',
|
|
'ethernet0.connectionType = "nat"',
|
|
'ethernet0.virtualDev = "vmxnet3"',
|
|
'ethernet0.wakeOnPcktRcv = "FALSE"',
|
|
'ethernet0.addressType = "generated"',
|
|
f'ide1:0.fileName = "{seed_iso_path}"',
|
|
'ide1:0.present = "TRUE"',
|
|
'ide1:0.deviceType = "cdrom-image"',
|
|
'ide1:0.startConnected = "TRUE"',
|
|
'pciBridge0.present = "TRUE"',
|
|
'pciBridge4.present = "TRUE"',
|
|
'pciBridge4.virtualDev = "pcieRootPort"',
|
|
'pciBridge4.functions = "8"',
|
|
'pciBridge5.present = "TRUE"',
|
|
'pciBridge5.virtualDev = "pcieRootPort"',
|
|
'pciBridge5.functions = "8"',
|
|
'pciBridge6.present = "TRUE"',
|
|
'pciBridge6.virtualDev = "pcieRootPort"',
|
|
'pciBridge6.functions = "8"',
|
|
'pciBridge7.present = "TRUE"',
|
|
'pciBridge7.virtualDev = "pcieRootPort"',
|
|
'pciBridge7.functions = "8"',
|
|
'floppy0.present = "FALSE"',
|
|
'usb.present = "FALSE"',
|
|
'sound.present = "FALSE"',
|
|
'mks.enable3d = "FALSE"',
|
|
'svga.vramSize = "4194304"',
|
|
'tools.syncTime = "TRUE"',
|
|
'uuid.action = "create"',
|
|
'sched.mem.pshare.enable = "FALSE"',
|
|
'MemAllowAutoScaleDown = "FALSE"',
|
|
'mainMem.useNamedFile = "FALSE"',
|
|
'isolation.tools.hgfs.disable = "TRUE"',
|
|
]
|
|
if guest_user_data_b64 and guest_meta_data_b64:
|
|
vmx_lines += [
|
|
f'guestinfo.userdata = "{guest_user_data_b64}"',
|
|
'guestinfo.userdata.encoding = "base64"',
|
|
f'guestinfo.metadata = "{guest_meta_data_b64}"',
|
|
'guestinfo.metadata.encoding = "base64"',
|
|
]
|
|
click.echo(
|
|
" [INFO] guestinfo cloud-init properties embedded in VMX (backup datasource)."
|
|
)
|
|
vmx_path.write_text("\n".join(vmx_lines) + "\n", encoding="ascii")
|
|
click.echo(f" [OK] VMX written: {vmx_path}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 6: power on VM
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step("Step 6: power on VM", 6, start_from_step)
|
|
if start_from_step <= 6:
|
|
gui_mode = "gui" if show_gui else "nogui"
|
|
proc = subprocess.Popen([vmrun, "-T", "ws", "start", str(vmx_path), gui_mode])
|
|
click.echo(f" [Deploy] VM power-on issued (pid={proc.pid}).")
|
|
time.sleep(10)
|
|
list_r = subprocess.run(
|
|
[vmrun, "-T", "ws", "list"], capture_output=True, text=True, check=False,
|
|
)
|
|
if str(vmx_path) not in (list_r.stdout or ""):
|
|
raise click.ClickException(
|
|
"[Deploy Step 6] VM not found in vmrun list after power-on."
|
|
)
|
|
click.echo(" [OK] VM appears in vmrun list.")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(
|
|
"Step 7: detect guest IP via vmrun getGuestIPAddress (max 10 min)", 7, start_from_step
|
|
)
|
|
if start_from_step <= 7:
|
|
guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=600.0)
|
|
if not guest_ip:
|
|
raise click.ClickException("[Deploy Step 7] Timed out waiting for guest IP.")
|
|
click.echo(f" [OK] Guest IP: {guest_ip}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 8: poll SSH port 22 until ready
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(f"Step 8: poll SSH port 22 (max {ssh_timeout}s)", 8, start_from_step)
|
|
if start_from_step <= 8:
|
|
if not guest_ip:
|
|
raise click.ClickException(
|
|
"[Deploy Step 8] Guest IP not available. Run from Step 7 or earlier."
|
|
)
|
|
if not _wait_ssh_port_deploy(
|
|
guest_ip, 22, float(ssh_timeout), float(ssh_poll_interval)
|
|
):
|
|
raise click.ClickException(
|
|
f"[Deploy Step 8] SSH port 22 not reachable on {guest_ip} "
|
|
f"within {ssh_timeout}s."
|
|
)
|
|
click.echo(f" [OK] SSH port 22 open on {guest_ip}")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 9: verify cloud-init completed (boot-finished flag)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(
|
|
"Step 9: verify cloud-init completed (boot-finished flag)", 9, start_from_step
|
|
)
|
|
if start_from_step <= 9:
|
|
if not guest_ip:
|
|
raise click.ClickException(
|
|
"[Deploy Step 9] Guest IP not available. Run from Step 7 or earlier."
|
|
)
|
|
click.echo(" Polling cloud-init completion (max 600s) ...")
|
|
if not _verify_cloud_init_done(
|
|
guest_ip, cloud_user, str(ssh_key_path), timeout=600.0
|
|
):
|
|
raise click.ClickException(
|
|
"[Deploy Step 9] Timed out (600s) waiting for cloud-init to complete."
|
|
)
|
|
click.echo(" [OK] cloud-init completed.")
|
|
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
# Step 10: remove seed ISO from VMX (prevent re-run on clone)
|
|
# ──────────────────────────────────────────────────────────────────────────
|
|
_deploy_step(
|
|
"Step 10: remove seed ISO from VMX (prevent re-run on clone)", 10, start_from_step
|
|
)
|
|
if start_from_step <= 10:
|
|
_patch_vmx_key(vmx_path, "ide1:0.present", "FALSE")
|
|
_patch_vmx_key(vmx_path, "ide1:0.startConnected", "FALSE")
|
|
click.echo(" [OK] ide1:0.present = FALSE")
|
|
click.echo(" [OK] ide1:0.startConnected = FALSE")
|
|
|
|
# Cleanup staging dir
|
|
if staging_dir.exists():
|
|
shutil.rmtree(staging_dir, ignore_errors=True)
|
|
|
|
# Summary
|
|
click.echo("\n=== Done ===\n")
|
|
click.echo("[Deploy] Summary")
|
|
click.echo(f" VMX: {vmx_path}")
|
|
click.echo(f" Guest IP: {guest_ip or '(not obtained)'}")
|
|
click.echo(f" SSH key: {ssh_key_path}")
|
|
click.echo(f" Snapshot name: {snapshot_name} (run template prepare-linux next)")
|
|
click.echo("")
|
|
if guest_ip:
|
|
click.echo(f" SSH: ssh -i {ssh_key_path} {cloud_user}@{guest_ip}")
|
|
click.echo("")
|
|
click.echo(" VM is powered on. cloud-init provisioning complete.")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template prepare-linux helpers
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
_PREPARE_LINUX_VALIDATION = (
|
|
'( id ci_build | grep -q sudo || sudo -l -U ci_build 2>/dev/null | grep -q NOPASSWD )'
|
|
' && echo "UserSudo:OK" || echo "UserSudo:FAIL"\n'
|
|
'sudo -n true 2>/dev/null && echo "SudoNoPass:OK" || echo "SudoNoPass:FAIL"\n'
|
|
'for t in gcc g++ clang cmake python3 git 7z; do'
|
|
' command -v "$t" >/dev/null && echo "Tool:$t:OK" || echo "Tool:$t:FAIL"; done\n'
|
|
'test -d /opt/ci/build && echo "Dir:build:OK" || echo "Dir:build:FAIL"\n'
|
|
'test -d /opt/ci/output && echo "Dir:output:OK" || echo "Dir:output:FAIL"\n'
|
|
'test -d /opt/ci/scripts && echo "Dir:scripts:OK" || echo "Dir:scripts:FAIL"\n'
|
|
'swapon --show | grep -q . && echo "Swap:ACTIVE:FAIL" || echo "Swap:off:OK"\n'
|
|
'systemctl is-active ssh >/dev/null && echo "SSHD:active:OK" || echo "SSHD:inactive:FAIL"\n'
|
|
'grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config'
|
|
' && echo "SSHPwdAuth:disabled:OK" || echo "SSHPwdAuth:enabled:FAIL"\n'
|
|
'systemctl is-enabled ci-report-ip.service >/dev/null 2>&1'
|
|
' && echo "CIReportIP:enabled:OK" || echo "CIReportIP:enabled:FAIL"\n'
|
|
'test -x /usr/local/bin/ci-report-ip.sh'
|
|
' && echo "CIReportIPScript:exists:OK" || echo "CIReportIPScript:exists:FAIL"\n'
|
|
)
|
|
|
|
|
|
def _pl_connect(
|
|
host: str, cloud_user: str, ssh_key: str, timeout: float = 10.0
|
|
) -> paramiko.SSHClient:
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
client.connect(
|
|
host,
|
|
username=cloud_user,
|
|
key_filename=ssh_key,
|
|
timeout=timeout,
|
|
auth_timeout=timeout,
|
|
banner_timeout=timeout * 2,
|
|
look_for_keys=False,
|
|
allow_agent=False,
|
|
)
|
|
return client
|
|
|
|
|
|
def _pl_run(
|
|
client: paramiko.SSHClient, cmd: str, timeout: float = 60.0
|
|
) -> tuple[int, str, str]:
|
|
_, stdout_fh, stderr_fh = client.exec_command(cmd, timeout=timeout)
|
|
out = stdout_fh.read().decode(errors="replace")
|
|
err = stderr_fh.read().decode(errors="replace")
|
|
rc = stdout_fh.channel.recv_exit_status()
|
|
return rc, out, err
|
|
|
|
|
|
def _pl_stream(client: paramiko.SSHClient, cmd: str) -> int:
|
|
"""Run cmd and stream stdout/stderr to console in real time. Returns exit code."""
|
|
transport = client.get_transport()
|
|
assert transport is not None, "SSH transport closed"
|
|
channel = transport.open_session()
|
|
channel.set_combine_stderr(False)
|
|
channel.exec_command(cmd)
|
|
while True:
|
|
if channel.recv_ready():
|
|
data = channel.recv(65536)
|
|
if data:
|
|
click.echo(data.decode(errors="replace"), nl=False)
|
|
if channel.recv_stderr_ready():
|
|
data = channel.recv_stderr(65536)
|
|
if data:
|
|
click.echo(data.decode(errors="replace"), nl=False, err=True)
|
|
if channel.exit_status_ready():
|
|
while channel.recv_ready():
|
|
data = channel.recv(65536)
|
|
if data:
|
|
click.echo(data.decode(errors="replace"), nl=False)
|
|
while channel.recv_stderr_ready():
|
|
data = channel.recv_stderr(65536)
|
|
if data:
|
|
click.echo(data.decode(errors="replace"), nl=False, err=True)
|
|
break
|
|
time.sleep(0.1)
|
|
return channel.recv_exit_status()
|
|
|
|
|
|
def _pl_upload(client: paramiko.SSHClient, local: Path, remote: str) -> None:
|
|
sftp = client.open_sftp()
|
|
try:
|
|
sftp.put(str(local), remote)
|
|
finally:
|
|
sftp.close()
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template prepare-linux command
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@template.command("prepare-linux")
|
|
@click.option(
|
|
"--vmx", "vmx_path_str", default=None, metavar="PATH",
|
|
help=(
|
|
"Template VM .vmx file. Enables auto power-on, IP detection, "
|
|
"and automatic shutdown + snapshot at the end."
|
|
),
|
|
)
|
|
@click.option(
|
|
"--ip", "guest_ip", default="",
|
|
help="Guest IP address (auto-detected via vmrun if --vmx is supplied).",
|
|
)
|
|
@click.option("--cloud-user", default="ci_build", show_default=True, help="SSH username in guest.")
|
|
@click.option(
|
|
"--ssh-key", "ssh_key_str", default=None, metavar="PATH",
|
|
help="SSH private key (default: from config ssh_key_path or <CI_KEYS>/ci_linux).",
|
|
)
|
|
@click.option(
|
|
"--snapshot", default="BaseClean-Linux", show_default=True,
|
|
help="Snapshot name (must match the name used by vm new).",
|
|
)
|
|
@click.option(
|
|
"--toolchain-script", "toolchain_script_str", default=None, metavar="PATH",
|
|
help=(
|
|
"Path to Install-CIToolchain-Linux2404.sh "
|
|
"(auto-detected from <repo-root>/template/ if omitted)."
|
|
),
|
|
)
|
|
@click.option("--skip-update", is_flag=True, help="Pass --skip-update to toolchain script.")
|
|
@click.option("--install-dotnet", is_flag=True, help="Pass --dotnet to toolchain script.")
|
|
@click.option("--no-mingw", is_flag=True, help="Pass --no-mingw to toolchain script.")
|
|
@click.option(
|
|
"--ssh-timeout", default=180, show_default=True,
|
|
help="Seconds to wait for SSH port 22 to become reachable.",
|
|
)
|
|
@click.option(
|
|
"--take-snapshot", is_flag=True,
|
|
help="Shut down and take snapshot after provisioning (always implied when --vmx is set).",
|
|
)
|
|
def template_prepare_linux(
|
|
vmx_path_str: str | None,
|
|
guest_ip: str,
|
|
cloud_user: str,
|
|
ssh_key_str: str | None,
|
|
snapshot: str,
|
|
toolchain_script_str: str | None,
|
|
skip_update: bool,
|
|
install_dotnet: bool,
|
|
no_mingw: bool,
|
|
ssh_timeout: int,
|
|
take_snapshot: bool,
|
|
) -> None:
|
|
"""Provision Linux template VM via SSH (ports Prepare-LinuxBuild2404.ps1).
|
|
|
|
Requires either --vmx (auto power-on + IP detection + snapshot) or
|
|
--ip (explicit IP, no auto VM lifecycle), or both.
|
|
"""
|
|
config = load_config()
|
|
|
|
# ── Resolve SSH key ────────────────────────────────────────────────────────
|
|
if ssh_key_str:
|
|
ssh_key_path = Path(ssh_key_str)
|
|
elif config.ssh_key_path:
|
|
ssh_key_path = config.ssh_key_path
|
|
else:
|
|
ssh_key_path = config.paths.keys / "ci_linux"
|
|
|
|
# ── Resolve VMX ───────────────────────────────────────────────────────────
|
|
vmx_path = Path(vmx_path_str).resolve() if vmx_path_str else None
|
|
|
|
# ── Resolve toolchain script ──────────────────────────────────────────────
|
|
if toolchain_script_str is None:
|
|
candidate = Path.cwd() / "template" / "Install-CIToolchain-Linux2404.sh"
|
|
if not candidate.is_file():
|
|
raise click.ClickException(
|
|
f"Toolchain script not found at {candidate}. "
|
|
"Run from the repo root or pass --toolchain-script."
|
|
)
|
|
toolchain_path = candidate
|
|
else:
|
|
toolchain_path = Path(toolchain_script_str)
|
|
if not toolchain_path.is_file():
|
|
raise click.ClickException(f"Toolchain script not found: {toolchain_path}")
|
|
|
|
if vmx_path is None and not guest_ip:
|
|
raise click.ClickException(
|
|
"Provide --vmx (auto power-on + IP detection) or --ip (explicit IP), or both."
|
|
)
|
|
|
|
do_snapshot = take_snapshot or vmx_path is not None
|
|
|
|
# ── Step 1: Pre-flight ────────────────────────────────────────────────────
|
|
click.echo("[prepare-linux] Step 1 — Pre-flight validation")
|
|
if not ssh_key_path.is_file():
|
|
raise click.ClickException(f"SSH private key not found: {ssh_key_path}")
|
|
click.echo(f" [OK] SSH key: {ssh_key_path}")
|
|
if vmx_path is not None and not vmx_path.is_file():
|
|
raise click.ClickException(f"VMX not found: {vmx_path}")
|
|
if vmx_path is not None:
|
|
click.echo(f" [OK] VMX: {vmx_path}")
|
|
click.echo(f" [OK] Toolchain script: {toolchain_path}")
|
|
|
|
# ── Step 2: Auto power-on + IP detection ──────────────────────────────────
|
|
vmrun = config.vmrun_path or shutil.which("vmrun") or "vmrun"
|
|
backend: WorkstationVmrunBackend | None = None
|
|
handle: VmHandle | None = None
|
|
|
|
if vmx_path is not None:
|
|
backend = WorkstationVmrunBackend(vmrun_path=config.vmrun_path)
|
|
handle = VmHandle(identifier=str(vmx_path), name=vmx_path.stem)
|
|
|
|
if not backend.is_running(handle):
|
|
click.echo(f"[prepare-linux] Starting VM: {vmx_path.name} ...")
|
|
backend.start(handle, headless=True)
|
|
else:
|
|
click.echo(f"[prepare-linux] VM already running: {vmx_path.name}")
|
|
# VM process alive but guest may be powered off (e.g. after a
|
|
# previous prepare run that shut down the guest). Do a quick IP
|
|
# probe; if nothing comes back, stop and cold-boot the VM.
|
|
if not guest_ip:
|
|
click.echo("[prepare-linux] Quick IP probe (30s) ...")
|
|
quick_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=30.0)
|
|
if not quick_ip:
|
|
click.echo(
|
|
"[prepare-linux] Guest not responding — stopping and restarting VM ..."
|
|
)
|
|
backend.stop(handle)
|
|
backend.start(handle, headless=True)
|
|
|
|
if not guest_ip:
|
|
click.echo("[prepare-linux] Detecting guest IP (vmrun getGuestIPAddress) ...")
|
|
detected = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=300.0)
|
|
if not detected:
|
|
raise click.ClickException(
|
|
"Could not get guest IP. Ensure open-vm-tools is running in the guest."
|
|
)
|
|
guest_ip = detected
|
|
click.echo(f"[prepare-linux] Guest IP: {guest_ip}")
|
|
|
|
# ── Step 3: Wait for SSH port 22 ──────────────────────────────────────────
|
|
click.echo(
|
|
f"[prepare-linux] Step 3 — Waiting for SSH port 22 on {guest_ip} "
|
|
f"(timeout {ssh_timeout}s) ..."
|
|
)
|
|
if not _wait_ssh_port_deploy(guest_ip, 22, float(ssh_timeout), 10.0):
|
|
raise click.ClickException(
|
|
f"SSH port 22 not reachable on {guest_ip} within {ssh_timeout}s."
|
|
)
|
|
click.echo(f" [OK] TCP/22 reachable on {guest_ip}")
|
|
|
|
# ── Step 4: SSH connectivity test ─────────────────────────────────────────
|
|
click.echo("[prepare-linux] Step 4 — Testing SSH connectivity ...")
|
|
try:
|
|
client = _pl_connect(guest_ip, cloud_user, str(ssh_key_path))
|
|
except Exception as exc:
|
|
raise click.ClickException(
|
|
f"SSH connection to {cloud_user}@{guest_ip} failed: {exc}"
|
|
) from exc
|
|
rc, out, _ = _pl_run(client, "echo connected")
|
|
if rc != 0 or "connected" not in out:
|
|
client.close()
|
|
raise click.ClickException("SSH connectivity test failed.")
|
|
click.echo(" [OK] SSH connectivity confirmed.")
|
|
|
|
try:
|
|
# ── Step 5: Upload toolchain script ───────────────────────────────────
|
|
click.echo(
|
|
f"[prepare-linux] Step 5 — Uploading {toolchain_path.name} to guest /tmp/ ..."
|
|
)
|
|
raw = toolchain_path.read_bytes()
|
|
if raw[:3] == b"\xef\xbb\xbf":
|
|
click.echo(" [WARN] BOM detected — stripping")
|
|
raw = raw[3:]
|
|
normalized = raw.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
|
|
remote_script = "/tmp/Install-CIToolchain-Linux2404.sh"
|
|
|
|
with tempfile.NamedTemporaryFile(suffix=".sh", delete=False) as tf:
|
|
tf.write(normalized)
|
|
tmp_local = Path(tf.name)
|
|
try:
|
|
_pl_upload(client, tmp_local, remote_script)
|
|
finally:
|
|
tmp_local.unlink(missing_ok=True)
|
|
|
|
rc, size_out, _ = _pl_run(client, f"stat -c %s {remote_script}")
|
|
remote_size = int(size_out.strip()) if size_out.strip().isdigit() else -1
|
|
local_size = len(normalized)
|
|
if abs(remote_size - local_size) > 2:
|
|
raise click.ClickException(
|
|
f"Upload size mismatch: local={local_size}, remote={remote_size}."
|
|
)
|
|
click.echo(f" [OK] {toolchain_path.name} uploaded ({local_size} bytes)")
|
|
|
|
# ── Step 6: Execute toolchain script ──────────────────────────────────
|
|
flags: list[str] = []
|
|
if skip_update:
|
|
flags.append("--skip-update")
|
|
if install_dotnet:
|
|
flags.append("--dotnet")
|
|
if no_mingw:
|
|
flags.append("--no-mingw")
|
|
flags_str = " ".join(flags)
|
|
remote_cmd = f"chmod +x {remote_script} && sudo {remote_script} {flags_str}".strip()
|
|
|
|
click.echo("[prepare-linux] Step 6 — Running toolchain script (may take several minutes)")
|
|
click.echo(f" Command: {remote_cmd}")
|
|
click.echo("-" * 60)
|
|
exit_code = _pl_stream(client, remote_cmd)
|
|
click.echo("-" * 60)
|
|
if exit_code != 0:
|
|
raise click.ClickException(
|
|
f"Toolchain script exited {exit_code} — see output above."
|
|
)
|
|
click.echo("[prepare-linux] Toolchain script completed (exit 0).")
|
|
|
|
# ── Step 7: Post-setup validation ─────────────────────────────────────
|
|
click.echo("[prepare-linux] Step 7 — Running post-setup validation ...")
|
|
_, val_out, _ = _pl_run(client, _PREPARE_LINUX_VALIDATION, timeout=120.0)
|
|
click.echo("[prepare-linux] Validation results:")
|
|
failures: list[str] = []
|
|
for line in val_out.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
click.echo(f" {line}")
|
|
if ":FAIL" in line:
|
|
failures.append(line)
|
|
if failures:
|
|
raise click.ClickException(
|
|
"Post-setup validation failed:\n"
|
|
+ "\n".join(f" {f}" for f in failures)
|
|
)
|
|
click.echo("[prepare-linux] All post-setup checks passed.")
|
|
|
|
# ── Step 7b: Clear machine-id (DHCP uniqueness for clones) ───────────
|
|
click.echo("[prepare-linux] Step 7b — Clearing machine-id ...")
|
|
rc2, _, err2 = _pl_run(
|
|
client,
|
|
"sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id",
|
|
)
|
|
if rc2 != 0:
|
|
raise click.ClickException(f"machine-id clear failed (exit {rc2}):\n{err2}")
|
|
_, size_out2, _ = _pl_run(client, "stat -c %s /etc/machine-id")
|
|
if size_out2.strip() != "0":
|
|
raise click.ClickException(
|
|
f"/etc/machine-id not empty after truncate (size={size_out2.strip()!r})."
|
|
)
|
|
click.echo("[prepare-linux] machine-id cleared — clones get unique DHCP ID.")
|
|
|
|
# ── Step 7c: Enable NTP time sync (clock-skew guard for clones) ──────
|
|
# Bake persistent NTP into the snapshot so every clone boots and
|
|
# corrects its clock automatically. Without this, a clone whose RTC
|
|
# drifted behind the host produces source mtimes "in the future" and
|
|
# make emits "Clock skew detected". This is the clean fix; the build
|
|
# path also force-syncs + touches sources as a runtime safety net.
|
|
click.echo("[prepare-linux] Step 7c — Enabling NTP time sync ...")
|
|
# set-local-rtc 0 forces the RTC to be interpreted as UTC. Without it
|
|
# a clone can boot hours ahead (RTC in local time read as UTC); NTP
|
|
# then steps the clock *backward* at runtime, which is what produces
|
|
# the "Clock skew detected" / future-mtime warnings in make.
|
|
rc_ntp, _, err_ntp = _pl_run(
|
|
client,
|
|
"sudo timedatectl set-local-rtc 0 2>/dev/null; "
|
|
"sudo systemctl enable --now systemd-timesyncd 2>/dev/null; "
|
|
"sudo timedatectl set-ntp true",
|
|
)
|
|
if rc_ntp != 0:
|
|
raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}")
|
|
click.echo("[prepare-linux] NTP time sync enabled.")
|
|
|
|
# ── Step 7d: Seal cloud-init (runs once at deploy, never in clones) ───
|
|
# cloud-init provisioned the image at deploy time. It must not run
|
|
# again on the template or on linked clones: the VMX carries the
|
|
# guestinfo seed (userdata/metadata), which every linked clone
|
|
# inherits, so cloud-init would re-run on each boot and rewrite
|
|
# /etc/netplan/50-cloud-init.yaml pinning the deploy-time MAC. That
|
|
# stale MAC match never matches a clone's NIC (nor a re-MAC'd
|
|
# template), which wedges systemd-networkd's DHCPv4 — the interface
|
|
# links up and gets an offer on the wire but no address is ever
|
|
# applied. We seal the image: drop the stale cloud-init netplan,
|
|
# leaving only the MAC-agnostic 99-ci-dhcp-allnics.yaml, regenerate
|
|
# networkd config, then disable cloud-init for all future boots.
|
|
click.echo("[prepare-linux] Step 7d — Sealing cloud-init (disable for clones) ...")
|
|
rc_seal, _, err_seal = _pl_run(
|
|
client,
|
|
"sudo rm -f /etc/netplan/50-cloud-init.yaml "
|
|
"/etc/netplan/50-cloud-init.yaml.bak && sudo netplan generate",
|
|
)
|
|
if rc_seal != 0:
|
|
raise click.ClickException(f"cloud-init seal (netplan) failed (exit {rc_seal}):\n{err_seal}")
|
|
# Disable cloud-init for every subsequent boot, then clean its state
|
|
# (best-effort: clean may exit non-zero if already clean).
|
|
_pl_run(client, "sudo touch /etc/cloud/cloud-init.disabled")
|
|
_pl_run(client, "sudo cloud-init clean --logs 2>/dev/null || true")
|
|
# Verify the stale netplan is gone and cloud-init is disabled.
|
|
_, seal_out, _ = _pl_run(
|
|
client,
|
|
"test -f /etc/cloud/cloud-init.disabled && echo DISABLED; "
|
|
"test ! -f /etc/netplan/50-cloud-init.yaml && echo NO50",
|
|
)
|
|
if "DISABLED" not in seal_out or "NO50" not in seal_out:
|
|
raise click.ClickException(
|
|
f"cloud-init seal verification failed (expected DISABLED+NO50):\n{seal_out}"
|
|
)
|
|
click.echo(
|
|
"[prepare-linux] cloud-init sealed — clones boot with static netplan only."
|
|
)
|
|
|
|
# ── Step 8: Graceful shutdown ─────────────────────────────────────────
|
|
if do_snapshot:
|
|
click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...")
|
|
with contextlib.suppress(Exception):
|
|
_pl_run(client, "sudo shutdown -h now", timeout=10.0)
|
|
else:
|
|
click.echo(
|
|
"[prepare-linux] --take-snapshot not set — VM left running. "
|
|
"Shut down and snapshot manually when ready."
|
|
)
|
|
|
|
finally:
|
|
client.close()
|
|
|
|
# ── Post-SSH: wait for power-off + snapshot ───────────────────────────────
|
|
if do_snapshot and vmx_path is not None and backend is not None and handle is not None:
|
|
click.echo("[prepare-linux] Waiting for VM to power off (120s) ...")
|
|
off_deadline = time.monotonic() + 120
|
|
while time.monotonic() < off_deadline:
|
|
if not backend.is_running(handle):
|
|
break
|
|
time.sleep(5)
|
|
click.echo(" ... waiting for shutdown ...")
|
|
else:
|
|
raise click.ClickException("VM did not power off within 120s.")
|
|
click.echo("[prepare-linux] VM powered off.")
|
|
|
|
existing = backend.list_snapshots(handle)
|
|
if snapshot in existing:
|
|
click.echo(f"[prepare-linux] Deleting existing snapshot '{snapshot}' ...")
|
|
subprocess.run(
|
|
[vmrun, "-T", "ws", "deleteSnapshot", str(vmx_path), snapshot],
|
|
check=True,
|
|
)
|
|
|
|
click.echo(f"[prepare-linux] Taking snapshot '{snapshot}' ...")
|
|
subprocess.run(
|
|
[vmrun, "-T", "ws", "snapshot", str(vmx_path), snapshot],
|
|
check=True,
|
|
)
|
|
click.echo(f"[prepare-linux] Snapshot '{snapshot}' created.")
|
|
|
|
# ── Summary ───────────────────────────────────────────────────────────────
|
|
click.echo("")
|
|
click.echo("[prepare-linux] Summary")
|
|
click.echo(f" Guest IP: {guest_ip}")
|
|
click.echo(f" SSH key: {ssh_key_path}")
|
|
if vmx_path:
|
|
click.echo(f" VMX: {vmx_path}")
|
|
if do_snapshot and vmx_path is not None:
|
|
click.echo(
|
|
f" Snapshot: {snapshot} — ready for "
|
|
f"`ci-orchestrator vm new --template {vmx_path.name}`"
|
|
)
|
|
else:
|
|
click.echo(f" Snapshot: not taken — name it '{snapshot}' when done manually.")
|
|
click.echo("[prepare-linux] Provisioning complete.")
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# template deploy-win command (Windows Server 2025, Linux host)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
#: Forward-compatible Workstation guestOS id for Windows Server 2025.
|
|
_WIN2025_GUEST_OS = "windows2019srvnext-64"
|
|
|
|
|
|
def _remove_vmx_lines(vmx_path: Path, prefix: str) -> int:
|
|
"""Remove every VMX line whose key starts with ``prefix``. Returns count."""
|
|
text = vmx_path.read_text(encoding="ascii", errors="replace")
|
|
kept: list[str] = []
|
|
removed = 0
|
|
pattern = re.compile(rf"^\s*{re.escape(prefix)}", re.IGNORECASE)
|
|
for line in text.splitlines():
|
|
if pattern.match(line):
|
|
removed += 1
|
|
else:
|
|
kept.append(line)
|
|
vmx_path.write_text("\n".join(kept) + "\n", encoding="ascii")
|
|
return removed
|
|
|
|
|
|
def _winbuild_vmx_lines(
|
|
*,
|
|
vm_name: str,
|
|
mem_mb: int,
|
|
vcpu_count: int,
|
|
cores_per_socket: int,
|
|
vmdk_name: str,
|
|
install_iso: str,
|
|
tools_iso: str,
|
|
) -> list[str]:
|
|
"""Build the WinBuild2025 VMX (ports Deploy-WinBuild2025.ps1 Step 5).
|
|
|
|
Boot NIC is e1000e; sata0:0 = patched install ISO, sata0:1 = VMware Tools
|
|
ISO. ``deploy-win`` removes the SATA/USB controllers and switches the NIC to
|
|
vmxnet3 after the unattended install drops install_complete.flag.
|
|
"""
|
|
return [
|
|
'.encoding = "windows-1252"',
|
|
'config.version = "8"',
|
|
'virtualHW.version = "21"',
|
|
f'displayName = "{vm_name}"',
|
|
f'guestOS = "{_WIN2025_GUEST_OS}"',
|
|
'firmware = "efi"',
|
|
'uefi.secureBoot.enabled = "FALSE"',
|
|
f'memSize = "{mem_mb}"',
|
|
f'numvcpus = "{vcpu_count}"',
|
|
f'cpuid.coresPerSocket = "{cores_per_socket}"',
|
|
'tools.syncTime = "TRUE"',
|
|
'tools.upgrade.policy = "manual"',
|
|
'powerType.powerOff = "soft"',
|
|
'powerType.powerOn = "soft"',
|
|
'powerType.suspend = "soft"',
|
|
'powerType.reset = "soft"',
|
|
'uuid.action = "create"',
|
|
'pciBridge0.present = "TRUE"',
|
|
'pciBridge4.present = "TRUE"',
|
|
'pciBridge4.virtualDev = "pcieRootPort"',
|
|
'pciBridge4.functions = "8"',
|
|
'pciBridge5.present = "TRUE"',
|
|
'pciBridge5.virtualDev = "pcieRootPort"',
|
|
'pciBridge5.functions = "8"',
|
|
'pciBridge6.present = "TRUE"',
|
|
'pciBridge6.virtualDev = "pcieRootPort"',
|
|
'pciBridge6.functions = "8"',
|
|
'pciBridge7.present = "TRUE"',
|
|
'pciBridge7.virtualDev = "pcieRootPort"',
|
|
'pciBridge7.functions = "8"',
|
|
'nvme0.present = "TRUE"',
|
|
'nvme0.pciSlotNumber = "224"',
|
|
'nvme0:0.present = "TRUE"',
|
|
f'nvme0:0.fileName = "{vmdk_name}"',
|
|
'nvme0:0.deviceType = "disk"',
|
|
'sata0.present = "TRUE"',
|
|
'sata0.pciSlotNumber = "32"',
|
|
'sata0:0.present = "TRUE"',
|
|
'sata0:0.deviceType = "cdrom-image"',
|
|
f'sata0:0.fileName = "{install_iso}"',
|
|
'sata0:0.startConnected = "TRUE"',
|
|
'sata0:1.present = "TRUE"',
|
|
'sata0:1.deviceType = "cdrom-image"',
|
|
f'sata0:1.fileName = "{tools_iso}"',
|
|
'sata0:1.startConnected = "TRUE"',
|
|
'ethernet0.present = "TRUE"',
|
|
'ethernet0.virtualDev = "e1000e"',
|
|
'ethernet0.connectionType = "nat"',
|
|
'ethernet0.addressType = "generated"',
|
|
'ethernet0.startConnected = "TRUE"',
|
|
'ethernet0.pciSlotNumber = "192"',
|
|
# No USB controllers: a headless CI build VM has no USB devices; input is
|
|
# the VMware PS/2 keyboard/mouse + SVGA console. Omitting usb/ehci/xhci
|
|
# trims device enumeration at boot. (Step 9 still strips any usb* line.)
|
|
'usb.present = "FALSE"',
|
|
'svga.present = "TRUE"',
|
|
'svga.graphicsMemoryKB = "262144"',
|
|
'floppy0.present = "FALSE"',
|
|
'sound.present = "FALSE"',
|
|
'vmci0.present = "TRUE"',
|
|
'vmci0.pciSlotNumber = "36"',
|
|
'hpet0.present = "TRUE"',
|
|
'mks.enable3d = "FALSE"',
|
|
'snapshot.action = "keep"',
|
|
]
|
|
|
|
|
|
def _vmrun_tools_running(vmrun: str, vmx: str) -> bool:
|
|
"""True when VMware Tools report 'running' inside the guest."""
|
|
proc = subprocess.run(
|
|
[vmrun, "-T", "ws", "checkToolsState", str(vmx)],
|
|
capture_output=True, text=True, timeout=30, check=False,
|
|
)
|
|
return proc.stdout.strip().lower() == "running"
|
|
|
|
|
|
def _wait_install_flag(
|
|
vmrun: str,
|
|
vmx: str,
|
|
*,
|
|
admin_user: str,
|
|
admin_password: str,
|
|
flag_path: str,
|
|
timeout_s: float,
|
|
poll_s: float,
|
|
) -> bool:
|
|
"""Poll until the guest drops ``flag_path`` (Tools must be running first)."""
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline:
|
|
if _vmrun_tools_running(vmrun, vmx):
|
|
proc = subprocess.run(
|
|
[vmrun, "-gu", admin_user, "-gp", admin_password,
|
|
"fileExistsInGuest", str(vmx), flag_path],
|
|
capture_output=True, text=True, timeout=60, check=False,
|
|
)
|
|
if "the file exists" in proc.stdout.strip().lower():
|
|
return True
|
|
click.echo(" ... Tools up; waiting for install_complete.flag ...")
|
|
else:
|
|
click.echo(" ... waiting for VMware Tools to report running ...")
|
|
time.sleep(poll_s)
|
|
return False
|
|
|
|
|
|
def _vmrun_soft_stop(vmrun: str, vmx: str, *, timeout_s: float = 300.0) -> None:
|
|
"""Soft-stop the VM, falling back to hard stop after ``timeout_s``."""
|
|
subprocess.run(
|
|
[vmrun, "-T", "ws", "stop", str(vmx), "soft"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline:
|
|
listing = subprocess.run(
|
|
[vmrun, "-T", "ws", "list"], capture_output=True, text=True, check=False,
|
|
)
|
|
if str(vmx) not in listing.stdout:
|
|
return
|
|
time.sleep(5)
|
|
subprocess.run(
|
|
[vmrun, "-T", "ws", "stop", str(vmx), "hard"],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
|
|
|
|
@template.command("deploy-win")
|
|
@click.option("--vmx", "vmx_path_str", default=None, metavar="PATH",
|
|
help="VMX to create (default: <CI_TEMPLATES>/WinBuild2025/WinBuild2025.vmx).")
|
|
@click.option("--vm-name", default="WinBuild2025", show_default=True, help="VM display name.")
|
|
@click.option("--build", "uup_build", default=None,
|
|
help="Pin a UUP build number (e.g. 26100.32860). Default: newest WS2025.")
|
|
@click.option("--win-iso", "win_iso_str", default=None, metavar="PATH",
|
|
help="Use this Windows ISO instead of downloading from UUP dump.")
|
|
@click.option("--edition", default="SERVERSTANDARD", show_default=True, help="UUP edition.")
|
|
@click.option("--lang", default="en-us", show_default=True, help="UUP language pack.")
|
|
@click.option("--image-index", default=2, show_default=True, help="install.wim image index.")
|
|
@click.option("--product-key", default="TVRH6-WHNXV-R9WG3-9XRFY-MY832", show_default=True)
|
|
@click.option("--admin-user", default="Administrator", show_default=True)
|
|
@click.option("--admin-password", default="WinBuild!", show_default=True)
|
|
@click.option("--computer-name", default="WINBUILD-2025", show_default=True)
|
|
@click.option("--locale-os", default="en-US", show_default=True)
|
|
@click.option("--locale-user", default="it-IT", show_default=True)
|
|
@click.option("--keyboard", default="0410:00000410", show_default=True)
|
|
@click.option("--timezone", default="W. Europe Standard Time", show_default=True)
|
|
@click.option("--disk-size-gb", default=80, show_default=True)
|
|
@click.option("--memory-mb", default=6144, show_default=True)
|
|
@click.option("--vcpu-count", default=4, show_default=True)
|
|
@click.option("--cores-per-socket", default=2, show_default=True)
|
|
@click.option("--tools-iso", "tools_iso_str", default=None, metavar="PATH",
|
|
help="VMware Tools ISO (default: <CI_ROOT>/iso/VMware-tools-windows.iso).")
|
|
@click.option("--snapshot-name", default="PostInstall", show_default=True)
|
|
@click.option("--gui/--nogui", "gui", default=False, show_default=True,
|
|
help="Start the VM with the Workstation window (needs an X display).")
|
|
@click.option("--flag-timeout", default=5400, show_default=True,
|
|
help="Seconds to wait for install_complete.flag.")
|
|
@click.option("--flag-poll", default=30, show_default=True, help="Flag poll interval (s).")
|
|
@click.option("--vdisk-manager", "vdisk_manager_str", default=None)
|
|
@click.option("--force", is_flag=True, default=False, help="Overwrite an existing VM directory.")
|
|
@click.option("--start-from-step", default=1, type=click.IntRange(1, 10), show_default=True)
|
|
def template_deploy_win(
|
|
vmx_path_str: str | None,
|
|
vm_name: str,
|
|
uup_build: str | None,
|
|
win_iso_str: str | None,
|
|
edition: str,
|
|
lang: str,
|
|
image_index: int,
|
|
product_key: str,
|
|
admin_user: str,
|
|
admin_password: str,
|
|
computer_name: str,
|
|
locale_os: str,
|
|
locale_user: str,
|
|
keyboard: str,
|
|
timezone: str,
|
|
disk_size_gb: int,
|
|
memory_mb: int,
|
|
vcpu_count: int,
|
|
cores_per_socket: int,
|
|
tools_iso_str: str | None,
|
|
snapshot_name: str,
|
|
gui: bool,
|
|
flag_timeout: int,
|
|
flag_poll: int,
|
|
vdisk_manager_str: str | None,
|
|
force: bool,
|
|
start_from_step: int,
|
|
) -> None:
|
|
"""Deploy the Windows Server 2025 build VM template (Linux VMware host).
|
|
|
|
Downloads the latest WS2025 ISO from UUP dump (deterministic, build-pinned),
|
|
patches it (inject autounattend.xml + post-install.ps1 + no-prompt boot),
|
|
runs the unattended install, swaps the NIC to vmxnet3 and snapshots the
|
|
template. Mirrors the legacy Deploy-WinBuild2025.ps1 ten-step flow.
|
|
"""
|
|
config = load_config()
|
|
vmx_path = (
|
|
Path(vmx_path_str).resolve()
|
|
if vmx_path_str
|
|
else config.paths.templates / "WinBuild2025" / "WinBuild2025.vmx"
|
|
)
|
|
vm_dir = vmx_path.parent
|
|
iso_dir = config.paths.root / "iso"
|
|
tools_iso = Path(tools_iso_str) if tools_iso_str else iso_dir / "VMware-tools-windows.iso"
|
|
vmdk_name = vmx_path.stem + ".vmdk"
|
|
vmdk_path = vm_dir / vmdk_name
|
|
patched_iso = vm_dir / "install-noprompt.iso"
|
|
flag_path = r"C:\Windows\Temp\install_complete.flag"
|
|
|
|
vmrun = config.vmrun_path or shutil.which("vmrun") or "vmrun"
|
|
vdisk_mgr = _find_vdisk_manager(vdisk_manager_str)
|
|
|
|
params = win_autounattend.WinUnattendParams(
|
|
admin_user=admin_user, admin_password=admin_password,
|
|
computer_name=computer_name, product_key=product_key,
|
|
image_index=image_index, locale_os=locale_os, locale_user=locale_user,
|
|
keyboard=keyboard, timezone=timezone,
|
|
)
|
|
|
|
if start_from_step > 1:
|
|
click.echo(f" [SKIP] Steps 1..{start_from_step - 1} -- starting at Step {start_from_step}")
|
|
|
|
# ── Step 1: preflight ──────────────────────────────────────────────────────
|
|
_deploy_step("Step 1: preflight", 1, start_from_step)
|
|
if start_from_step <= 1:
|
|
try:
|
|
win_autounattend.validate_computer_name(computer_name)
|
|
except ValueError as exc:
|
|
raise click.ClickException(f"[PreFlight] {exc}") from exc
|
|
checks: list[tuple[str, bool]] = [
|
|
(f"vmrun found: {vmrun}", bool(shutil.which(vmrun) or Path(vmrun).is_file())),
|
|
(f"vmware-vdiskmanager found: {vdisk_mgr}", Path(vdisk_mgr).is_file()),
|
|
("xorriso in PATH", shutil.which("xorriso") is not None),
|
|
("7z in PATH", any(shutil.which(n) for n in ("7z", "7za", "7zr"))),
|
|
(f"VMware Tools ISO: {tools_iso}", tools_iso.is_file()),
|
|
]
|
|
for label, ok in checks:
|
|
if not ok:
|
|
raise click.ClickException(f"[PreFlight] {label}")
|
|
click.echo(f" [OK] {label}")
|
|
if win_iso_str is None:
|
|
missing = win_uup.check_tools()
|
|
if missing:
|
|
raise click.ClickException(
|
|
"[PreFlight] missing UUP tools: " + ", ".join(missing)
|
|
)
|
|
click.echo(" [OK] UUP conversion tools present")
|
|
if vm_dir.exists():
|
|
if not force:
|
|
raise click.ClickException(
|
|
f"[PreFlight] VM directory exists: {vm_dir}\n Use --force to overwrite."
|
|
)
|
|
click.echo(f" [WARN] --force -- reusing VM directory: {vm_dir}")
|
|
_clean_force_artifacts(vm_dir, vmx_path.stem)
|
|
else:
|
|
vm_dir.mkdir(parents=True, exist_ok=True)
|
|
click.echo(f" [OK] VM directory ready: {vm_dir}")
|
|
|
|
# ── Step 2: render answer files ────────────────────────────────────────────
|
|
_deploy_step("Step 2: render autounattend.xml + post-install.ps1", 2, start_from_step)
|
|
autounattend_xml = win_autounattend.render_autounattend(params)
|
|
post_install_ps1 = win_autounattend.render_post_install(params)
|
|
if start_from_step <= 2:
|
|
click.echo(" [OK] answer files rendered")
|
|
|
|
# ── Step 3: acquire base ISO (UUP dump) ────────────────────────────────────
|
|
_deploy_step("Step 3: acquire Windows Server 2025 ISO", 3, start_from_step)
|
|
if win_iso_str is not None:
|
|
base_iso = Path(win_iso_str)
|
|
if start_from_step <= 3 and not base_iso.is_file():
|
|
raise click.ClickException(f"--win-iso not found: {base_iso}")
|
|
click.echo(f" [OK] using supplied ISO: {base_iso}")
|
|
else:
|
|
build = win_uup.resolve_build(build=uup_build)
|
|
click.echo(f" resolved build: {build.build} ({build.uuid})")
|
|
base_iso = win_uup.build_iso(build, dest_dir=iso_dir, lang=lang, edition=edition)
|
|
|
|
# ── Step 4: patch ISO (inject answer files + no-prompt boot) ───────────────
|
|
_deploy_step("Step 4: patch ISO (inject autounattend + no-prompt boot)", 4, start_from_step)
|
|
if start_from_step <= 4:
|
|
key = _patched_iso_inputs_key(base_iso, autounattend_xml, post_install_ps1)
|
|
sidecar = patched_iso.with_suffix(patched_iso.suffix + ".inputs")
|
|
cached = (
|
|
patched_iso.is_file()
|
|
and sidecar.is_file()
|
|
and sidecar.read_text(encoding="utf-8").strip() == key
|
|
)
|
|
if cached:
|
|
click.echo(f" [OK] patched ISO up to date — skipping repack: {patched_iso}")
|
|
else:
|
|
win_iso_patch.patch_iso(
|
|
base_iso,
|
|
autounattend_xml=autounattend_xml,
|
|
post_install_ps1=post_install_ps1,
|
|
out_iso=patched_iso,
|
|
work_dir=vm_dir,
|
|
)
|
|
sidecar.write_text(key, encoding="utf-8")
|
|
|
|
# ── Step 5: create VMDK ────────────────────────────────────────────────────
|
|
_deploy_step("Step 5: create VMDK", 5, start_from_step)
|
|
if start_from_step <= 5:
|
|
if vmdk_path.exists():
|
|
vmdk_path.unlink()
|
|
subprocess.run(
|
|
[vdisk_mgr, "-c", "-s", f"{disk_size_gb}GB", "-a", "lsilogic", "-t", "0",
|
|
str(vmdk_path)],
|
|
check=True,
|
|
)
|
|
click.echo(f" [OK] VMDK created: {vmdk_path}")
|
|
|
|
# ── Step 6: write VMX ──────────────────────────────────────────────────────
|
|
_deploy_step("Step 6: write VMX", 6, start_from_step)
|
|
if start_from_step <= 6:
|
|
lines = _winbuild_vmx_lines(
|
|
vm_name=vm_name, mem_mb=memory_mb, vcpu_count=vcpu_count,
|
|
cores_per_socket=cores_per_socket, vmdk_name=vmdk_name,
|
|
install_iso=str(patched_iso), tools_iso=str(tools_iso),
|
|
)
|
|
vmx_path.write_text("\n".join(lines) + "\n", encoding="ascii")
|
|
click.echo(f" [OK] VMX written: {vmx_path}")
|
|
|
|
# ── Step 7: power on for unattended install ────────────────────────────────
|
|
_deploy_step("Step 7: power on (unattended install)", 7, start_from_step)
|
|
if start_from_step <= 7:
|
|
gui_mode = "gui" if gui else "nogui"
|
|
subprocess.Popen(
|
|
[vmrun, "-T", "ws", "start", str(vmx_path), gui_mode]
|
|
)
|
|
time.sleep(10)
|
|
click.echo(" [OK] VM powered on")
|
|
|
|
# ── Step 8: wait for install_complete.flag ─────────────────────────────────
|
|
_deploy_step("Step 8: wait for install_complete.flag", 8, start_from_step)
|
|
if start_from_step <= 8:
|
|
ok = _wait_install_flag(
|
|
vmrun, str(vmx_path), admin_user=admin_user, admin_password=admin_password,
|
|
flag_path=flag_path, timeout_s=float(flag_timeout), poll_s=float(flag_poll),
|
|
)
|
|
if not ok:
|
|
raise click.ClickException(
|
|
f"[Deploy Step 8] install_complete.flag not seen within {flag_timeout}s."
|
|
)
|
|
click.echo(" [OK] install_complete.flag detected")
|
|
|
|
# ── Step 9: stop, strip install media, swap NIC to vmxnet3 ─────────────────
|
|
_deploy_step("Step 9: strip install media + swap NIC to vmxnet3", 9, start_from_step)
|
|
if start_from_step <= 9:
|
|
_vmrun_soft_stop(vmrun, str(vmx_path))
|
|
for prefix in ("sata0", "usb", "ehci", "usb_xhci"):
|
|
_remove_vmx_lines(vmx_path, prefix)
|
|
_patch_vmx_key(vmx_path, "ethernet0.virtualDev", "vmxnet3")
|
|
click.echo(" [OK] SATA/USB removed, NIC -> vmxnet3")
|
|
|
|
# ── Step 10: verify vmxnet3 IP, then cold snapshot ─────────────────────────
|
|
_deploy_step("Step 10: verify network + snapshot", 10, start_from_step)
|
|
if start_from_step <= 10:
|
|
gui_mode = "gui" if gui else "nogui"
|
|
subprocess.Popen(
|
|
[vmrun, "-T", "ws", "start", str(vmx_path), gui_mode]
|
|
)
|
|
guest_ip = _poll_guest_ip_deploy(vmrun, str(vmx_path), timeout=600.0)
|
|
if not guest_ip:
|
|
raise click.ClickException("[Deploy Step 10] no guest IP after vmxnet3 swap.")
|
|
click.echo(f" [OK] vmxnet3 guest IP: {guest_ip}")
|
|
_vmrun_soft_stop(vmrun, str(vmx_path))
|
|
subprocess.run(
|
|
[vmrun, "-T", "ws", "snapshot", str(vmx_path), snapshot_name], check=True,
|
|
)
|
|
listing = subprocess.run(
|
|
[vmrun, "-T", "ws", "listSnapshots", str(vmx_path)],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
if snapshot_name not in listing.stdout:
|
|
raise click.ClickException(
|
|
f"[Deploy Step 10] snapshot '{snapshot_name}' not found after creation."
|
|
)
|
|
click.echo(f" [OK] snapshot '{snapshot_name}' created (cold)")
|
|
|
|
click.echo("\n=== deploy-win complete ===")
|
|
click.echo(f" VMX: {vmx_path}")
|
|
click.echo(f" Snapshot: {snapshot_name}")
|
|
click.echo(f" Next: ci template prepare-win --vmx {vmx_path}")
|