Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(deploy-win): drop UUP dump, take ISO via --iso-url / --win-iso

Browse Source 
On a Linux host the UUP converter only produces the RTM install.wim (build
26100.1), so Windows Update had to install the entire cumulative anyway —
the UUP download/convert machinery added a slow ~5 GB step and several host
tools (aria2/wimtools/cabextract/chntpw) for no benefit.

Replace it: deploy-win now takes the install ISO directly via `--iso-url`
(urllib download + cache by file name under <CI_ROOT>/iso/, optional
--iso-sha256) or a local `--win-iso` (exactly one required). Patch + deploy
flow is unchanged. Supply an already-updated ISO (e.g. a massgrave link) to
skip the WU step entirely.

- Remove winimage/uup.py and its tests.
- Trim setup-host-linux.sh deps to p7zip-full + xorriso + genisoimage.
- Update RUNBOOK §5 and the Windows-host port plan. Coverage 93.6%.
This commit is contained in:
Simone
2026-06-09 18:42:22 +02:00
parent cd88dec15d
commit c257c931c3
8 changed files with 108 additions and 445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ci_orchestrator/commands/template.py
+42 -24
View File
@@ -68,6 +68,7 @@ import sys
import tempfile
import threading
import time
import urllib.parse
import urllib.request
from datetime import UTC, datetime
from pathlib import Path
@@ -82,7 +83,6 @@ from ci_orchestrator.transport.errors import TransportError
from ci_orchestrator.transport.winrm import WinRmResult, WinRmTransport
from ci_orchestrator.winimage import autounattend as win_autounattend
from ci_orchestrator.winimage import iso_patch as win_iso_patch
from ci_orchestrator.winimage import uup as win_uup
@click.group()
@@ -2086,12 +2086,13 @@ def _vmrun_soft_stop(vmrun: str, vmx: str, *, timeout_s: float = 300.0) -> None:
@click.option("--vmx", "vmx_path_str", default=None, metavar="PATH",
help="VMX to create (default: <CI_TEMPLATES>/WinBuild2025/WinBuild2025.vmx).")
@click.option("--vm-name", default="WinBuild2025", show_default=True, help="VM display name.")
@click.option("--build", "uup_build", default=None,
help="Pin a UUP build number (e.g. 26100.32860). Default: newest WS2025.")
@click.option("--iso-url", "iso_url", default=None, metavar="URL",
help="Direct download URL of the Windows Server 2025 install ISO. "
"Cached under <CI_ROOT>/iso/. Mutually exclusive with --win-iso.")
@click.option("--iso-sha256", "iso_sha256", default="",
help="Expected SHA256 of the downloaded ISO (empty = skip verification).")
@click.option("--win-iso", "win_iso_str", default=None, metavar="PATH",
help="Use this Windows ISO instead of downloading from UUP dump.")
@click.option("--edition", default="SERVERSTANDARD", show_default=True, help="UUP edition.")
@click.option("--lang", default="en-us", show_default=True, help="UUP language pack.")
help="Use this local Windows ISO instead of --iso-url.")
@click.option("--image-index", default=2, show_default=True, help="install.wim image index.")
@click.option("--product-key", default="TVRH6-WHNXV-R9WG3-9XRFY-MY832", show_default=True)
@click.option("--admin-user", default="Administrator", show_default=True)
@@ -2119,10 +2120,9 @@ def _vmrun_soft_stop(vmrun: str, vmx: str, *, timeout_s: float = 300.0) -> None:
def template_deploy_win(
vmx_path_str: str | None,
vm_name: str,
uup_build: str | None,
iso_url: str | None,
iso_sha256: str,
win_iso_str: str | None,
edition: str,
lang: str,
image_index: int,
product_key: str,
admin_user: str,
@@ -2147,10 +2147,11 @@ def template_deploy_win(
) -> None:
"""Deploy the Windows Server 2025 build VM template (Linux VMware host).
Downloads the latest WS2025 ISO from UUP dump (deterministic, build-pinned),
patches it (inject autounattend.xml + post-install.ps1 + no-prompt boot),
runs the unattended install, swaps the NIC to vmxnet3 and snapshots the
template. Mirrors the legacy Deploy-WinBuild2025.ps1 ten-step flow.
Downloads the install ISO from ``--iso-url`` (cached under <CI_ROOT>/iso/)
or uses a local ``--win-iso``, patches it (inject autounattend.xml +
post-install.ps1 + no-prompt boot), runs the unattended install, swaps the
NIC to vmxnet3 and snapshots the template. Mirrors the legacy
Deploy-WinBuild2025.ps1 ten-step flow.
"""
config = load_config()
vmx_path = (
@@ -2197,13 +2198,11 @@ def template_deploy_win(
if not ok:
raise click.ClickException(f"[PreFlight] {label}")
click.echo(f" [OK] {label}")
if win_iso_str is None:
missing = win_uup.check_tools()
if missing:
raise click.ClickException(
"[PreFlight] missing UUP tools: " + ", ".join(missing)
)
click.echo(" [OK] UUP conversion tools present")
if bool(iso_url) == bool(win_iso_str):
raise click.ClickException(
"[PreFlight] provide exactly one of --iso-url or --win-iso."
)
click.echo(" [OK] ISO source: " + (f"url {iso_url}" if iso_url else f"file {win_iso_str}"))
if vm_dir.exists():
if not force:
raise click.ClickException(
@@ -2222,7 +2221,7 @@ def template_deploy_win(
if start_from_step <= 2:
click.echo(" [OK] answer files rendered")
# ── Step 3: acquire base ISO (UUP dump) ────────────────────────────────────
# ── Step 3: acquire base ISO (download from URL, or local file) ────────────
_deploy_step("Step 3: acquire Windows Server 2025 ISO", 3, start_from_step)
if win_iso_str is not None:
base_iso = Path(win_iso_str)
@@ -2230,9 +2229,28 @@ def template_deploy_win(
raise click.ClickException(f"--win-iso not found: {base_iso}")
click.echo(f" [OK] using supplied ISO: {base_iso}")
else:
build = win_uup.resolve_build(build=uup_build)
click.echo(f" resolved build: {build.build} ({build.uuid})")
base_iso = win_uup.build_iso(build, dest_dir=iso_dir, lang=lang, edition=edition)
assert iso_url is not None # guaranteed by Step 1 source validation
iso_name = Path(urllib.parse.urlparse(iso_url).path).name or "windows-install.iso"
base_iso = iso_dir / iso_name
if start_from_step <= 3:
if base_iso.is_file():
click.echo(f" [OK] ISO cached, reusing: {base_iso}")
else:
iso_dir.mkdir(parents=True, exist_ok=True)
try:
_download_vmdk(iso_url, base_iso)
except Exception as exc:
raise click.ClickException(f"[Deploy] ISO download failed: {exc}") from exc
click.echo(f" [OK] downloaded: {base_iso}")
if iso_sha256:
click.echo(" Verifying ISO SHA256 ...")
actual = _sha256_file(base_iso)
if actual != iso_sha256.upper():
raise click.ClickException(
f"[Deploy] ISO SHA256 mismatch!\n Expected: {iso_sha256.upper()}\n"
f" Actual: {actual}"
)
click.echo(f" [OK] ISO SHA256: {actual}")
# ── Step 4: patch ISO (inject answer files + no-prompt boot) ───────────────
_deploy_step("Step 4: patch ISO (inject autounattend + no-prompt boot)", 4, start_from_step)
src/ci_orchestrator/winimage/__init__.py
+4 -4
View File
@@ -1,9 +1,9 @@
"""Windows install-image tooling for ``ci template deploy-win``.
Linux-host implementation: acquire a Windows Server 2025 ISO from UUP dump
(``uup.py``), render the unattended answer file + guest post-install script
(``autounattend.py``), and patch the ISO with a no-prompt boot image plus those
files injected at the root (``iso_patch.py``).
Linux-host implementation: render the unattended answer file + guest
post-install script (``autounattend.py``) and patch a Windows install ISO with a
no-prompt boot image plus those files injected at the root (``iso_patch.py``).
The base ISO is supplied directly (``deploy-win --iso-url`` / ``--win-iso``).
The heavy / OS-specific logic lives here so ``commands/template.py`` stays a thin
orchestrator. The Windows-host port is tracked in ``plans/deploy-win-windows-host.md``.
src/ci_orchestrator/winimage/uup.py
-233
View File
@@ -1,233 +0,0 @@
"""Acquire a Windows Server 2025 install ISO from UUP dump (Linux host).
UUP dump is deterministic on its *input*: a build is pinned by build number and
the component files come MS-signed from the official Windows Update CDN. The
final ISO is assembled locally (its hash is not canonical), so we cache the
result per build number under ``<CI_ROOT>/iso/ws2025-<build>.iso``.
Flow (mirrors what uupdump.net's generated package does):
1. ``resolve_build`` — query the uupdump API for Windows Server 2025 builds and
pick the newest (or match a pinned build string).
2. ``build_iso`` — download the per-set conversion package, run its
``uup_download_linux.sh`` (aria2 download + wimlib conversion), then move the
produced ISO into the cache.
The downloader requires ``aria2c``, ``cabextract``, ``wimlib-imagex``,
``chntpw`` and ``genisoimage``/``mkisofs`` on PATH (installed by
``setup-host-linux.sh``).
"""
from __future__ import annotations
import json
import shutil
import subprocess
import urllib.parse
import urllib.request
import zipfile
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import click
API_BASE = "https://api.uupdump.net"
GET_URL = "https://uupdump.net/get.php"
_USER_AGENT = "ci-orchestrator/1.0 (+deploy-win)"
#: Tools the UUP → ISO conversion needs (uup_download_linux.sh checks these).
REQUIRED_TOOLS = ("aria2c", "cabextract", "wimlib-imagex", "chntpw")
#: Title prefix that identifies a Windows Server 2025 set (excludes "version 23H2").
_WS2025_PREFIX = "Windows Server 2025"
@dataclass(frozen=True)
class UupBuild:
"""A UUP dump build set."""
uuid: str
build: str # e.g. "26100.32860"
title: str
arch: str
def _http_json(url: str, *, timeout: float = 60.0) -> dict[str, Any]:
"""GET ``url`` and parse the JSON body."""
req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT, "Accept": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8")
data: Any = json.loads(raw)
if not isinstance(data, dict):
raise click.ClickException(f"uupdump API returned unexpected JSON from {url}")
return data
def _build_sort_key(build: str) -> tuple[int, ...]:
"""Numeric sort key for a build string like ``26100.32860``."""
parts: list[int] = []
for chunk in build.split("."):
try:
parts.append(int(chunk))
except ValueError:
parts.append(0)
return tuple(parts)
def list_ws2025_builds(*, arch: str = "amd64", timeout: float = 60.0) -> list[UupBuild]:
"""Return all Windows Server 2025 builds for ``arch``, newest first."""
url = f"{API_BASE}/listid.php?search={urllib.parse.quote(_WS2025_PREFIX)}"
data = _http_json(url, timeout=timeout)
builds_raw = data.get("response", {}).get("builds", {})
if not isinstance(builds_raw, dict):
raise click.ClickException("uupdump API: missing 'builds' in listid response")
out: list[UupBuild] = []
for entry in builds_raw.values():
if not isinstance(entry, dict):
continue
title = str(entry.get("title", ""))
if not title.startswith(_WS2025_PREFIX):
continue
if str(entry.get("arch", "")) != arch:
continue
out.append(
UupBuild(
uuid=str(entry["uuid"]),
build=str(entry["build"]),
title=title,
arch=arch,
)
)
out.sort(key=lambda b: _build_sort_key(b.build), reverse=True)
return out
def resolve_build(
*, build: str | None = None, arch: str = "amd64", timeout: float = 60.0
) -> UupBuild:
"""Resolve the newest WS2025 build, or the one matching ``build`` if pinned."""
builds = list_ws2025_builds(arch=arch, timeout=timeout)
if not builds:
raise click.ClickException(
f"No Windows Server 2025 ({arch}) builds found on uupdump."
)
if build is None:
return builds[0]
for b in builds:
if b.build == build:
return b
available = ", ".join(b.build for b in builds)
raise click.ClickException(
f"WS2025 build {build!r} not found on uupdump. Available: {available}"
)
def check_tools() -> list[str]:
"""Return the list of missing conversion tools (empty when all present)."""
missing = [t for t in REQUIRED_TOOLS if shutil.which(t) is None]
if shutil.which("genisoimage") is None and shutil.which("mkisofs") is None:
missing.append("genisoimage|mkisofs")
return missing
def cache_path(dest_dir: Path, build: UupBuild) -> Path:
"""Path of the cached ISO for ``build`` under ``dest_dir``."""
return dest_dir / f"ws2025-{build.build}.iso"
def _download_package(build: UupBuild, lang: str, edition: str, dest: Path, *, timeout: float) -> None:
"""Download the uupdump conversion package zip for ``build``."""
url = (
f"{GET_URL}?id={build.uuid}"
f"&pack={urllib.parse.quote(lang)}"
f"&edition={urllib.parse.quote(edition)}"
f"&autodl=2"
)
req = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
with urllib.request.urlopen(req, timeout=timeout) as resp, dest.open("wb") as fh:
shutil.copyfileobj(resp, fh)
if dest.stat().st_size < 1024:
raise click.ClickException(f"uupdump package download too small: {dest}")
def build_iso(
build: UupBuild,
*,
dest_dir: Path,
lang: str = "en-us",
edition: str = "SERVERSTANDARD",
work_dir: Path | None = None,
keep_work: bool = False,
) -> Path:
"""Build (or reuse a cached) WS2025 ISO for ``build`` and return its path.
Downloads the uupdump conversion package, runs its ``uup_download_linux.sh``
(aria2 download of the MS-signed UUP set + wimlib conversion to ISO), then
moves the produced ISO to ``<dest_dir>/ws2025-<build>.iso``.
"""
dest_dir.mkdir(parents=True, exist_ok=True)
cached = cache_path(dest_dir, build)
if cached.is_file():
click.echo(f" [uup] cached ISO present, reusing: {cached}")
return cached
missing = check_tools()
if missing:
raise click.ClickException(
"Missing UUP conversion tools: "
+ ", ".join(missing)
+ ". Install them (setup-host-linux.sh installs aria2 wimtools "
"cabextract chntpw genisoimage)."
)
work = work_dir or (dest_dir / f".uup-{build.build}")
if work.exists():
shutil.rmtree(work)
work.mkdir(parents=True)
pkg = work / "uup-package.zip"
click.echo(f" [uup] downloading conversion package for build {build.build} ({build.uuid})")
_download_package(build, lang, edition, pkg, timeout=120.0)
with zipfile.ZipFile(pkg) as zf:
zf.extractall(work)
script = work / "uup_download_linux.sh"
if not script.is_file():
raise click.ClickException(f"uup_download_linux.sh missing from package: {work}")
script.chmod(0o755)
click.echo(" [uup] running converter (downloads ~5 GB + builds ISO; this is slow) ...")
proc = subprocess.run(
["bash", str(script)],
cwd=str(work),
check=False,
)
if proc.returncode != 0:
raise click.ClickException(
f"UUP conversion failed (exit {proc.returncode}). See {work}/aria2_download.log."
)
produced = _find_produced_iso(work)
if produced is None:
raise click.ClickException(f"UUP conversion produced no ISO under {work}")
shutil.move(str(produced), str(cached))
click.echo(f" [uup] ISO ready: {cached}")
if not keep_work:
shutil.rmtree(work, ignore_errors=True)
return cached
def _find_produced_iso(work: Path) -> Path | None:
"""Find the ISO the converter produced (largest *.iso, excluding the package)."""
candidates = [
p
for p in work.rglob("*.iso")
if p.name != "uup-package.zip" and p.is_file()
]
candidates += [p for p in work.rglob("*.ISO") if p.is_file()]
if not candidates:
return None
return max(candidates, key=lambda p: p.stat().st_size)