Compare commits
95 Commits
v2.0.0-phaseA
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| c18c5bd254 | |||
| 005662e359 | |||
| 6f6a304bbe | |||
| 0e963f2194 | |||
| 461eb3f8de | |||
| f13b24956e | |||
| 2241d6af7f | |||
| 04ca316f17 | |||
| decdf73202 | |||
| 962a69d287 | |||
| d23ce1939e | |||
| d19099a257 | |||
| 88168e37cb | |||
| d63fca5967 | |||
| 8f1b3b7eb1 | |||
| 4366ba6b04 | |||
| f965db71b3 | |||
| f22b142c61 | |||
| 66cdd4866f | |||
| ab0c13007e | |||
| c12fcbc434 | |||
| 89299b91c7 | |||
| aadab2d49b | |||
| 5bff7b4f0c | |||
| 210b4f5f16 | |||
| 6f51d44f92 | |||
| e91e55f20e | |||
| 2b0e98774f | |||
| 6104811e7e | |||
| 4c80ff2528 | |||
| 3eb29b404f | |||
| d0f67ce807 | |||
| 9053064119 | |||
| c0f31e375a | |||
| 8dc7a4fb99 | |||
| 0a1c455e54 | |||
| 189b865a52 | |||
| 1a1560ec3d | |||
| 58d04d3bc1 | |||
| 5d1ec39b1d | |||
| 9c84e66dc3 | |||
| a190282757 | |||
| 6d205386dc | |||
| 64f800019d | |||
| 138fe8d19f | |||
| c9549c1db2 | |||
| e538c0c037 | |||
| 87d440b672 | |||
| f090365d57 | |||
| faf5b9e29b | |||
| f96b9953c5 | |||
| d5f362b8dc | |||
| f6f8ffc3d2 | |||
| 5d3214d10c | |||
| 258ef3a7e7 | |||
| 483dd51a3b | |||
| 84746d2232 | |||
| 9e84d89823 | |||
| 6b76115b6a | |||
| de6858e3df | |||
| 04b716149c | |||
| de33e9f723 | |||
| 41a0a113df | |||
| 1bc3ec601c | |||
| 927d6927fd | |||
| 203515f5d7 | |||
| 855e9d7b3f | |||
| 4f9149f7e2 | |||
| c1147c5b19 | |||
| 77f72f71b9 | |||
| 0b69155b02 | |||
| 40aae7ad16 | |||
| 5982a88041 | |||
| 10998247a0 | |||
| 568740ed17 | |||
| 5d59446114 | |||
| ce84afd901 | |||
| c1137bc5cf | |||
| defb95bd22 | |||
| 758f86c11c | |||
| 0d6486b19b | |||
| 222c41dd16 | |||
| 915a04b5c3 | |||
| 53649b40af | |||
| a906f4696f | |||
| 9688e7e2f8 | |||
| bbd376f057 | |||
| 6db20d66bf | |||
| 43a69b82db | |||
| 536fd688e8 | |||
| b2803c38ef | |||
| a6aa9260b3 | |||
| 3921758392 | |||
| b9d6994c85 | |||
| 573d11691d |
@@ -20,6 +20,7 @@
|
||||
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
||||
# artifact-source: 'dist'
|
||||
# submodules: 'true'
|
||||
# # xvfb: 'true' # Linux: run the build under a headless X server
|
||||
#
|
||||
# Runner label determines the guest OS template:
|
||||
# windows-build -> WinBuild2025 (WinRM/HTTPS transport)
|
||||
@@ -150,7 +151,7 @@ inputs:
|
||||
description: >
|
||||
JSON object of extra environment variables to inject into the build VM
|
||||
guest before the build command runs. Use for signing keys and secrets.
|
||||
Example: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","GPG_KEY_ID":"0xABCD1234"}'
|
||||
Example: '{"SIGN_PASS":"<secrets.SIGN_PASS>","GPG_KEY_ID":"0xABCD1234"}'
|
||||
Values are passed via ExtraGuestEnv hashtable in Invoke-CIJob.ps1 and
|
||||
are never echoed to the log.
|
||||
required: false
|
||||
@@ -190,6 +191,15 @@ inputs:
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
xvfb:
|
||||
description: >
|
||||
Linux only. Set to "true" to run the build command under xvfb-run, a
|
||||
headless X server providing $DISPLAY for GUI toolkits (e.g. GTK4 /
|
||||
PyGObject). Requires xvfb in the Linux template (installed by
|
||||
Install-CIToolchain-Linux2404.sh). Ignored on Windows guests.
|
||||
required: false
|
||||
default: 'false'
|
||||
|
||||
outputs:
|
||||
artifact-path:
|
||||
description: >
|
||||
@@ -208,7 +218,7 @@ runs:
|
||||
# ── Invoke the CI orchestrator ─────────────────────────────────────────────
|
||||
- name: Invoke CI Job
|
||||
id: invoke-ci
|
||||
shell: powershell
|
||||
shell: pwsh
|
||||
# Inputs are forwarded via environment variables to avoid shell injection
|
||||
# from user-supplied build-command or path strings.
|
||||
env:
|
||||
@@ -228,6 +238,7 @@ runs:
|
||||
INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }}
|
||||
INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }}
|
||||
INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }}
|
||||
INPUT_XVFB: ${{ inputs.xvfb }}
|
||||
INPUT_GUEST_CPU: ${{ inputs.guest-cpu }}
|
||||
INPUT_GUEST_MEMORY_MB: ${{ inputs.guest-memory-mb }}
|
||||
run: |
|
||||
@@ -236,9 +247,12 @@ runs:
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Resolve Python interpreter from venv (Phase A4 — Python orchestrator).
|
||||
# Override with CI_VENV_PYTHON env var; default matches Setup-Host.ps1 layout.
|
||||
# Override with CI_VENV_PYTHON env var; OS-specific defaults otherwise.
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
if ($IsLinux) { $venvPython = '/opt/ci/venv/bin/python' }
|
||||
else { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $venvPython)) {
|
||||
throw "Python interpreter not found at '$venvPython'. Set CI_VENV_PYTHON or run Setup-Host.ps1."
|
||||
}
|
||||
@@ -246,7 +260,8 @@ runs:
|
||||
# Build job ID — append suffix when provided (matrix disambiguation)
|
||||
$jobId = '${{ github.run_id }}-${{ github.run_attempt }}'
|
||||
if ($env:INPUT_JOB_ID_SUFFIX) { $jobId = "$jobId-$($env:INPUT_JOB_ID_SUFFIX)" }
|
||||
$artifactPath = "F:\CI\Artifacts\$jobId"
|
||||
if ($IsLinux) { $artifactPath = "/var/lib/ci/artifacts/$jobId" }
|
||||
else { $artifactPath = "F:\CI\Artifacts\$jobId" }
|
||||
|
||||
# Resolve artifact name: caller-supplied or default to build-{run_id}-{sha}
|
||||
if ($env:INPUT_ARTIFACT_NAME) {
|
||||
@@ -329,10 +344,10 @@ runs:
|
||||
'--guest-artifact-source', $env:INPUT_ARTIFACT_SOURCE,
|
||||
'--guest-os', $resolvedGuestOS,
|
||||
'--configuration', $env:INPUT_CONFIGURATION,
|
||||
'--template-path', $resolvedTemplatePath,
|
||||
'--snapshot-name', $resolvedSnapshotName
|
||||
)
|
||||
if ($repoCommit) { $pyArgs += @('--commit', $repoCommit) }
|
||||
if ($resolvedTemplatePath) { $pyArgs += @('--template-path', $resolvedTemplatePath) }
|
||||
if ($repoCommit) { $pyArgs += @('--commit', $repoCommit) }
|
||||
|
||||
# Submodules and transport default to ON; pass the explicit
|
||||
# negative flag when the caller opted out so the Python default
|
||||
@@ -343,6 +358,7 @@ runs:
|
||||
else { $pyArgs += '--use-git-clone' }
|
||||
if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $pyArgs += '--use-shared-cache' }
|
||||
if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $pyArgs += '--skip-artifact' }
|
||||
if ($env:INPUT_XVFB -eq 'true') { $pyArgs += '--xvfb' }
|
||||
# VMX overrides: forward only when > 0 (0 = keep template value).
|
||||
if ([int]($env:INPUT_GUEST_CPU) -gt 0) { $pyArgs += @('--guest-cpu', $env:INPUT_GUEST_CPU) }
|
||||
if ([int]($env:INPUT_GUEST_MEMORY_MB) -gt 0) { $pyArgs += @('--guest-memory-mb', $env:INPUT_GUEST_MEMORY_MB) }
|
||||
@@ -374,7 +390,7 @@ runs:
|
||||
# directly (see build-nsInnoUnp.yml release job).
|
||||
- name: Report artifact location
|
||||
if: success() && inputs.skip-artifact != 'true'
|
||||
shell: powershell
|
||||
shell: pwsh
|
||||
run: |
|
||||
$p = '${{ steps.invoke-ci.outputs.artifact-path }}'
|
||||
if (-not (Test-Path -LiteralPath $p) -or
|
||||
@@ -387,7 +403,7 @@ runs:
|
||||
|
||||
- name: Report diagnostic log location on failure
|
||||
if: failure()
|
||||
shell: powershell
|
||||
shell: pwsh
|
||||
run: |
|
||||
$p = '${{ steps.invoke-ci.outputs.artifact-path }}'
|
||||
if ($p -and (Test-Path -LiteralPath $p)) {
|
||||
|
||||
+26
-55
@@ -1,8 +1,13 @@
|
||||
# PSScriptAnalyzer + Python lint — runs on every push/PR that touches code files.
|
||||
# Requires PSScriptAnalyzer installed on the runner host:
|
||||
# Install-Module PSScriptAnalyzer -Scope AllUsers -Force
|
||||
# Python lint runs on every push/PR (linux-build runner) and blocks the PR.
|
||||
#
|
||||
# Failures block the PR. Fix warnings with:
|
||||
# PSScriptAnalyzer (pssa) runs on the Windows runner (windows-build) on-demand
|
||||
# only — triggered via workflow_dispatch — because the .ps1/.psm1 targets are
|
||||
# Windows-native (PS 5.1). It is intentionally NOT queued on push/PR so it does
|
||||
# not block the pipeline when no Windows runner is registered (dual-boot Passo 9
|
||||
# may be down). This keeps the Linux host/runner free of pwsh (Phase C goal).
|
||||
# Requires PSScriptAnalyzer installed on the Windows runner host:
|
||||
# Install-Module PSScriptAnalyzer -Scope AllUsers -Force
|
||||
# Fix warnings with:
|
||||
# Invoke-ScriptAnalyzer -Path scripts\ -Recurse -Fix
|
||||
|
||||
|
||||
@@ -27,6 +32,9 @@ on:
|
||||
|
||||
jobs:
|
||||
pssa:
|
||||
# On-demand only: runs on the Windows runner (PS 5.1), never on push/PR,
|
||||
# so it cannot block the pipeline when no Windows runner is registered.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
runs-on: windows-build
|
||||
timeout-minutes: 10
|
||||
|
||||
@@ -64,17 +72,12 @@ jobs:
|
||||
}
|
||||
|
||||
python:
|
||||
runs-on: windows-build
|
||||
runs-on: linux-build
|
||||
timeout-minutes: 20
|
||||
|
||||
env:
|
||||
PYTHONIOENCODING: utf-8
|
||||
# Job-local, ephemeral venv. MUST NOT be the production venv
|
||||
# (F:\CI\python\venv): an editable install here would repoint the
|
||||
# shared orchestrator package at this run's transient checkout and
|
||||
# break every later smoke/build job with "No module named
|
||||
# ci_orchestrator". github.workspace is discarded after the job.
|
||||
VENV_DIR: ${{ github.workspace }}\.venv-lint
|
||||
VENV_DIR: ${{ github.workspace }}/.venv-lint
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -83,57 +86,25 @@ jobs:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup venv + install package
|
||||
shell: powershell
|
||||
shell: bash
|
||||
run: |
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Production venv lives at $env:VENV_DIR (F:\CI\python\venv per AGENTS.md).
|
||||
# The system Python used to bootstrap it is configured once in
|
||||
# runner/config.yaml as CI_PYTHON_LAUNCHER (single source of truth).
|
||||
if (-not $env:CI_PYTHON_LAUNCHER) {
|
||||
throw "CI_PYTHON_LAUNCHER not set. Configure it in runner/config.yaml (runner.envs) and restart act_runner."
|
||||
}
|
||||
if (-not (Test-Path $env:CI_PYTHON_LAUNCHER)) {
|
||||
throw "CI_PYTHON_LAUNCHER points to '$env:CI_PYTHON_LAUNCHER' which does not exist."
|
||||
}
|
||||
|
||||
$venvPy = Join-Path $env:VENV_DIR 'Scripts\python.exe'
|
||||
|
||||
# Always build a fresh, job-local venv. Never reuse or repair a
|
||||
# shared one — lint must be fully isolated from the production
|
||||
# orchestrator venv.
|
||||
if (Test-Path $env:VENV_DIR) {
|
||||
Remove-Item $env:VENV_DIR -Recurse -Force
|
||||
}
|
||||
& $env:CI_PYTHON_LAUNCHER -m venv $env:VENV_DIR
|
||||
if ($LASTEXITCODE -ne 0) { throw "venv creation failed" }
|
||||
|
||||
Write-Host "Using venv python: $venvPy"
|
||||
& $venvPy --version
|
||||
if ($LASTEXITCODE -ne 0) { throw "python --version failed" }
|
||||
|
||||
& $venvPy -m pip install --upgrade pip
|
||||
if ($LASTEXITCODE -ne 0) { throw "pip upgrade failed" }
|
||||
& $venvPy -m pip install -e ".[dev]"
|
||||
if ($LASTEXITCODE -ne 0) { throw "pip install failed" }
|
||||
set -euo pipefail
|
||||
rm -rf "$VENV_DIR"
|
||||
python3 -m venv "$VENV_DIR"
|
||||
"$VENV_DIR/bin/python" -m pip install --upgrade pip
|
||||
"$VENV_DIR/bin/python" -m pip install -e ".[dev]"
|
||||
|
||||
- name: ruff
|
||||
shell: powershell
|
||||
shell: bash
|
||||
run: |
|
||||
$venvPy = Join-Path $env:VENV_DIR 'Scripts\python.exe'
|
||||
& $venvPy -m ruff check src tests/python
|
||||
if ($LASTEXITCODE -ne 0) { throw "ruff failed" }
|
||||
"$VENV_DIR/bin/python" -m ruff check src tests/python
|
||||
|
||||
- name: mypy --strict
|
||||
shell: powershell
|
||||
shell: bash
|
||||
run: |
|
||||
$venvPy = Join-Path $env:VENV_DIR 'Scripts\python.exe'
|
||||
& $venvPy -m mypy --strict src
|
||||
if ($LASTEXITCODE -ne 0) { throw "mypy failed" }
|
||||
"$VENV_DIR/bin/python" -m mypy --strict src
|
||||
|
||||
- name: pytest (coverage >= 90%)
|
||||
shell: powershell
|
||||
shell: bash
|
||||
run: |
|
||||
$venvPy = Join-Path $env:VENV_DIR 'Scripts\python.exe'
|
||||
& $venvPy -m pytest tests/python --cov=ci_orchestrator --cov-fail-under=90
|
||||
if ($LASTEXITCODE -ne 0) { throw "pytest failed" }
|
||||
"$VENV_DIR/bin/python" -m pytest tests/python --cov=ci_orchestrator --cov-fail-under=90
|
||||
|
||||
+10
@@ -10,6 +10,7 @@ secrets/
|
||||
# Log e output temporanei
|
||||
logs/
|
||||
*.log
|
||||
*.swp
|
||||
|
||||
# Artifact di build (storti su F:\CI\Artifacts, non nel repo)
|
||||
artifacts/
|
||||
@@ -50,6 +51,15 @@ $RECYCLE.BIN/
|
||||
# AI/strumenti locali
|
||||
.claude/
|
||||
|
||||
# Graphify generated output (knowledge graph, AST/semantic cache)
|
||||
graphify-out/
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Python dev/test artifacts
|
||||
.venv/
|
||||
.coverage
|
||||
|
||||
cache/
|
||||
@@ -226,3 +226,4 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali:
|
||||
10. **`vmrun getGuestIPAddress` per check "VM running"** — richiede VMware Tools attivi nel guest; in headless Tools puo' impiegare 30-60s a rispondere e il check appare bloccato. Usare `vmrun list` e cercare il path del VMX nell'output per `is_running`. Per recuperare l'IP usare `getGuestIPAddress -wait` (blocca internamente finché Tools registrano l'IP) e filtrare le righe che iniziano con `"Error:"` perché vmrun in modalità headless può tornare exit code 0 con `"Error: The VMware Tools are not running..."` su stdout invece di usare un exit code non-zero.
|
||||
11. **Machine-id identico tra clone Linux** — le cloud image Ubuntu hanno un `/etc/machine-id` fisso. VMware DHCP assegna l'IP tramite DHCP client-id derivato da machine-id: se tutti i clone hanno lo stesso machine-id ottengono lo stesso IP e si bloccano per "IP collision". Fix nel template (prima di prendere lo snapshot): `sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id && sudo shutdown -h now`, poi ricreare lo snapshot `BaseClean-Linux` da stato powered-off.
|
||||
12. **`& nativecmd 2>$null` NON sopprime stderr in PS 5.1 con `$ErrorActionPreference='Stop'`** — la stderr del processo nativo viene convertita in ErrorRecord nel pipeline di PowerShell; sotto `'Stop'` questo diventa un errore terminante PRIMA che `2>$null` possa scartarlo. Esempio: `ssh-keygen -R <ip> -f <file>` stampa "Host X not found in file" su stderr → eccezione terminante. Fix: usare `$ErrorActionPreference = 'SilentlyContinue'` attorno alla chiamata oppure `2>&1 | Out-Null`. Per i job CI su VM ephemere locali, usare direttamente `StrictHostKeyChecking=no UserKnownHostsFile=NUL` (default in `_Transport.psm1` con `KnownHostsFile=''`) — nessun `ssh-keygen` necessario.
|
||||
13. **Quote WinRM/desktop-heap troppo basse per builds Windows paralleli** — sintomo: `MSBuild error MSB6003: CL.exe could not be run. System.ComponentModel.Win32Exception (0x80004005): Not enough quota is available to process this command` (oppure le altre tool MSVC: link.exe, mt.exe) quando il build interno fa fan-out parallelo (es. matrix x86/x64/x86-ansi, MSBuild `/maxcpucount`). Causa: il plugin endpoint Microsoft.PowerShell del guest ha `MaxMemoryPerShellMB=150` di default e l'heap del desktop non-interattivo (Session 0) ha `SharedSection 3rd field=768 KB` — entrambi insufficienti per più CL.exe concorrenti via WinRM. **Non emerge** in pre-migration Win→Win perché `Invoke-Command` su `PSSession` localhost-style eredita quote più larghe; emerge appena il path passa per `pypsrp` Linux→Win. Fix nel template (Deploy-WinBuild2025.ps1 / 2022 lo fa, Install-CIToolchain lo valida): `Set-Item WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB 2048`, `Set-Item WSMan:\localhost\Shell\MaxProcessesPerShell 100`, registry `HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems\Windows` patch del 3° campo `SharedSection` a 4096, **reboot** (csrss legge SharedSection solo al boot), nuovo `BaseClean`. Dettagli e procedura completa in `docs/WINDOWS-TEMPLATE-SETUP.md § "WinRM quotas e desktop heap"`.
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with this repository.
|
||||
|
||||
---
|
||||
|
||||
## Project Overview
|
||||
|
||||
Self-hosted CI/CD system running on a Windows 11 host with VMware Workstation. Each build job runs inside an ephemeral VM (linked clone from a frozen template snapshot), communicates via WinRM (Windows guests) or SSH (Linux guests), and is destroyed after the job.
|
||||
|
||||
**Current state**: The Python orchestrator (`src/ci_orchestrator/`) is the production entry point. The PowerShell scripts in `scripts/` are 3-line shims that forward to the Python CLI. All new logic goes in Python.
|
||||
|
||||
**Phase C (PowerShell-free Linux host)**: the manual burn-in / benchmark / smoke / host-validation / credential-setup tooling that used to run on the Linux host via `pwsh` is now native Python: `bench run`, `bench measure`, `smoke run`, `validate host`, `validate guest`, `creds set`. The Linux host no longer needs `pwsh` for any operation — it may be uninstalled (`setup-host-linux.sh` no longer installs it by default; opt in with `--with-pwsh`). PSScriptAnalyzer lint runs on a Windows runner on-demand, not on the Linux runner.
|
||||
|
||||
---
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Python (primary)
|
||||
|
||||
The production venv is at `/opt/ci/venv` (Linux host). To invoke the CLI as the service user:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help
|
||||
```
|
||||
|
||||
```bash
|
||||
# Dev venv setup (one-time)
|
||||
python -m venv .venv
|
||||
.\.venv\Scripts\python.exe -m pip install -e ".[dev]"
|
||||
|
||||
# Run all tests with coverage (gate: ≥90%)
|
||||
python -m pytest tests/python -q --cov=ci_orchestrator --cov-fail-under=90
|
||||
|
||||
# Run a single test file
|
||||
python -m pytest tests/python/test_commands_job.py -q
|
||||
|
||||
# Lint
|
||||
python -m ruff check src tests/python
|
||||
|
||||
# Type-check (strict)
|
||||
python -m mypy --strict src
|
||||
|
||||
# CLI
|
||||
python -m ci_orchestrator --help
|
||||
```
|
||||
|
||||
**Important**: The production venv at `F:\CI\python\venv` must be installed **non-editable** (`pip install .`, not `-e`), because act_runner runs as LocalSystem and cannot resolve editable `.pth` files from a transient RunnerWork path. Re-run `pip install .` into the production venv after every code change. The dev `.venv` may use `-e`.
|
||||
|
||||
### PowerShell linting
|
||||
|
||||
```powershell
|
||||
Invoke-ScriptAnalyzer -Path scripts\ -Recurse -Settings PSScriptAnalyzerSettings.psd1 -Severity Error,Warning
|
||||
# Auto-fix formatting:
|
||||
Invoke-ScriptAnalyzer -Path scripts\ -Recurse -Fix
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
|
||||
### Pipeline flow
|
||||
|
||||
```
|
||||
git push → Gitea Actions → act_runner (Windows service) → python -m ci_orchestrator job
|
||||
→ vm new (linked clone)
|
||||
→ wait-ready (WinRM poll or SSH poll)
|
||||
→ build run (WinRM Invoke-Command or SSH+SFTP)
|
||||
→ artifacts collect (WinRM Copy-Item or SCP)
|
||||
→ vm remove (vmrun stop + deleteVM)
|
||||
```
|
||||
|
||||
The host **never executes build tools directly** — it only orchestrates VM lifecycle.
|
||||
|
||||
### Python package layout (`src/ci_orchestrator/`)
|
||||
|
||||
| Module | Role |
|
||||
|--------|------|
|
||||
| `__main__.py` | click entry point; registers all sub-command groups |
|
||||
| `config.py` | `Config` dataclass + TOML/env loader; resolution: OS defaults → `config.toml` → env vars |
|
||||
| `credentials.py` | `CredentialStore` Protocol + `KeyringCredentialStore` |
|
||||
| `backends/protocol.py` | `VmBackend` Protocol + `VmHandle` + `VmState` (hypervisor-agnostic) |
|
||||
| `backends/workstation.py` | `WorkstationVmrunBackend` — wraps `vmrun.exe` subprocesses |
|
||||
| `backends/__init__.py` | `load_backend(config)` factory — the only import application code should use |
|
||||
| `transport/winrm.py` | WinRM transport via `pypsrp` |
|
||||
| `transport/ssh.py` | SSH/SFTP transport via `paramiko` |
|
||||
| `transport/errors.py` | `TransportConnectError`, `TransportCommandError` |
|
||||
| `commands/job.py` | Full end-to-end orchestrator (auto-detects guest OS from VMX `guestOS` field) |
|
||||
| `commands/vm.py` | `vm new / remove / cleanup` |
|
||||
| `commands/build.py` | `build run` |
|
||||
| `commands/artifacts.py` | `artifacts collect` |
|
||||
| `commands/wait.py` | `wait-ready` |
|
||||
| `commands/monitor.py` | `monitor disk / runner` |
|
||||
| `commands/report.py` | `report job` |
|
||||
|
||||
**Key design rule**: Application code (especially `commands/job.py`) must import `backends.load_backend` only — never `WorkstationVmrunBackend` directly. This keeps the Phase D ESXi extension path open.
|
||||
|
||||
### Transport selection
|
||||
|
||||
`commands/job.py` reads `guestOS = "..."` from the cloned VMX. `ubuntu-*` → Linux/SSH (`transport/ssh.py`); anything else → Windows/WinRM (`transport/winrm.py`).
|
||||
|
||||
### VM templates and snapshots
|
||||
|
||||
| Template | VMX path | Snapshot | Transport |
|
||||
|----------|----------|----------|-----------|
|
||||
| WinBuild2025 | `F:\CI\Templates\WinBuild2025\WinBuild2025.vmx` | `BaseClean` | WinRM HTTPS/5986 |
|
||||
| WinBuild2022 | `F:\CI\Templates\WinBuild2022\WinBuild2022.vmx` | `BaseClean` | WinRM HTTPS/5986 |
|
||||
| LinuxBuild2404 | `F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx` | `BaseClean-Linux` | SSH/22 (`ci_build`, key `F:\CI\keys\ci_linux`) |
|
||||
|
||||
Snapshots **must be taken from powered-off state** — presence of `.vmem` in the template dir means the snapshot captured running memory and `vmrun clone ... linked` will fail.
|
||||
|
||||
### Configuration
|
||||
|
||||
The config file is auto-discovered at `$CI_ROOT/config.toml` (Linux default: `/var/lib/ci/config.toml`; Windows: `F:\CI\config.toml`). Set `$CI_CONFIG` to override. The file at `/var/lib/ci/config.toml` is the live config on the Linux host — keep its `[paths]` section pointing to `/var/lib/ci/...` (not the old Windows `F:/CI/...` paths). Environment variables `CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS` override the file. `CI_PYTHON_LAUNCHER` in `runner/config.yaml` is the single source of truth for the system Python path used to bootstrap the production venv.
|
||||
|
||||
---
|
||||
|
||||
## PowerShell 5.1 Constraints
|
||||
|
||||
These constraints apply to the `.ps1` / `.psm1` scripts that still run **inside Windows guests** and on the optional Windows runner. The **Linux host itself no longer runs any `.ps1`** (Phase C) — do not add PowerShell to the Linux host orchestration path.
|
||||
|
||||
All `.ps1` / `.psm1` scripts run on **Windows PowerShell 5.1** (not Core/7). Every script must start with:
|
||||
|
||||
```powershell
|
||||
#Requires -Version 5.1
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
```
|
||||
|
||||
Forbidden PS 7+ syntax (use the PS 5.1 alternatives):
|
||||
|
||||
| Forbidden | Use instead |
|
||||
|-----------|-------------|
|
||||
| `$x ??= 'val'` | `if ($null -eq $x) { $x = 'val' }` |
|
||||
| `$a ?? $b` | `if ($null -ne $a) { $a } else { $b }` |
|
||||
| `cmd1 && cmd2` | `cmd1; if ($LASTEXITCODE -eq 0) { cmd2 }` |
|
||||
| Ternary `$a ? $b : $c` | `if ($a) { $b } else { $c }` |
|
||||
| `ForEach-Object -Parallel` | `foreach` or `Start-Job` |
|
||||
|
||||
Always check `$LASTEXITCODE` after `vmrun`, `ssh`, `scp` and `throw` if non-zero.
|
||||
|
||||
---
|
||||
|
||||
## Python Conventions
|
||||
|
||||
- `click.echo` everywhere — no bare `print()`.
|
||||
- No `subprocess.run(..., shell=True)`. Use `paramiko` for SSH, `pypsrp` for WinRM, `keyring` for credentials.
|
||||
- No global variables — state lives in `Config` and objects passed explicitly.
|
||||
- All public functions and exported helpers must have type hints. Avoid `Any` except in pytest fixtures/mocks.
|
||||
- `ruff` + `mypy --strict` are enforced in CI. The coverage gate is **90%**.
|
||||
|
||||
---
|
||||
|
||||
## Known Gotchas (from `AGENTS.md`)
|
||||
|
||||
- **`vmrun getGuestIPAddress`** can return exit code 0 with `"Error: The VMware Tools are not running..."` on stdout. Filter lines starting with `"Error:"`.
|
||||
- **Linux machine-id**: Ubuntu cloud images ship with a fixed `/etc/machine-id`. VMware DHCP assigns IPs by machine-id — clones get the same IP and collide. The `BaseClean-Linux` snapshot must be taken after truncating machine-id.
|
||||
- **`& nativecmd 2>$null` under `$ErrorActionPreference='Stop'`**: native stderr is converted to an ErrorRecord before `2>$null` discards it, causing a terminating error. Wrap with `$ErrorActionPreference = 'SilentlyContinue'` or `2>&1 | Out-Null`.
|
||||
- **Snapshot with VM running**: never take a template snapshot while the VM is powered on.
|
||||
- **WinRM vs SSH separation**: do not modify the Windows WinRM branch and the Linux SSH branch in the same commit/PR. Keep `if ($GuestOS -eq 'Linux')` guards clean.
|
||||
|
||||
---
|
||||
|
||||
## Key Files Reference
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `AGENTS.md` | Environment reference + error registry — read before writing any script |
|
||||
| `docs/ARCHITECTURE.md` | Full system diagram and component descriptions |
|
||||
| `docs/CI-FLOW.md` | Step-by-step pipeline description |
|
||||
| `docs/RUNBOOK.md` | Common incident triage |
|
||||
| `TODO.md` | Roadmap, audit trail, backlog |
|
||||
| `plans/` | Phase closeout docs (A1–A5, B5) and implementation plans |
|
||||
| `config.example.toml` | Annotated config template |
|
||||
| `tests/python/test_agents_errors.py` | Regression tests for `AGENTS.md` errors #9–#12 — do not remove |
|
||||
@@ -299,6 +299,19 @@ Cooldown: max 3 restart/h via JSON state file `F:\CI\State\runner-restart-log.js
|
||||
Oltre il limite: EventId 1004 (Error) + webhook `:sos:` senza restart. Webhook opzionale
|
||||
Discord/Gitea identico a `Watch-DiskSpace.ps1`.
|
||||
|
||||
### 2.8 [P2] [ ] IP-pool auto-riconciliazione vs clone reali — APERTO (2026-06-07)
|
||||
|
||||
`ip-pool.json` non viene riconciliato contro i clone vivi all'avvio. Un job
|
||||
killato mid-flight (SIGKILL/crash) **leaka il lease in modo permanente**; con
|
||||
soli 4 IP (`192.168.79.201–204`) per parallelism 4, un singolo leak esaurisce
|
||||
il pool e ogni round successivo fallisce con `IP pool exhausted after 60s`.
|
||||
Emerso durante il burn-in B7 (RUNBOOK §9) — pool inquinato da lease del 25 mag
|
||||
mai rilasciati. Workaround manuale: reset a `{}` con `build-vms/` vuoto.
|
||||
|
||||
**Fix proposto**: all'allocazione (o in `vm cleanup`), rilasciare i lease il
|
||||
cui clone-dir non esiste più in `CloneBaseDir`. Opzionale: allargare il pool
|
||||
(es. 201–210) per dare margine oltre parallelism. Aggiungere test.
|
||||
|
||||
---
|
||||
|
||||
## 3. Performance & throughput
|
||||
@@ -820,6 +833,35 @@ da workflow. Attualmente fisso a `numvcpus=4, memsize=6144` — subottimo per bu
|
||||
**Rifiori quando**: Performance profiling mostra contentione CPU/RAM è bottleneck per la
|
||||
workload target. Per home lab, 4 vCPU/6 GB è ragionevole compromesso.
|
||||
|
||||
### 3.6 [P2] [ ] Stabilità WinRM sotto concorrenza 4× su host Windows
|
||||
File: [src/ci_orchestrator/transport/winrm.py](src/ci_orchestrator/transport/winrm.py),
|
||||
[docs/RUNBOOK.md §13](docs/RUNBOOK.md).
|
||||
|
||||
Burn-in §13 (2026-06-07) su host Windows: guest **Windows 36/40** (round 6–8
|
||||
hanno perso 1–2 job per fault WinRM transitori — `ConnectTimeoutError` su 5986 e
|
||||
`WSManFault 2150858843` "shell not found" — con round inflazionati a 269/266/622 s).
|
||||
Guest **Linux 40/40** (SSH stabile). Stesso hardware: host Linux §9 = Win 40/40.
|
||||
Auto-recupero OK (slot IP rilasciati, 0 orfani), ma OVERALL FAIL.
|
||||
|
||||
Causa probabile: oversubscription vCPU (4 VM × 4 vCPU = 16) + overhead host-OS
|
||||
Windows che manda in timeout WinRM concorrente (RAM non era il vincolo: 13.8 GB
|
||||
liberi su 63.7). **Mitigazioni**: ridurre vCPU/VM (lega a §3.5), `-Parallelism 3`,
|
||||
alzare connect-timeout pypsrp + retry a livello job in `_windows_build`.
|
||||
**Rifiori quando**: si vuole concorrenza 4× Windows-guest affidabile su host Windows.
|
||||
|
||||
### 3.7 [P3] [x] Shim Invoke-CIJob.ps1 — `$IsWindows` non guardato (PS 5.1 + StrictMode) — RISOLTO 2026-06-07 (Fase C)
|
||||
File: [scripts/Invoke-CIJob.ps1:37](scripts/Invoke-CIJob.ps1).
|
||||
|
||||
`if ($null -ne $IsWindows -and $IsWindows -eq $false)` dereferenzia `$IsWindows`,
|
||||
inesistente su Windows PowerShell 5.1 → con `Set-StrictMode -Version Latest`
|
||||
lancia `VariableIsUndefined` e il job fallisce subito. Si manifestava solo quando lo
|
||||
shim girava sotto PS 5.1 **senza** `CI_VENV_PYTHON` settato (il ramo auto-detect),
|
||||
cioè nelle invocazioni manuali/burn-in.
|
||||
**Risoluzione**: il percorso burn-in manuale non passa più dallo shim — `bench run`
|
||||
(Fase C) lancia i job concorrenti in-process via `concurrent.futures`, senza
|
||||
`Start-Job`/`pwsh`/`$IsWindows`. Lo shim resta solo per il runner Windows, dove
|
||||
`CI_VENV_PYTHON` è sempre settato. Il bug non è più raggiungibile sull'host Linux.
|
||||
|
||||
### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com)
|
||||
File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`).
|
||||
|
||||
|
||||
@@ -32,3 +32,24 @@ keys = "F:/CI/keys"
|
||||
# Phase C hook: backend selector. Only "workstation" is implemented in Phase A.
|
||||
[backend]
|
||||
type = "workstation"
|
||||
|
||||
# Optional: pre-assign static IPs to build VMs via VMX guestinfo injection.
|
||||
# Eliminates VMware-Tools IP-detect polling (20–180 s variance) — the host
|
||||
# injects guestinfo.ip-assignment into each cloned VMX before vmrun start,
|
||||
# and the guest boot script applies the static IP immediately.
|
||||
#
|
||||
# Prerequisite: install the guest-side startup script from
|
||||
# template/guest-setup/linux/ci-report-ip.sh (Linux guests)
|
||||
# template/guest-setup/windows/ci-static-ip.ps1 (Windows guests)
|
||||
# and re-take the BaseClean / BaseClean-Linux snapshot.
|
||||
#
|
||||
# Pool size should equal runner capacity (default 4).
|
||||
# Addresses must be outside the VMnet8 DHCP range (check
|
||||
# /etc/vmware/vmnet8/dhcpd/dhcpd.conf — typically .128–.254).
|
||||
#
|
||||
# [ip_pool]
|
||||
# addresses = ["192.168.79.201", "192.168.79.202",
|
||||
# "192.168.79.203", "192.168.79.204"]
|
||||
# netmask = "255.255.255.0"
|
||||
# gateway = "192.168.79.2" # VMnet8 gateway; leave empty to skip default route
|
||||
# lease_file = "/var/lib/ci/ip-pool.json"
|
||||
|
||||
@@ -10,33 +10,10 @@ installate su un host Linux Mint / Ubuntu LTS, con il package
|
||||
| Windows Task Scheduler | systemd unit | Cadenza | Comando |
|
||||
| ----------------------- | ---------------------------------- | ----------------------- | --------------------------------------------------------------- |
|
||||
| `CI-CleanupOrphans` | `ci-cleanup-orphans.timer` | every 6h + at boot | `python -m ci_orchestrator vm cleanup --max-age-hours 6` |
|
||||
| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `pwsh Invoke-RetentionPolicy.ps1` (vedi nota PowerShell) |
|
||||
| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `python -m ci_orchestrator retention run` |
|
||||
| `CI-DiskSpaceAlert` | `ci-watch-disk-space.timer` | every 15 min | `python -m ci_orchestrator monitor disk` |
|
||||
| `CI-RunnerHealth` | `ci-watch-runner-health.timer` | every 15 min | `python -m ci_orchestrator monitor runner` |
|
||||
| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `pwsh Backup-CITemplate.ps1` (vedi nota PowerShell) |
|
||||
|
||||
## Nota PowerShell su Linux
|
||||
|
||||
Due unit (`ci-retention-policy`, `ci-backup-template`) invocano script
|
||||
PowerShell che, per scelta documentata in `AGENTS.md` ("Mappatura script
|
||||
PowerShell → Python"), restano in PowerShell. Su Linux questo richiede
|
||||
**PowerShell Core (`pwsh`)**:
|
||||
|
||||
```bash
|
||||
# Ubuntu / Mint LTS
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y wget apt-transport-https software-properties-common
|
||||
wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb
|
||||
sudo dpkg -i packages-microsoft-prod.deb
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y powershell
|
||||
pwsh --version # verifica
|
||||
```
|
||||
|
||||
Se preferisci non installare `pwsh`, porta `Invoke-RetentionPolicy.ps1` e
|
||||
`Backup-CITemplate.ps1` come sub-comandi `python -m ci_orchestrator
|
||||
retention run` / `template backup` (out of scope B5 per il piano corrente
|
||||
— vedi `plans/implementation-plan-A-B.md`).
|
||||
| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `python -m ci_orchestrator template backup --all-templates` |
|
||||
|
||||
## Prerequisiti
|
||||
|
||||
@@ -55,8 +32,12 @@ retention run` / `template backup` (out of scope B5 per il piano corrente
|
||||
CI_KEYS=/etc/ci/keys
|
||||
PYTHONIOENCODING=utf-8
|
||||
```
|
||||
4. `Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` presenti in
|
||||
`/opt/ci/local-ci-cd-system/scripts/` (clone del repo).
|
||||
4. `ci_orchestrator` installato nel venv (punto 2) — nessuna dipendenza PowerShell
|
||||
richiesta; `retention run` e `template backup` sono comandi Python puri.
|
||||
|
||||
> **Nota**: `ci-backup-template.service` gira come `root` (non `ci-runner`)
|
||||
> perché chiama `systemctl stop/start act-runner.service` prima e dopo il
|
||||
> backup. Tutti gli altri service girano come `ci-runner`.
|
||||
|
||||
## Installazione
|
||||
|
||||
|
||||
@@ -5,13 +5,15 @@ After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=ci-runner
|
||||
Group=ci-runner
|
||||
User=root
|
||||
EnvironmentFile=/etc/ci/environment
|
||||
WorkingDirectory=/opt/ci/local-ci-cd-system
|
||||
# NOTE: requires PowerShell Core (`pwsh`) on the Linux host.
|
||||
# See deploy/systemd/README.md "Nota PowerShell su Linux".
|
||||
ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Backup-CITemplate.ps1
|
||||
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator template backup --all-templates
|
||||
# Safety net: bring act-runner.service back up on any exit path (success,
|
||||
# failure, signal). The Python `finally` block can be skipped if the
|
||||
# service is killed mid-run, and a runner left stopped by a killed
|
||||
# backup would cause the next backup to skip the restart too.
|
||||
ExecStopPost=-/bin/systemctl start act-runner.service
|
||||
TimeoutStartSec=4h
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
@@ -20,7 +22,7 @@ ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
NoNewPrivileges=true
|
||||
ReadWritePaths=/var/lib/ci /var/backups/ci
|
||||
ReadWritePaths=/var/lib/ci
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
@@ -9,9 +9,7 @@ User=ci-runner
|
||||
Group=ci-runner
|
||||
EnvironmentFile=/etc/ci/environment
|
||||
WorkingDirectory=/opt/ci/local-ci-cd-system
|
||||
# NOTE: requires PowerShell Core (`pwsh`) on the Linux host.
|
||||
# See deploy/systemd/README.md "Nota PowerShell su Linux".
|
||||
ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Invoke-RetentionPolicy.ps1
|
||||
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator retention run
|
||||
TimeoutStartSec=1h
|
||||
Nice=15
|
||||
ProtectSystem=strict
|
||||
|
||||
@@ -10,7 +10,7 @@ Group=ci-runner
|
||||
EnvironmentFile=/etc/ci/environment
|
||||
# `monitor runner` enforces internal rate-limit: max 3 restarts/h
|
||||
# (matches Windows CI-RunnerHealth scheduled task semantics).
|
||||
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner
|
||||
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner --service-name act-runner
|
||||
TimeoutStartSec=5min
|
||||
Nice=15
|
||||
ProtectSystem=strict
|
||||
|
||||
@@ -5,6 +5,30 @@ Open backlog items remain in [TODO.md](../TODO.md).
|
||||
|
||||
---
|
||||
|
||||
## 2026-06-06 — Linux template: GUI toolchain, headless builds, deploy hardening
|
||||
|
||||
- **Toolchain** (`Install-CIToolchain-Linux2404.sh`): new Step 4a installs
|
||||
`python3-gi gir1.2-gtk-4.0 xvfb` (headless PyGObject/GTK4 GUI tests). Step 1
|
||||
now always runs `apt-get update`; `--skip-update` only skips the full upgrade
|
||||
(previously it skipped the index refresh too, so newly-added packages could
|
||||
not be located).
|
||||
- **`build run` / `job`**: new `-Xvfb` / `--xvfb` switch (Linux only) runs the
|
||||
build under `xvfb-run` so GUI toolkits get a `$DISPLAY`. Ignored for Windows.
|
||||
See [WORKFLOW-AUTHORING.md](WORKFLOW-AUTHORING.md) §5/§8.2.
|
||||
- **`deploy-linux --force`**: now clears stale snapshot-chain artifacts
|
||||
(`-NNNNNN.vmdk`, `.vmsn`, `.vmsd`, `.lck`) before the Step 4 resize, fixing
|
||||
*"disk is part of a snapshot chain"*.
|
||||
- **`prepare-linux`**: Step 7d seals cloud-init (drops the stale
|
||||
`50-cloud-init.yaml`, keeps the MAC-agnostic `99-ci-dhcp-allnics`, disables
|
||||
cloud-init) so it never re-runs on linked clones; deploy user-data sets
|
||||
`package_update/upgrade: false` and the Step 7 IP-detect timeout is raised to
|
||||
600 s.
|
||||
- **Postmortem** [postmortem-2026-06-06-linux-template-no-ip.md](postmortem-2026-06-06-linux-template-no-ip.md):
|
||||
intermittent deploy/prepare "no IP" traced to a wedged host `vmnet-dhcpd`;
|
||||
recovery is `vmware-networks --stop/--start`.
|
||||
|
||||
---
|
||||
|
||||
## 2026-05-13 — Sprint 16: Medium/Low hardening (final-master-plan)
|
||||
|
||||
- **Watch-DiskSpace.ps1**: Event Log source renamed `CI-DiskAlert` → `CI-DiskSpaceAlert`
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# CI/CD Job Lifecycle
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A([git push]) --> B[Gitea Actions\ntrigger workflow]
|
||||
B --> C[act_runner\nWindows service]
|
||||
C --> D["python -m ci_orchestrator job\njob.py · job()"]
|
||||
|
||||
D --> SRC{use-git-clone?}
|
||||
|
||||
SRC -->|false — default| HOST_CLONE["host: git clone\n+ Compress-Archive\n+ WinRM copy zip\n+ Expand-Archive in VM"]
|
||||
SRC -->|true — opt-in| VM_CLONE["skip host clone\nVM: git clone --recurse-submodules\nhttps://PAT@gitea/repo\n~25.7% faster"]
|
||||
|
||||
HOST_CLONE --> E
|
||||
VM_CLONE --> E
|
||||
|
||||
D --> LB
|
||||
LB --> E
|
||||
|
||||
E["vm_new()\nlinked clone from template snapshot"]
|
||||
E --> F{guestOS in VMX?}
|
||||
|
||||
F -->|ubuntu-*| G_WAIT["wait_ready()\nSSH poll · port 22"]
|
||||
F -->|Windows| W_WAIT["wait_ready()\nWinRM poll · port 5986"]
|
||||
|
||||
G_WAIT --> G_PROBE["_probe_transport()\nSshTransport"]
|
||||
W_WAIT --> W_PROBE["_probe_transport()\nWinRmTransport"]
|
||||
|
||||
G_PROBE --> G_BUILD["_linux_build()\nSSH + SFTP · paramiko"]
|
||||
W_PROBE --> W_BUILD["_windows_build()\nWinRM · pypsrp"]
|
||||
|
||||
G_BUILD --> G_ART["_linux_collect()\nSCP artifacts"]
|
||||
W_BUILD --> W_ART["_windows_collect()\nWinRM Copy-Item"]
|
||||
|
||||
G_ART --> DESTROY
|
||||
W_ART --> DESTROY
|
||||
|
||||
DESTROY["_destroy_clone()\nvmrun stop + deleteVM"]
|
||||
DESTROY --> DONE([Job complete])
|
||||
|
||||
subgraph Templates
|
||||
T1["WinBuild2025\nBaseClean · WinRM"]
|
||||
T2["WinBuild2022\nBaseClean · WinRM"]
|
||||
T3["LinuxBuild2404\nBaseClean-Linux · SSH"]
|
||||
end
|
||||
|
||||
E -.->|clone from| T1
|
||||
E -.->|clone from| T2
|
||||
E -.->|clone from| T3
|
||||
|
||||
subgraph Backend
|
||||
LB["load_backend()\nWorkstationVmrunBackend\nwraps vmrun.exe"]
|
||||
end
|
||||
```
|
||||
|
||||
## Phase breakdown
|
||||
|
||||
| Phase | Module | Transport |
|
||||
|-------|--------|-----------|
|
||||
| Clone VM | `commands/vm.py · vm_new()` | `vmrun.exe` (host) |
|
||||
| Source delivery | workflow input `use-git-clone` | WinRM (in-VM git clone) or host zip |
|
||||
| Wait ready | `commands/wait.py · wait_ready()` | SSH/22 or WinRM/5986 |
|
||||
| Build | `commands/build.py · build_run()` | `paramiko` (Linux) / `pypsrp` (Windows) |
|
||||
| Collect artifacts | `commands/artifacts.py · artifacts_collect()` | SCP (Linux) / WinRM Copy-Item (Windows) |
|
||||
| Destroy VM | `commands/job.py · _destroy_clone()` | `vmrun.exe` (host) |
|
||||
|
||||
## Transport selection
|
||||
|
||||
`job.py` reads `guestOS` from the cloned `.vmx`:
|
||||
- `ubuntu-*` → `SshTransport` (paramiko, key auth, `ci_build` user)
|
||||
- anything else → `WinRmTransport` (pypsrp, HTTPS/5986)
|
||||
|
||||
## Source delivery modes
|
||||
|
||||
| Mode | Trigger | Flow | Notes |
|
||||
|------|---------|------|-------|
|
||||
| Standard | `use-git-clone: false` (default) | host clone → zip → WinRM copy → expand | No credentials in VM |
|
||||
| UseGitClone | `use-git-clone: true` | VM clones directly via PAT | 25.7% faster, PAT never persisted |
|
||||
@@ -229,11 +229,13 @@ Eseguito da Prepare via SSH come `ci_build` con sudo.
|
||||
Header: `#!/usr/bin/env bash` + `set -euo pipefail`
|
||||
Helper `assert_step()`: stampa `[OK] [Step N] desc` su successo, `exit 1` su fallimento.
|
||||
|
||||
- [x] **Step 1 apt update + upgrade** (saltabile con `--skip-update`)
|
||||
- [x] **Step 1 apt update (+ upgrade)**
|
||||
```bash
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq # sempre
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq # saltato da --skip-update
|
||||
```
|
||||
> `apt-get update` gira **sempre** (l'indice deve essere fresco per trovare i
|
||||
> pacchetti). `--skip-update` salta solo l'`upgrade` (lento).
|
||||
|
||||
- [x] **Step 2 Toolchain build essentials**
|
||||
```bash
|
||||
@@ -257,6 +259,15 @@ Helper `assert_step()`: stampa `[OK] [Step N] desc` su successo, `exit 1` su fal
|
||||
```
|
||||
- assert: `command -v git`, `command -v 7z`
|
||||
|
||||
- [x] **Step 4a GTK4 / GObject + Xvfb** (test GUI headless, es. PyGObject)
|
||||
```bash
|
||||
sudo apt-get install -y -qq python3-gi gir1.2-gtk-4.0 xvfb
|
||||
```
|
||||
- assert: `python3 -c "import gi"`, `command -v Xvfb`
|
||||
- Usato a runtime dai build con il flag `--xvfb` / `-Xvfb` (vedi
|
||||
[WORKFLOW-AUTHORING.md](WORKFLOW-AUTHORING.md) §5): il comando di build gira
|
||||
sotto `xvfb-run` con un `$DISPLAY` virtuale.
|
||||
|
||||
- [x] **Step 4b .NET SDK** (opzionale, flag `--dotnet`)
|
||||
```bash
|
||||
curl -fsSL https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh \
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
# RUNBOOK — manual CLI reference (Linux host, pwsh-free)
|
||||
|
||||
Simplified but **complete** reference for every `ci_orchestrator` command you can
|
||||
run by hand on the Linux host. All commands run as the **`ci-runner`** service
|
||||
user against the live config (`/var/lib/ci/config.toml`).
|
||||
|
||||
Two notations are used:
|
||||
|
||||
- **Full form:** `sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator <cmd>`
|
||||
- **Short form (alias):** `ci <cmd>` — after installing the helper (see §0).
|
||||
|
||||
Examples below use the **short form**. Drop `ci` for the literal command.
|
||||
|
||||
---
|
||||
|
||||
## 0. Setup (do once)
|
||||
|
||||
### 0.1 Refresh the production venv (mandatory after a code change/merge)
|
||||
|
||||
```bash
|
||||
cd /opt/ci/local-ci-cd-system
|
||||
sudo /opt/ci/venv/bin/python -m pip install . # non-editable, see CLAUDE.md
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help | grep -E 'bench|smoke|validate|creds'
|
||||
```
|
||||
|
||||
### 0.2 Install the `ci` alias
|
||||
|
||||
```bash
|
||||
./scripts/install-ci-alias.sh # into ~/.bashrc + ~/.zshrc (idempotent)
|
||||
# --system install /etc/profile.d/ci-alias.sh for all users (needs sudo)
|
||||
# --uninstall remove it
|
||||
source ~/.bashrc # or open a new shell
|
||||
ci validate host # smoke test the alias
|
||||
```
|
||||
|
||||
The script defines two helpers:
|
||||
|
||||
| Helper | Runs as | Use for |
|
||||
|---------|---------------|---------|
|
||||
| `ci` | `ci-runner` (sudo) | everything (writes keyring, starts VMs) |
|
||||
| `ci-me` | current user | read-only checks where sudo is noise |
|
||||
|
||||
### 0.3 Live environment
|
||||
|
||||
| Thing | Value |
|
||||
|-------|-------|
|
||||
| Service user | `ci-runner` · Prod venv `/opt/ci/venv/bin/python` |
|
||||
| Config | `/var/lib/ci/config.toml` |
|
||||
| Linux template | `/var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx` (snap `BaseClean-Linux`) |
|
||||
| Windows templates | `/var/lib/ci/templates/WinBuild2025` · `WinBuild2022` (snap `BaseClean`) |
|
||||
| Clones / Artifacts | `/var/lib/ci/build-vms` · `/var/lib/ci/artifacts` |
|
||||
| Linux SSH key | `/var/lib/ci/keys/ci_linux` · cred target `BuildVMGuest` |
|
||||
|
||||
---
|
||||
|
||||
## 1. Quick reference — every command
|
||||
|
||||
| Command | Does | Section |
|
||||
|---------|------|---------|
|
||||
| `ci validate host` | host prerequisites check (no VM) | §2 |
|
||||
| `ci validate guest --host IP` | probe a running guest's transport | §2 |
|
||||
| `ci monitor disk` | free-space alert | §2 |
|
||||
| `ci monitor runner` | watch + auto-restart act_runner | §2 |
|
||||
| `ci report job` | job summaries from JSONL logs | §2 |
|
||||
| `ci job ...` | **full pipeline** clone→build→collect→destroy | §3 |
|
||||
| `ci vm new ...` | linked-clone a template | §4 |
|
||||
| `ci wait-ready --vmx ...` | poll until SSH/WinRM ready | §4 |
|
||||
| `ci build run ...` | run a build in a running guest | §4 |
|
||||
| `ci artifacts collect ...` | copy artifacts out of a guest | §4 |
|
||||
| `ci vm remove --vmx ...` | stop + delete one clone | §4 |
|
||||
| `ci vm cleanup` | sweep orphaned clones + stale locks | §4 / §6 |
|
||||
| `ci vmgui --vmx ...` | open a VMX in the VMware Workstation GUI | §4 |
|
||||
| `ci creds set --user ...` | store guest credentials in keyring | §5 |
|
||||
| `ci template deploy-linux` | build the Linux template VM | §5 |
|
||||
| `ci template prepare-linux` | provision Linux template over SSH | §5 |
|
||||
| `ci template prepare-win` | provision Windows template over WinRM | §5 |
|
||||
| `ci template backup` | timestamped copy of template dirs | §5 |
|
||||
| `ci retention run` | purge old artifacts/logs | §6 |
|
||||
| `ci bench measure ...` | phase-timing benchmark | §7 |
|
||||
| `ci bench run ...` | concurrency burn-in | §7 |
|
||||
| `ci smoke run ...` | one E2E job + assertions | §7 |
|
||||
|
||||
---
|
||||
|
||||
## 2. Health, monitoring & reporting (safe, no VM boot)
|
||||
|
||||
### `validate host` — run this first, every day
|
||||
```bash
|
||||
ci validate host ; echo "exit=$?"
|
||||
```
|
||||
Checks vmrun, template snapshots, keyring, `/var/lib/ci` permissions, `act-runner`
|
||||
unit. Exit `0` + all `[OK ]`; a `[FAIL]` line names the broken check.
|
||||
|
||||
### `validate guest` — diagnose a running guest's transport
|
||||
```bash
|
||||
ci validate guest --host 192.168.79.200 --ssh --guest-os linux # SSH
|
||||
ci validate guest --host 192.168.79.201 --winrm --guest-os windows # WinRM
|
||||
```
|
||||
Connects + runs a trivial command, printing the explicit cause on failure
|
||||
(connect vs auth vs command). The guest must already be booted.
|
||||
|
||||
### `monitor disk` — free-space alert
|
||||
```bash
|
||||
ci monitor disk --min-free-gb 50
|
||||
ci monitor disk --min-free-gb 50 --json # machine-readable one-liner
|
||||
ci monitor disk --min-free-gb 50 --webhook-url "$DISCORD_WEBHOOK"
|
||||
```
|
||||
|
||||
### `monitor runner` — keep act_runner alive
|
||||
```bash
|
||||
ci monitor runner --service-name act-runner --max-restarts 3
|
||||
```
|
||||
Restarts the runner up to N times per window; optional `--webhook-url`,
|
||||
`--gitea-url`, `--gitea-credential-target`.
|
||||
|
||||
### `report job` — read job history
|
||||
```bash
|
||||
ci report job --last 10 # 10 most recent, table
|
||||
ci report job --failed # only failures
|
||||
ci report job --job-id smoke-20260607-2030 # phase breakdown for one job
|
||||
ci report job --last 20 --json # JSON out
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Full pipeline (`job`) — the normal way to run a build
|
||||
|
||||
One command does everything: clone → wait-ready → build → collect → destroy.
|
||||
This is what the runner invokes per push; you can run it by hand too.
|
||||
|
||||
```bash
|
||||
# Linux build, in-guest git clone, no artifact packaging
|
||||
ci job \
|
||||
--job-id manual-$(date +%s) \
|
||||
--repo-url https://gitea.emulab.it/Simone/myrepo.git \
|
||||
--branch main \
|
||||
--guest-os linux \
|
||||
--build-command "make all" \
|
||||
--use-git-clone
|
||||
```
|
||||
|
||||
Key flags: `--commit`, `--configuration Release`, `--guest-artifact-source dist`,
|
||||
`--submodules/--no-submodules`, `--use-git-clone/--host-clone`, `--use-shared-cache`,
|
||||
`--skip-artifact`, `--xvfb` (Linux GUI builds), `--guest-cpu N`, `--guest-memory-mb N`,
|
||||
`--template-path`, `--snapshot-name`, `--ready-timeout`, `--extra-env-json '{"K":"V"}'`,
|
||||
`--gitea-credential-target` (private repos). Required: `--job-id --repo-url --branch`.
|
||||
|
||||
> For a zero-config end-to-end check, prefer `smoke run` (§7) over hand-rolling `job`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Manual VM lifecycle (the steps `job` runs internally)
|
||||
|
||||
Use these to drive a build stage-by-stage, e.g. when debugging a single phase.
|
||||
|
||||
### 4.1 Clone a VM
|
||||
```bash
|
||||
ci vm new \
|
||||
--template /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx \
|
||||
--snapshot BaseClean-Linux \
|
||||
--clone-base-dir /var/lib/ci/build-vms \
|
||||
--job-id dbg-1 \
|
||||
--guest-os linux \
|
||||
--start # prints the clone VMX path on stdout
|
||||
```
|
||||
|
||||
### 4.2 Wait until reachable
|
||||
```bash
|
||||
ci wait-ready \
|
||||
--vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx \
|
||||
--guest-os linux \
|
||||
--ssh-user ci_build \
|
||||
--ssh-key-path /var/lib/ci/keys/ci_linux \
|
||||
--timeout 180
|
||||
```
|
||||
Prints the guest IP when ready. `--ip-address` skips polling; `--credential-target`
|
||||
for WinRM guests.
|
||||
|
||||
### 4.3 Run the build (guest already up)
|
||||
```bash
|
||||
ci build run \
|
||||
--ip-address 192.168.79.200 \
|
||||
--guest-os linux \
|
||||
--ssh-user ci_build --ssh-key-path /var/lib/ci/keys/ci_linux \
|
||||
--clone-url https://gitea.emulab.it/Simone/myrepo.git --clone-branch main \
|
||||
--build-command "make all" \
|
||||
--guest-artifact-source dist
|
||||
```
|
||||
Windows guests use `--credential-target BuildVMGuest` instead of the SSH flags.
|
||||
Repeatable `--extra-env KEY=VALUE`; `--xvfb` Linux GUI; `--use-shared-cache`.
|
||||
|
||||
### 4.4 Collect artifacts
|
||||
```bash
|
||||
ci artifacts collect \
|
||||
--ip-address 192.168.79.200 \
|
||||
--guest-os linux \
|
||||
--ssh-user ci_build --ssh-key-path /var/lib/ci/keys/ci_linux \
|
||||
--guest-artifact-path dist \
|
||||
--host-artifact-dir /var/lib/ci/artifacts/dbg-1 \
|
||||
--include-logs --job-id dbg-1
|
||||
```
|
||||
|
||||
### 4.5 Destroy the clone
|
||||
```bash
|
||||
ci vm remove --vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx
|
||||
ci vm remove --vmx <path> --force # skip soft stop
|
||||
```
|
||||
|
||||
### 4.6 Sweep orphans (also a maintenance task, §6)
|
||||
```bash
|
||||
ci vm cleanup --what-if # list only
|
||||
ci vm cleanup --max-age-hours 4 --lock-file /var/lib/ci/vm-start.lock
|
||||
```
|
||||
|
||||
### 4.7 Open a VM in the Workstation GUI (interactive debugging)
|
||||
Launches the **interactive** VMware Workstation GUI (not the headless vmrun path)
|
||||
so you can watch/poke a clone by hand.
|
||||
|
||||
**Required once per desktop session** — authorise `ci-runner` on your X server
|
||||
(run as YOUR logged-in user, not ci-runner):
|
||||
```bash
|
||||
xhost +SI:localuser:ci-runner
|
||||
```
|
||||
Then open the GUI as ci-runner, forcing the display:
|
||||
```bash
|
||||
ci vmgui --vmx /var/lib/ci/build-vms/Clone_dbg-1_*/Clone_dbg-1_*.vmx --display :0
|
||||
ci vmgui --vmx <path> --display :0 --power-on # also power it on (-x)
|
||||
ci vmgui --vmx <path> --display :0 --fullscreen # power on + fullscreen (-X)
|
||||
ci vmgui --display :0 # bare GUI, no VM loaded
|
||||
```
|
||||
**If nothing opens**: you skipped the `xhost` step. `vmgui` now pre-flights the
|
||||
display (`xdpyinfo`) and fails with the exact `xhost` command instead of dying
|
||||
silently; it also drops an unreadable inherited `XAUTHORITY` and logs the GUI's
|
||||
output to `/tmp/vmgui-*.log`. The GUI needs an X display — `sudo -u ci-runner`
|
||||
loses your cookie, so `xhost` + `--display :0` is the reliable combo. Other
|
||||
flags: `--new-window` (`-n`), `--vmware-path`. Spawns detached, prints PID + log.
|
||||
|
||||
---
|
||||
|
||||
## 5. Credentials & template provisioning
|
||||
|
||||
### `creds set` — store the guest login (replaces Set-CIGuestCredential.ps1)
|
||||
```bash
|
||||
ci creds set --target BuildVMGuest --user ci_build # hidden prompt
|
||||
printf '%s' 'SuperSecret123' | ci creds set --target BuildVMGuest --user ci_build --password-stdin
|
||||
```
|
||||
Password is never on the command line. Verify via `validate host` keyring check.
|
||||
(Leading space before `printf` keeps the secret out of shell history.)
|
||||
|
||||
### Template management
|
||||
```bash
|
||||
# Build the Linux template VM from scratch (Ubuntu 24.04)
|
||||
ci template deploy-linux
|
||||
|
||||
# Provision an existing Linux template over SSH (installs toolchain, sets snapshot prep)
|
||||
ci template prepare-linux
|
||||
|
||||
# Provision a Windows template over WinRM (Linux-host driven)
|
||||
ci template prepare-win
|
||||
|
||||
# Timestamped backup of the template directories
|
||||
ci template backup
|
||||
```
|
||||
Run `ci template <sub> --help` for the (many) per-template options.
|
||||
|
||||
---
|
||||
|
||||
## 6. Maintenance
|
||||
|
||||
### `retention run` — purge old artifacts & logs
|
||||
```bash
|
||||
ci retention run --what-if # dry run first
|
||||
ci retention run --retention-days 30 --aggressive-retention-days 7 --min-free-gb 50
|
||||
```
|
||||
Switches to aggressive retention automatically when free space drops below
|
||||
`--min-free-gb`.
|
||||
|
||||
### Orphan sweep — see §4.6 (`vm cleanup`).
|
||||
|
||||
---
|
||||
|
||||
## 7. Phase C testing (burn-in / benchmark / smoke)
|
||||
|
||||
### `smoke run` — one E2E job + assertions (replaces Test-Smoke.ps1)
|
||||
```bash
|
||||
ci smoke run --guest-os linux # no-op marker job (fastest)
|
||||
ci smoke run --guest-os windows
|
||||
ci smoke run --guest-os linux --preset ns7zip # real Linux build E2E
|
||||
```
|
||||
Asserts: exit 0 + artifact dir under `/var/lib/ci/artifacts/<job-id>/` +
|
||||
`job/success` event in `invoke-ci.jsonl`.
|
||||
|
||||
### `bench measure` — phase timings (replaces Measure-CIBenchmark.ps1)
|
||||
```bash
|
||||
ci bench measure --guest-os linux --iterations 1
|
||||
ci bench measure --guest-os linux --iterations 4 # match historical baselines
|
||||
sudo -u ci-runner tail -n1 /var/lib/ci/artifacts/benchmark.jsonl
|
||||
```
|
||||
Times clone/start/IP/transport/destroy; appends to `benchmark.jsonl`
|
||||
(legacy field names preserved for trend continuity).
|
||||
|
||||
### `bench run` — concurrency burn-in (replaces Test-CapacityBurnIn.ps1)
|
||||
```bash
|
||||
ci bench run --guest-os linux --concurrency 2 --rounds 2 # quick confidence
|
||||
ci bench run --guest-os linux --concurrency 4 --rounds 10 # standard gate
|
||||
ci bench run --guest-os windows --concurrency 3 --rounds 10 # WinRM unstable at 4x (TODO §3.6)
|
||||
sudo -u ci-runner cat "$(ls -t /var/lib/ci/artifacts/bench/*.json | head -1)"
|
||||
```
|
||||
Per round asserts: all jobs exit 0, no orphan clone dir, no stale `vm-start.lock`.
|
||||
JSON report under `/var/lib/ci/artifacts/bench/<ts>.json`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Cutover checklist & uninstalling pwsh
|
||||
|
||||
Repeat for ≥1 week with pwsh **still installed** (A/B safety net):
|
||||
|
||||
- [ ] Normal `git push` CI pipeline green.
|
||||
- [ ] `ci validate host` green daily.
|
||||
- [ ] `ci bench run --guest-os linux --concurrency 4 --rounds 10` green (no orphans/locks).
|
||||
- [ ] `ci smoke run` green on `--guest-os linux` **and** `--guest-os windows`.
|
||||
- [ ] `ci bench measure` timings comparable to historical baselines.
|
||||
- [ ] No procedure forced a fallback to a `.ps1` on the Linux host.
|
||||
|
||||
Then — only after a fully green week:
|
||||
|
||||
```bash
|
||||
pwsh --version
|
||||
sudo apt-get remove powershell
|
||||
ci validate host # must still exit 0 without pwsh
|
||||
```
|
||||
Rollback: `sudo ./setup-host-linux.sh --with-pwsh` reinstalls pwsh; the original
|
||||
`.ps1` scripts remain in `scripts/`.
|
||||
|
||||
---
|
||||
|
||||
## 9. Troubleshooting quick map
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---------|-------|-----|
|
||||
| `No such command 'bench'` | prod venv not refreshed | §0.1 `pip install .` |
|
||||
| `validate host` keyring `[FAIL]` | credential missing/wrong user | §5 `creds set` |
|
||||
| `validate guest` connect failure | VM not booted / wrong IP / firewall | confirm VM running + IP |
|
||||
| `bench run` round FAIL: orphan | clone not destroyed | `ci vm cleanup`; check vmrun + `/var/lib/ci/build-vms` |
|
||||
| WinRM jobs drop at concurrency 4 | known instability (TODO §3.6) | use `--concurrency 3` |
|
||||
| IP acquire timeout (Linux) | vmnet8 DHCP wedged | `sudo systemctl restart vmware-networks` |
|
||||
| `vmrun` "operation was canceled" | vmmon/vmnet unbuilt after kernel bump | `sudo vmware-modconfig --console --install-all` (or reboot; systemd guard rebuilds) |
|
||||
| `ci: command not found` | alias not loaded | `source ~/.bashrc` or re-run §0.2 |
|
||||
+519
@@ -198,6 +198,46 @@ Estimated time: 2-4 hours including Windows Update.
|
||||
|
||||
---
|
||||
|
||||
## Windows host baseline
|
||||
|
||||
**Data**: 2026-05-17 (Phase A closure — commit `36913ab6`)
|
||||
|
||||
**Hardware**: Intel i9-10900X, 64 GB RAM, NVMe SSD
|
||||
|
||||
**Versioni**:
|
||||
- `ci_orchestrator`: v2.0.0-phaseA (SHA `b4ca7f3`), Python 3.13.3
|
||||
- `act_runner`: v1.0.2
|
||||
|
||||
**Benchmark infra — VM lifecycle** (`Measure-CIBenchmark.ps1`, 4 iter, template Windows):
|
||||
|
||||
| Phase | iter 1 | iter 2 | iter 3 | iter 4 | **media** |
|
||||
| --------- | ------ | ------ | ------ | ------ | --------- |
|
||||
| Clone (s) | 0.63 | 0.63 | 0.62 | 0.61 | **0.62** |
|
||||
| Start (s) | 1.75 | 1.89 | 1.72 | 1.72 | **1.77** |
|
||||
| IP (s) | 66.57 | 20.21 | 85.07 | 60.97 | **58.2** |
|
||||
| WinRM (s) | 0.01 | 0.01 | 0.01 | 0.01 | **0.01** |
|
||||
| Destroy(s)| 4.81 | 6.39 | 4.50 | 4.20 | **4.98** |
|
||||
| Boot tot | 68.96 | 22.74 | 87.42 | 63.31 | **60.6** |
|
||||
|
||||
> Fase IP (20–85 s) = costo dominante e variabile (detect IP via VMware Tools). Normale, non bloccante.
|
||||
|
||||
**Tempo medio per job** (smoke `self-test.yml`, Gitea Actions, Passo 6 Phase A):
|
||||
|
||||
| Template | Transport | Wall time |
|
||||
| -------------- | --------------------- | --------- |
|
||||
| WinBuild2025 | in-guest git clone | 26 s |
|
||||
| WinBuild2025 | host-side clone + zip | 28 s |
|
||||
| LinuxBuild2404 | in-guest git clone | 51 s |
|
||||
| LinuxBuild2404 | host-side clone + zip | 46 s |
|
||||
| **media Win** | | **27 s** |
|
||||
| **media Linux**| | **49 s** |
|
||||
|
||||
**Success rate**: 100 % — burn-in 12/12 (3 round × 4 job concorrenti, Passo 7 Phase A).
|
||||
|
||||
> Questi valori sono il baseline di riferimento per il criterio B7 ("tempo medio entro ±20% baseline Windows"). Margini: Win ≤ 32 s, Linux ≤ 59 s.
|
||||
|
||||
---
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Symptom | First command |
|
||||
@@ -348,4 +388,483 @@ Restart-Service act_runner
|
||||
# The prior snapshot is still in the template — jobs will use it immediately.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Windows host pre-migration baseline (reference for B7)
|
||||
|
||||
Recorded 2026-05-17 — `Measure-CIBenchmark.ps1 × 4 iterations`, Python
|
||||
orchestrator post-Phase-A, Windows 11 + VMware Workstation Pro,
|
||||
template `WinBuild2025` / snapshot `BaseClean`.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | WinRM (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1 | 0.63 | 1.75 | 66.57 | 0.01 | 4.81 | 68.96 |
|
||||
| 2 | 0.63 | 1.89 | 20.21 | 0.01 | 6.39 | 22.74 |
|
||||
| 3 | 0.62 | 1.72 | 85.07 | 0.01 | 4.50 | 87.42 |
|
||||
| 4 | 0.61 | 1.72 | 60.97 | 0.01 | 4.20 | 63.31 |
|
||||
| **avg** | **0.62** | **1.77** | **58.20** | **0.01** | **4.98** | **60.61** |
|
||||
|
||||
**Key finding**: IP-acquire phase dominates total time and is highly variable
|
||||
(20–85 s) due to VMware Tools guest IP detection latency. Clone/Start/WinRM
|
||||
are negligible and stable.
|
||||
|
||||
**B7 comparison guidance** (tolerance ±20%):
|
||||
|
||||
| Metric | Windows baseline | ±20% range |
|
||||
| ---------------- | ---------------- | ----------- |
|
||||
| Clone | 0.62 s | 0.50–0.74 s |
|
||||
| Start | 1.77 s | 1.42–2.12 s |
|
||||
| Destroy | 4.98 s | 3.98–5.98 s |
|
||||
| Boot total (avg) | 60.6 s | 48.5–72.7 s |
|
||||
|
||||
IP-acquire variance on Windows (σ ≈ 26 s) means boot-total comparison
|
||||
requires ≥10 samples on Linux to be meaningful. If Linux avg boot total
|
||||
exceeds 72.7 s, open an issue in `TODO.md` with per-phase breakdown before
|
||||
declaring B7 failed — check whether IP-acquire increased or non-IP phases
|
||||
regressed.
|
||||
|
||||
---
|
||||
|
||||
## 7. Linux host post-migration baseline (B7 result)
|
||||
|
||||
Recorded 2026-05-24 — `Measure-CIBenchmark.ps1 × 4 iterations`, Linux Mint
|
||||
host + VMware Workstation Pro Linux, template `WinBuild2025` / snapshot
|
||||
`BaseClean`. Ready column = WinRM/5986 TCP probe.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | Ready (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1 | 0.42 | 1.89 | 53.06 | 0.03 | 4.55 | 55.40 |
|
||||
| 2 | 0.40 | 1.89 | 129.76 | 0.00 | 4.96 | 132.05 |
|
||||
| 3 | 0.52 | 2.81 | 176.83 | 0.00 | 5.67 | 180.16 |
|
||||
| 4 | 0.40 | 1.90 | 39.17 | 0.00 | 4.51 | 41.47 |
|
||||
| **avg** | **0.44** | **2.12** | **99.71** | **0.01** | **4.92** | **102.27** |
|
||||
|
||||
**Phase verdict vs Windows baseline (±20%):**
|
||||
|
||||
| Metric | Windows | Linux avg | In range? |
|
||||
| ------- | ------- | --------- | ---------------------------------- |
|
||||
| Clone | 0.62 s | 0.44 s | ✓ (faster) |
|
||||
| Start | 1.77 s | 2.12 s | ✓ (at upper edge) |
|
||||
| Destroy | 4.98 s | 4.92 s | ✓ |
|
||||
| IP avg | 58.2 s | 99.7 s | ✗ outside — IP variance (39–177 s) |
|
||||
| Ready | 0.01 s | 0.01 s | ✓ |
|
||||
|
||||
**Key finding**: Clone/Start/Ready/Destroy within ±20%. IP-acquire dominates
|
||||
and is highly variable on Linux host (σ ≈ 57 s, range 39–177 s) — wider than
|
||||
Windows (σ ≈ 26 s). This is VMware Tools DHCP/guestinfo reporting latency,
|
||||
not a regression in orchestrator logic. With 4 samples the avg is not stable;
|
||||
additional runs may close the gap. No non-IP phase regressed.
|
||||
|
||||
---
|
||||
|
||||
## 8. Static IP baseline — WinBuild2025 with ip_pool (B8 result)
|
||||
|
||||
Recorded 2026-05-25 — `Measure-CIBenchmark.ps1 -StaticIP 192.168.79.200 -Iterations 4`,
|
||||
Linux Mint host, template `WinBuild2025` / snapshot `BaseClean`.
|
||||
`guestinfo.ip-assignment` injected into cloned VMX before start;
|
||||
`ci-static-ip.ps1` scheduled task applies IP at boot and writes back
|
||||
`guestinfo.ci-ip`. IP column = time until `guestinfo.ci-ip` readable via
|
||||
`vmrun readVariable` (`-GuestInfoOnly` mode — DHCP fallback disabled).
|
||||
Ready column = WinRM/5986 TCP probe after IP known.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | Ready (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1 | 0.41 | 1.86 | 21.80 | 0.00 | 9.75 | 24.07 |
|
||||
| 2 | 0.40 | 1.88 | 21.74 | 0.00 | 9.81 | 24.02 |
|
||||
| 3 | 0.40 | 1.90 | 21.75 | 0.00 | 12.26 | 24.05 |
|
||||
| 4 | 0.40 | 1.89 | 21.77 | 0.00 | 9.77 | 24.06 |
|
||||
| **avg** | **0.40** | **1.88** | **21.77** | **0.00** | **10.40** | **24.05** |
|
||||
|
||||
**Three-way comparison — B6 Windows DHCP / B7 Linux DHCP / B8 Linux static IP:**
|
||||
|
||||
| Metric | B6 Win DHCP avg | B7 Lin DHCP avg | B8 Lin static avg | B8 vs B6 | B8 vs B7 |
|
||||
| ---------- | --------------- | --------------- | ----------------- | ---------------- | ---------------- |
|
||||
| Clone | 0.62 s | 0.44 s | 0.40 s | −35% | ≈ same |
|
||||
| Start | 1.77 s | 2.12 s | 1.88 s | +6% | ≈ same |
|
||||
| IP acquire | 58.2 s | 99.7 s | 21.8 s | **−63%** | **−78%** |
|
||||
| Ready | 0.01 s | 0.01 s | 0.00 s | ≈ same | ≈ same |
|
||||
| Boot total | 60.6 s | 102.3 s | 24.1 s | **−60%** | **−76%** |
|
||||
| IP σ | ~26 s | ~57 s | <0.03 s | **deterministic**| **deterministic**|
|
||||
|
||||
**Key findings**:
|
||||
- Static IP beats even the Windows DHCP baseline by 60% on boot total.
|
||||
- IP acquire drops from 58 s (Win) / 100 s (Linux) DHCP to a deterministic
|
||||
21.8 s — variance eliminated entirely (σ < 0.03 s).
|
||||
- Ready = 0 s: WinRM is already listening on the static IP by the time
|
||||
`guestinfo.ci-ip` is written — no additional TCP probe wait.
|
||||
- `ci-static-ip.ps1` startup latency (~21.8 s) is the new floor; it reflects
|
||||
Windows boot + Task Scheduler + NIC reconfiguration time.
|
||||
- Clone/Start/Ready unchanged across all three baselines — static IP has no
|
||||
side effects on non-IP phases.
|
||||
- Destroy is slower than B6/B7 (~10 s vs ~5 s) — likely disk pressure or
|
||||
clone state at test time, unrelated to static IP.
|
||||
|
||||
---
|
||||
|
||||
## 9. Concurrent capacity burn-in (B7 — 4 × 10)
|
||||
|
||||
Recorded 2026-06-07 — `Test-CapacityBurnIn.ps1 -Parallelism 4 -Rounds 10`,
|
||||
Linux Mint host + VMware Workstation Pro Linux, run as `ci-runner`, static IP
|
||||
pool (`192.168.79.201–204`). Repo `Simone/burnin-dummy`. Each "round" is the
|
||||
wall-clock time for 4 concurrent end-to-end jobs (clone → boot → IP →
|
||||
transport → build → artifacts → destroy). This is the first concurrent
|
||||
capacity burn-in (the §6–§8 baselines are single-job `Measure-CIBenchmark`).
|
||||
|
||||
| Template | Snapshot | Build cmd | Jobs PASS | Round time (min–max) | Round avg |
|
||||
| ----------------- | ---------------- | --------------- | --------- | -------------------- | --------- |
|
||||
| WinBuild2025 | `BaseClean` | `build.ps1` | 40 / 40 | 68–81 s | ~78.6 s |
|
||||
| LinuxBuild2404 | `BaseClean-Linux`| `build.sh` | 40 / 40 | 70–71 s | ~70.2 s |
|
||||
|
||||
**Result: OVERALL PASS** — 80/80 jobs, 20/20 rounds, zero orphaned clones,
|
||||
zero leaked IP leases between rounds.
|
||||
|
||||
**Key findings**:
|
||||
- Per-round wall time is near-deterministic (Win σ ≈ 1.3 s, Linux σ ≈ 0.4 s),
|
||||
confirming the static-IP pool eliminates the DHCP IP-acquire variance that
|
||||
dominated the §6/§7 single-job baselines (σ 26–57 s).
|
||||
- 4-way concurrency adds no contention penalty: round time ≈ single-job boot
|
||||
total (§8 static ≈ 24 s) plus build + serialized destroy, well within the
|
||||
4-IP pool capacity.
|
||||
- Linux rounds (~70 s) slightly faster and tighter than Windows (~79 s).
|
||||
- IP-pool release-on-completion works under concurrency — pool returned to
|
||||
all-free (`null`) after every round.
|
||||
|
||||
**Operational note (pool not auto-reconciled)**: `ip-pool.json` is *not*
|
||||
reconciled against live clones at startup. A job killed mid-flight (SIGKILL,
|
||||
crash) leaks its lease permanently; with only 4 IPs for parallelism 4, a
|
||||
single leak exhausts the pool and every subsequent round fails fast with
|
||||
`IP pool exhausted after 60s`. Recovery: stop all jobs, confirm
|
||||
`build-vms/` empty, then reset the pool:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -c \
|
||||
"import pathlib; pathlib.Path('/var/lib/ci/ip-pool.json').write_text('{}\n')"
|
||||
```
|
||||
|
||||
See `TODO.md` (IP-pool auto-reconciliation) for the proposed fix.
|
||||
|
||||
---
|
||||
|
||||
## 10. Linux guest single-job baseline — LinuxBuild2404 (B7 follow-up)
|
||||
|
||||
Recorded 2026-06-07 — `Measure-CIBenchmark.ps1 -GuestOS Linux -Iterations 10`,
|
||||
Linux Mint host + VMware Workstation Pro Linux, run as `ci-runner`, template
|
||||
`LinuxBuild2404` / snapshot `BaseClean-Linux`, **DHCP** (no static IP — the
|
||||
guest reads its DHCP lease and publishes it via `guestinfo.ci-ip`; the
|
||||
Windows-only `ci-static-ip.ps1` task does not apply). Ready = SSH/22 probe.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | Ready (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1–10 | 0.22 | 1.14–1.16 | 7.44–7.48 | 0.00–0.01 | 4.90–4.99 | 8.80–8.86 |
|
||||
| **avg** | **0.22** | **1.14** | **7.45** | **0.00** | **4.94** | **8.82** |
|
||||
|
||||
**Key finding**: the Linux guest is near-deterministic *even on DHCP*
|
||||
(σ ≈ 0.01 s on IP-acquire) — no static-IP injection needed. IP-acquire is
|
||||
7.45 s vs Windows-guest 58 s (DHCP, §6) / 21.8 s (static, §8). Boot total
|
||||
8.82 s — ~2.7× faster than the Windows static-IP guest.
|
||||
|
||||
**Guest-vs-guest comparison (Linux host, single-job):**
|
||||
|
||||
| Metric | Win guest static (§8) | Lin guest DHCP (§10) | Lin advantage |
|
||||
| ---------- | --------------------- | -------------------- | ------------- |
|
||||
| Clone | 0.40 s | 0.22 s | −45% |
|
||||
| Start | 1.88 s | 1.14 s | −39% |
|
||||
| IP acquire | 21.77 s | 7.45 s | **−66%** |
|
||||
| Ready | 0.00 s | 0.00 s | ≈ same |
|
||||
| Destroy | 10.40 s | 4.94 s | −53% |
|
||||
| Boot total | 24.05 s | 8.82 s | **−63%** |
|
||||
|
||||
Caveat: different IP modes (Win static-IP task vs Lin DHCP+guestinfo) — the
|
||||
comparison reflects the *as-deployed* path for each guest, not an
|
||||
IP-mode-controlled A/B. The Win guest's static-IP floor (~21.8 s) is dominated
|
||||
by Windows boot + Task Scheduler + NIC reconfig; the Linux guest needs no such
|
||||
in-guest agent.
|
||||
|
||||
---
|
||||
|
||||
## 11. Static IP baseline — WinBuild2025 on Windows host (pairs §8)
|
||||
|
||||
Recorded 2026-06-07 — `Measure-CIBenchmark.ps1 -StaticIP 192.168.79.200
|
||||
-Iterations 10`, **Windows 11 host** + VMware Workstation Pro, template
|
||||
`WinBuild2025` / snapshot `BaseClean`, `guestinfo.ip-assignment` injected
|
||||
before start; the in-guest `ci-static-ip.ps1` task applies the IP and writes
|
||||
back `guestinfo.ci-ip`. IP column = time to `guestinfo.ci-ip` readable
|
||||
(`-GuestInfoOnly`). Ready = WinRM/5986 TCP probe. This is the Windows-host
|
||||
counterpart to the Linux-host §8.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | Ready (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1 | 0.70 | 1.71 | 26.65 | 0.45 | 10.47 | 29.51 |
|
||||
| 2 | 0.62 | 1.75 | 24.29 | 2.01 | 24.29 | 28.67 |
|
||||
| 3 | 0.63 | 1.76 | 21.83 | 6.01 | 24.32 | 30.23 |
|
||||
| 4 | 0.62 | 1.57 | 21.92 | 6.02 | 10.80 | 30.13 |
|
||||
| 5 | 0.62 | 1.56 | 24.24 | 2.02 | 24.21 | 28.44 |
|
||||
| 6 | 0.62 | 1.55 | 21.94 | 6.01 | 24.28 | 30.12 |
|
||||
| 7 | 0.63 | 1.55 | 21.85 | 6.01 | 24.28 | 30.04 |
|
||||
| 8 | 0.64 | 1.74 | 21.73 | 6.01 | 11.25 | 30.12 |
|
||||
| 9 | 0.62 | 1.57 | 21.90 | 6.01 | 24.28 | 30.10 |
|
||||
| 10 | 0.62 | 1.74 | 21.72 | 6.02 | 10.76 | 30.10 |
|
||||
| **avg** | **0.63** | **1.65** | **22.81** | **4.66** | **18.89** | **29.75** |
|
||||
|
||||
**Host comparison — §8 (Linux host) vs §11 (Windows host), Win guest static:**
|
||||
|
||||
| Metric | §8 Linux host | §11 Windows host | Δ (Win vs Lin) |
|
||||
| ---------- | ------------- | ---------------- | ------------------------- |
|
||||
| Clone | 0.40 s | 0.63 s | +58 % (NTFS linked clone) |
|
||||
| Start | 1.88 s | 1.65 s | −12 % |
|
||||
| IP acquire | 21.77 s | 22.81 s | +5 % (≈ same) |
|
||||
| Ready | 0.00 s | 4.66 s | +4.7 s |
|
||||
| Destroy | 10.40 s | 18.89 s | +82 % (bimodal ~11/24 s) |
|
||||
| Boot total | 24.05 s | 29.75 s | +24 % |
|
||||
|
||||
**Key findings**:
|
||||
- The **static-IP floor (~21.8 s) is host-independent** — exactly as the plan
|
||||
predicted. It is set by Windows boot + Task Scheduler + NIC reconfig inside
|
||||
the guest, identical across both hosts (Win 22.81 s vs Lin 21.77 s, +5 %).
|
||||
- **Ready ≠ 0 on the Windows host** (avg 4.66 s, mostly ~6 s) where it was 0 on
|
||||
Linux. After `ci-static-ip.ps1` rewrites the NIC, WinRM/5986 needs a few
|
||||
seconds to re-bind on the new address before the TCP probe succeeds; on the
|
||||
Linux host WinRM was already listening when `ci-ip` was written.
|
||||
- Clone is slower on NTFS (+58 %); destroy is slower and bimodal (~11 s vs
|
||||
~24 s) — the ~24 s cases hit the 10 s graceful-stop timeout before deleteVM.
|
||||
- Net boot-total penalty of the Windows host at constant guest+mode: **+24 %**.
|
||||
|
||||
**Pre-req gotcha (template parity)**: the first §11 attempt timed out at 300 s
|
||||
every iteration — `guestinfo.ci-ip` never appeared, guest stayed on DHCP.
|
||||
Cause: the `F:\CI\Templates\WinBuild2025` `BaseClean` snapshot did **not**
|
||||
contain the in-guest `ci-static-ip` agent (added to the Linux-host template
|
||||
during B8/§8, never propagated to the Windows-host copy). Host-side guestinfo
|
||||
injection works, but with no in-guest consumer the static IP is never applied.
|
||||
Fixed by copying the Linux-host template set onto the Windows host (template
|
||||
parity), after which all 10 iterations passed. If §11 ever regresses to 300 s
|
||||
timeouts, confirm `C:\CI\ci-static-ip.ps1` + the `CI-StaticIp` startup task
|
||||
exist inside a clone (see [§13 op-notes](#13-concurrent-capacity-burn-in-on-windows-host-pairs-9)).
|
||||
|
||||
---
|
||||
|
||||
## 12. Linux guest single-job baseline — Windows host (pairs §10)
|
||||
|
||||
Recorded 2026-06-07 — `Measure-CIBenchmark.ps1 -GuestOS Linux -Iterations 10`,
|
||||
**Windows 11 host** + VMware Workstation Pro, template `LinuxBuild2404` /
|
||||
snapshot `BaseClean-Linux`, **DHCP** (guest publishes its lease via
|
||||
`guestinfo.ci-ip`). Ready = SSH/22 probe. Windows-host counterpart to §10.
|
||||
|
||||
| Iter | Clone (s) | Start (s) | IP acquire (s) | Ready (s) | Destroy (s) | Boot total (s) |
|
||||
| ------- | --------- | --------- | -------------- | --------- | ----------- | -------------- |
|
||||
| 1–10 | 0.62–0.69 | 1.37–1.81 | 11.26–11.42 | 0.00–0.02 | 6.02–6.21 | 13.43–13.83 |
|
||||
| **avg** | **0.64** | **1.50** | **11.36** | **0.00** | **6.13** | **13.51** |
|
||||
|
||||
**Host comparison — §10 (Linux host) vs §12 (Windows host), Lin guest DHCP:**
|
||||
|
||||
| Metric | §10 Linux host | §12 Windows host | Δ (Win vs Lin) |
|
||||
| ---------- | -------------- | ---------------- | -------------------------- |
|
||||
| Clone | 0.22 s | 0.64 s | +191 % (NTFS linked clone) |
|
||||
| Start | 1.14 s | 1.50 s | +32 % |
|
||||
| IP acquire | 7.45 s | 11.36 s | +52 % |
|
||||
| Ready | 0.00 s | 0.00 s | ≈ same |
|
||||
| Destroy | 4.94 s | 6.13 s | +24 % |
|
||||
| Boot total | 8.82 s | 13.51 s | **+53 %** |
|
||||
|
||||
**Key findings**:
|
||||
- The Linux guest stays near-deterministic on the Windows host too
|
||||
(IP σ ≈ 0.05 s) — no static-IP agent needed; DHCP+guestinfo is stable.
|
||||
- The **host penalty is larger for the Linux guest (+53 % boot total) than for
|
||||
the Windows guest (+24 %, §11)** because the Linux guest's boot is so fast
|
||||
(8.82 s) that fixed Windows-host overheads — NTFS clone (+0.4 s) and the
|
||||
vmnet8/NAT DHCP+guestinfo round-trip (+3.9 s on IP-acquire) — are a larger
|
||||
*fraction* of a small total. In absolute terms the host adds ~4.7 s either way.
|
||||
- IP-acquire delta (7.45 → 11.36 s) isolates the **host networking effect**
|
||||
(Windows vmnet8 NAT + VMware Tools guestinfo path) on an otherwise identical
|
||||
guest.
|
||||
|
||||
---
|
||||
|
||||
## 13. Concurrent capacity burn-in on Windows host (pairs §9)
|
||||
|
||||
Recorded 2026-06-07 — `Test-CapacityBurnIn.ps1 -Parallelism 4 -Rounds 10`,
|
||||
**Windows 11 host** + VMware Workstation Pro, static IP pool
|
||||
`192.168.79.201–204` (added to `F:\CI\config.toml` `[ip_pool]` for parity with
|
||||
the Linux-host §9). Repo `Simone/burnin-dummy`. Windows guests use the static
|
||||
pool; Linux guests skip it (DHCP+guestinfo). Windows-host counterpart to §9.
|
||||
|
||||
| Template | Snapshot | Build cmd | Jobs PASS | Rounds OK | Round time (PASS rounds) | Round avg (PASS) |
|
||||
| -------------- | ----------------- | ----------- | --------- | --------- | ------------------------ | ---------------- |
|
||||
| WinBuild2025 | `BaseClean` | `build.ps1` | 36 / 40 | 7 / 10 | 88–93 s | ~91.6 s |
|
||||
| LinuxBuild2404 | `BaseClean-Linux` | `build.sh` | 40 / 40 | 10 / 10 | 88–112 s | ~96.4 s |
|
||||
|
||||
**Result: PARTIAL** — Linux 40/40 (OVERALL PASS); Windows 36/40 (OVERALL FAIL,
|
||||
3 rounds with transient WinRM faults). 76/80 jobs overall.
|
||||
|
||||
**Host comparison vs §9 (Linux host):**
|
||||
|
||||
| Guest | §9 Linux host | §13 Windows host | Δ round avg |
|
||||
| ----- | ---------------- | ---------------------------- | ---------------------- |
|
||||
| Win | 40/40, ~78.6 s | 36/40, ~91.6 s (PASS rounds) | +16 % + WinRM instability |
|
||||
| Lin | 40/40, ~70.2 s | 40/40, ~96.4 s | +37 % |
|
||||
|
||||
**Key findings**:
|
||||
- **Linux-guest concurrency is stable on the Windows host** (40/40, 10/10) just
|
||||
as on the Linux host — SSH transport showed no faults. Round avg +37 % vs §9,
|
||||
consistent with the §12 single-job host penalty.
|
||||
- **Windows-guest concurrency is *not* robust on the Windows host.** Rounds 1–5,
|
||||
9–10 were clean (~91 s); rounds 6–8 each lost 1–2 jobs to **transient WinRM
|
||||
faults** under 4× load and ballooned to 269 / 266 / 622 s (30 s connect
|
||||
timeouts + pypsrp retries). Two failure signatures:
|
||||
- `ConnectTimeoutError` to 5986 (WinRM listener briefly unreachable);
|
||||
- `WSManFault 2150858843` "the shell was not found on the server" (the WinRM
|
||||
shell was recycled mid-build).
|
||||
Failures **self-recovered**: each failed job released its IP slot, no clones
|
||||
were orphaned, the IP pool returned to all-free, and rounds 9–10 were clean
|
||||
again. So this is bursty WinRM contention, not a monotonic collapse or a leak.
|
||||
- Likely cause: vCPU oversubscription (4 VMs × 4 vCPU = 16 vCPU) plus the higher
|
||||
host-OS overhead of Windows-host VMware tips concurrent WinRM into timeouts;
|
||||
RAM was not the constraint (13.8 GB free of 63.7 GB during the run). The Linux
|
||||
host on the same hardware sustained 40/40 (§9). **Recommendation for sustained
|
||||
4× Windows-guest concurrency on the Windows host**: lower per-VM vCPU, reduce
|
||||
parallelism to 3, or raise WinRM/pypsrp connect timeouts + add a job-level
|
||||
retry. Tracked in `TODO.md`.
|
||||
|
||||
**Operational notes (Windows-host benchmark prerequisites)** — discovered while
|
||||
running §11–§13; required for any Windows-host orchestration:
|
||||
1. **Template parity** — the Windows-host templates must carry the same in-guest
|
||||
agents as the Linux-host templates (`ci-static-ip.ps1` + `CI-StaticIp`
|
||||
startup task for Windows guests; `ci-report-ip` for Linux). Without the
|
||||
static-IP agent, static-IP jobs hang for the full IP timeout (see §11).
|
||||
2. **Production venv must be current** — `F:\CI\python\venv` predated the
|
||||
`ip_pool` feature; `[ip_pool]` in config is silently ignored by a stale venv.
|
||||
Re-run `F:\CI\python\venv\Scripts\python.exe -m pip install .` (non-editable)
|
||||
after pulling new code, then verify `import ci_orchestrator.ip_pool` succeeds.
|
||||
3. **Set `CI_VENV_PYTHON`** before invoking `Test-CapacityBurnIn.ps1` /
|
||||
`Invoke-CIJob.ps1` interactively. The shim's auto-detect branch dereferences
|
||||
`$IsWindows`, which is undefined under Windows PowerShell 5.1 + `StrictMode`
|
||||
and throws `VariableIsUndefined`; setting `CI_VENV_PYTHON` skips that branch.
|
||||
Production sets it via the runner (`CI_PYTHON_LAUNCHER`). *Note (Phase C): on
|
||||
the Linux host this is moot — burn-in now runs via `bench run` (below), which
|
||||
never touches the shim.*
|
||||
|
||||
### Phase C — Python equivalents (run these on the Linux host; no `pwsh`)
|
||||
|
||||
The manual `.ps1` procedures above are superseded by native CLI commands. Invoke
|
||||
as the service user (`sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator ...`):
|
||||
|
||||
| Old PowerShell (Linux host) | Python command |
|
||||
| --------------------------------------------------- | ------------------------------------------------ |
|
||||
| `Test-CapacityBurnIn.ps1 -Parallelism 4 -Rounds 10` | `bench run --concurrency 4 --rounds 10 --guest-os linux` |
|
||||
| `Measure-CIBenchmark.ps1 -Iterations N` | `bench measure --iterations N --guest-os linux` |
|
||||
| `Test-Smoke.ps1 -GuestOS Linux` | `smoke run --guest-os linux` |
|
||||
| `Validate-HostState.ps1` (host checks) | `validate host` |
|
||||
| `Test-CIGuestWinRM.ps1` (transport diag) | `validate guest --host <IP> [--winrm\|--ssh]` |
|
||||
| `Set-CIGuestCredential.ps1` | `creds set --target BuildVMGuest --user <u> --password-stdin` |
|
||||
|
||||
`bench run` writes a JSON report to `CI_ARTIFACTS/bench/`; `bench measure` appends
|
||||
to `benchmark.jsonl` (same fields as before, so the §6–§13 trend lines stay
|
||||
comparable). The benchmark records in §6–§13 above are kept as historical
|
||||
provenance — they were captured with the `.ps1` tools before the cutover.
|
||||
|
||||
---
|
||||
|
||||
## 14. Host × guest × IP-mode matrix (complete)
|
||||
|
||||
With §11–§13 the Linux-host / Windows-host comparison is symmetric:
|
||||
|
||||
| Host \ (Guest, mode) | Win/DHCP | Win/static | Lin/DHCP | burn-in 4×10 |
|
||||
| --------------------- | -------- | ---------- | -------- | --------------------------- |
|
||||
| **Linux** (boot/avg) | §7 102 s | §8 24.1 s | §10 8.8 s | §9 Win 78.6 s / Lin 70.2 s |
|
||||
| **Windows** (boot/avg)| §6 60.6 s| §11 29.8 s | §12 13.5 s| §13 Win ~91.6 s / Lin 96.4 s |
|
||||
|
||||
Read **down a column** = host effect at constant guest+mode; **across a row** =
|
||||
guest/IP-mode effect at constant host. Headline: the Windows host adds **+24 %**
|
||||
(Win-guest static) to **+53 %** (Lin-guest DHCP) on single-job boot-to-ready,
|
||||
the static-IP floor (~22 s) is host-independent, and Windows-guest WinRM is the
|
||||
only path that loses jobs under sustained 4× concurrency (Linux/SSH is 40/40 on
|
||||
both hosts).
|
||||
|
||||
> Note on §6 vs §7: §6 (Win host DHCP) 60.6 s is *faster* than §7 (Lin host
|
||||
> DHCP) 102 s, the reverse of every other column. Both are 4-iteration DHCP
|
||||
> baselines whose IP-acquire is dominated by high-variance VMware-Tools polling
|
||||
> (σ 26–57 s); the 10-iteration static (§8/§11) and Linux (§10/§12) rows are the
|
||||
> reliable host-effect signal.
|
||||
|
||||
---
|
||||
|
||||
## 15. Dual-boot operation (Linux ⇄ Windows host)
|
||||
|
||||
The CI machine is **dual-boot** on one piece of hardware: Linux Mint and
|
||||
Windows 11 never run at the same time. Each OS carries a full CI stack
|
||||
(orchestrator + runner + templates under its own root). Booting Linux brings
|
||||
up the Linux runner; booting Windows brings up the Windows runner. Templates
|
||||
and the `burnin-dummy` repo are kept at **template parity** across both roots.
|
||||
|
||||
| | Linux host | Windows host |
|
||||
| --- | --- | --- |
|
||||
| CI root | `/var/lib/ci/` | `F:\CI\` |
|
||||
| Runner | `act-runner.service` (systemd) | `actions-runner` (Windows service / NSSM) |
|
||||
| Transport to guests | WinRM (Win guest) / SSH (Linux guest) | same |
|
||||
| vmrun | `/usr/bin/vmrun` | `C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe` |
|
||||
|
||||
### Current GRUB behaviour
|
||||
|
||||
`GRUB_DEFAULT=0`, `GRUB_TIMEOUT=0`, `GRUB_TIMEOUT_STYLE=hidden` — the machine
|
||||
boots **straight into Linux Mint with no menu**. Boot entries present:
|
||||
`Linux Mint 22.3 Cinnamon` (default) and
|
||||
`Windows Boot Manager (on /dev/nvme0n1p1)`.
|
||||
|
||||
### Switch Linux → Windows
|
||||
|
||||
With the hidden/zero-timeout GRUB, hold **Esc** (or **Shift**) during early
|
||||
boot to reveal the GRUB menu, then pick `Windows Boot Manager`. For a planned,
|
||||
unattended switch, set a one-shot next-boot target instead:
|
||||
|
||||
```bash
|
||||
# Requires GRUB_DEFAULT=saved (see "Optional GRUB tweak" below).
|
||||
sudo grub-reboot "Windows Boot Manager (on /dev/nvme0n1p1)"
|
||||
sudo reboot
|
||||
```
|
||||
|
||||
Before rebooting out of Linux: confirm no job is mid-flight
|
||||
(`sudo -u ci-runner ls /var/lib/ci/build-vms/` empty) and optionally pause the
|
||||
Linux runner in the Gitea UI so queued jobs wait for the Windows runner.
|
||||
|
||||
### Switch Windows → Linux
|
||||
|
||||
Reboot; with `GRUB_DEFAULT=0` the machine returns to Linux automatically (no
|
||||
key needed). If `grub-reboot` / `GRUB_DEFAULT=saved` is in use, Linux is the
|
||||
saved default after a normal Linux shutdown.
|
||||
|
||||
### Post-boot verification (either OS)
|
||||
|
||||
Linux:
|
||||
```bash
|
||||
systemctl is-active act-runner # → active
|
||||
systemctl --failed # → 0 loaded units
|
||||
systemctl list-timers 'ci-*' # all ci-* timers scheduled
|
||||
# then confirm the runner shows online in the Gitea UI (Admin → Runners)
|
||||
```
|
||||
|
||||
Windows (PowerShell):
|
||||
```powershell
|
||||
Get-Service actions-runner # Status Running
|
||||
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' list # vmrun OK
|
||||
# then confirm the runner shows online in the Gitea UI
|
||||
```
|
||||
|
||||
### Optional GRUB tweak for everyday dual-boot
|
||||
|
||||
To get a short pick menu and remember the last choice (handy if Windows is used
|
||||
often), edit `/etc/default/grub`:
|
||||
|
||||
```bash
|
||||
GRUB_DEFAULT=saved
|
||||
GRUB_SAVEDEFAULT=true
|
||||
GRUB_TIMEOUT=5
|
||||
GRUB_TIMEOUT_STYLE=menu
|
||||
```
|
||||
|
||||
```bash
|
||||
sudo update-grub
|
||||
```
|
||||
|
||||
This is **optional** — the default boots-to-Linux behaviour is intentional so
|
||||
the machine comes back as the Linux CI host after any unattended reboot
|
||||
(power-loss, kernel update). Leave it as-is unless Windows becomes the daily
|
||||
driver.
|
||||
|
||||
@@ -197,7 +197,7 @@ Final Pre-snapshot gate — 7 check cross-cutting + no installer leftover
|
||||
| ----- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| 1 | Assert | `Get-NetFirewallProfile -Profile Domain/Private/Public → Enabled=False` |
|
||||
| 2 | Assert | GPO DisableAntiSpyware=1 (set da Deploy) |
|
||||
| 3 | Assert | WinRM Running+Automatic, AllowUnencrypted=false (HTTPS-only), Auth/Basic=true, MaxMem≥2048 |
|
||||
| 3 | Assert | WinRM Running+Automatic, AllowUnencrypted=false (HTTPS-only), Auth/Basic=true, Shell+Plugin MaxMem≥2048, MaxProcesses≥100, desktop heap SharedSection 3rd≥4096 |
|
||||
| 4 | Operativo | User exists, enabled, PasswordNeverExpires, member of Administrators |
|
||||
| 4b | Assert | EnableLUA=0, LocalAccountTokenFilterPolicy=1 |
|
||||
| 5 | Operativo | Ogni dir Test-Path -PathType Container |
|
||||
@@ -290,6 +290,55 @@ toolchain (.NET, VS Build Tools, Python).
|
||||
|
||||
---
|
||||
|
||||
## WinRM quotas e desktop heap (build paralleli)
|
||||
|
||||
I template Windows escono di default da Windows con quote dimensionate per uso
|
||||
interattivo. Da Linux/`pypsrp` queste quote si rivelano strette appena il build
|
||||
guest fa fan-out di processi nativi (compilatori, linker) in parallelo. Tre
|
||||
configurazioni che il template **deve** applicare prima dello snapshot
|
||||
`BaseClean` (gestite automaticamente da `Deploy-WinBuild2025.ps1` / `2022`):
|
||||
|
||||
| Setting | Default | Valore template | Motivo |
|
||||
| ---------------------------------------------------------------------------------------- | ------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `WSMan:\localhost\Shell\MaxMemoryPerShellMB` | 150 | 2048 | Limite globale memoria per shell WinRM (winrs) |
|
||||
| `WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB` | 150 | 2048 | **pypsrp da Linux usa l'endpoint plugin Microsoft.PowerShell**, non lo Shell. Senza questo, builds paralleli falliscono con `MSB6003 / Not enough quota` anche se Shell è a 2048 |
|
||||
| `WSMan:\localhost\Shell\MaxProcessesPerShell` | 25 | 100 | MSBuild + cl + link + mt + altri tool si moltiplicano in fan-out parallelo |
|
||||
| `HKLM\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems\Windows SharedSection` | `…,768` | `…,4096` | Heap del desktop **non-interattivo** (Session 0). CreateProcess fallisce con `ERROR_NOT_ENOUGH_QUOTA` (1816) quando 3+ cl.exe partono insieme. Richiede **reboot** per essere letto da `csrss.exe`. |
|
||||
|
||||
**Importante**: la modifica `SharedSection` è una scrittura di registry che il
|
||||
kernel legge **solo al boot** (`csrss.exe`). Lo `shutdown /s` finale di
|
||||
`post-install.ps1` fa sì che il prossimo avvio del template (per ulteriore
|
||||
manutenzione, o l'invocazione di `Install-CIToolchain`) parta con il valore
|
||||
corretto. Il `BaseClean` snapshot va preso **dopo** un boot completo per essere
|
||||
certi che il valore sia attivo.
|
||||
|
||||
**Perché pre-migrazione (Win host) funzionava**: il path Windows→Windows usava
|
||||
`Invoke-Command` su `PSSession` con la stessa identità del processo locale (act_runner
|
||||
come LocalSystem), ereditando spesso quote più generose tramite GPO e default
|
||||
del client locale. Il path Linux→Windows (`pypsrp`) attinge ai default puri
|
||||
dell'endpoint plugin Microsoft.PowerShell del guest — se il template non li ha
|
||||
bumpati, sono **150 MB / 25 processi / 768 KB heap**.
|
||||
|
||||
**Validazione**: `Install-CIToolchain-WinBuild2025.ps1` Step 3 controlla tutti e
|
||||
quattro i valori e fallisce hard se anche solo uno è sotto soglia. Per
|
||||
template già esistenti, applicare il blocco PowerShell sotto, reboot, shutdown,
|
||||
nuovo snapshot `BaseClean`:
|
||||
|
||||
```powershell
|
||||
# Una tantum su template legacy
|
||||
Set-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB 2048 -Force
|
||||
Set-Item 'WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB' 2048 -Force
|
||||
Set-Item WSMan:\localhost\Shell\MaxProcessesPerShell 100 -Force
|
||||
$k = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems'
|
||||
$cur = (Get-ItemProperty -Path $k -Name Windows).Windows
|
||||
$new = $cur -replace 'SharedSection=(\d+),(\d+),\d+', 'SharedSection=$1,$2,4096'
|
||||
Set-ItemProperty -Path $k -Name Windows -Value $new
|
||||
Restart-Service WinRM -Force
|
||||
Restart-Computer -Force # necessario per il desktop heap
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Sintomo | Diagnosi / Fix |
|
||||
@@ -301,6 +350,7 @@ toolchain (.NET, VS Build Tools, Python).
|
||||
| `Python version match` fail | `$PythonVersion` ≠ versione installata. Aggiorna param o reinstalla |
|
||||
| Post-setup `MSBuild not found` | VS install fallita silenziosa. Check `C:\CI\vs_log.txt` |
|
||||
| Pre-snapshot Final check fail | Stato VM rotto — non prendere snapshot. Re-run o ricomincia da zero |
|
||||
| MSBuild `error MSB6003: CL.exe could not be run` + `Win32Exception (0x80004005): Not enough quota` su build paralleli | Quote WinRM/desktop-heap non bumpate nel template (vedi §"WinRM quotas e desktop heap" sotto). Tipico quando il build interno fa fan-out (es. 3× cl.exe paralleli) via pypsrp Linux→Win |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -134,6 +134,7 @@ Script location (fixed on host): `N:\Code\Workspace\Local-CI-CD-System\scripts\I
|
||||
| `-GuestOS` | string | `Auto` | `Windows`, `Linux`, or `Auto` (detected from VMX guestOS field). |
|
||||
| `-BuildCommand` | string | `''` | Shell command to run inside VM. Empty = `dotnet build`. |
|
||||
| `-GuestArtifactSource` | string | `dist` | Path inside VM guest from which artifacts are collected. |
|
||||
| `-Xvfb` | switch | off | Linux only: run the build under `xvfb-run` (headless X for GUI toolkits, e.g. GTK4/PyGObject). Needs `xvfb` in the template (installed by the toolchain). Ignored for Windows. |
|
||||
| `-Submodules` | switch | off | Pass to clone with `--recurse-submodules`. |
|
||||
| `-UseGitClone` | switch | off | Skip host-side clone; let the guest VM clone directly via HTTPS. |
|
||||
| `-GiteaCredentialTarget` | string | `GiteaPAT` | Credential Manager target for Gitea PAT (UseGitClone only). |
|
||||
@@ -321,6 +322,15 @@ Notes:
|
||||
- `-SshKeyPath $env:GITEA_CI_SSH_KEY_PATH` uses the injected key path.
|
||||
- The default snapshot name for Linux templates is `BaseClean-Linux`. If you rely on
|
||||
`Auto`, add `-SnapshotName 'BaseClean-Linux'` explicitly.
|
||||
- For builds or tests that need a display (GUI toolkits such as GTK4/PyGObject),
|
||||
add the `-Xvfb` switch — the build command then runs under `xvfb-run` with a
|
||||
headless X server providing `$DISPLAY`. Example:
|
||||
```powershell
|
||||
-BuildCommand 'xvfb-aware-tests.sh' `
|
||||
-Xvfb `
|
||||
```
|
||||
The Ubuntu template ships `python3-gi`, `gir1.2-gtk-4.0` and `xvfb`
|
||||
(installed by the toolchain). `-Xvfb` is ignored for Windows guests.
|
||||
|
||||
### 8.3 Windows build (UseGitClone mode)
|
||||
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,182 @@
|
||||
# Postmortem — Template Linux: deploy/prepare "no IP" (2026-06-06)
|
||||
|
||||
## Sintesi
|
||||
|
||||
Durante l'aggiunta di dipendenze GUI al toolchain del template Linux
|
||||
(`python3-gi gir1.2-gtk-4.0 xvfb`), una serie di fallimenti ha aperto una lunga
|
||||
sessione di debug. Sono emersi **due problemi distinti**, più una causa
|
||||
ambientale che ha confuso la diagnosi:
|
||||
|
||||
1. **Origine reale della sessione** — `prepare-linux` falliva allo Step 4a con
|
||||
`Unable to locate package gir1.2-gtk-4.0`. Causa: `--skip-update` saltava
|
||||
anche `apt-get update`, lasciando l'indice apt vecchio. **Non c'entrava la
|
||||
rete.** Fix banale, applicato.
|
||||
|
||||
2. **Bug `deploy-linux --force`** — il resize allo Step 4 falliva con
|
||||
*"disk is part of a snapshot chain"* perché `--force` riusava la directory
|
||||
senza rimuovere la snapshot chain precedente. Bug deterministico, colpito
|
||||
due volte. Fix applicato.
|
||||
|
||||
3. **Saga "no IP" (ambientale)** — il rilevamento IP via open-vm-tools andava in
|
||||
timeout in modo **intermittente**. Causa finale: il servizio DHCP/NAT di
|
||||
vmnet8 sull'host si era **inceppato** a causa della churn di debug (decine di
|
||||
`vmrun start/stop`, `tcpdump` promiscuo, DISCOVER ripetuti del guest).
|
||||
Risolto con un **restart di `vmware-networks`**. Non è un bug del codice né
|
||||
del CI normale.
|
||||
|
||||
Stato finale: template `BaseClean-Linux` ricostruito da powered-off, con gtk/xvfb
|
||||
e cloud-init sigillato. Tutti i test passano (coverage 94.4%).
|
||||
|
||||
---
|
||||
|
||||
## Cronologia
|
||||
|
||||
| Ora (host) | Evento |
|
||||
|-----------|--------|
|
||||
| ~10:50 | `prepare-linux --skip-update` → fallisce allo Step 4a (`gir1.2-gtk-4.0` non trovato) per indice apt stale |
|
||||
| ~11:00 | Dopo fix apt + revert snapshot: `prepare-linux` riesce, guest ottiene IP `.153`, snapshot ripreso |
|
||||
| ~11:03+ | I boot successivi dello snapshot non ottengono IP utile → inizio debug rete |
|
||||
| 11:20 | Cattura sul filo: DHCP OFFER/ACK presenti, ma il guest non applica l'IPv4 |
|
||||
| 12:06 | Deploy fresco: il guest **ottiene** `.143` (cloud-init), ma deploy Step 7 va in timeout sui tools |
|
||||
| 12:26 | Reboot: stesso guest **non** ottiene IP (intermittenza) |
|
||||
| ~12:30 | `vmware-networks --stop/--start` → l'IP torna stabile. Sblocco confermato |
|
||||
| dopo | Implementati e testati i fix; template rigenerato |
|
||||
|
||||
---
|
||||
|
||||
## Analisi cause
|
||||
|
||||
### 2.1 Step 4a apt (origine della sessione)
|
||||
|
||||
`Install-CIToolchain-Linux2404.sh` faceva `apt-get update` **solo** se non era
|
||||
passato `--skip-update`. Ma `--skip-update` dovrebbe saltare il solo *upgrade*
|
||||
(lento), non il refresh dell'indice. Con indice vecchio, i pacchetti nuovi
|
||||
(`gir1.2-gtk-4.0`, `xvfb`) non venivano trovati.
|
||||
|
||||
A riprova che a quel punto la rete funzionava: quel `prepare-linux` era entrato
|
||||
in SSH nel guest senza problemi.
|
||||
|
||||
### 2.2 deploy-linux --force e la snapshot chain
|
||||
|
||||
`deploy-linux --force` riusa la directory della VM ma non rimuoveva gli artefatti
|
||||
di un'eventuale snapshot precedente (`-NNNNNN.vmdk`, `.vmsn`, `.vmsd`). Con la
|
||||
chain presente, `vmware-vdiskmanager` rifiuta il resize dello Step 4:
|
||||
|
||||
```
|
||||
This disk is part of a snapshot chain in '.../LinuxBuild2404.vmx'.
|
||||
The selected operation can only be executed on a disk with no snapshots.
|
||||
```
|
||||
|
||||
### 2.3 La saga "no IP" — vmnet8 DHCP inceppato
|
||||
|
||||
Sintomo: `vmrun getGuestIPAddress` → *"VMware Tools are not running"* /
|
||||
*"Unable to get the IP address"*, **in modo intermittente**. Il guest in realtà:
|
||||
|
||||
- bootava correttamente (login prompt),
|
||||
- aveva open-vm-tools installato e attivo,
|
||||
- ma `systemd-networkd` a volte **non applicava** un lease DHCPv4 (link up,
|
||||
solo IPv6LL, `networkd-wait-online` in timeout).
|
||||
|
||||
Sul filo il DHCP a volte completava (`.143` applicato e raggiungibile), a volte
|
||||
no. Il server rispondeva (altre VM ottenevano lease). La causa finale: il demone
|
||||
`vmnet-dhcpd`/`vmnet-natd` di vmnet8 era andato in uno stato degradato dopo la
|
||||
churn di debug. Il **restart** `vmware-networks --stop/--start` lo ha sbloccato
|
||||
e l'IP è tornato stabile e immediato.
|
||||
|
||||
#### Ipotesi scartate (vicoli ciechi)
|
||||
- **Controller pvscsi vs lsilogic** — il boot funzionava con pvscsi; irrilevante.
|
||||
- **Naming NIC `ens192` vs `ens160`** — cloud-init si adattava; l'IP arrivava lo
|
||||
stesso quando la rete funzionava.
|
||||
- **Saturazione del pool DHCP** — il pool era di fatto libero (0 lease attivi);
|
||||
i MAC multipli per IP nel lease DB sono storia normale di ISC dhcpd.
|
||||
- **Seal cloud-init / machine-id** — difensivi ma non erano la causa del no-IP.
|
||||
|
||||
---
|
||||
|
||||
## Fix applicati (branch `fix/linux-template-toolchain-deploy`)
|
||||
|
||||
### Commit 1 — toolchain
|
||||
`template/Install-CIToolchain-Linux2404.sh`
|
||||
- **Step 4a**: installa `python3-gi gir1.2-gtk-4.0 xvfb` con assert su
|
||||
`import gi` e `Xvfb` (richiesta originale).
|
||||
- **Step 1**: `apt-get update` ora gira **sempre**; `--skip-update` salta solo
|
||||
l'upgrade. Risolve il fallimento Step 4a.
|
||||
|
||||
### Commit 2 — robustezza deploy/prepare
|
||||
`src/ci_orchestrator/commands/template.py` (+ test)
|
||||
- **`_clean_force_artifacts`**: con `--force` rimuove snapshot delta, `.vmsn`,
|
||||
`.vmsd` e `.lck` prima del resize → niente più errore snapshot-chain.
|
||||
Coperto da 3 unit test.
|
||||
- **prepare-linux Step 7d (seal cloud-init)**: rimuove la `50-cloud-init.yaml`
|
||||
stale, rigenera netplan (resta solo il `99-ci-dhcp-allnics` MAC-agnostic),
|
||||
disabilita cloud-init (`/etc/cloud/cloud-init.disabled`) così non rigira sui
|
||||
cloni.
|
||||
- **deploy user-data**: `package_update/upgrade: false` per non tenere il lock
|
||||
dpkg al primo boot e non ritardare open-vm-tools oltre il poll dello Step 7.
|
||||
- **Step 7 timeout**: `300s → 600s` come rete di sicurezza per primi boot lenti.
|
||||
|
||||
Verifica: `ruff` ✅, `mypy --strict` (23 file) ✅, suite completa ✅,
|
||||
coverage **94.4%** (gate 90%). Prod venv reinstallato (non-editable).
|
||||
|
||||
---
|
||||
|
||||
## Procedura di recovery: deploy/prepare non ottiene IP
|
||||
|
||||
Quando `deploy-linux` (Step 7) o `prepare-linux` non riescono a rilevare l'IP
|
||||
**ma il guest sembra sano** (booting, tools attivi), prima di sospettare
|
||||
immagine o codice, riavvia il networking VMware sull'host:
|
||||
|
||||
```bash
|
||||
sudo /usr/bin/vmware-networks --stop
|
||||
sudo /usr/bin/vmware-networks --start
|
||||
```
|
||||
|
||||
Flush opzionale dei lease (backup prima), **solo se nessuna VM è accesa**:
|
||||
|
||||
```bash
|
||||
sudo cp /etc/vmware/vmnet8/dhcpd/dhcpd.leases /tmp/dhcpd.leases.bak
|
||||
sudo sh -c ': > /etc/vmware/vmnet8/dhcpd/dhcpd.leases'
|
||||
```
|
||||
|
||||
Distinto da:
|
||||
- **Tools-not-running sul template** → revert a `BaseClean-Linux`
|
||||
(vedi `template-live-state-recovery`).
|
||||
- **vmmon/vmnet non compilati dopo bump kernel** → il guard systemd ricostruisce
|
||||
i moduli al boot (vedi `vmware-modules-kernel-upgrade`).
|
||||
|
||||
---
|
||||
|
||||
## Raccomandazioni (NON applicate di proposito)
|
||||
|
||||
### Cosa NON fare
|
||||
- **Restart/flush automatico ad ogni clone/deploy/prepare.** `vmware-networks`
|
||||
è globale (bridge + hostonly + NAT insieme): un restart sotto un job in corso
|
||||
rompe la rete di **tutte** le VM in esecuzione → job concorrenti falliti. Il
|
||||
flush dei lease con VM attive causa conflitti IP al rinnovo. Sproporzionato
|
||||
rispetto a un problema non dimostrato ricorrente.
|
||||
|
||||
### Cosa fare
|
||||
- **Monitorare.** L'inceppamento è nato dalla churn anomala di debug, non dal CI
|
||||
normale. Tenere la recovery manuale e agire solo se ricapita con dati reali.
|
||||
|
||||
### Se servisse hardening preventivo (a freddo, con cautela)
|
||||
- Abbassare `max-lease-time` (i cloni vivono minuti → IP liberati in fretta; i
|
||||
job lunghi rinnovano da soli via networkd).
|
||||
- Allargare il range DHCP per più margine.
|
||||
- **Caveat**: `/etc/vmware/vmnet8/dhcpd/dhcpd.conf` è **auto-generato da VMware**
|
||||
("DO NOT MODIFY", validato con `CFG_HASH` in `/etc/vmware/networking`). Edit a
|
||||
mano fragili: rischio rigenerazione al prossimo `--configure`/update. Da fare
|
||||
solo se il problema si dimostra ricorrente.
|
||||
|
||||
---
|
||||
|
||||
## Lezioni
|
||||
|
||||
- L'attività di debug intensa sulla rete host (start/stop ripetuti, sniffing
|
||||
promiscuo) può degradare `vmnet-dhcpd`: in futuro, minimizzare la churn e
|
||||
preferire osservazione mirata.
|
||||
- Tre cause distinte si sono sovrapposte; separarle prima ha richiesto tempo. Il
|
||||
guest era fondamentalmente sano per quasi tutta la sessione — l'errore di
|
||||
rilevamento IP non implicava un guest rotto.
|
||||
- Non sovrastimare: i fix committati coprono i bug **veri e provati**; il DHCP è
|
||||
stato lasciato invariato per evitare over-engineering.
|
||||
Executable
+200
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env bash
|
||||
# fix-ci-storage-layout.sh — Normalizza il layout /var/lib/ci dopo migrazione da Windows
|
||||
#
|
||||
# Il disco conteneva già i dati CI con nomi Windows (BuildVMs, Templates, ecc.).
|
||||
# setup-host-linux.sh ha creato le versioni lowercase Linux (build-vms, templates, ecc.).
|
||||
# Questo script unisce i duplicati e rinomina le directory Windows-only.
|
||||
#
|
||||
# MODALITÀ DRY-RUN per default: mostra cosa farebbe senza toccare nulla.
|
||||
# Usa --apply per eseguire le modifiche.
|
||||
#
|
||||
# Uso:
|
||||
# sudo bash fix-ci-storage-layout.sh [--apply] [--root /var/lib/ci]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CI_ROOT="/var/lib/ci"
|
||||
DRY_RUN=true
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN} ✓${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW} !${NC} $*"; }
|
||||
plan() { echo -e "${CYAN} →${NC} $*"; }
|
||||
die() { echo -e "${RED}ERRORE: $*${NC}" >&2; exit 1; }
|
||||
|
||||
run() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
plan "[DRY-RUN] $*"
|
||||
else
|
||||
eval "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Parse args ────────────────────────────────────────────────────────────────
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--apply) DRY_RUN=false; shift ;;
|
||||
--root) CI_ROOT="$2"; shift 2 ;;
|
||||
*) echo "Uso: sudo bash $0 [--apply] [--root <path>]"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ $EUID -ne 0 ]] && die "Eseguire come root: sudo bash $0 ..."
|
||||
[[ ! -d "$CI_ROOT" ]] && die "$CI_ROOT non esiste"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════╗"
|
||||
echo "║ DRY-RUN — nessuna modifica verrà apportata ║"
|
||||
echo "║ Riesegui con --apply per applicare le fix ║"
|
||||
echo "╚══════════════════════════════════════════════════╝"
|
||||
else
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════╗"
|
||||
echo "║ APPLICAZIONE MODIFICHE ║"
|
||||
echo "╚══════════════════════════════════════════════════╝"
|
||||
fi
|
||||
echo " CI_ROOT: $CI_ROOT"
|
||||
echo ""
|
||||
|
||||
# ── Funzione merge: sposta contenuto di SRC in DST ───────────────────────────
|
||||
# Gestisce conflitti a livello top-level (non ricorsivo: i subdir ci/sono distinti)
|
||||
merge_into() {
|
||||
local src="$CI_ROOT/$1"
|
||||
local dst="$CI_ROOT/$2"
|
||||
local had_conflict=false
|
||||
|
||||
echo "── Merge: $1/ → $2/"
|
||||
if [[ ! -d "$src" ]]; then
|
||||
warn "$1/ non presente — skip"
|
||||
return
|
||||
fi
|
||||
|
||||
shopt -s dotglob nullglob
|
||||
local items=("$src"/*)
|
||||
shopt -u dotglob nullglob
|
||||
|
||||
if [[ ${#items[@]} -eq 0 ]]; then
|
||||
warn "$1/ è vuota — rimuovo solo la directory"
|
||||
run "rmdir '$src'"
|
||||
return
|
||||
fi
|
||||
|
||||
for item in "${items[@]}"; do
|
||||
local name
|
||||
name=$(basename "$item")
|
||||
if [[ -e "$dst/$name" ]]; then
|
||||
warn " CONFLITTO: $name già esiste in $2/ — skip (controlla manualmente)"
|
||||
had_conflict=true
|
||||
else
|
||||
run "mv '$item' '$dst/'"
|
||||
info " $1/$name → $2/$name"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$had_conflict" == false ]]; then
|
||||
run "rmdir '$src'"
|
||||
info " Rimossa: $1/ (vuota dopo merge)"
|
||||
else
|
||||
warn " $1/ NON rimossa: contiene ancora file in conflitto"
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ── Funzione rename: rinomina directory Windows-only ─────────────────────────
|
||||
rename_dir() {
|
||||
local src="$CI_ROOT/$1"
|
||||
local dst="$CI_ROOT/$2"
|
||||
|
||||
echo "── Rinomina: $1/ → $2/"
|
||||
if [[ ! -d "$src" ]]; then
|
||||
warn "$1/ non presente — skip"
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
if [[ -d "$dst" ]]; then
|
||||
warn "$2/ esiste già — usa merge invece di rename"
|
||||
merge_into "$1" "$2"
|
||||
return
|
||||
fi
|
||||
run "mv '$src' '$dst'"
|
||||
info "$1/ → $2/"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ── Funzione remove: rimuove directory non necessaria su Linux ────────────────
|
||||
remove_dir() {
|
||||
local target="$CI_ROOT/$1"
|
||||
local reason="$2"
|
||||
|
||||
echo "── Rimuovi: $1/ ($reason)"
|
||||
if [[ ! -d "$target" ]]; then
|
||||
warn "$1/ non presente — skip"
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
local count
|
||||
count=$(find "$target" -mindepth 1 | wc -l)
|
||||
if [[ "$count" -gt 0 ]]; then
|
||||
warn "$1/ contiene $count file/dir — rimozione manuale consigliata"
|
||||
warn " Esegui: sudo rm -rf '$target'"
|
||||
else
|
||||
run "rmdir '$target'"
|
||||
info "Rimossa: $1/ (vuota)"
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# 1. MERGE: directory con doppio (Windows uppercase + Linux lowercase)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
echo "━━━ FASE 1: Merge duplicati (Windows → Linux) ━━━"
|
||||
echo ""
|
||||
merge_into "Artifacts" "artifacts"
|
||||
merge_into "BuildVMs" "build-vms"
|
||||
merge_into "Logs" "logs"
|
||||
merge_into "Templates" "templates"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# 2. RINOMINA: directory Windows-only utili su Linux
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
echo "━━━ FASE 2: Rinomina directory Windows-only ━━━"
|
||||
echo ""
|
||||
rename_dir "ISO" "iso"
|
||||
rename_dir "Cache" "cache"
|
||||
rename_dir "State" "state" # stato orchestrator — potrebbe contenere dati utili
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# 3. RIMUOVI: directory non necessarie su Linux
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
echo "━━━ FASE 3: Directory non necessarie su Linux ━━━"
|
||||
echo ""
|
||||
# RunnerWork: working dir di act_runner su Windows; su Linux è /var/lib/ci/runner/
|
||||
remove_dir "RunnerWork" "working dir act_runner Windows — su Linux è runner/"
|
||||
# python: venv Windows (F:\CI\python\venv); su Linux è /opt/ci/venv
|
||||
remove_dir "python" "venv Python Windows — su Linux è /opt/ci/venv"
|
||||
# act_runner: binario act_runner Windows; su Linux è /opt/ci/act_runner/
|
||||
remove_dir "act_runner" "binario act_runner Windows — su Linux è /opt/ci/act_runner/"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# 4. RIEPILOGO
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
echo "━━━ Layout atteso dopo fix ━━━"
|
||||
echo ""
|
||||
echo " /var/lib/ci/"
|
||||
for d in build-vms artifacts templates keys logs runner iso cache state; do
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
[[ -d "$CI_ROOT/$d" ]] && echo " ├── $d/" || echo " ├── $d/ ← MANCANTE"
|
||||
else
|
||||
echo " ├── $d/"
|
||||
fi
|
||||
done
|
||||
echo " ├── config.toml"
|
||||
echo " └── lost+found (lasciato intatto)"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "Riesegui con --apply per applicare tutte le modifiche."
|
||||
else
|
||||
echo "Fix completato. Verifica con: sudo ls -la $CI_ROOT/"
|
||||
fi
|
||||
@@ -1,76 +0,0 @@
|
||||
# B5 — Closeout (parziale: artefatti repo)
|
||||
|
||||
Fase B5 di `plans/implementation-plan-A-B.md`: conversione scheduled task
|
||||
Windows in coppie `*.service` + `*.timer` systemd su Linux.
|
||||
|
||||
## Cosa è stato fatto in questa sessione (su host Windows dev)
|
||||
|
||||
- [x] Inventario dei 4 task in `scripts/Register-CIScheduledTasks.ps1` (cadenze + comandi)
|
||||
- [x] 5 coppie `*.service` + `*.timer` create in `deploy/systemd/`:
|
||||
- `ci-cleanup-orphans.{service,timer}` — every 6h + at boot → `python -m ci_orchestrator vm cleanup --max-age-hours 6`
|
||||
- `ci-retention-policy.{service,timer}` — daily 03:00 + 30min jitter → `pwsh Invoke-RetentionPolicy.ps1`
|
||||
- `ci-watch-disk-space.{service,timer}` — every 15min → `python -m ci_orchestrator monitor disk`
|
||||
- `ci-watch-runner-health.{service,timer}` — every 15min → `python -m ci_orchestrator monitor runner`
|
||||
- `ci-backup-template.{service,timer}` — weekly Sun 02:00 + 1h jitter → `pwsh Backup-CITemplate.ps1` (NEW, non era in Register-CIScheduledTasks.ps1)
|
||||
- [x] `deploy/systemd/README.md` con: tabella mapping, prerequisiti (utente, venv, env file), istruzioni install/test/rollback, nota PowerShell Core su Linux
|
||||
|
||||
## Note sulle cadenze
|
||||
|
||||
Le cadenze del piano (`OnCalendar=hourly`, `*:0/15`, ecc.) sono state
|
||||
adattate per fedeltà semantica all'originale Windows:
|
||||
|
||||
| Task | Original Windows | systemd timer scelto |
|
||||
| --------------------- | --------------------------------------------- | ----------------------------------------------------- |
|
||||
| cleanup-orphans | every 6h + at startup | `OnBootSec=2min, OnUnitActiveSec=6h, Persistent` |
|
||||
| retention-policy | daily 03:00 + 30min random delay | `OnCalendar=*-*-* 03:00:00, RandomizedDelaySec=30min` |
|
||||
| watch-disk-space | every 15min | `OnBootSec=5min, OnUnitActiveSec=15min` |
|
||||
| watch-runner-health | every 15min | `OnBootSec=3min, OnUnitActiveSec=15min` |
|
||||
| backup-template | *(non in Register-CIScheduledTasks.ps1)* | `OnCalendar=Sun *-*-* 02:00:00, RandomizedDelaySec=1h`|
|
||||
|
||||
`Persistent=true` ovunque per non perdere esecuzioni se la macchina è
|
||||
spenta al momento del trigger.
|
||||
|
||||
## Decisione — script che restano in PowerShell
|
||||
|
||||
`Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` non sono stati
|
||||
portati a Python (resta scelta documentata in `AGENTS.md` "Mappatura").
|
||||
Su Linux quindi le rispettive `.service` invocano `pwsh` (PowerShell
|
||||
Core). Il README documenta l'installazione `apt install powershell` da
|
||||
repo Microsoft. Se in futuro si deciderà di portarli a Python (sub-comandi
|
||||
`retention run` / `template backup`), basterà sostituire `ExecStart=`
|
||||
nelle due unit senza toccare il timer.
|
||||
|
||||
## Definizione di "fatto" B5 (dal piano)
|
||||
|
||||
| Criterio | Stato |
|
||||
| --------------------------------------------------- | ------------------------------------ |
|
||||
| Tutti i task periodici come timer systemd attivi | ⏳ unit pronti; install richiede host Linux |
|
||||
| Trigger manuale di ognuno PASS | ⏳ richiede host Linux |
|
||||
| Mapping documentato | ✅ `deploy/systemd/README.md` |
|
||||
|
||||
## Cosa resta a carico utente (sul nuovo host Linux)
|
||||
|
||||
Tutti i passi B1–B4 e B6–B8 sono operazioni hardware/host non eseguibili
|
||||
da questa sessione. Per chiudere B5 una volta sull'host Linux:
|
||||
|
||||
```bash
|
||||
# Prerequisito: aver completato B1 (ci-runner user, /opt/ci/venv, /etc/ci/environment)
|
||||
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \
|
||||
ci-watch-runner-health ci-backup-template; do
|
||||
sudo systemctl enable --now "${t}.timer"
|
||||
done
|
||||
systemctl list-timers --all 'ci-*'
|
||||
|
||||
# Trigger manuale di smoke
|
||||
sudo systemctl start ci-cleanup-orphans.service
|
||||
journalctl -u ci-cleanup-orphans.service -n 50 --no-pager
|
||||
```
|
||||
|
||||
## Riferimenti
|
||||
|
||||
- Spec systemd timer: `man systemd.timer`, `man systemd.time`
|
||||
- Original PS task scheduler: `scripts/Register-CIScheduledTasks.ps1`
|
||||
- Mapping completo: `deploy/systemd/README.md`
|
||||
- Piano: `plans/implementation-plan-A-B.md` sezione B5
|
||||
@@ -1,484 +0,0 @@
|
||||
# Fase B — Checklist finale per l'utente
|
||||
|
||||
La Fase B è la migrazione del runner CI dall'host Windows attuale a un
|
||||
nuovo host **Linux Mint LTS** con VMware Workstation Pro Linux.
|
||||
|
||||
> **Stato repo**: solo lo step **B5** (unit `systemd` per i task
|
||||
> periodici) è già committato in `deploy/systemd/` sul branch
|
||||
> `feature/python-rewrite-and-linux-migration` (commit `50d37b5`).
|
||||
> Tutti gli altri step (B1–B4, B6–B8) sono **operazioni hardware**
|
||||
> sull'host Linux nuovo e devono essere eseguiti dall'utente.
|
||||
|
||||
> **Pre-requisito**: la Fase A deve essere `done` (vedi
|
||||
> `plans/PhaseA-user-checklist.md` Passo 9, merge + tag).
|
||||
> B1, B2, B3 possono partire in parallelo a A3/A4/A5; B4–B8 sono
|
||||
> sequenziali e iniziano solo dopo "A done".
|
||||
|
||||
> **Tempo stimato totale**: 1–2 giorni di lavoro distribuiti su una
|
||||
> finestra di 1–2 settimane (per consentire il burn-in tra B6 e B8).
|
||||
|
||||
---
|
||||
|
||||
## Passo 1 — Setup host Linux Mint (B1)
|
||||
|
||||
**Obiettivo**: hardware target con Linux Mint LTS, VMware Workstation
|
||||
Pro Linux, layout storage e venv Python pronti.
|
||||
|
||||
- [ ] Installare Linux Mint LTS sull'hardware target.
|
||||
- [ ] `sudo apt update && sudo apt full-upgrade -y && sudo reboot`
|
||||
- [ ] Scaricare e installare VMware Workstation Pro Linux (bundle
|
||||
`.bundle` da Broadcow):
|
||||
|
||||
```bash
|
||||
sudo bash VMware-Workstation-Full-*.bundle
|
||||
vmrun -T ws --version # deve ritornare versione coerente
|
||||
```
|
||||
|
||||
- [ ] Smoke test Workstation (UI): creare VM trivia, clone, start,
|
||||
stop, delete.
|
||||
- [ ] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato a
|
||||
Windows) editando `/etc/vmware/vmnet8/dhcpd/dhcpd.conf` o via
|
||||
`vmware-netcfg`.
|
||||
- [ ] Creare utente di servizio `ci-runner`:
|
||||
|
||||
```bash
|
||||
sudo useradd -r -m -s /bin/bash ci-runner
|
||||
```
|
||||
|
||||
- [ ] Creare layout storage:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /var/lib/ci/{build-vms,artifacts,templates,keys}
|
||||
sudo chown -R ci-runner:ci-runner /var/lib/ci
|
||||
sudo chmod 750 /var/lib/ci /var/lib/ci/*
|
||||
sudo setfacl -d -m u:ci-runner:rwX /var/lib/ci/build-vms
|
||||
```
|
||||
|
||||
- [ ] Installare Python 3.11+ e creare venv produzione:
|
||||
|
||||
```bash
|
||||
sudo apt install -y python3.11 python3.11-venv git
|
||||
sudo mkdir -p /opt/ci
|
||||
sudo python3.11 -m venv /opt/ci/venv
|
||||
sudo chown -R ci-runner:ci-runner /opt/ci
|
||||
```
|
||||
|
||||
- [ ] Clonare il repo e installare il package nel venv:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner git clone https://gitea.emulab.it/Simone/local-ci-cd-system.git /opt/ci/local-ci-cd-system
|
||||
sudo -u ci-runner /opt/ci/venv/bin/pip install -e /opt/ci/local-ci-cd-system
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help
|
||||
```
|
||||
|
||||
Atteso: lista degli 11 sub-comandi.
|
||||
|
||||
- [ ] Installare PowerShell Core (richiesto da `Invoke-RetentionPolicy.ps1`
|
||||
e `Backup-CITemplate.ps1`):
|
||||
|
||||
```bash
|
||||
sudo apt install -y wget apt-transport-https software-properties-common
|
||||
source /etc/os-release
|
||||
wget -q https://packages.microsoft.com/config/ubuntu/${VERSION_ID}/packages-microsoft-prod.deb
|
||||
sudo dpkg -i packages-microsoft-prod.deb
|
||||
sudo apt update && sudo apt install -y powershell
|
||||
pwsh --version
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Passo 2 — Trasferimento template VM (B2)
|
||||
|
||||
**Obiettivo**: copiare i template VMware da `F:\CI\Templates\` su
|
||||
`/var/lib/ci/templates/` mantenendo gli snapshot.
|
||||
|
||||
> ⚠️ AGENTS.md errore #9: i template devono essere **fully
|
||||
> powered-off** prima della copia. Verificare assenza di `*.vmem` /
|
||||
> `*.vmsn` di runtime.
|
||||
|
||||
- [ ] Sull'host Windows:
|
||||
|
||||
```powershell
|
||||
Get-ChildItem 'F:\CI\Templates' -Recurse -Include *.vmem,*.vmsn
|
||||
# atteso: nessun risultato
|
||||
```
|
||||
|
||||
- [ ] Sull'host Linux, eseguire rsync via SSH dall'host Windows
|
||||
(richiede OpenSSH Server attivo su Windows o, in alternativa,
|
||||
`scp -r` lanciato da Windows verso Linux):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner rsync -av --progress \
|
||||
user@windows-host:/cygdrive/f/CI/Templates/ \
|
||||
/var/lib/ci/templates/
|
||||
```
|
||||
|
||||
- [ ] Validare integrità snapshot:
|
||||
|
||||
```bash
|
||||
find /var/lib/ci/templates -name '*.vmx' -exec vmrun -T ws listSnapshots {} \;
|
||||
```
|
||||
|
||||
Atteso: `BaseClean` su `WinBuild2025.vmx` e `WinBuild2022.vmx`,
|
||||
`BaseClean-Linux` su `LinuxBuild2404.vmx`.
|
||||
|
||||
- [ ] Aprire ciascun `.vmx` su Workstation Linux per registrarli;
|
||||
al prompt rispondere "**I copied it**".
|
||||
- [ ] Smoke test pipeline VM:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm new \
|
||||
--template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \
|
||||
--snapshot BaseClean --name smoke-b2-win
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator wait-ready \
|
||||
--vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx \
|
||||
--guest-os windows --timeout 180
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm remove \
|
||||
--vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx --force
|
||||
```
|
||||
|
||||
Ripetere per `LinuxBuild2404` (snapshot `BaseClean-Linux`,
|
||||
`--guest-os linux`).
|
||||
|
||||
---
|
||||
|
||||
## Passo 3 — Trasferimento credenziali e chiavi (B3)
|
||||
|
||||
**Obiettivo**: chiavi SSH e credenziali guest disponibili al user
|
||||
`ci-runner` headless.
|
||||
|
||||
- [ ] Copiare le chiavi SSH guest Linux:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /etc/ci/keys
|
||||
sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux /etc/ci/keys/
|
||||
sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux.pub /etc/ci/keys/
|
||||
sudo chown ci-runner:ci-runner /etc/ci/keys/*
|
||||
sudo chmod 600 /etc/ci/keys/ci_linux
|
||||
sudo chmod 644 /etc/ci/keys/ci_linux.pub
|
||||
```
|
||||
|
||||
- [ ] Re-store credenziale guest Windows (sostituire la password al
|
||||
prompt — **NON** inserirla in chat o issue):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner secret-tool store --label='BuildVMGuest' service ci target BuildVMGuest
|
||||
```
|
||||
|
||||
- [ ] Re-store Gitea PAT:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner secret-tool store --label='GiteaPAT' service ci target GiteaPAT
|
||||
```
|
||||
|
||||
- [ ] Validare lettura interattiva:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -c \
|
||||
"import keyring; print(keyring.get_credential('BuildVMGuest', None))"
|
||||
```
|
||||
|
||||
- [ ] **PoC headless** (critico): verificare lettura keyring da
|
||||
contesto systemd (no D-Bus user session):
|
||||
|
||||
```bash
|
||||
sudo systemd-run --uid=ci-runner --pipe \
|
||||
/opt/ci/venv/bin/python -c \
|
||||
"import keyring; print(keyring.get_credential('BuildVMGuest', None))"
|
||||
```
|
||||
|
||||
Se fallisce → implementare backend keyring file-based con `age` o
|
||||
`sops` come fallback. Documentare la scelta finale in
|
||||
`docs/HOST-SETUP.md` prima di proseguire.
|
||||
|
||||
---
|
||||
|
||||
## Passo 4 — Setup act_runner come systemd service (B4)
|
||||
|
||||
**Obiettivo**: act_runner Linux registrato verso Gitea, gestito da
|
||||
systemd.
|
||||
|
||||
- [ ] Scaricare il binario act_runner Linux ≥ v1.0.2:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /opt/ci/act_runner
|
||||
sudo wget -O /opt/ci/act_runner/act_runner \
|
||||
https://gitea.com/gitea/act_runner/releases/download/v1.0.2/act_runner-1.0.2-linux-amd64
|
||||
sudo chmod +x /opt/ci/act_runner/act_runner
|
||||
sudo chown -R ci-runner:ci-runner /opt/ci/act_runner
|
||||
```
|
||||
|
||||
- [ ] Generare token registrazione su Gitea (Admin → Runners → Create new runner).
|
||||
- [ ] Registrare il runner (sostituire `<TOKEN>`):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner mkdir -p /var/lib/ci/runner
|
||||
cd /var/lib/ci/runner
|
||||
sudo -u ci-runner /opt/ci/act_runner/act_runner register \
|
||||
--no-interactive \
|
||||
--instance https://gitea.emulab.it \
|
||||
--token <TOKEN> \
|
||||
--name ci-linux \
|
||||
--labels windows-build:host,linux-build:host
|
||||
```
|
||||
|
||||
> ⚠️ Lasciare il runner in stato **paused** lato Gitea UI fino al
|
||||
> cutover (Passo 6), per non intercettare job di produzione.
|
||||
|
||||
- [ ] Creare `/etc/systemd/system/act-runner.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Gitea Act Runner
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ci-runner
|
||||
WorkingDirectory=/var/lib/ci/runner
|
||||
Environment="PYTHONIOENCODING=utf-8"
|
||||
Environment="CI_ROOT=/var/lib/ci"
|
||||
Environment="CI_TEMPLATES=/var/lib/ci/templates"
|
||||
Environment="CI_BUILD_VMS=/var/lib/ci/build-vms"
|
||||
Environment="CI_ARTIFACTS=/var/lib/ci/artifacts"
|
||||
Environment="CI_KEYS=/etc/ci/keys"
|
||||
ExecStart=/opt/ci/act_runner/act_runner daemon
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
- [ ] Creare `/etc/ci/environment` (letto dalle unit dei task periodici
|
||||
di B5; vedi `deploy/systemd/README.md`):
|
||||
|
||||
```bash
|
||||
sudo tee /etc/ci/environment <<'EOF'
|
||||
PYTHONIOENCODING=utf-8
|
||||
CI_ROOT=/var/lib/ci
|
||||
CI_TEMPLATES=/var/lib/ci/templates
|
||||
CI_BUILD_VMS=/var/lib/ci/build-vms
|
||||
CI_ARTIFACTS=/var/lib/ci/artifacts
|
||||
CI_KEYS=/etc/ci/keys
|
||||
EOF
|
||||
sudo chmod 644 /etc/ci/environment
|
||||
```
|
||||
|
||||
- [ ] Abilitare e avviare:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now act-runner.service
|
||||
sudo systemctl status act-runner
|
||||
sudo journalctl -u act-runner -f # Ctrl+C dopo conferma OK
|
||||
```
|
||||
|
||||
- [ ] Smoke job: triggerare manualmente `self-test.yml` selezionando
|
||||
il runner Linux (mettere temporaneamente in pausa quello Windows
|
||||
o usare label distinte). Atteso: PASS.
|
||||
|
||||
---
|
||||
|
||||
## Passo 5 — Installare i timer systemd (B5)
|
||||
|
||||
**Obiettivo**: attivare le coppie `.service`+`.timer` già committate
|
||||
in `deploy/systemd/` per replicare i task periodici di
|
||||
`Register-CIScheduledTasks.ps1`.
|
||||
|
||||
- [ ] Copiare le unit:
|
||||
|
||||
```bash
|
||||
cd /opt/ci/local-ci-cd-system
|
||||
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
- [ ] Abilitare e avviare tutti i timer:
|
||||
|
||||
```bash
|
||||
for t in ci-cleanup-orphans ci-retention-policy \
|
||||
ci-watch-disk-space ci-watch-runner-health \
|
||||
ci-backup-template; do
|
||||
sudo systemctl enable --now "${t}.timer"
|
||||
done
|
||||
```
|
||||
|
||||
- [ ] Verificare lo schedule:
|
||||
|
||||
```bash
|
||||
systemctl list-timers --all 'ci-*'
|
||||
```
|
||||
|
||||
Atteso: 5 timer in stato `active`.
|
||||
|
||||
- [ ] Trigger manuale di ciascun service per validare l'esecuzione
|
||||
one-shot:
|
||||
|
||||
```bash
|
||||
for s in ci-cleanup-orphans ci-watch-disk-space ci-watch-runner-health; do
|
||||
sudo systemctl start "${s}.service"
|
||||
sudo systemctl status "${s}.service" --no-pager
|
||||
done
|
||||
```
|
||||
|
||||
> `ci-retention-policy` e `ci-backup-template` chiamano `pwsh`:
|
||||
> eseguirli solo dopo aver verificato che `pwsh --version` funzioni
|
||||
> sotto user `ci-runner`.
|
||||
|
||||
- [ ] Per maggiori dettagli (mapping Windows→Linux, troubleshooting,
|
||||
rollback) vedi `deploy/systemd/README.md`.
|
||||
|
||||
---
|
||||
|
||||
## Passo 6 — Cutover (B6)
|
||||
|
||||
**Obiettivo**: spostare la produzione dall'host Windows a Linux in una
|
||||
finestra di manutenzione concordata.
|
||||
|
||||
> ⚠️ Eseguire solo dopo che i Passi 1–5 sono tutti `[x]` PASS.
|
||||
|
||||
- [ ] Annunciare la finestra di manutenzione (durata stimata: 30–60 min).
|
||||
- [ ] Verificare nessun job in coda lato Gitea (Admin → Actions → Tasks).
|
||||
- [ ] **Stop** act_runner sull'host Windows:
|
||||
|
||||
```powershell
|
||||
Stop-Service actions-runner
|
||||
Get-Service actions-runner # atteso: Stopped
|
||||
```
|
||||
|
||||
- [ ] **Disabilitare** scheduled task Windows:
|
||||
|
||||
```powershell
|
||||
Get-ScheduledTask -TaskName 'CI-*' | Unregister-ScheduledTask -Confirm:$false
|
||||
```
|
||||
|
||||
- [ ] Rsync incrementale finale degli artifact:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner rsync -av --delete \
|
||||
user@windows-host:/cygdrive/f/CI/Artifacts/ \
|
||||
/var/lib/ci/artifacts/
|
||||
```
|
||||
|
||||
- [ ] Sbloccare il runner Linux su Gitea UI (rimuovere `paused` o
|
||||
ripristinare le label di produzione `windows-build:host,linux-build:host`).
|
||||
- [ ] Verificare runner Linux online su Gitea UI.
|
||||
- [ ] Smoke trigger:
|
||||
- [ ] `self-test.yml` → PASS dal runner Linux
|
||||
- [ ] `build-ns7zip.yml` matrix Win+Linux → PASS dal runner Linux
|
||||
- [ ] Monitorare per ≥30 min:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u act-runner -f --since "30min ago"
|
||||
```
|
||||
|
||||
Nessun errore di severità critica atteso.
|
||||
|
||||
> **Rollback (entro 30 min se PASS critico fallisce)**:
|
||||
> 1. `sudo systemctl stop act-runner` su Linux
|
||||
> 2. `Start-Service actions-runner` su Windows
|
||||
> 3. Re-registrare scheduled task con `scripts\Register-CIScheduledTasks.ps1`
|
||||
> 4. Documentare incidente in `TODO.md` e tornare a B1–B5 per fix
|
||||
|
||||
---
|
||||
|
||||
## Passo 7 — Capacity burn-in (B7)
|
||||
|
||||
**Obiettivo**: validare il carico target con tempi paragonabili al
|
||||
baseline Windows (entro ±20%).
|
||||
|
||||
- [ ] Burn-in 4 job concorrenti × 10 round su `WinBuild2025`
|
||||
(lo script `Test-CapacityBurnIn.ps1` ora delega via shim alla CLI
|
||||
Python `job` e gira anche da `pwsh` su Linux):
|
||||
|
||||
```bash
|
||||
pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \
|
||||
-Concurrency 4 -Rounds 10 -Template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx
|
||||
```
|
||||
|
||||
- [ ] Burn-in 4 × 10 su `LinuxBuild2404` (analogo, `--Template`
|
||||
sul VMX Linux).
|
||||
- [ ] Misurare:
|
||||
- [ ] Tempo medio per job, confronto con baseline A5
|
||||
- [ ] Tutti i 80 job (2 × 4 × 10) PASS
|
||||
- [ ] Zero VM orfane in `/var/lib/ci/build-vms/`:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --dry-run
|
||||
```
|
||||
|
||||
- [ ] Spazio disco `/var/lib/ci/build-vms/` torna al baseline post-cleanup
|
||||
- [ ] Documentare i risultati in `docs/RUNBOOK.md` (sezione "Linux
|
||||
host baseline").
|
||||
- [ ] Se delta > 20% vs baseline Windows: aprire issue in `TODO.md`
|
||||
con dettagli per profiling (probabile candidato: filesystem ext4
|
||||
vs NTFS — valutare XFS/BTRFS).
|
||||
|
||||
---
|
||||
|
||||
## Passo 8 — Stabilità ≥1 settimana (gate per B8)
|
||||
|
||||
**Obiettivo**: verificare ≥1 settimana di esercizio Linux senza
|
||||
incidenti critici prima di dismettere l'host Windows.
|
||||
|
||||
- [ ] Esercizio normale per ≥7 giorni con runner Linux primario.
|
||||
- [ ] Verifica giornaliera (bastano 2 minuti):
|
||||
|
||||
```bash
|
||||
sudo systemctl --failed # nessun service in failed
|
||||
systemctl list-timers --all 'ci-*' # tutti gli ultimi run OK
|
||||
sudo journalctl -u act-runner --since "24h ago" -p err
|
||||
```
|
||||
|
||||
- [ ] Nessun rollback effettuato durante la settimana.
|
||||
|
||||
---
|
||||
|
||||
## Passo 9 — Decommissioning host Windows (B8, opzionale)
|
||||
|
||||
> ⚠️ Procedere solo dopo Passo 8 `[x]`. L'host Windows va lasciato
|
||||
> spento ma intatto per ≥1 mese come rollback.
|
||||
|
||||
- [ ] Backup finale di `F:\CI\` su archivio esterno:
|
||||
|
||||
```powershell
|
||||
$stamp = Get-Date -Format 'yyyyMMdd'
|
||||
$archive = "E:\Backup\CI-$stamp.tar"
|
||||
& 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\'
|
||||
Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256"
|
||||
```
|
||||
|
||||
- [ ] Verifica integrità (estrazione di prova in tmp):
|
||||
|
||||
```powershell
|
||||
$test = "$env:TEMP\ci-restore-test"
|
||||
& 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20
|
||||
```
|
||||
|
||||
- [ ] Spegnere host Windows (`shutdown /s /t 0`).
|
||||
- [ ] Lasciare hardware integro e spento per ≥1 mese.
|
||||
- [ ] Aggiornare `docs/RUNBOOK.md` con la procedura di riaccensione +
|
||||
ri-registrazione scheduled task come rollback.
|
||||
- [ ] Dopo 1 mese senza rollback: riallocare hardware.
|
||||
|
||||
---
|
||||
|
||||
## Tracciamento globale
|
||||
|
||||
| Passo | Step | Descrizione | Stato |
|
||||
| ----- | ---- | -------------------------------------------------- | ----- |
|
||||
| 1 | B1 | Setup host Linux Mint (OS, Workstation, venv, ACL) | [ ] |
|
||||
| 2 | B2 | Trasferimento template VM + smoke `vm new` | [ ] |
|
||||
| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [ ] |
|
||||
| 4 | B4 | act_runner systemd service registrato | [ ] |
|
||||
| 5 | B5 | Timer systemd installati e attivi | [ ] |
|
||||
| 6 | B6 | Cutover (stop Windows, runner Linux primario) | [ ] |
|
||||
| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux) | [ ] |
|
||||
| 8 | — | ≥1 settimana di stabilità senza rollback | [ ] |
|
||||
| 9 | B8 | Decommissioning host Windows (opzionale) | [ ] |
|
||||
|
||||
Quando tutti i passi sono `[x]`, la **Fase B è chiusa** e si può
|
||||
valutare l'apertura della Fase C (backend ESXi, vedi
|
||||
`plans/idea-3-esxi-support.md`).
|
||||
@@ -0,0 +1,40 @@
|
||||
# B1 — Closeout
|
||||
|
||||
Passo B1 di `plans/PhaseB-user-checklist.md`: setup hardware target con
|
||||
Linux Mint LTS, VMware Workstation Pro Linux, layout storage e venv Python.
|
||||
|
||||
## Stato
|
||||
|
||||
✅ **COMPLETATO** — tutti i prerequisiti hardware e software soddisfatti.
|
||||
|
||||
## Cosa è stato fatto
|
||||
|
||||
- [x] Linux Mint LTS installato sull'hardware target (`ws1-mint`).
|
||||
- [x] Sistema aggiornato (`apt full-upgrade`) e riavviato.
|
||||
- [x] VMware Workstation Pro Linux installato via bundle `.bundle`; `vmrun -T ws --version` OK.
|
||||
- [x] Smoke test Workstation UI: VM trivia creata, clonata, avviata, spenta, eliminata.
|
||||
- [x] `vmnet8` NAT configurato su `192.168.79.0/24` (stesso range di Windows) via `vmware-netcfg`.
|
||||
- [x] Utente di servizio `ci-runner` creato (`useradd -r -m -s /bin/bash`).
|
||||
- [x] `setup-host-linux.sh -d /dev/sdf1` eseguito:
|
||||
- `/dev/sdf1` montata su `/var/lib/ci` via fstab con boot automatico.
|
||||
- Layout directory creato: `build-vms/`, `artifacts/`, `templates/`, `keys/`, `logs/`, `runner/`.
|
||||
- Python 3.12 installato, `/opt/ci/venv` creato.
|
||||
- Repo clonato in `/opt/ci/local-ci-cd-system`, package installato in modalità editabile.
|
||||
- PowerShell Core (`pwsh`) installato (necessario per B5 nel piano originale;
|
||||
sarà rimosso nella Fase C).
|
||||
- [x] Verifica finale: `sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help`
|
||||
lista tutti i sub-comandi.
|
||||
|
||||
## Deviazioni / gotcha
|
||||
|
||||
- Lo script `setup-host-linux.sh` è stato scritto e committato durante questa
|
||||
fase (commit `b9d6994`) — non era pre-esistente nel repo.
|
||||
- La partizione `/dev/sdf1` conteneva una sottocartella `ci/` residua dalla
|
||||
configurazione Windows; lo script la ha rilevata e spostata alla root prima
|
||||
del mount definitivo.
|
||||
- Python 3.12 (non 3.11 come da piano) installato perché è la versione LTS
|
||||
disponibile su Linux Mint LTS — il package è compatibile.
|
||||
|
||||
## Riferimenti commit
|
||||
|
||||
- `b9d6994` — feat: add setup-host-linux.sh script for CI host bootstrap
|
||||
@@ -0,0 +1,46 @@
|
||||
# B2 — Closeout
|
||||
|
||||
Passo B2 di `plans/PhaseB-user-checklist.md`: trasferimento template VM da
|
||||
`F:\CI\Templates\` a `/var/lib/ci/templates/` con integrità snapshot.
|
||||
|
||||
## Stato
|
||||
|
||||
✅ **COMPLETATO** — 3 template trasferiti, snapshot validati, smoke test PASS.
|
||||
|
||||
## Cosa è stato fatto
|
||||
|
||||
- [x] Verificato assenza di `*.vmem`/`*.vmsn` di runtime sui template Windows
|
||||
prima della copia (VM spente).
|
||||
- [x] Trasferimento via `rsync` SSH da Windows → Linux per tutti e 3 i template:
|
||||
- `WinBuild2025` (snapshot `BaseClean`)
|
||||
- `WinBuild2022` (snapshot `BaseClean`)
|
||||
- `LinuxBuild2404` (snapshot `BaseClean-Linux`)
|
||||
- [x] Snapshot validati con `vmrun -T ws listSnapshots` su ciascun `.vmx`.
|
||||
- [x] Ogni `.vmx` aperto in Workstation Linux GUI; risposto "**I copied it**"
|
||||
al prompt di registrazione.
|
||||
- [x] Keyring file-based configurato anticipatamente per `ci-runner`
|
||||
(prerequisito del smoke test Windows; il setup completo è in B3):
|
||||
- `keyrings.alt.file.PlaintextKeyring` scelto al posto di `secret-tool`
|
||||
(vedi Deviazioni).
|
||||
- Credenziale `BuildVMGuest` salvata con `tools/ci-cred-set.py`.
|
||||
- [x] Smoke test `vm new` + `wait-ready` + `vm remove` PASS su:
|
||||
- `WinBuild2025` (WinRM HTTPS/5986)
|
||||
- `LinuxBuild2404` (SSH/22)
|
||||
|
||||
## Deviazioni / gotcha
|
||||
|
||||
- **`secret-tool` scartato**: il piano originale usava `secret-tool` (GNOME
|
||||
Secret Service) per le credenziali headless. Non funziona senza una D-Bus
|
||||
session attiva, che non è disponibile quando il processo gira come servizio
|
||||
systemd (`act-runner.service`). Soluzione adottata: `keyrings.alt` con
|
||||
`PlaintextKeyring` (file `~/.local/share/python_keyring/keyring_pass.cfg`),
|
||||
che non dipende da D-Bus e funziona correttamente in contesto headless.
|
||||
Questa scelta è documentata in B3-closeout.md.
|
||||
- **AGENTS.md errore #9 (machine-id)**: lo snapshot `BaseClean-Linux` era già
|
||||
stato preso con `machine-id` troncato — nessuna collisione DHCP rilevata
|
||||
durante lo smoke test.
|
||||
|
||||
## Riferimenti commit
|
||||
|
||||
- `3921758` — feat: add credential management scripts (`tools/ci-cred-set.py`,
|
||||
`tools/ci-cred-check.py`)
|
||||
@@ -0,0 +1,52 @@
|
||||
# B3 — Closeout
|
||||
|
||||
Passo B3 di `plans/PhaseB-user-checklist.md`: trasferimento credenziali
|
||||
e chiavi SSH al user `ci-runner` headless.
|
||||
|
||||
## Stato
|
||||
|
||||
✅ **COMPLETATO** — chiavi SSH, credenziale guest Windows e GiteaPAT
|
||||
disponibili a `ci-runner`; PoC headless verificato.
|
||||
|
||||
## Cosa è stato fatto
|
||||
|
||||
- [x] Chiavi SSH guest Linux già presenti in `/var/lib/ci/keys/` (copiate
|
||||
con i template in B2). Permessi corretti:
|
||||
- `ci_linux`: `600` (owner `ci-runner`)
|
||||
- `ci_linux.pub`: `644` (owner `ci-runner`)
|
||||
- [x] Credenziale guest Windows `BuildVMGuest` già salvata come
|
||||
prerequisito del Passo 2 con `tools/ci-cred-set.py`.
|
||||
- Backend: `keyrings.alt.file.PlaintextKeyring`
|
||||
(`~/.local/share/python_keyring/keyring_pass.cfg`)
|
||||
- Verificata con `tools/ci-cred-check.py BuildVMGuest`
|
||||
→ `OK (file) username='WINBUILD-2025\ci_build'`
|
||||
- [x] Gitea PAT salvato:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python \
|
||||
/opt/ci/local-ci-cd-system/tools/ci-cred-set.py GiteaPAT ci-runner-linux
|
||||
```
|
||||
|
||||
Verificato con `ci-cred-check.py GiteaPAT` → `OK (file)`.
|
||||
- [x] PoC headless confermato: il backend `PlaintextKeyring` non dipende
|
||||
da D-Bus e funziona sia da `sudo -u ci-runner` che dall'interno di
|
||||
`act-runner.service` (verificato durante smoke test B2 e B4).
|
||||
|
||||
## Deviazioni / gotcha
|
||||
|
||||
- **`secret-tool` scartato**: il piano originale usava `secret-tool`
|
||||
(GNOME Secret Service). Non funziona senza una D-Bus session attiva,
|
||||
che non è disponibile in contesti systemd headless. Alternativa adottata:
|
||||
`keyrings.alt.file.PlaintextKeyring` — nessuna dipendenza da D-Bus,
|
||||
file in chiaro in `~/.local/share/python_keyring/keyring_pass.cfg`
|
||||
(accessibile solo a `ci-runner`). Configurazione in
|
||||
`~/.local/share/python_keyring/keyringrc.cfg`.
|
||||
- **Credenziale B2**: il prerequisito del Passo 2 anticipava la parte
|
||||
minima di B3 (solo `BuildVMGuest`) per sbloccare lo smoke test Windows.
|
||||
In B3 si è completato aggiungendo il GiteaPAT e fissando i permessi
|
||||
chiavi SSH.
|
||||
|
||||
## Riferimenti commit
|
||||
|
||||
- `3921758` — feat: add credential management scripts (`tools/ci-cred-set.py`,
|
||||
`tools/ci-cred-check.py`)
|
||||
@@ -0,0 +1,65 @@
|
||||
# B4 — Closeout
|
||||
|
||||
Passo B4 di `plans/PhaseB-user-checklist.md`: act_runner Linux
|
||||
registrato verso Gitea e gestito da systemd.
|
||||
|
||||
## Stato
|
||||
|
||||
✅ **COMPLETATO** — `act-runner.service` attivo, runner registrato,
|
||||
`config.yaml` corretto, smoke job PASS.
|
||||
|
||||
## Cosa è stato fatto
|
||||
|
||||
- [x] Binario act_runner Linux v1.0.4 scaricato in `/opt/ci/act_runner/`
|
||||
con permessi `ci-runner`.
|
||||
- [x] Runner registrato verso `https://gitea.emulab.it` con nome
|
||||
`ci-linux` e label `windows-build:host`, `linux-build:host`.
|
||||
- [x] `/etc/systemd/system/act-runner.service` creato con:
|
||||
- `User=ci-runner`, `WorkingDirectory=/var/lib/ci/runner`
|
||||
- Variabili d'ambiente CI (`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`,
|
||||
`CI_ARTIFACTS`, `CI_KEYS`, `PYTHONIOENCODING`)
|
||||
- `ExecStart=/opt/ci/act_runner/act_runner daemon --config /var/lib/ci/runner/config.yaml`
|
||||
- `Restart=on-failure`, `RestartSec=10`
|
||||
- [x] `/etc/ci/environment` creato con le stesse variabili CI (usato dai
|
||||
timer systemd di B5).
|
||||
- [x] `config.yaml` generato con `generate-config` e poi patchato via
|
||||
`sed` per:
|
||||
- Rimuovere i label `ubuntu-*:docker://` generati di default e
|
||||
impostare `windows-build:host` + `linux-build:host`
|
||||
- Sostituire le variabili placeholder `A_TEST_ENV_NAME_*` con:
|
||||
- `CI_PYTHON_LAUNCHER: /opt/ci/venv/bin/python`
|
||||
- `GITEA_CI_TEMPLATE_PATH: /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx`
|
||||
- `GITEA_CI_LINUX_TEMPLATE_PATH: /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx`
|
||||
- [x] `act-runner.service` abilitato e avviato; log confermano connessione
|
||||
Gitea senza errori Docker.
|
||||
- [x] Smoke job `self-test.yml` triggerato manualmente → PASS dal runner
|
||||
Linux.
|
||||
|
||||
## Deviazioni / gotcha
|
||||
|
||||
- **Nome servizio `act-runner` (non `act_runner`)**: il binario Linux
|
||||
si registra come `act-runner.service` (trattino), mentre su Windows
|
||||
il servizio si chiama `act_runner` (underscore). Questa differenza ha
|
||||
causato un bug nel comando `monitor runner`, che usava `act_runner` come
|
||||
default — corretto aggiungendo `--service-name act-runner` nell'`ExecStart`
|
||||
di `ci-watch-runner-health.service`.
|
||||
- **`generate-config` scrive label Docker**: `act_runner generate-config`
|
||||
produce label `ubuntu-latest:docker://...` che fanno cercare Docker (non
|
||||
installato). È necessario patchare il file con `sed` dopo la generazione.
|
||||
Le istruzioni sono nel Passo 4 del checklist B.
|
||||
- **`GITEA_CI_TEMPLATE_PATH` nel service unit prima dello smoke test**:
|
||||
se le variabili template non sono presenti in `config.yaml` o nel service
|
||||
unit, il job fallisce con `"Got unexpected extra argument"` (i path non
|
||||
vengono passati al CLI Python). Prerequisito obbligatorio prima di
|
||||
trigggerare lo smoke job.
|
||||
- **`config.yaml` separato da `.runner`**: il file `.runner` contiene il
|
||||
token di registrazione (non toccare). `config.yaml` contiene solo la
|
||||
configurazione operativa del runner.
|
||||
|
||||
## Riferimenti commit
|
||||
|
||||
- `536fd68` — feat: update runner configuration steps and add config.yaml
|
||||
generation instructions
|
||||
- `a6aa926` — fix: improve Python interpreter resolution for OS compatibility
|
||||
- `b2803c3` — fix: correct order of arguments for template path in local
|
||||
CI build action
|
||||
@@ -0,0 +1,78 @@
|
||||
# B5 — Closeout
|
||||
|
||||
Passo B5 di `plans/PhaseB-user-checklist.md`: conversione scheduled task
|
||||
Windows in coppie `*.service` + `*.timer` systemd su Linux.
|
||||
|
||||
## Stato
|
||||
|
||||
✅ **COMPLETATO** — 5 timer attivi, tutti e 5 i service PASS al trigger
|
||||
manuale.
|
||||
|
||||
## Cosa è stato fatto
|
||||
|
||||
- [x] Inventario dei 4 task in `scripts/Register-CIScheduledTasks.ps1`
|
||||
(cadenze + comandi) + aggiunta `ci-backup-template` (non era in PS).
|
||||
- [x] `Invoke-RetentionPolicy.ps1` portato in Python come sub-comando
|
||||
`retention run` (`src/ci_orchestrator/commands/retention.py`).
|
||||
- [x] `Backup-CITemplate.ps1` portato in Python come sub-comando
|
||||
`template backup` (`src/ci_orchestrator/commands/template.py`) con
|
||||
compressione `7z a -mx=1 -mmt=on` → archivi `.7z` in
|
||||
`/var/lib/ci/backups/`.
|
||||
- [x] 5 coppie `*.service` + `*.timer` deployate in
|
||||
`/etc/systemd/system/`:
|
||||
|
||||
| Unit | Cadenza | Comando |
|
||||
| --------------------------- | ---------------------------------------- | ------------------------------------------------------- |
|
||||
| `ci-cleanup-orphans` | ogni 6h + al boot | `ci_orchestrator vm cleanup --max-age-hours 6` |
|
||||
| `ci-retention-policy` | daily 03:00 + 30min jitter | `ci_orchestrator retention run` |
|
||||
| `ci-watch-disk-space` | ogni 15min | `ci_orchestrator monitor disk` |
|
||||
| `ci-watch-runner-health` | ogni 15min | `ci_orchestrator monitor runner --service-name act-runner` |
|
||||
| `ci-backup-template` | domenica 02:00 + 1h jitter | `ci_orchestrator template backup --all-templates` |
|
||||
|
||||
- [x] Prerequisiti creati sull'host prima del deploy:
|
||||
```bash
|
||||
sudo mkdir -p /var/log/ci /var/lib/ci/backups
|
||||
sudo chown ci-runner:ci-runner /var/log/ci
|
||||
```
|
||||
- [x] `systemctl list-timers --all 'ci-*'` → 5 timer `active`. ✅ PASS
|
||||
- [x] Trigger manuale di tutti e 5 i service → ✅ tutti PASS.
|
||||
- [x] `deploy/systemd/README.md` aggiornato: tabella mapping, prerequisiti,
|
||||
istruzioni install/test/rollback — senza riferimenti a PowerShell.
|
||||
|
||||
## Deviazioni / gotcha
|
||||
|
||||
- **pwsh non richiesto**: `Invoke-RetentionPolicy.ps1` e
|
||||
`Backup-CITemplate.ps1` sono stati portati integralmente a Python in
|
||||
questa fase, eliminando la dipendenza da PowerShell Core per tutti e 5
|
||||
i timer. Questo è il primo passo verso la Fase C
|
||||
(`plans/idea-3-powershell-removal.md`).
|
||||
- **`--service-name act-runner`**: il default del comando `monitor runner`
|
||||
era `act_runner` (nome Windows). Il service Linux si chiama `act-runner`
|
||||
(trattino). Il fix è nel parametro `ExecStart` di
|
||||
`ci-watch-runner-health.service`.
|
||||
- **`ci-backup-template` gira come root**: il backup richiede
|
||||
`systemctl stop/start act-runner.service` (gestito con try/finally in
|
||||
Python). Gli altri 4 service girano come `ci-runner`.
|
||||
- **Prerequisiti directory**: `ReadWritePaths` in `ProtectSystem=strict`
|
||||
richiede che le directory esistano *prima* che la unit venga avviata.
|
||||
`ci-watch-runner-health` e `ci-cleanup-orphans` usano `/var/log/ci`;
|
||||
`ci-backup-template` usa `/var/lib/ci/backups`. Entrambe vanno create
|
||||
prima del deploy (vedi sopra).
|
||||
- **Bug pre-esistente in `vm.py`** (`_list_orphans`): `is_dir()` era
|
||||
chiamato fuori dal blocco `try/except OSError`, causando un'eccezione
|
||||
su symlink rotti. Corretto spostando `is_dir()` all'interno del try.
|
||||
|
||||
## Cadenze systemd (vs Windows originale)
|
||||
|
||||
| Task | Windows originale | systemd timer |
|
||||
| --------------------- | -------------------------- | ----------------------------------------------------- |
|
||||
| cleanup-orphans | ogni 6h + al boot | `OnBootSec=2min, OnUnitActiveSec=6h, Persistent` |
|
||||
| retention-policy | daily 03:00 + 30min delay | `OnCalendar=*-*-* 03:00:00, RandomizedDelaySec=30min` |
|
||||
| watch-disk-space | ogni 15min | `OnBootSec=5min, OnUnitActiveSec=15min` |
|
||||
| watch-runner-health | ogni 15min | `OnBootSec=3min, OnUnitActiveSec=15min` |
|
||||
| backup-template | *(non era in PS)* | `OnCalendar=Sun *-*-* 02:00:00, RandomizedDelaySec=1h`|
|
||||
|
||||
## Riferimenti commit
|
||||
|
||||
- `43a69b8` — feat: B5 — port retention/template-backup to Python, deploy
|
||||
systemd timers
|
||||
+42
-67
@@ -20,21 +20,21 @@ codice Python) né in [PhaseB-user-checklist.md](PhaseB-user-checklist.md)
|
||||
Senza un baseline registrato, il confronto richiesto da B7 ("tempo
|
||||
medio entro ±20% baseline Windows") è impossibile.
|
||||
|
||||
- [ ] Eseguire benchmark sull'host Windows (post Fase A, runner Python attivo):
|
||||
- [x] Eseguire benchmark sull'host Windows (post Fase A, runner Python attivo):
|
||||
|
||||
```powershell
|
||||
cd <path-locale-repo>
|
||||
.\scripts\Measure-CIBenchmark.ps1 | Tee-Object -FilePath "baseline-windows-$(Get-Date -Format 'yyyyMMdd').txt"
|
||||
```
|
||||
|
||||
- [ ] Annotare in `docs/RUNBOOK.md` (sezione nuova "Windows host baseline"):
|
||||
- [x] Annotare in `docs/RUNBOOK.md` (sezione nuova "Windows host baseline"):
|
||||
- Data
|
||||
- Tempo medio per job (Windows e Linux template)
|
||||
- Success rate
|
||||
- Versione `act_runner` e `ci_orchestrator` (output di `python -m ci_orchestrator --version` se disponibile, altrimenti SHA del commit)
|
||||
- Hardware: i9-10900X, 64 GB RAM, NVMe SSD (da `AGENTS.md`)
|
||||
|
||||
- [ ] Committare:
|
||||
- [x] Committare:
|
||||
|
||||
```powershell
|
||||
git add docs/RUNBOOK.md
|
||||
@@ -51,7 +51,7 @@ presuppongono già pronti.
|
||||
|
||||
### 2.1 Inventario chiavi SSH
|
||||
|
||||
- [ ] Verificare presenza chiavi:
|
||||
- [x] Verificare presenza chiavi:
|
||||
|
||||
```powershell
|
||||
Get-ChildItem F:\CI\keys\ci_linux*
|
||||
@@ -68,8 +68,8 @@ La password salvata in Windows Credential Manager **non è
|
||||
esportabile**. Devi conoscerla per re-inserirla con `secret-tool` su
|
||||
Linux (Passo 3 di Fase B).
|
||||
|
||||
- [ ] Verificare di averla nel password manager personale.
|
||||
- [ ] Se non la ricordi: bootare il template `WinBuild2025` (e `WinBuild2022`),
|
||||
- [x] Verificare di averla nel password manager personale.
|
||||
- [X] Se non la ricordi: bootare il template `WinBuild2025` (e `WinBuild2022`),
|
||||
resettare la password dell'utente CI (`net user <user> <newpass>`),
|
||||
ricatturare snapshot `BaseClean` da stato powered-off (vedi
|
||||
`AGENTS.md` errore #9), aggiornare il Credential Manager Windows con
|
||||
@@ -77,10 +77,10 @@ Linux (Passo 3 di Fase B).
|
||||
|
||||
### 2.3 Generazione Gitea PAT per il nuovo runner
|
||||
|
||||
- [ ] Aprire Gitea → User Settings → Applications → Generate New Token.
|
||||
- [ ] Nome: `ci-runner-linux`. Scope: `read:repository` + `write:package`
|
||||
- [x] Aprire Gitea → User Settings → Applications → Generate New Token.
|
||||
- [x] Nome: `ci-runner-linux`. Scope: `read:repository` + `write:package`
|
||||
(o gli stessi del runner Windows attuale).
|
||||
- [ ] Annotare il token in modo sicuro (verrà inserito al Passo 3 di
|
||||
- [x] Annotare il token in modo sicuro (verrà inserito al Passo 3 di
|
||||
Fase B con `secret-tool store ... target GiteaPAT`).
|
||||
|
||||
### 2.4 OpenSSH Server su Windows (per rsync pull dal Linux)
|
||||
@@ -89,7 +89,7 @@ Necessario solo se al Passo 2 di Fase B (transfer template) preferisci
|
||||
fare `rsync pull` dal Linux. Alternativa: `rsync push` da Windows con
|
||||
WSL/Cygwin/MSYS2.
|
||||
|
||||
- [ ] Installare OpenSSH Server:
|
||||
- [x] Installare OpenSSH Server:
|
||||
|
||||
```powershell
|
||||
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
|
||||
@@ -99,16 +99,17 @@ WSL/Cygwin/MSYS2.
|
||||
-Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
|
||||
```
|
||||
|
||||
- [ ] Validare connessione da un altro host: `ssh user@<windows-ip> "whoami"`.
|
||||
- [ ] (Opzionale) Configurare `authorized_keys` per il futuro user
|
||||
- [x] Validare connessione da un altro host: `ssh user@<windows-ip> "whoami"`.
|
||||
- [x] (Opzionale) Configurare `authorized_keys` per il futuro user
|
||||
`ci-runner` del Linux box.
|
||||
|
||||
### 2.5 Template fully powered-off (gate per B2)
|
||||
|
||||
> AGENTS.md errore #9: snapshot catturato con VM accesa produce
|
||||
> `*.vmem` / `*.vmsn` che rompono il clone su host diverso.
|
||||
> `*.vmem` che rompono il clone su host diverso. `*.vmsn` è sempre
|
||||
> presente (metadata snapshot) anche con VM spenta — non è un problema.
|
||||
|
||||
- [ ] Verificare che nessun template sia running:
|
||||
- [x] Verificare che nessun template sia running:
|
||||
|
||||
```powershell
|
||||
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' list
|
||||
@@ -116,15 +117,16 @@ WSL/Cygwin/MSYS2.
|
||||
|
||||
Atteso: nessun path sotto `F:\CI\Templates\` nell'output.
|
||||
|
||||
- [ ] Verificare assenza file di runtime:
|
||||
- [x] Verificare assenza file di runtime:
|
||||
|
||||
```powershell
|
||||
Get-ChildItem F:\CI\Templates -Recurse -Include *.vmem,*.vmsn,*.lck
|
||||
Get-ChildItem F:\CI\Templates -Recurse -Include *.vmem,*.lck
|
||||
```
|
||||
|
||||
Atteso: nessun risultato. Se ci sono `.vmem`/`.vmsn`, lo snapshot
|
||||
non è clean → ricatturare `BaseClean` / `BaseClean-Linux` da stato
|
||||
fully powered-off.
|
||||
Atteso: nessun risultato. Se ci sono `.vmem`, lo snapshot non è
|
||||
clean (VM era accesa) → ricatturare `BaseClean` / `BaseClean-Linux`
|
||||
da stato fully powered-off. `.lck` indica VM ancora in esecuzione →
|
||||
spegnerla prima.
|
||||
|
||||
### 2.6 Backup precauzionale di `F:\CI\`
|
||||
|
||||
@@ -134,45 +136,19 @@ buona pratica.
|
||||
- [ ] Eseguire backup completo:
|
||||
|
||||
```powershell
|
||||
$stamp = Get-Date -Format 'yyyyMMdd'
|
||||
$archive = "E:\Backup\CI-pre-migration-$stamp.tar"
|
||||
& 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\'
|
||||
$stamp = Get-Date -Format 'yyyyMMdd'
|
||||
$archive = "F:\BACKUP\CI-pre-migration-$stamp.7z"
|
||||
& 'C:\Program Files\7-Zip\7z.exe' a -mx=3 -xr!ISO -xr!RunnerWork $archive 'F:\CI\'
|
||||
Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256"
|
||||
```
|
||||
|
||||
- [ ] Verifica integrità (lista contenuto, primi 20 record):
|
||||
- [x] Verifica integrità (lista contenuto, primi 20 record):
|
||||
|
||||
```powershell
|
||||
& 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20
|
||||
& 'C:\Program Files\7-Zip\7z.exe' l $archive | Select-Object -First 20
|
||||
```
|
||||
|
||||
- [ ] Annotare path archivio + checksum SHA256.
|
||||
|
||||
### 2.7 Annotare cadenze scheduled task (se custom)
|
||||
|
||||
- [ ] Esportare l'inventario corrente:
|
||||
|
||||
```powershell
|
||||
Get-ScheduledTask -TaskName 'CI-*' |
|
||||
Select-Object TaskName,
|
||||
@{n='Trigger';e={ $_.Triggers | ForEach-Object { $_.PSObject.Properties | Where-Object Name -in 'StartBoundary','RepetitionInterval','DaysOfWeek' | ForEach-Object { "$($_.Name)=$($_.Value)" } } -join '; ' }},
|
||||
@{n='Action'; e={ "$($_.Actions.Execute) $($_.Actions.Arguments)" }} |
|
||||
Format-List | Tee-Object -FilePath "scheduled-tasks-inventory-$(Get-Date -Format 'yyyyMMdd').txt"
|
||||
```
|
||||
|
||||
- [ ] Confrontare con i `.timer` in `deploy/systemd/`:
|
||||
|
||||
| Task Windows | systemd timer | Cadenza prevista |
|
||||
| ----------------------------- | ------------------------------------- | ---------------------- |
|
||||
| `CI-CleanupOrphanedBuildVMs` | `ci-cleanup-orphans.timer` | ogni 6h + boot |
|
||||
| `CI-Invoke-RetentionPolicy` | `ci-retention-policy.timer` | giornaliero 03:00 |
|
||||
| `CI-Watch-DiskSpace` | `ci-watch-disk-space.timer` | ogni 15min |
|
||||
| `CI-Watch-RunnerHealth` | `ci-watch-runner-health.timer` | ogni 15min |
|
||||
| (nuovo) | `ci-backup-template.timer` | settimanale Dom 02:00 |
|
||||
|
||||
- [ ] Se le cadenze Windows sono diverse, aggiornare gli `OnCalendar` /
|
||||
`OnUnitActiveSec` in `deploy/systemd/*.timer` **prima** del Passo
|
||||
5 di Fase B. Committare la modifica.
|
||||
- [x] Annotare path archivio + checksum SHA256.
|
||||
|
||||
---
|
||||
|
||||
@@ -180,32 +156,31 @@ buona pratica.
|
||||
|
||||
Questi non sono comandi, ma checkpoint prima di iniziare B1 e B6.
|
||||
|
||||
- [ ] **Hardware Linux disponibile**: il box target è acquisito,
|
||||
- [x] **Hardware Linux disponibile**: il box target è acquisito,
|
||||
racked, raggiungibile in rete dall'host Windows e da Gitea.
|
||||
- [ ] **DNS / IP statico** assegnato al nuovo host Linux (per
|
||||
- [x] **DNS / IP statico** assegnato al nuovo host Linux (per
|
||||
registrazione runner + raggiungibilità da admin).
|
||||
- [ ] **Finestra di manutenzione per cutover B6 concordata**: 30–60 min
|
||||
- [x] **Finestra di manutenzione per cutover B6 concordata**: 30–60 min
|
||||
di stop Gitea Actions, in un orario a basso traffico.
|
||||
- [ ] **Comunicazione utenti del repo**: avviso che durante la
|
||||
- [x] **Comunicazione utenti del repo**: avviso che durante la
|
||||
finestra B6 i workflow saranno temporaneamente fermi.
|
||||
- [ ] **Piano di rollback compreso**: rileggere la sezione "Rollback"
|
||||
- [x] **Piano di rollback compreso**: rileggere la sezione "Rollback"
|
||||
del Passo 6 di [PhaseB-user-checklist.md](PhaseB-user-checklist.md).
|
||||
|
||||
---
|
||||
|
||||
## Tracciamento globale
|
||||
|
||||
| Sezione | Descrizione | Stato |
|
||||
| ------- | ---------------------------------------------- | ----- |
|
||||
| 1 | Baseline benchmark salvato in `docs/RUNBOOK.md` | [ ] |
|
||||
| 2.1 | Chiavi SSH presenti | [ ] |
|
||||
| 2.2 | Password `BuildVMGuest` recuperata | [ ] |
|
||||
| 2.3 | Gitea PAT generato per `ci-runner-linux` | [ ] |
|
||||
| 2.4 | OpenSSH Server attivo (o alt. rsync) | [ ] |
|
||||
| 2.5 | Template fully powered-off, no `.vmem` | [ ] |
|
||||
| 2.6 | Backup `F:\CI\` con SHA256 | [ ] |
|
||||
| 2.7 | Cadenze scheduled task confrontate con timer | [ ] |
|
||||
| 3 | Hardware + DNS + finestra + comunicazione OK | [ ] |
|
||||
| Sezione | Descrizione | Stato |
|
||||
| ------- | ----------------------------------------------- | ----- |
|
||||
| 1 | Baseline benchmark salvato in `docs/RUNBOOK.md` | [x] |
|
||||
| 2.1 | Chiavi SSH presenti | [x] |
|
||||
| 2.2 | Password `BuildVMGuest` recuperata | [x] |
|
||||
| 2.3 | Gitea PAT generato per `ci-runner-linux` | [x] |
|
||||
| 2.4 | OpenSSH Server attivo (o alt. rsync) | [x] |
|
||||
| 2.5 | Template fully powered-off, no `.vmem` | [x] |
|
||||
| 2.6 | Backup `F:\CI\` con SHA256 | [x] |
|
||||
| 3 | Hardware + DNS + finestra + comunicazione OK | [x] |
|
||||
|
||||
Quando tutte le righe sono `[x]`, puoi iniziare il Passo 1 di
|
||||
[PhaseB-user-checklist.md](PhaseB-user-checklist.md).
|
||||
@@ -202,13 +202,23 @@ Costo: codebase divisa in due ambienti, maggiore overhead operativo.
|
||||
|
||||
## 8. Definizione di "fatto" (Fase B)
|
||||
|
||||
- [ ] act_runner gira come systemd service `act-runner.service` su Linux Mint
|
||||
- [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`)
|
||||
> **Stato 2026-06-07**: Fase B COMPLETATA. B1–B7 + stabilità ≥1 settimana
|
||||
> (host live dal cutover ~24 mag, >2 settimane senza incidenti). Tracking di
|
||||
> dettaglio in [PhaseB-user-checklist.md](../2026-06-07/PhaseB-user-checklist.md)
|
||||
> (Passi 1–9 `[x]`, archiviato). Dual-boot in RUNBOOK §15.
|
||||
|
||||
- [x] act_runner gira come systemd service `act-runner.service` su Linux Mint
|
||||
- [x] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`)
|
||||
operativi dal nuovo host con snapshot integri
|
||||
- [ ] Workflow `build-ns7zip.yml` (matrix Win+Linux) PASS dal nuovo host
|
||||
- [ ] Burn-in capacity 4 job concorrenti PASS, tempi entro ±20% del baseline
|
||||
- [ ] Storage migrato a `/var/lib/ci/`, host Windows in stand-by come rollback
|
||||
- [ ] Tutti i timer systemd attivi e schedulati
|
||||
- [ ] Credenziali guest accessibili da `ci-runner` headless
|
||||
- [ ] `README.md`, `AGENTS.md`, `docs/HOST-SETUP.md` aggiornati per host Linux
|
||||
- [ ] Almeno 1 settimana di esercizio in produzione senza incidenti
|
||||
- [x] Build Win+Linux PASS dal nuovo host — burn-in B7 concorrente
|
||||
(WinBuild2025 + LinuxBuild2404, 80/80 job PASS); vedi RUNBOOK §9
|
||||
- [x] Burn-in capacity 4 job concorrenti PASS, tempi near-deterministici
|
||||
(static-IP pool, σ < 1.5 s) — RUNBOOK §9
|
||||
- [x] Storage migrato a `/var/lib/ci/`; host Windows in stand-by come rollback
|
||||
- [x] Tutti i timer systemd attivi e schedulati (watch-disk-space,
|
||||
watch-runner-health, cleanup-orphans, backup-template, retention-policy)
|
||||
- [x] Credenziali guest accessibili da `ci-runner` headless
|
||||
(backend `PlaintextKeyring` file-based, no D-Bus)
|
||||
- [x] `README.md`, `AGENTS.md`, `docs/HOST-SETUP.md` aggiornati per host Linux
|
||||
- [x] Almeno 1 settimana di esercizio in produzione senza incidenti
|
||||
(>2 settimane dal cutover ~24 mag; salute 7 giu OK)
|
||||
+88
-69
@@ -23,7 +23,12 @@ Sintesi fasi:
|
||||
- **Stato: pronta** (gating "A done" soddisfatto). Esecuzione via
|
||||
`plans/PhaseB-user-checklist.md`; design/rischi/cronoprogramma in
|
||||
§2/§3/§5/§6 di questo piano.
|
||||
- **Fase C — (Hook only, non implementata) backend ESXi via `pyVmomi`**
|
||||
- **Fase C — Eliminazione dipendenza `pwsh` dall'host Linux**
|
||||
- Output verificabile: `bench run`, `validate host`, `smoke run` portati
|
||||
in Python; `pwsh` disinstallabile senza impatti.
|
||||
- Stato: non in scope (da avviare dopo B stabile). Design in
|
||||
`plans/idea-3-powershell-removal.md`.
|
||||
- **Fase D — (Hook only, non implementata) backend ESXi via `pyVmomi`**
|
||||
- Output verificabile: Protocol `VmBackend` rispettato; `[backend]`
|
||||
selector in `config.toml`.
|
||||
- Stato: non in scope.
|
||||
@@ -77,30 +82,30 @@ A4; cambiano solo path/env vars in Fase B).
|
||||
- [x] [A5] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout package
|
||||
- [x] [A5] Aggiornare `README.md` con setup Python
|
||||
- [x] [A5] Eseguire burn-in 4 job concorrenti PASS (Start-BurnInTest: 3 round × 4 = 12/12, 0 orfani)
|
||||
- [ ] [B1] Installare Linux Mint LTS + aggiornamenti su hardware target
|
||||
- [ ] [B1] Installare VMware Workstation Pro Linux (bundle ufficiale Broadcom) e validare `vmrun` su VM di test
|
||||
- [ ] [B1] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato all'host Windows)
|
||||
- [ ] [B1] Creare utente `ci-runner` (uid dedicato) e storage `/var/lib/ci/{build-vms,artifacts,templates,keys}` con ownership `ci-runner:ci-runner` mode 750
|
||||
- [ ] [B1] Installare Python 3.11+ e creare venv in `/opt/ci/venv/` con `pip install -e .` del package portato in Fase A
|
||||
- [ ] [B1] Validare `python -m ci_orchestrator --help` da utente `ci-runner`
|
||||
- [ ] [B2] Spegnere fully powered-off i template VM sull'host Windows
|
||||
- [ ] [B2] Trasferire `F:\CI\Templates\` → `/var/lib/ci/templates/` con `rsync` (preservare snapshot `BaseClean` / `BaseClean-Linux`)
|
||||
- [ ] [B2] Registrare i `.vmx` su Workstation Linux e validare snapshot via `vmrun listSnapshots`
|
||||
- [ ] [B2] Smoke test manuale: `vm new` + `wait-ready` + `vm remove` da host Linux
|
||||
- [ ] [B3] Copiare `F:\CI\keys\ci_linux*` → `/etc/ci/keys/` con perms 600 owner `ci-runner`
|
||||
- [ ] [B3] Re-store credenziali `BuildVMGuest` e `GiteaPAT` con `secret-tool` nel keyring Linux
|
||||
- [ ] [B3] PoC accesso headless al keyring sotto systemd (file vault `age`/`sops` come fallback)
|
||||
- [ ] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host`
|
||||
- [ ] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8`
|
||||
- [ ] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f`
|
||||
- [x] [B1] Installare Linux Mint LTS + aggiornamenti su hardware target
|
||||
- [x] [B1] Installare VMware Workstation Pro Linux (bundle ufficiale Broadcom) e validare `vmrun` su VM di test
|
||||
- [x] [B1] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato all'host Windows)
|
||||
- [x] [B1] Creare utente `ci-runner` (uid dedicato) e storage `/var/lib/ci/{build-vms,artifacts,templates,keys}` con ownership `ci-runner:ci-runner` mode 750
|
||||
- [x] [B1] Installare Python 3.11+ e creare venv in `/opt/ci/venv/` con `pip install -e .` del package portato in Fase A
|
||||
- [x] [B1] Validare `python -m ci_orchestrator --help` da utente `ci-runner`
|
||||
- [x] [B2] Spegnere fully powered-off i template VM sull'host Windows
|
||||
- [x] [B2] Trasferire `F:\CI\Templates\` → `/var/lib/ci/templates/` con `rsync` (preservare snapshot `BaseClean` / `BaseClean-Linux`)
|
||||
- [x] [B2] Registrare i `.vmx` su Workstation Linux e validare snapshot via `vmrun listSnapshots`
|
||||
- [x] [B2] Smoke test manuale: `vm new` + `wait-ready` + `vm remove` da host Linux
|
||||
- [x] [B3] Copiare `F:\CI\keys\ci_linux*` → `/var/lib/ci/keys/` con perms 600 owner `ci-runner`
|
||||
- [x] [B3] Re-store credenziali `BuildVMGuest` e `GiteaPAT` con `keyrings.alt.file.PlaintextKeyring` (headless, no D-Bus — `secret-tool` scartato)
|
||||
- [x] [B3] PoC accesso headless al keyring sotto systemd — `PlaintextKeyring` verificato PASS
|
||||
- [x] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host`
|
||||
- [x] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8`; `config.yaml` con `runner.envs` per i path template
|
||||
- [x] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f`
|
||||
- [x] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`)
|
||||
- [ ] [B5] Abilitare tutti i timer con `systemctl enable --now` e validare `systemctl list-timers`
|
||||
- [ ] [B6] Stop act_runner sull'host Windows e verifica coda Gitea vuota
|
||||
- [ ] [B6] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/`
|
||||
- [ ] [B6] Trigger smoke workflow + `build-ns7zip.yml` matrix da host Linux PASS
|
||||
- [ ] [B7] Eseguire burn-in 4 job concorrenti × 10 round su template Windows e Linux con tempi entro ±20% baseline
|
||||
- [ ] [B8] Backup finale di `F:\CI\` su archivio
|
||||
- [ ] [B8] Lasciare host Windows spento ma reinstallato per ≥1 mese come rollback
|
||||
- [ ] [B8] (Facoltativo) Backup periodico di `F:\CI\`
|
||||
- [x] [B8] ~~Decommissioning host Windows~~ **N/A** — la macchina è dual-boot: Windows e Linux sullo stesso hardware, non girano contemporaneamente. Nessuna dismissione.
|
||||
- [ ] [X1] Aggiornare `lint.yml` per girare ruff + mypy + pytest oltre a PSSA
|
||||
- [ ] [X2] Aggiungere observability: log strutturato `logging` + journald handler in Fase B
|
||||
- [ ] [X3] Documentare gestione credenziali headless (DPAPI machine scope su Win, vault `age` su Linux)
|
||||
@@ -177,7 +182,7 @@ WinRM/SSH e credential store funzionanti contro l'ambiente reale via PoC
|
||||
- [x] Test pytest unitari: `test_vmrun.py` (mock subprocess, casi `vmrun list` con e senza VMX target — copre errore #10), `test_winrm.py` (mock `pypsrp`), `test_ssh.py` (mock `paramiko`, copre errore #12), `test_credentials.py` (mock `keyring`)
|
||||
- [x] Aggiungere job `python` a `gitea/workflows/lint.yml`: setup venv → `ruff check src/ tests/` → `mypy --strict src/` → `pytest --cov=ci_orchestrator --cov-fail-under=70`
|
||||
|
||||
**Hook futuri Fase C**: il Protocol `VmBackend` deve usare nomi neutri
|
||||
**Hook futuri Fase D**: il Protocol `VmBackend` deve usare nomi neutri
|
||||
(`clone_linked`, non `vmrun_clone`), accettare `template`/`snapshot`/`name`
|
||||
come stringhe opache (non path Windows), separare `clone_linked` da
|
||||
`start` (ESXi richiede `PowerOn` come task distinto). Niente assunzioni
|
||||
@@ -230,9 +235,9 @@ stato con l'orchestratore, sostituendoli con shim minimi.
|
||||
- [ ] Convertire `tests/Remove-BuildVM.Tests.ps1` in `tests/test_commands_vm_remove.py`
|
||||
- [ ] Verificare che gli scheduled task esistenti (`Register-CIScheduledTasks.ps1`) continuino a funzionare invocando gli shim PS
|
||||
|
||||
**Hook futuri Fase C**: `vm cleanup` non deve assumere scan locale del
|
||||
**Hook futuri Fase D**: `vm cleanup` non deve assumere scan locale del
|
||||
filesystem — la firma deve accettare un `VmBackend` come dipendenza,
|
||||
così che in Fase C la stessa logica interroghi l'API ESXi (folder-scoped
|
||||
così che in Fase D la stessa logica interroghi l'API ESXi (folder-scoped
|
||||
list).
|
||||
|
||||
**Test / validazione**:
|
||||
@@ -275,7 +280,7 @@ Python preservando il comportamento dei `.ps1` esistenti.
|
||||
- [x] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path)
|
||||
- [ ] Validare end-to-end: clone WinBuild2025 → build script PowerShell trivial → collect artifact ZIP
|
||||
|
||||
**Hook futuri Fase C**: `build run` deve ricevere un `VmHandle` opaco,
|
||||
**Hook futuri Fase D**: `build run` deve ricevere un `VmHandle` opaco,
|
||||
non un path VMX (ESXi non ha path locale al control plane). Adattare le
|
||||
CLI a accettare entrambe (`--vmx` per workstation, `--vm-id` futuro per
|
||||
ESXi) con dispatcher nel layer `commands/`.
|
||||
@@ -317,7 +322,7 @@ Gitea per chiamare la CLI Python direttamente.
|
||||
- [ ] Eseguire workflow `self-test.yml` PASS
|
||||
- [ ] Eseguire workflow `lint.yml` PASS
|
||||
|
||||
**Hook futuri Fase C**: la selezione del backend deve avvenire in
|
||||
**Hook futuri Fase D**: la selezione del backend deve avvenire in
|
||||
`job.py` leggendo `config.toml` `[backend].type` (default `workstation`).
|
||||
Niente import diretto di `WorkstationVmrunBackend` in `job.py` — usare
|
||||
factory `backends.load_backend(config)`.
|
||||
@@ -359,7 +364,7 @@ aggiornare documentazione, eseguire burn-in finale.
|
||||
- [ ] Eseguire burn-in 4 job concorrenti × 10 round PASS sull'host Windows — richiede VM reali, a carico utente
|
||||
- [ ] Aggiornare `docs/RUNBOOK.md` con sezione "Operare da Python CLI" — deferito a chiusura Fase B (RUNBOOK riscritto end-to-end in B-finale)
|
||||
|
||||
**Hook futuri Fase C**: documentare in `docs/ARCHITECTURE.md` il
|
||||
**Hook futuri Fase D**: documentare in `docs/ARCHITECTURE.md` il
|
||||
contratto `VmBackend` come "extension point" e il selettore `[backend]`
|
||||
in `config.toml` come unica via per aggiungere ESXi.
|
||||
|
||||
@@ -384,6 +389,16 @@ qualche caller esterno non ancora migrato emerge.
|
||||
|
||||
## 2. Fase B — Migrazione host Linux Mint
|
||||
|
||||
> **VINCOLO CATEGORICO (non negoziabile): compatibilità host Windows
|
||||
> permanente.** Fase B **aggiunge** il supporto host Linux **accanto**
|
||||
> a Windows — non lo sostituisce né lo degrada. Windows host resta un
|
||||
> target di **prima classe** e supportato, NON solo rollback. In
|
||||
> concreto: **B8 NON va eseguito come rimozione reale** (l'host Windows
|
||||
> resta operativo/usabile); nessun assunto Linux-only nel codice
|
||||
> condiviso; la rimozione shim (B5) deve lasciare un modo Windows-nativo
|
||||
> di invocare la CLI Python; ogni modifica Fase B deve dichiarare
|
||||
> l'impatto su host Windows e preferire parità dual-host.
|
||||
>
|
||||
> **Parallelizzazione**: B1 può iniziare durante A3/A4 (setup hardware
|
||||
> indipendente dal codice). B2, B3 possono iniziare appena B1 è done e
|
||||
> A5 è done. B4–B6 sono **strettamente sequenziali** dopo "A done".
|
||||
@@ -411,7 +426,7 @@ parallelo a A3/A4. Riferimento `idea-2-linux-host.md` §4.
|
||||
- [ ] Validare `sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help`
|
||||
- [ ] Aggiungere `[paths.linux]` a `config.example.toml` con i path POSIX
|
||||
|
||||
**Hook futuri Fase C**: lo stesso host Linux farà da control plane se
|
||||
**Hook futuri Fase D**: lo stesso host Linux farà da control plane se
|
||||
si attiverà ESXi. Non installare nulla che assuma "VM girano in locale"
|
||||
(es. condivisioni `vmware-shared-folders` host-only).
|
||||
|
||||
@@ -431,12 +446,12 @@ si attiverà ESXi. Non installare nulla che assuma "VM girano in locale"
|
||||
**Rollback**: il host Linux è fisicamente separato. Spegnerlo. Nessun
|
||||
impatto su produzione Windows.
|
||||
|
||||
**Definizione di fatto step B1**:
|
||||
**Definizione di fatto step B1**: ✅ COMPLETATO (2026-05-20)
|
||||
|
||||
- [ ] Workstation Pro Linux operativa
|
||||
- [ ] Storage layout creato con ACL
|
||||
- [ ] Python venv pronto e package installato
|
||||
- [ ] `vmnet8` configurato
|
||||
- [x] Workstation Pro Linux operativa (`/usr/bin/vmrun`)
|
||||
- [x] Storage layout `/var/lib/ci/` creato con ACL, owner `ci-runner`
|
||||
- [x] Python venv `/opt/ci/venv/` pronto, `ci_orchestrator` installato
|
||||
- [x] `vmnet8` configurato su `192.168.79.0/24`
|
||||
|
||||
### B2 — Trasferimento template VM
|
||||
|
||||
@@ -456,7 +471,7 @@ mantenendo snapshot integri.
|
||||
- [ ] Validare snapshot `BaseClean-Linux` su `LinuxBuild2404.vmx`
|
||||
- [ ] Smoke test: `python -m ci_orchestrator vm new --template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx --snapshot BaseClean --name smoke1` + `wait-ready` + `vm remove`
|
||||
|
||||
**Hook futuri Fase C**: i template restano in formato VMX. Per ESXi
|
||||
**Hook futuri Fase D**: i template restano in formato VMX. Per ESXi
|
||||
serviranno OVF (`ovftool`) — non fare nulla in B2, ma documentare che
|
||||
il dataset attuale è la base per export futuro.
|
||||
|
||||
@@ -474,11 +489,11 @@ il dataset attuale è la base per export futuro.
|
||||
**Rollback**: i template originali su `F:\CI\Templates\` sono intatti.
|
||||
Spegnere host Linux. Eventualmente eliminare `/var/lib/ci/templates/`.
|
||||
|
||||
**Definizione di fatto step B2**:
|
||||
**Definizione di fatto step B2**: ✅ COMPLETATO (2026-05-21)
|
||||
|
||||
- [ ] Tutti i template registrati su Workstation Linux
|
||||
- [ ] Snapshot integri verificati
|
||||
- [ ] Smoke `vm new` PASS per Win e Linux
|
||||
- [x] Tutti i template registrati su Workstation Linux
|
||||
- [x] Snapshot integri verificati
|
||||
- [x] Smoke `vm new` PASS per Win e Linux
|
||||
|
||||
### B3 — Trasferimento credenziali e chiavi
|
||||
|
||||
@@ -497,7 +512,7 @@ guest e i token Gitea, accessibili da `ci-runner` headless.
|
||||
- [ ] PoC headless: testare lettura keyring da contesto systemd (no sessione D-Bus utente). Se fallisce, implementare backend `keyring` file-based con vault `age` o `sops` come fallback documentato
|
||||
- [ ] Documentare la scelta finale in `docs/HOST-SETUP.md`
|
||||
|
||||
**Hook futuri Fase C**: anche ESXi userà credenziali da `keyring` (user
|
||||
**Hook futuri Fase D**: anche ESXi userà credenziali da `keyring` (user
|
||||
vSphere). Lo stesso `KeyringCredentialStore` deve poter caricare `EsxiHost`
|
||||
target — niente hard-coding di nomi credenziali.
|
||||
|
||||
@@ -514,11 +529,11 @@ target — niente hard-coding di nomi credenziali.
|
||||
**Rollback**: rimuovere `/etc/ci/keys/`, `secret-tool clear` per ogni
|
||||
target. Credenziali su host Windows intatte.
|
||||
|
||||
**Definizione di fatto step B3**:
|
||||
**Definizione di fatto step B3**: ✅ COMPLETATO (2026-05-21)
|
||||
|
||||
- [ ] Chiavi SSH copiate con perms corretti
|
||||
- [ ] Credenziali nel keyring leggibili headless
|
||||
- [ ] Strategia documentata in `docs/HOST-SETUP.md`
|
||||
- [x] Chiavi SSH copiate con perms corretti
|
||||
- [x] Credenziali nel keyring leggibili headless (`PlaintextKeyring` — `secret-tool`/D-Bus scartato)
|
||||
- [ ] Strategia documentata in `docs/HOST-SETUP.md` (pendente X3/X4)
|
||||
|
||||
### B4 — Setup act_runner come systemd service
|
||||
|
||||
@@ -541,7 +556,7 @@ systemd, equivalente al servizio Windows attuale.
|
||||
- [ ] Validare `journalctl -u act-runner -f` mostra connessione a Gitea OK
|
||||
- [ ] **NON** avviare ancora workflow di produzione — il runner Windows è ancora primario
|
||||
|
||||
**Hook futuri Fase C**: env vars `CI_*` restano valide; per ESXi si
|
||||
**Hook futuri Fase D**: env vars `CI_*` restano valide; per ESXi si
|
||||
aggiungeranno solo `CI_BACKEND=esxi` + sezione `[backend.esxi]` in
|
||||
`config.toml`. `act-runner.service` non cambia.
|
||||
|
||||
@@ -559,12 +574,12 @@ aggiungeranno solo `CI_BACKEND=esxi` + sezione `[backend.esxi]` in
|
||||
**Rollback**: `systemctl disable --now act-runner`, de-registrare
|
||||
runner da Gitea. Runner Windows resta primario.
|
||||
|
||||
**Definizione di fatto step B4**:
|
||||
**Definizione di fatto step B4**: ✅ COMPLETATO (2026-05-21)
|
||||
|
||||
- [ ] act-runner.service attivo e abilitato
|
||||
- [ ] Runner online in Gitea
|
||||
- [ ] `self-test.yml` PASS dal nuovo runner
|
||||
- [ ] Logging via journald validato
|
||||
- [x] act-runner.service attivo e abilitato
|
||||
- [x] Runner online in Gitea
|
||||
- [x] `self-test.yml` PASS dal nuovo runner (Win + Linux, entrambi i transport)
|
||||
- [x] Logging via journald validato
|
||||
|
||||
### B5 — Conversione scheduled tasks → systemd timers
|
||||
|
||||
@@ -586,7 +601,7 @@ per l'inventario dei task.
|
||||
- [ ] Validare `systemctl list-timers` mostra tutti i timer schedulati
|
||||
- [x] Documentare il mapping in `docs/HOST-SETUP.md`
|
||||
|
||||
**Hook futuri Fase C**: `vm cleanup` con backend ESXi userà la stessa
|
||||
**Hook futuri Fase D**: `vm cleanup` con backend ESXi userà la stessa
|
||||
unit — il selettore backend è in `config.toml`, non nel comando.
|
||||
|
||||
**Rischi specifici**:
|
||||
@@ -627,7 +642,7 @@ una finestra di manutenzione concordata.
|
||||
- [ ] Trigger manuale `build-ns7zip.yml` matrix Win + Linux PASS
|
||||
- [ ] Monitorare `journalctl -u act-runner -f` per ≥30 min sotto carico reale
|
||||
|
||||
**Hook futuri Fase C**: nessuno specifico in B6. Cutover non tocca
|
||||
**Hook futuri Fase D**: nessuno specifico in B6. Cutover non tocca
|
||||
backend.
|
||||
|
||||
**Rischi specifici**:
|
||||
@@ -672,7 +687,7 @@ tempi paragonabili.
|
||||
- [ ] Misurare uso disco `/var/lib/ci/build-vms/` durante e dopo burn-in (cleanup automatico)
|
||||
- [ ] Documentare risultati in `docs/RUNBOOK.md`
|
||||
|
||||
**Hook futuri Fase C**: i risultati di B7 sono il baseline contro cui
|
||||
**Hook futuri Fase D**: i risultati di B7 sono il baseline contro cui
|
||||
confrontare un eventuale backend ESXi (criterio di "C done" in
|
||||
`idea-3-esxi-support.md` §8).
|
||||
|
||||
@@ -697,22 +712,25 @@ fallisce, rollback a Windows (vedi B6 rollback).
|
||||
- [ ] Tempi documentati
|
||||
- [ ] Nessuna VM orfana
|
||||
|
||||
### B8 — Decommissioning host Windows (opzionale)
|
||||
### B8 — ~~Decommissioning host Windows~~ — ANNULLATO (vincolo categorico)
|
||||
|
||||
**Obiettivo**: dopo ≥1 settimana di stabilità, dismettere host Windows.
|
||||
> **B8 NON va eseguito.** Vincolo categorico: l'host Windows resta
|
||||
> supportato di prima classe in modo permanente (non solo rollback).
|
||||
> Niente dismissione, niente riallocazione hardware che rimuova la
|
||||
> capacità di girare su host Windows. Questo step resta solo come
|
||||
> record storico del piano originale.
|
||||
|
||||
**Input / dipendenze**: B7 done + ≥1 settimana di esercizio Linux
|
||||
senza incidenti.
|
||||
**Obiettivo (annullato)**: ~~dopo ≥1 settimana di stabilità, dismettere
|
||||
host Windows.~~ → Host Windows mantenuto operativo a regime.
|
||||
|
||||
**Attività**:
|
||||
**Attività (NON eseguire la dismissione; resta solo, se utile):**
|
||||
|
||||
- [ ] Backup finale di `F:\CI\` su archivio esterno (`tar` + checksum)
|
||||
- [ ] Spegnimento runner Windows (`Stop-Service actions-runner`, runner già fermo da B6)
|
||||
- [ ] Lasciare host Windows installato ma spento per ≥1 mese come rollback
|
||||
- [ ] Documentare procedura di riaccensione in `docs/RUNBOOK.md`
|
||||
- [ ] Dopo 1 mese: riallocare hardware se non ci sono stati rollback
|
||||
- [ ] (Facoltativo) Backup periodico di `F:\CI\` su archivio esterno
|
||||
- [ ] Host Windows **resta installato e utilizzabile** (run dual-host)
|
||||
- [ ] Documentare in `docs/RUNBOOK.md` come usare entrambi gli host
|
||||
- [ ] ~~Riallocare hardware~~ — NON applicabile (Windows host permanente)
|
||||
|
||||
**Hook futuri Fase C**: nessuno.
|
||||
**Hook futuri Fase D**: nessuno.
|
||||
|
||||
**Rischi specifici**:
|
||||
|
||||
@@ -749,7 +767,7 @@ re-registrare runner Windows verso Gitea.
|
||||
- [ ] Configurare `PSScriptAnalyzerSettings.psd1` per coprire `scripts/`, `template/`, `tests/` (già presente in repo)
|
||||
- [ ] Configurare `pyproject.toml` con `[tool.ruff]`, `[tool.mypy]`, `[tool.pytest.ini_options]`
|
||||
|
||||
**Hook futuri Fase C**: nessuno specifico.
|
||||
**Hook futuri Fase D**: nessuno specifico.
|
||||
|
||||
**Test / validazione**:
|
||||
|
||||
@@ -776,7 +794,7 @@ re-registrare runner Windows verso Gitea.
|
||||
- [ ] In B4: configurare service unit per inviare stdout/stderr a journald (default systemd) e validare `journalctl -u act-runner -o json`
|
||||
- [ ] Documentare query journald utili in `docs/RUNBOOK.md`
|
||||
|
||||
**Hook futuri Fase C**: log strutturati facilitano correlazione job ↔ backend.
|
||||
**Hook futuri Fase D**: log strutturati facilitano correlazione job ↔ backend.
|
||||
|
||||
**Test / validazione**:
|
||||
|
||||
@@ -805,7 +823,7 @@ systemd (Linux) senza sessione utente interattiva.
|
||||
- [ ] Implementare backend `keyring` file-based come opzione `KeyringFileBackend(vault_path, age_key_path)` se serve
|
||||
- [ ] Test pytest che simulano contesto headless (mock `keyring.get_credential` ritorna `None` → fallback a file vault)
|
||||
|
||||
**Hook futuri Fase C**: stesso `CredentialStore` userà credenziali
|
||||
**Hook futuri Fase D**: stesso `CredentialStore` userà credenziali
|
||||
vSphere — mantenere API generica.
|
||||
|
||||
**Test / validazione**:
|
||||
@@ -839,7 +857,7 @@ Linux + Python).
|
||||
- [ ] Aggiornare `AGENTS.md`: sezione "Python development" + nota errore #12 N/A in Python
|
||||
- [ ] Aggiornare `README.md`: quick-start Python + Linux
|
||||
|
||||
**Hook futuri Fase C**: lasciare placeholder "Backend ESXi (futuro)" in
|
||||
**Hook futuri Fase D**: lasciare placeholder "Backend ESXi (futuro)" in
|
||||
`docs/ARCHITECTURE.md` e `docs/HOST-SETUP.md`.
|
||||
|
||||
**Test / validazione**:
|
||||
@@ -855,7 +873,7 @@ Linux + Python).
|
||||
- [ ] Link interni validi
|
||||
- [ ] Quick-start testato da operatore esterno
|
||||
|
||||
## 4. Hook architetturali per Fase C
|
||||
## 4. Hook architetturali per Fase D
|
||||
|
||||
Punti di design da rispettare durante A e B per non bloccare un futuro
|
||||
backend ESXi senza implementarlo:
|
||||
@@ -942,7 +960,7 @@ Ogni voce: **Rischio** — Fase / Severità / Mitigazione / Owner action item.
|
||||
- Mitigazione: strategia shim minimizza la finestra; `lint.yml`
|
||||
dual-stack.
|
||||
- Owner: X1 attività complete.
|
||||
- **Astrazione `VmBackend` over-engineered se Fase C non parte mai**
|
||||
- **Astrazione `VmBackend` over-engineered se Fase D non parte mai**
|
||||
- Fase: A1 — Severità: Bassa
|
||||
- Mitigazione: 1 implementazione concreta + Protocol = ~50 LOC extra;
|
||||
costo trascurabile.
|
||||
@@ -1037,7 +1055,7 @@ Ogni voce: **Rischio** — Fase / Severità / Mitigazione / Owner action item.
|
||||
Domande di auto-verifica a cui il documento risponde "sì":
|
||||
|
||||
- **Ogni voce della checklist master compare almeno una volta come `- [ ]` dentro lo step corrispondente?** Sì: voci `[A1]…[A5]` mappate alle attività in §1, `[B1]…[B8]` in §2, `[X1]…[X4]` in §3.
|
||||
- **Ogni step `A<n>`/`B<n>`/`X<n>` ha tutte le sottosezioni richieste?** Sì: Obiettivo, Input/dipendenze, Attività, Hook futuri Fase C, Test/validazione, Rollback, Definizione di fatto presenti per ogni step (B*: in più rischi specifici).
|
||||
- **Ogni step `A<n>`/`B<n>`/`X<n>` ha tutte le sottosezioni richieste?** Sì: Obiettivo, Input/dipendenze, Attività, Hook futuri Fase D, Test/validazione, Rollback, Definizione di fatto presenti per ogni step (B*: in più rischi specifici).
|
||||
- **I 12 errori frequenti di `AGENTS.md` sono stati indirizzati o dichiarati non applicabili?**
|
||||
- #1 (sintassi PS 7+): si applica solo agli shim PS residui in A2/A3 — vincolo PS 5.1 mantenuto
|
||||
- #2 (`$LASTEXITCODE` non controllato): N/A in Python (`subprocess.returncode` esplicito)
|
||||
@@ -1059,7 +1077,8 @@ Domande di auto-verifica a cui il documento risponde "sì":
|
||||
- [plans/ideas-overview.md](ideas-overview.md) — razionale ordine fasi, criteri "fase completata", rollback
|
||||
- [plans/idea-1-python-rewrite.md](idea-1-python-rewrite.md) — Fase A: design, mappa traduzione, step A1–A5, layout repo, rischi
|
||||
- [plans/idea-2-linux-host.md](idea-2-linux-host.md) — Fase B: hypervisor, mappa adattamento, step B1–B8, rischi
|
||||
- [plans/idea-3-esxi-support.md](idea-3-esxi-support.md) — Fase C: §2 architettura, §3 backend pyVmomi (solo per hook)
|
||||
- [plans/idea-3-powershell-removal.md](idea-3-powershell-removal.md) — Fase C: eliminazione `pwsh` dall'host Linux
|
||||
- [plans/idea-3-esxi-support.md](idea-3-esxi-support.md) — Fase D: §2 architettura, §3 backend pyVmomi (solo per hook)
|
||||
- [docs/ARCHITECTURE.md](../docs/ARCHITECTURE.md) — layout attuale del sistema
|
||||
- [docs/CI-FLOW.md](../docs/CI-FLOW.md) — flusso job CI corrente
|
||||
- [docs/HOST-SETUP.md](../docs/HOST-SETUP.md) — setup host Windows attuale
|
||||
@@ -0,0 +1,602 @@
|
||||
# Fase B — Checklist finale per l'utente
|
||||
|
||||
La Fase B è la migrazione del runner CI dall'host Windows attuale a un
|
||||
nuovo host **Linux Mint LTS** con VMware Workstation Pro Linux.
|
||||
|
||||
> **Stato repo**: solo lo step **B5** (unit `systemd` per i task
|
||||
> periodici) è già committato in `deploy/systemd/` sul branch
|
||||
> `feature/python-rewrite-and-linux-migration` (commit `50d37b5`).
|
||||
> Tutti gli altri step (B1–B4, B6–B8) sono **operazioni hardware**
|
||||
> sull'host Linux nuovo e devono essere eseguiti dall'utente.
|
||||
>
|
||||
> **Avanzamento**: B1 ✅ B2 ✅ B3 ✅ B4 ✅ — in corso: **B5** (timer systemd).
|
||||
|
||||
> **Pre-requisito**: la Fase A deve essere `done` (vedi
|
||||
> `plans/PhaseA-user-checklist.md` Passo 9, merge + tag).
|
||||
> B1, B2, B3 possono partire in parallelo a A3/A4/A5; B4–B8 sono
|
||||
> sequenziali e iniziano solo dopo "A done".
|
||||
|
||||
> **Tempo stimato totale**: 1–2 giorni di lavoro distribuiti su una
|
||||
> finestra di 1–2 settimane (per consentire il burn-in tra B6 e B8).
|
||||
|
||||
---
|
||||
|
||||
## Passo 1 — Setup host Linux Mint (B1)
|
||||
|
||||
**Obiettivo**: hardware target con Linux Mint LTS, VMware Workstation
|
||||
Pro Linux, layout storage e venv Python pronti.
|
||||
|
||||
- [x] Installare Linux Mint LTS sull'hardware target.
|
||||
- [x] `sudo apt update && sudo apt full-upgrade -y && sudo reboot`
|
||||
- [x] Scaricare e installare VMware Workstation Pro Linux (bundle
|
||||
`.bundle` da Broadcow):
|
||||
|
||||
```bash
|
||||
sudo bash VMware-Workstation-Full-*.bundle
|
||||
vmrun -T ws --version # deve ritornare versione coerente
|
||||
```
|
||||
|
||||
- [x] Smoke test Workstation (UI): creare VM trivia, clone, start,
|
||||
stop, delete.
|
||||
- [x] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato a
|
||||
Windows) editando `/etc/vmware/vmnet8/dhcpd/dhcpd.conf` o via
|
||||
`vmware-netcfg`.
|
||||
- [x] Creare utente di servizio `ci-runner`:
|
||||
|
||||
```bash
|
||||
sudo useradd -r -m -s /bin/bash ci-runner
|
||||
```
|
||||
|
||||
- [x] Eseguire `setup-host-linux.sh` (copre mount disco, layout, Python venv,
|
||||
clone repo, PowerShell Core):
|
||||
|
||||
```bash
|
||||
# Clonare il repo prima di eseguire lo script
|
||||
git clone https://gitea.emulab.it/Simone/local-ci-cd-system.git /tmp/local-ci-cd-system
|
||||
|
||||
sudo bash /tmp/local-ci-cd-system/setup-host-linux.sh -d /dev/sdf1
|
||||
```
|
||||
|
||||
Lo script:
|
||||
- Monta `/dev/sdf1` su `/var/lib/ci` via fstab (boot automatico).
|
||||
Se la partizione ha una sottocartella `ci/`, ne sposta i contenuti
|
||||
alla root della partizione prima del mount definitivo.
|
||||
- Crea il layout `build-vms/`, `artifacts/`, `templates/`, `keys/`,
|
||||
`logs/`, `runner/` con permessi e ACL corretti.
|
||||
- Installa Python 3.11, crea `/opt/ci/venv`, clona il repo in
|
||||
`/opt/ci/local-ci-cd-system` e installa il package.
|
||||
- Installa PowerShell Core.
|
||||
|
||||
Flag utili se alcune dipendenze sono già presenti:
|
||||
```bash
|
||||
sudo bash setup-host-linux.sh -d /dev/sdf1 --skip-pwsh --skip-clone
|
||||
```
|
||||
|
||||
Atteso a fine script: `ci_orchestrator --help` lista gli 11 sub-comandi.
|
||||
|
||||
---
|
||||
|
||||
## Passo 2 — Trasferimento template VM (B2)
|
||||
|
||||
**Obiettivo**: copiare i template VMware da `F:\CI\Templates\` su
|
||||
`/var/lib/ci/templates/` mantenendo gli snapshot.
|
||||
|
||||
> ⚠️ AGENTS.md errore #9: i template devono essere **fully
|
||||
> powered-off** prima della copia. Verificare assenza di `*.vmem` /
|
||||
> `*.vmsn` di runtime.
|
||||
|
||||
- [x] Sull'host Windows:
|
||||
|
||||
```powershell
|
||||
Get-ChildItem 'F:\CI\Templates' -Recurse -Include *.vmem,*.vmsn
|
||||
# atteso: nessun risultato
|
||||
```
|
||||
|
||||
- [x] Sull'host Linux, eseguire rsync via SSH dall'host Windows
|
||||
(richiede OpenSSH Server attivo su Windows o, in alternativa,
|
||||
`scp -r` lanciato da Windows verso Linux):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner rsync -av --progress \
|
||||
user@windows-host:/cygdrive/f/CI/Templates/ \
|
||||
/var/lib/ci/templates/
|
||||
```
|
||||
|
||||
- [x] Validare integrità snapshot:
|
||||
|
||||
```bash
|
||||
find /var/lib/ci/templates -name '*.vmx' -exec vmrun -T ws listSnapshots {} \;
|
||||
```
|
||||
|
||||
Atteso: `BaseClean` su `WinBuild2025.vmx` e `WinBuild2022.vmx`,
|
||||
`BaseClean-Linux` su `LinuxBuild2404.vmx`.
|
||||
|
||||
- [x] Aprire ciascun `.vmx` su Workstation Linux per registrarli;
|
||||
al prompt rispondere "**I copied it**".
|
||||
- [x] **Prerequisito smoke test Windows**: il keyring file-based deve essere
|
||||
configurato e la credenziale `BuildVMGuest` deve essere presente
|
||||
**prima** di eseguire `wait-ready` (che prova WinRM). Questo è
|
||||
formalmente il Passo 3 (B3), ma il minimo indispensabile per lo
|
||||
smoke test è:
|
||||
|
||||
```bash
|
||||
# Configurare il backend file keyring per ci-runner (una sola volta)
|
||||
sudo -u ci-runner mkdir -p ~/.local/share/python_keyring
|
||||
sudo -u ci-runner bash -c "cat > ~/.local/share/python_keyring/keyringrc.cfg << 'EOF'
|
||||
[backend]
|
||||
default-keyring=keyrings.alt.file.PlaintextKeyring
|
||||
EOF"
|
||||
sudo -u ci-runner /opt/ci/venv/bin/pip install keyrings.alt -q
|
||||
|
||||
# Salvare la credenziale guest Windows
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-set.py BuildVMGuest "WINBUILD-2025\\ci_build"
|
||||
|
||||
# Verificare
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-check.py BuildVMGuest
|
||||
# atteso: OK (file) username='WINBUILD-2025\ci_build' password=****
|
||||
```
|
||||
|
||||
> Il setup completo delle credenziali (chiavi SSH, GiteaPAT,
|
||||
> PoC headless systemd) è nel Passo 3.
|
||||
|
||||
- [x] Smoke test pipeline VM:
|
||||
|
||||
```bash
|
||||
# Windows smoke
|
||||
CLONE_VMX=$(sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm new \
|
||||
--template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \
|
||||
--snapshot BaseClean \
|
||||
--clone-base-dir /var/lib/ci/build-vms \
|
||||
--job-id smoke-b2-win \
|
||||
--start)
|
||||
echo "Clone VMX: $CLONE_VMX"
|
||||
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator wait-ready \
|
||||
--vmx "$CLONE_VMX" --guest-os windows --timeout 300
|
||||
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm remove \
|
||||
--vmx "$CLONE_VMX" --force
|
||||
```
|
||||
|
||||
Ripetere per `LinuxBuild2404` (snapshot `BaseClean-Linux`,
|
||||
`--guest-os linux`, `--job-id smoke-b2-linux`).
|
||||
|
||||
---
|
||||
|
||||
## Passo 3 — Trasferimento credenziali e chiavi (B3)
|
||||
|
||||
**Obiettivo**: chiavi SSH e credenziali guest disponibili al user
|
||||
`ci-runner` headless.
|
||||
|
||||
- [x] Chiavi SSH guest Linux già presenti in `/var/lib/ci/keys/` (copiate
|
||||
insieme ai template). Correggere solo i permessi:
|
||||
|
||||
```bash
|
||||
sudo chown ci-runner:ci-runner /var/lib/ci/keys/ci_linux /var/lib/ci/keys/ci_linux.pub
|
||||
sudo chmod 600 /var/lib/ci/keys/ci_linux
|
||||
sudo chmod 644 /var/lib/ci/keys/ci_linux.pub
|
||||
# verifica
|
||||
ls -la /var/lib/ci/keys/
|
||||
```
|
||||
|
||||
- [x] Credenziale guest Windows `BuildVMGuest` già salvata nel
|
||||
prerequisito del Passo 2 con `keyrings.alt.file.PlaintextKeyring`
|
||||
(schema a due entry — `secret-tool` scartato perché non funziona
|
||||
headless senza D-Bus session).
|
||||
|
||||
Per riscrivere la credenziale se necessario:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-set.py BuildVMGuest "WINBUILD-2025\\ci_build"
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-check.py BuildVMGuest
|
||||
```
|
||||
|
||||
- [x] Re-store Gitea PAT con lo stesso schema:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-set.py GiteaPAT ci-runner-linux
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python /opt/ci/local-ci-cd-system/tools/ci-cred-check.py GiteaPAT
|
||||
```
|
||||
|
||||
- [x] **PoC headless**: il backend `PlaintextKeyring` non usa D-Bus —
|
||||
funziona già da `sudo -u ci-runner` e funzionerà da
|
||||
`act-runner.service`. Verificato durante smoke test Passo 2.
|
||||
|
||||
---
|
||||
|
||||
## Passo 4 — Setup act_runner come systemd service (B4)
|
||||
|
||||
**Obiettivo**: act_runner Linux registrato verso Gitea, gestito da
|
||||
systemd.
|
||||
|
||||
- [x] Scaricare il binario act_runner Linux ≥ v1.0.4:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /opt/ci/act_runner
|
||||
sudo wget -O /opt/ci/act_runner/act_runner \
|
||||
https://gitea.com/gitea/runner/releases/download/v1.0.4/gitea-runner-1.0.4-linux-amd64
|
||||
sudo chmod +x /opt/ci/act_runner/act_runner
|
||||
sudo chown -R ci-runner:ci-runner /opt/ci/act_runner
|
||||
```
|
||||
|
||||
- [x] Generare token registrazione su Gitea (Admin → Runners → Create new runner).
|
||||
- [x] Registrare il runner (sostituire `<TOKEN>`):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner mkdir -p /var/lib/ci/runner
|
||||
cd /var/lib/ci/runner
|
||||
sudo -u ci-runner /opt/ci/act_runner/act_runner register \
|
||||
--no-interactive \
|
||||
--instance https://gitea.emulab.it \
|
||||
--token <TOKEN> \
|
||||
--name ci-linux \
|
||||
--labels windows-build:host,linux-build:host
|
||||
```
|
||||
|
||||
> ⚠️ Lasciare il runner in stato **paused** lato Gitea UI fino al
|
||||
> cutover (Passo 6), per non intercettare job di produzione.
|
||||
|
||||
- [x] Creare `/etc/systemd/system/act-runner.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=Gitea Act Runner
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ci-runner
|
||||
WorkingDirectory=/var/lib/ci/runner
|
||||
Environment="PYTHONIOENCODING=utf-8"
|
||||
Environment="CI_ROOT=/var/lib/ci"
|
||||
Environment="CI_TEMPLATES=/var/lib/ci/templates"
|
||||
Environment="CI_BUILD_VMS=/var/lib/ci/build-vms"
|
||||
Environment="CI_ARTIFACTS=/var/lib/ci/artifacts"
|
||||
Environment="CI_KEYS=/etc/ci/keys"
|
||||
ExecStart=/opt/ci/act_runner/act_runner daemon --config /var/lib/ci/runner/config.yaml
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
- [x] Creare `/etc/ci/environment` (letto dalle unit dei task periodici
|
||||
di B5; vedi `deploy/systemd/README.md`):
|
||||
|
||||
```bash
|
||||
sudo tee /etc/ci/environment <<'EOF'
|
||||
PYTHONIOENCODING=utf-8
|
||||
CI_ROOT=/var/lib/ci
|
||||
CI_TEMPLATES=/var/lib/ci/templates
|
||||
CI_BUILD_VMS=/var/lib/ci/build-vms
|
||||
CI_ARTIFACTS=/var/lib/ci/artifacts
|
||||
CI_KEYS=/etc/ci/keys
|
||||
EOF
|
||||
sudo chmod 644 /etc/ci/environment
|
||||
```
|
||||
|
||||
- [x] Abilitare e avviare:
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now act-runner.service
|
||||
sudo systemctl status act-runner
|
||||
sudo journalctl -u act-runner -f # Ctrl+C dopo conferma OK
|
||||
```
|
||||
|
||||
- [x] Generare e configurare `config.yaml` del runner:
|
||||
|
||||
```bash
|
||||
# 1. Generare il file di default (conserva la registrazione in .runner)
|
||||
sudo -u ci-runner /opt/ci/act_runner/act_runner generate-config \
|
||||
| sudo -u ci-runner tee /var/lib/ci/runner/config.yaml > /dev/null
|
||||
|
||||
# 2. Sostituire i label Docker generati di default con quelli host corretti
|
||||
# (generate-config scrive label ubuntu-*:docker:// che fanno cercare Docker)
|
||||
sudo -u ci-runner sed -i \
|
||||
-e 's| - "ubuntu-latest:docker://.*"| - "windows-build:host"|' \
|
||||
-e '/ubuntu-24.04:docker:\/\//d' \
|
||||
-e '/ubuntu-22.04:docker:\/\//d' \
|
||||
/var/lib/ci/runner/config.yaml
|
||||
sudo -u ci-runner sed -i \
|
||||
'/ - "windows-build:host"/a\ - "linux-build:host"' \
|
||||
/var/lib/ci/runner/config.yaml
|
||||
|
||||
# 3. Sostituire le variabili di esempio nella sezione runner.envs con le variabili CI
|
||||
# (generate-config scrive due righe A_TEST_ENV_NAME_* come placeholder)
|
||||
sudo -u ci-runner sed -i \
|
||||
-e 's/ A_TEST_ENV_NAME_1: a_test_env_value_1/ CI_PYTHON_LAUNCHER: \/opt\/ci\/venv\/bin\/python/' \
|
||||
-e 's/ A_TEST_ENV_NAME_2: a_test_env_value_2/ GITEA_CI_TEMPLATE_PATH: \/var\/lib\/ci\/templates\/WinBuild2025\/WinBuild2025.vmx\n GITEA_CI_LINUX_TEMPLATE_PATH: \/var\/lib\/ci\/templates\/LinuxBuild2404\/LinuxBuild2404.vmx/' \
|
||||
/var/lib/ci/runner/config.yaml
|
||||
|
||||
# 4. Verificare
|
||||
grep -A3 'labels:' /var/lib/ci/runner/config.yaml | head -5
|
||||
grep -A4 'envs:' /var/lib/ci/runner/config.yaml
|
||||
```
|
||||
|
||||
Atteso:
|
||||
```yaml
|
||||
labels:
|
||||
- "windows-build:host"
|
||||
- "linux-build:host"
|
||||
envs:
|
||||
CI_PYTHON_LAUNCHER: /opt/ci/venv/bin/python
|
||||
GITEA_CI_TEMPLATE_PATH: /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx
|
||||
GITEA_CI_LINUX_TEMPLATE_PATH: /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx
|
||||
```
|
||||
|
||||
> Le variabili di sistema (`CI_ROOT`, `PYTHONIOENCODING`, ecc.) restano
|
||||
> nel service unit. `runner.envs` è per la configurazione specifica
|
||||
> del runner CI (path template, launcher Python).
|
||||
|
||||
```bash
|
||||
# 5. Ricaricare e riavviare
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl restart act-runner.service
|
||||
sudo journalctl -u act-runner -n 20 # confermare avvio OK — nessun errore Docker
|
||||
```
|
||||
|
||||
- [x] Smoke job: triggerare manualmente `self-test.yml` selezionando
|
||||
il runner Linux (mettere temporaneamente in pausa quello Windows
|
||||
o usare label distinte). Atteso: PASS.
|
||||
|
||||
> ⚠️ Prerequisito: le variabili `GITEA_CI_TEMPLATE_PATH` e
|
||||
> `GITEA_CI_LINUX_TEMPLATE_PATH` devono essere presenti nel service
|
||||
> unit **prima** di avviare lo smoke test, altrimenti il job fallisce
|
||||
> con "Got unexpected extra argument".
|
||||
|
||||
---
|
||||
|
||||
## Passo 5 — Installare i timer systemd (B5)
|
||||
|
||||
**Obiettivo**: attivare le coppie `.service`+`.timer` già committate
|
||||
in `deploy/systemd/` per replicare i task periodici di
|
||||
`Register-CIScheduledTasks.ps1`.
|
||||
|
||||
> **Nota**: `ci-retention-policy` e `ci-backup-template` sono stati portati
|
||||
> in Python puro (`retention run` / `template backup`) — **pwsh non è
|
||||
> richiesto**. I backup template vengono compressi con `7z -mx=1` in
|
||||
> `/var/lib/ci/backups/`. `ci-watch-runner-health` usa `--service-name
|
||||
> act-runner` (nome Linux, non Windows).
|
||||
|
||||
> **Prerequisiti creati sull'host**:
|
||||
> ```bash
|
||||
> sudo mkdir -p /var/log/ci /var/lib/ci/backups
|
||||
> sudo chown ci-runner:ci-runner /var/log/ci
|
||||
> ```
|
||||
|
||||
- [x] Copiare le unit:
|
||||
|
||||
```bash
|
||||
cd /opt/ci/local-ci-cd-system
|
||||
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
- [x] Abilitare e avviare tutti i timer:
|
||||
|
||||
```bash
|
||||
for t in ci-cleanup-orphans ci-retention-policy \
|
||||
ci-watch-disk-space ci-watch-runner-health \
|
||||
ci-backup-template; do
|
||||
sudo systemctl enable --now "${t}.timer"
|
||||
done
|
||||
```
|
||||
|
||||
- [x] Verificare lo schedule:
|
||||
|
||||
```bash
|
||||
systemctl list-timers --all 'ci-*'
|
||||
```
|
||||
|
||||
Atteso: 5 timer in stato `active`. ✅ PASS
|
||||
|
||||
- [x] Trigger manuale di ciascun service per validare l'esecuzione
|
||||
one-shot:
|
||||
|
||||
```bash
|
||||
for s in ci-cleanup-orphans ci-retention-policy \
|
||||
ci-watch-disk-space ci-watch-runner-health \
|
||||
ci-backup-template; do
|
||||
sudo systemctl start "${s}.service"
|
||||
sudo systemctl status "${s}.service" --no-pager
|
||||
done
|
||||
```
|
||||
|
||||
✅ Tutti e 5 PASS.
|
||||
|
||||
- [x] Per maggiori dettagli (mapping Windows→Linux, troubleshooting,
|
||||
rollback) vedi `deploy/systemd/README.md`.
|
||||
|
||||
---
|
||||
|
||||
## Passo 6 — Cutover (B6)
|
||||
|
||||
**Obiettivo**: passare all'uso normale di Linux come OS primario di
|
||||
produzione CI. Il cutover è un semplice riavvio — i due sistemi non
|
||||
coesistono mai.
|
||||
|
||||
> ⚠️ Eseguire solo dopo che i Passi 1–5 sono tutti `[x]` PASS.
|
||||
|
||||
- [x] Verificare nessun job in coda lato Gitea (Admin → Actions → Tasks).
|
||||
- [x] Attendere la fine di eventuali job in esecuzione su Windows.
|
||||
- [x] Sbloccare il runner Linux su Gitea UI (rimuovere `paused`).
|
||||
- [x] Riavviare la macchina in Linux.
|
||||
- [x] Verificare runner Linux online su Gitea UI e act-runner attivo:
|
||||
|
||||
```bash
|
||||
sudo systemctl status act-runner
|
||||
sudo journalctl -u act-runner -n 20
|
||||
```
|
||||
|
||||
- [x] Smoke trigger:
|
||||
- [x] `self-test.yml` → PASS dal runner Linux
|
||||
- [x] `build-ns7zip.yml` matrix Win+Linux → PASS dal runner Linux
|
||||
- [x] Monitorare per ≥30 min:
|
||||
|
||||
```bash
|
||||
sudo journalctl -u act-runner -f
|
||||
```
|
||||
|
||||
> **Rollback**: riavviare in Windows — il runner Windows riprende
|
||||
> automaticamente. Mettere in pausa il runner Linux su Gitea UI,
|
||||
> documentare l'incidente in `TODO.md`.
|
||||
|
||||
---
|
||||
|
||||
## Passo 7 — Capacity burn-in (B7)
|
||||
|
||||
**Obiettivo**: validare il carico target con tempi paragonabili al
|
||||
baseline Windows (entro ±20%). I due burn-in sono sequenziali
|
||||
(dual-boot: un OS alla volta).
|
||||
|
||||
> **Eseguire come `ci-runner`** (utente di produzione — ha già accesso
|
||||
> a `/var/lib/ci/`, keyring e ip-pool; evita i perm-hack del path `simone`).
|
||||
> Verificato 2026-06-07.
|
||||
>
|
||||
> **Prerequisiti una-tantum**:
|
||||
> - Repo `burnin-dummy` esistente in Gitea (`Simone/burnin-dummy`,
|
||||
> privato — l'auth del clone in-VM usa il PAT da keyring `GiteaPAT`).
|
||||
> Deve contenere `build.ps1` + `build.sh`. Push iniziale:
|
||||
> ```bash
|
||||
> cd /opt/ci/local-ci-cd-system/gitea/burnin-dummy
|
||||
> git init && git add . && git commit -m "initial"
|
||||
> git remote add origin http://10.10.20.11:3100/Simone/burnin-dummy.git
|
||||
> git push -u origin main
|
||||
> ```
|
||||
> - **ip-pool pulito** prima di ogni burn-in: un run precedente killato
|
||||
> può lasciare lease stale (il pool NON si auto-riconcilia, vedi
|
||||
> RUNBOOK §9). Con `build-vms/` vuoto, resettare:
|
||||
> ```bash
|
||||
> sudo -u ci-runner /opt/ci/venv/bin/python -c \
|
||||
> "import pathlib; pathlib.Path('/var/lib/ci/ip-pool.json').write_text('{}\n')"
|
||||
> ```
|
||||
>
|
||||
> **(Path alternativo `simone`)**: se proprio si vuole girare come `simone`,
|
||||
> oltre a group `ci-runner` + copia keyring serve anche
|
||||
> `sudo chmod g+w /var/lib/ci` (la **root**, non solo
|
||||
> `artifacts/`+`build-vms/` — `ip-pool.tmp` è scritto lì). Sconsigliato.
|
||||
|
||||
- [x] **Avviato in Linux** — burn-in 4 job concorrenti × 10 round su
|
||||
`WinBuild2025` (esito: 40/40 PASS, round ~78.6 s):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner -H pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \
|
||||
-Parallelism 4 -Rounds 10 \
|
||||
-TemplatePath /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \
|
||||
-RepoUrl 'http://10.10.20.11:3100/Simone/burnin-dummy.git' \
|
||||
-CloneBaseDir /var/lib/ci/build-vms/ \
|
||||
-ArtifactBaseDir /var/lib/ci/artifacts/ \
|
||||
-VmrunPath /usr/bin/vmrun \
|
||||
-BuildCommand 'powershell -ExecutionPolicy Bypass -File .\build.ps1'
|
||||
```
|
||||
|
||||
- [x] **Avviato in Linux** — burn-in 4 × 10 su `LinuxBuild2404` (esito:
|
||||
40/40 PASS, round ~70.2 s):
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner -H pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \
|
||||
-Parallelism 4 -Rounds 10 \
|
||||
-TemplatePath /var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx \
|
||||
-SnapshotName BaseClean-Linux \
|
||||
-RepoUrl 'http://10.10.20.11:3100/Simone/burnin-dummy.git' \
|
||||
-CloneBaseDir /var/lib/ci/build-vms/ \
|
||||
-ArtifactBaseDir /var/lib/ci/artifacts/ \
|
||||
-VmrunPath /usr/bin/vmrun \
|
||||
-GuestOS Linux \
|
||||
-BuildCommand 'bash build.sh'
|
||||
```
|
||||
> **Nota**: `-SnapshotName BaseClean-Linux` è **obbligatorio** — il default
|
||||
> dello script è `BaseClean` (Windows); senza, fallisce con
|
||||
> `Error: Invalid snapshot name 'BaseClean'`.
|
||||
- [x] Misurare:
|
||||
- [x] Tempo medio per round, vs baseline §6–§8 — round near-deterministico
|
||||
(σ < 1.5 s), in linea con static-IP floor (RUNBOOK §9)
|
||||
- [x] Tutti i 80 job (2 × 4 × 10) PASS
|
||||
- [x] Zero VM orfane in `/var/lib/ci/build-vms/`:
|
||||
|
||||
```bash
|
||||
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --what-if
|
||||
```
|
||||
|
||||
- [x] Spazio disco `/var/lib/ci/build-vms/` torna al baseline post-cleanup
|
||||
- [x] Documentare i risultati in `docs/RUNBOOK.md` (§9 — Concurrent
|
||||
capacity burn-in).
|
||||
- [ ] Se delta > 20% vs baseline Windows: aprire issue in `TODO.md`
|
||||
con dettagli per profiling — **N/A**: nessun delta, burn-in PASS.
|
||||
|
||||
---
|
||||
|
||||
## Passo 8 — Stabilità ≥1 settimana
|
||||
|
||||
**Obiettivo**: verificare ≥1 settimana di esercizio con la macchina
|
||||
avviata abitualmente in Linux senza incidenti critici.
|
||||
|
||||
> **Esito 2026-06-07**: host live dal cutover B6 (~24 mag) → >2 settimane
|
||||
> di esercizio. Salute al 7 giu: `systemctl --failed` vuoto, tutti i timer
|
||||
> `ci-*` con ultimo run OK, nessun errore reale in `act-runner` (solo
|
||||
> `NamedPipeIPC_ServerListenerError` cosmetici su exit dei processi pwsh).
|
||||
> Nessun rollback. Le sessioni del 6 giu sono state hardening template
|
||||
> Linux (no-IP/DHCP), non instabilità dell'host.
|
||||
|
||||
- [x] Esercizio normale per ≥7 giorni con boot in Linux.
|
||||
- [x] Verifica giornaliera (bastano 2 minuti):
|
||||
|
||||
```bash
|
||||
sudo systemctl --failed # nessun service in failed
|
||||
systemctl list-timers --all 'ci-*' # tutti gli ultimi run OK
|
||||
sudo journalctl -u act-runner --since "24h ago" -p err
|
||||
```
|
||||
|
||||
- [x] Nessun rollback effettuato durante la settimana.
|
||||
|
||||
---
|
||||
|
||||
## Passo 9 — Gestione dual-boot
|
||||
|
||||
> La macchina è **dual-boot**: Windows e Linux coesistono sullo stesso
|
||||
> hardware, non girano mai contemporaneamente. Non c'è dismissione —
|
||||
> Windows resta pienamente funzionale e si usa riavviando nel boot
|
||||
> entry corrispondente.
|
||||
|
||||
- [x] Aggiornare `docs/RUNBOOK.md` con la procedura dual-boot: vedi
|
||||
**RUNBOOK §15** (switch Linux⇄Windows, verifica post-boot per
|
||||
entrambi gli OS, tweak GRUB opzionale).
|
||||
- [x] (Facoltativo) GRUB verificato: `GRUB_DEFAULT=0`, `GRUB_TIMEOUT=0`,
|
||||
`GRUB_TIMEOUT_STYLE=hidden` → boot diretto in Linux (intenzionale:
|
||||
la macchina torna host Linux dopo ogni reboot unattended). Per
|
||||
switch a Windows: tasto Esc al boot, oppure `grub-reboot` con
|
||||
`GRUB_DEFAULT=saved`. Tweak documentato in RUNBOOK §15, **non
|
||||
applicato** (default Linux voluto).
|
||||
|
||||
```bash
|
||||
sudo nano /etc/default/grub # GRUB_DEFAULT, GRUB_TIMEOUT
|
||||
sudo update-grub
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Tracciamento globale
|
||||
|
||||
| Passo | Step | Descrizione | Stato |
|
||||
| ----- | ---- | -------------------------------------------------- | ----- |
|
||||
| 1 | B1 | Setup host Linux Mint (OS, Workstation, venv, ACL) | [x] |
|
||||
| 2 | B2 | Trasferimento template VM + smoke `vm new` | [x] |
|
||||
| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [x] |
|
||||
| 4 | B4 | act_runner systemd service + `self-test.yml` PASS | [x] |
|
||||
| 5 | B5 | Timer systemd installati e attivi | [x] |
|
||||
| 6 | B6 | Cutover (riavvio in Linux, runner Linux attivo) | [x] |
|
||||
| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux, sequenziale) | [x] |
|
||||
| 8 | — | ≥1 settimana di stabilità (boot Linux abituale) | [x] |
|
||||
| 9 | — | Gestione dual-boot + RUNBOOK aggiornato (§15) | [x] |
|
||||
|
||||
> **Architettura**: la macchina è **dual-boot** (stesso hardware).
|
||||
> Avviata in Linux → runner Linux attivo. Avviata in Windows → runner
|
||||
> Windows attivo. I due sistemi non girano mai contemporaneamente.
|
||||
|
||||
Quando i passi 1-8 sono `[x]`, la **Fase B è chiusa** (B8 annullato per
|
||||
vincolo) e si può valutare l'apertura della Fase C (eliminazione `pwsh`
|
||||
dall'host Linux, vedi `plans/idea-3-powershell-removal.md`). La Fase D
|
||||
(backend ESXi, `plans/idea-3-esxi-support.md`) viene dopo.
|
||||
@@ -0,0 +1,151 @@
|
||||
# Plan — Windows-host benchmarks (comparable to Linux-host §6–§10)
|
||||
|
||||
> **Scopo**: la macchina è dual-boot (Windows ⇄ Linux, stesso hardware).
|
||||
> Le baseline RUNBOOK §6–§10 sono quasi tutte sul **Linux host**. Questo
|
||||
> plan raccoglie i numeri mancanti sul **Windows host** per chiudere la
|
||||
> matrice host × guest × IP-mode e rendere il confronto Linux-vs-Windows
|
||||
> completo e simmetrico.
|
||||
>
|
||||
> **Prerequisito**: riavviare la macchina nel boot entry **Windows**.
|
||||
> Tutto gira su Windows PowerShell 5.1 o pwsh 7; usare la stessa
|
||||
> `Measure-CIBenchmark.ps1` / `Test-CapacityBurnIn.ps1` del repo (sono
|
||||
> cross-platform, i default path passano a `F:\CI\...` quando `$IsWindows`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Cosa esiste già e cosa manca
|
||||
|
||||
| RUNBOOK § | Host | Guest | IP mode | Stato |
|
||||
| --------- | ----- | -------------- | --------------------- | ------------ |
|
||||
| §6 | Win | WinBuild2025 | DHCP | ✓ 2026-05-17 |
|
||||
| §7 | Linux | WinBuild2025 | DHCP | ✓ |
|
||||
| §8 | Linux | WinBuild2025 | static | ✓ |
|
||||
| §9 | Linux | Win + Linux | burn-in (static pool) | ✓ |
|
||||
| §10 | Linux | LinuxBuild2404 | DHCP | ✓ |
|
||||
|
||||
**Mancanti sul Windows host** (target di questo plan):
|
||||
|
||||
| Nuovo § | Host | Guest | IP mode | Pair-with |
|
||||
| ------- | ---- | -------------- | ------------ | --------- |
|
||||
| §11 | Win | WinBuild2025 | static | §8 |
|
||||
| §12 | Win | LinuxBuild2404 | DHCP | §10 |
|
||||
| §13 | Win | Win + Linux | burn-in 4×10 | §9 |
|
||||
|
||||
§6 copre già Win×Win×DHCP. Opzionale: ri-eseguirlo nella stessa sessione per
|
||||
coerenza (stesso disco/fragmentazione del momento).
|
||||
|
||||
---
|
||||
|
||||
## 2. Prerequisiti sul Windows host
|
||||
|
||||
- [x] Boot in Windows; verificare VMware Workstation + `vmrun.exe` in
|
||||
`C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe`.
|
||||
- [x] Template fully powered-off con snapshot integri:
|
||||
```powershell
|
||||
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' -T ws listSnapshots 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx'
|
||||
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' -T ws listSnapshots 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx'
|
||||
```
|
||||
Attesi: `BaseClean` e `BaseClean-Linux`. ✓ (anche `PostInstall` su Win)
|
||||
- [x] Credenziali in **Windows Credential Manager** (non keyring file):
|
||||
target `BuildVMGuest` (guest WinRM, user `ci_build`) e `GiteaPAT`
|
||||
(user `Simone`). Verificato.
|
||||
- [x] Chiave SSH guest Linux presente: `F:\CI\keys\ci_linux` (per §12/§13
|
||||
lato LinuxBuild2404).
|
||||
- [x] Gitea raggiungibile (`http://10.10.20.11:3100`, HTTP 200) e repo
|
||||
`Simone/burnin-dummy` con `build.ps1` + `build.sh` (verificato via API).
|
||||
- [x] `F:\CI\BuildVMs\` privo di cloni orfani e `F:\CI\ip-pool.json` assente
|
||||
(pulito). Reset se serve:
|
||||
```powershell
|
||||
Set-Content -Path 'F:\CI\ip-pool.json' -Value '{}' -NoNewline
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. §11 — WinBuild2025 static IP (pair §8)
|
||||
|
||||
```powershell
|
||||
.\scripts\Measure-CIBenchmark.ps1 `
|
||||
-TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
||||
-SnapshotName BaseClean `
|
||||
-StaticIP 192.168.79.200 -Netmask 255.255.255.0 `
|
||||
-Iterations 10
|
||||
```
|
||||
- [x] 10 iter completate, tabella per-fase stampata. **Boot total avg 29.75 s.**
|
||||
- [x] Confronto vs §8 (Linux host static): IP-floor confermato host-independent
|
||||
(22.81 s vs 21.77 s, +5 %). Penalità host: clone +58 % (NTFS), Ready +4.7 s
|
||||
(WinRM re-bind dopo NIC reconfig), destroy +82 % (bimodale), boot +24 %.
|
||||
Vedi RUNBOOK §11. **Nota**: primo tentativo fallito (timeout 300 s/iter) —
|
||||
template Windows privo dell'agent `ci-static-ip`; risolto copiando i
|
||||
template dal Linux host (parità template).
|
||||
|
||||
## 4. §12 — LinuxBuild2404 DHCP (pair §10)
|
||||
|
||||
```powershell
|
||||
.\scripts\Measure-CIBenchmark.ps1 `
|
||||
-TemplatePath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
||||
-SnapshotName BaseClean-Linux `
|
||||
-GuestOS Linux `
|
||||
-Iterations 10
|
||||
```
|
||||
- [x] 10 iter completate. **Boot total avg 13.51 s.**
|
||||
- [x] Confronto vs §10 (Linux host, 8.82 s): **+53 %** host-side — clone +191 %
|
||||
(NTFS), IP-acquire +52 % (NAT vmnet8 Windows + guestinfo). Guest Linux resta
|
||||
deterministico (IP σ ≈ 0.05 s). Vedi RUNBOOK §12.
|
||||
|
||||
## 5. §13 — Concurrent burn-in 4×10 (pair §9)
|
||||
|
||||
```powershell
|
||||
# WinBuild2025
|
||||
.\scripts\Test-CapacityBurnIn.ps1 `
|
||||
-Parallelism 4 -Rounds 10 `
|
||||
-TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
||||
-RepoUrl 'http://10.10.20.11:3100/Simone/burnin-dummy.git' `
|
||||
-CloneBaseDir 'F:\CI\BuildVMs\' `
|
||||
-ArtifactBaseDir 'F:\CI\Artifacts\' `
|
||||
-VmrunPath 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||
-BuildCommand 'powershell -ExecutionPolicy Bypass -File .\build.ps1'
|
||||
|
||||
# LinuxBuild2404 (snapshot OBBLIGATORIO, default è BaseClean=Windows)
|
||||
.\scripts\Test-CapacityBurnIn.ps1 `
|
||||
-Parallelism 4 -Rounds 10 `
|
||||
-TemplatePath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
||||
-SnapshotName BaseClean-Linux `
|
||||
-RepoUrl 'http://10.10.20.11:3100/Simone/burnin-dummy.git' `
|
||||
-CloneBaseDir 'F:\CI\BuildVMs\' `
|
||||
-ArtifactBaseDir 'F:\CI\Artifacts\' `
|
||||
-VmrunPath 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||
-GuestOS Linux `
|
||||
-BuildCommand 'bash build.sh'
|
||||
```
|
||||
- [~] **Linux 40/40 PASS (10/10 round). Windows 36/40 (7/10 round).** I 4 fail
|
||||
Windows = fault WinRM transitori (round 6–8: connect-timeout 5986 +
|
||||
WSManFault "shell not found") sotto 4× concorrenza; auto-recuperati (slot IP
|
||||
rilasciati, 0 cloni orfani, round 9–10 di nuovo puliti). Linux/SSH 0 fault.
|
||||
- [x] Round avg vs §9 (Win ~78.6 s, Lin ~70.2 s): Windows host Win ~91.6 s
|
||||
(round PASS) **+16 %** + instabilità WinRM; Lin 96.4 s **+37 %**.
|
||||
Vedi RUNBOOK §13. **Pre-req scoperti**: (1) parità template, (2) prod venv
|
||||
F:\CI\python\venv stale (pre-`ip_pool`) → reinstallato `pip install .`,
|
||||
(3) `CI_VENV_PYTHON` da settare (bug shim `$IsWindows` su PS 5.1+StrictMode).
|
||||
|
||||
---
|
||||
|
||||
## 6. Output e documentazione
|
||||
|
||||
- Tutti i run appendono a `F:\CI\Logs\benchmark.jsonl` (single-job) e
|
||||
stampano summary (burn-in).
|
||||
- [x] Risultati aggiunti a `docs/RUNBOOK.md` §11–§13 (+ §14 matrice completa),
|
||||
con confronto host Linux-vs-Windows per ciascuna coppia.
|
||||
- [x] Pulizia post-run: `F:\CI\BuildVMs\` vuoto, artifact `burnin-*`
|
||||
rimossi, `F:\CI\ip-pool.json` reset.
|
||||
|
||||
---
|
||||
|
||||
## 7. Matrice finale attesa (dopo questo plan)
|
||||
|
||||
| Host \ (Guest, mode) | Win/DHCP | Win/static | Lin/DHCP | burn-in 4×10 |
|
||||
| -------------------- | -------- | ---------- | -------- | ------------ |
|
||||
| **Linux** | §7 | §8 | §10 | §9 |
|
||||
| **Windows** | §6 | §11 | §12 | §13 |
|
||||
|
||||
Confronto colonna-per-colonna = **effetto host** a guest+mode costanti.
|
||||
Confronto riga-per-riga = **effetto guest/IP-mode** a host costante.
|
||||
@@ -0,0 +1,185 @@
|
||||
# Fase C — Piano di applicazione (eliminazione `pwsh` dall'host Linux)
|
||||
|
||||
> Piano operativo per [idea-3-powershell-removal.md](idea-3-powershell-removal.md).
|
||||
> Generato il 2026-06-07. Eseguire i passi **in ordine**: ogni passo è
|
||||
> autonomo, testabile e con rollback banale (i `.ps1` non vengono rimossi
|
||||
> finché C9 non chiude il burn-in di validazione).
|
||||
|
||||
---
|
||||
|
||||
## Summary — checklist master
|
||||
|
||||
Stato passi (spunta man mano):
|
||||
|
||||
- [x] **C1** — Scaffolding gruppi CLI `bench` / `smoke` / `validate` / `creds` + test skeleton
|
||||
- [x] **C2** — `bench run`: porting burn-in concorrenza (`Test-CapacityBurnIn` + `Start-BurnInTest*`)
|
||||
- [x] **C3** — `bench measure`: porting timing fasi (`Measure-CIBenchmark`) + retire `_Common.psm1`/`_Transport.psm1`/`_Common.Tests.ps1`
|
||||
- [x] **C4** — `smoke run`: porting smoke + build E2E (`Test-Smoke`, `Test-*Build-Linux`)
|
||||
- [x] **C5** — `validate host`: equivalente Linux di `Validate-HostState`
|
||||
- [x] **C6** — `creds set` (rimpiazza `Set-CIGuestCredential`) + `validate guest` (diag transport)
|
||||
- [x] **C7** — Lint CI: risolvere dipendenza `pwsh` di `lint.yml` (decisione §4 dell'analisi)
|
||||
- [x] **C8** — `setup-host-linux.sh`: `pwsh` non più installato di default (`--with-pwsh` opt-in)
|
||||
- [~] **C9** — Docs ✅ + cutover/burn-in ≥1 settimana ⏳ + disinstallazione `pwsh` ⏳ (operativi, vedi sotto)
|
||||
|
||||
**Definition of done globale**: tutti i criteri del §7 dell'analisi
|
||||
verificati; `pwsh` rimosso dall'host Linux; pytest verde ≥90%.
|
||||
|
||||
---
|
||||
|
||||
## Convenzioni trasversali (valgono per ogni passo)
|
||||
|
||||
- Codice nuovo solo in `src/ci_orchestrator/`. `click.echo`, no `print()`,
|
||||
no `subprocess(..., shell=True)`, type hints completi, `mypy --strict`.
|
||||
- Importare **solo** `backends.load_backend` + transport esistenti
|
||||
(`transport/ssh.py`, `transport/winrm.py`). Mai `WorkstationVmrunBackend`.
|
||||
- Ogni nuovo comando ha test in `tests/python/test_commands_<gruppo>.py`
|
||||
con backend e transport mockati (no VM reali in unit test).
|
||||
- Gate coverage 90% per ogni PR. `ruff` + `mypy --strict` verdi.
|
||||
- Un passo = una PR. Non toccare il branch WinRM e SSH nello stesso commit
|
||||
(gotcha CLAUDE.md): se un comando ha entrambi i rami, separarli o
|
||||
isolare dietro `if guest_os == "linux"`.
|
||||
|
||||
---
|
||||
|
||||
## C1 — Scaffolding
|
||||
|
||||
**Obiettivo**: registrare i gruppi vuoti e i file, senza logica reale,
|
||||
per fissare l'interfaccia CLI e i test.
|
||||
|
||||
- [ ] Creare `commands/bench.py` con `@click.group("bench")` e i comandi
|
||||
stub `run` / `measure` (firma completa, corpo `raise NotImplementedError`
|
||||
dietro `--dry-run` per i test).
|
||||
- [ ] Creare `commands/smoke.py` (`@click.group("smoke")` → `run`).
|
||||
- [ ] Creare `commands/validate.py` (`@click.group("validate")` → `host`, `guest`).
|
||||
- [ ] Creare `commands/creds.py` (`@click.group("creds")` → `set`).
|
||||
- [ ] Registrarli in `__main__.py` (`cli.add_command(...)`).
|
||||
- [ ] `tests/python/test_commands_bench.py` ecc.: verificare `--help` e
|
||||
registrazione comandi (smoke test CLI).
|
||||
- **Done**: `python -m ci_orchestrator bench --help` ecc. funzionano;
|
||||
pytest verde.
|
||||
|
||||
## C2 — `bench run` (burn-in concorrenza)
|
||||
|
||||
**Sostituisce**: `Test-CapacityBurnIn.ps1`, `Start-BurnInTest.ps1`,
|
||||
`Start-BurnInTest-Linux.ps1`.
|
||||
|
||||
- [ ] Implementare il lancio di N job concorrenti **in-process**
|
||||
(`concurrent.futures` su `commands/job.py`) o come N subprocess del
|
||||
CLI Python — **niente `Start-Job`/`pwsh`/`$IsWindows`** (chiude il bug
|
||||
shim f22b142).
|
||||
- [ ] Per ogni round, asserire: tutti i job exit 0; nessuna clone-dir
|
||||
orfana in `CI_BUILD_VMS`; nessun `vm-start.lock` stale (>5 min).
|
||||
- [ ] Report JSON in `CI_ARTIFACTS/bench/` + tabella riepilogo a video.
|
||||
- [ ] Default lab: `--concurrency 4 --rounds 10`, build-command no-op
|
||||
(`ping`/`sleep ~30s`) come negli wrapper.
|
||||
- [ ] Test: round con job mockati (PASS/FAIL), rilevamento orfani e lock.
|
||||
- **Done**: output equivalente a `Test-CapacityBurnIn.ps1` su Win e Linux.
|
||||
|
||||
## C3 — `bench measure` (timing fasi) + retire moduli PS
|
||||
|
||||
**Sostituisce**: `Measure-CIBenchmark.ps1` (+ `_Common.psm1`,
|
||||
`_Transport.psm1`, `tests/_Common.Tests.ps1`).
|
||||
|
||||
- [ ] Misurare fasi clone / start / IP acquire / transport ready / destroy
|
||||
via `load_backend` + transport, su 1+ VM effimere.
|
||||
- [ ] Append a `benchmark.jsonl` (stesso formato, per continuità trend).
|
||||
- [ ] Test con backend/transport mockati che simulano i tempi di fase.
|
||||
- [ ] **Rimuovere** `scripts/_Common.psm1`, `scripts/_Transport.psm1`,
|
||||
`tests/_Common.Tests.ps1` (nessun altro consumatore — verificato).
|
||||
- [ ] Aggiornare `lint.yml` se elencava esplicitamente quei path.
|
||||
- **Done**: `bench measure` riproduce le metriche; moduli PS rimossi;
|
||||
pytest verde.
|
||||
|
||||
## C4 — `smoke run`
|
||||
|
||||
**Sostituisce**: `Test-Smoke.ps1`, `Test-Ns7zipBuild-Linux.ps1`,
|
||||
`Test-NsinnounpBuild-Linux.ps1` (e i corrispettivi Win restano per fallback).
|
||||
|
||||
- [ ] `smoke run` esegue un job no-op (marker file) end-to-end e verifica:
|
||||
exit 0, dir artifact creata, evento `job-success` nel JSONL.
|
||||
- [ ] `--guest-os windows|linux`; preset build-command per le build reali
|
||||
ns7zip / nsinnounp (opzionali, dietro flag o `--build-command`).
|
||||
- [ ] Test: pipeline mockata, asserzioni su artifact dir + evento.
|
||||
- **Done**: `smoke run --guest-os linux` e `--guest-os windows` passano.
|
||||
|
||||
## C5 — `validate host`
|
||||
|
||||
**Nuovo** equivalente Linux (non porting 1:1 di `Validate-HostState.ps1`).
|
||||
|
||||
- [ ] Verificare: `vmrun` presente/eseguibile; snapshot template presenti
|
||||
(`BaseClean` / `BaseClean-Linux`); keyring file-based leggibile
|
||||
(`keyring.get_credential`); permessi `/var/lib/ci/...`; unit
|
||||
`act-runner` / `ci-*` attive (`systemctl is-active`).
|
||||
- [ ] Exit 0 = tutto ok; non-zero + report per check fallito.
|
||||
- [ ] Test: ogni check mockato (pass/fail) → exit code corretto.
|
||||
- **Done**: `validate host` esce 0 su host configurato.
|
||||
|
||||
## C6 — `creds set` + `validate guest`
|
||||
|
||||
**Sostituisce**: `Set-CIGuestCredential.ps1` (host Linux) e
|
||||
`Test-CIGuestWinRM.ps1` (parte diagnostica host-side).
|
||||
|
||||
- [ ] `creds set --target ... --user ... --password-stdin`: scrive nel
|
||||
backend `keyring` letto da `ci_orchestrator`
|
||||
(`keyrings.alt.file.PlaintextKeyring`). Password mai su command line.
|
||||
- [ ] `validate guest --host IP [--winrm|--ssh]`: tenta connessione +
|
||||
comando banale, riporta causa esplicita su fallimento (la sonda
|
||||
`is_ready` ingoia gli errori — vedi gotcha).
|
||||
- [ ] Test: scrittura/lettura credenziale (keyring mockato); diag con
|
||||
transport mockato (connect ok / auth fail / connect fail).
|
||||
- **Done**: credenziali settabili e transport diagnosticabile senza `pwsh`.
|
||||
|
||||
## C7 — Lint CI (decisione `pwsh`)
|
||||
|
||||
**Risolve dipendenza #5**. Applicare l'opzione scelta nel §4 dell'analisi.
|
||||
|
||||
- [ ] Decidere: (1) spostare job `pssa` su runner Windows / on-demand
|
||||
[consigliata]; (2) isolare `pwsh` nel solo runner di lint; (3)
|
||||
rimuovere PSSA da CI.
|
||||
- [ ] Aggiornare `.gitea/workflows/lint.yml` di conseguenza.
|
||||
- [ ] Verificare che nessun job che gira su `runs-on: linux-build` invochi
|
||||
più `shell: pwsh`.
|
||||
- **Done**: pipeline lint verde senza `pwsh` su runner Linux.
|
||||
|
||||
## C8 — `setup-host-linux.sh`
|
||||
|
||||
- [ ] Invertire lo step 6: non installare `pwsh` di default; aggiungere
|
||||
flag `--with-pwsh` per opt-in (mantenere `--skip-pwsh` come alias
|
||||
no-op deprecato o rimuoverlo).
|
||||
- [ ] Aggiornare l'help e gli esempi d'uso nello script.
|
||||
- **Done**: `setup-host-linux.sh` su host pulito non installa `pwsh`.
|
||||
|
||||
## C9 — Docs, cutover, burn-in, disinstallazione
|
||||
|
||||
Parte **docs** completata in codice (2026-06-07). Parte **operativa** (cutover
|
||||
+ burn-in di validazione + disinstallazione) resta da eseguire sull'host reale.
|
||||
|
||||
- [x] `docs/RUNBOOK.md`: aggiunta tabella «Phase C — Python equivalents» + nota
|
||||
che i record §6–§13 restano provenance storica; op-note §13 punto 3 aggiornato.
|
||||
- [x] `CLAUDE.md`: nota "host Linux non usa più `pwsh`" (constraints PS 5.1
|
||||
restano per script guest Windows + runner Windows).
|
||||
- [x] `TODO.md`: chiusa voce bug shim `$IsWindows` (§3.7, risolto-per-rimpiazzo).
|
||||
- [ ] **Cutover** (operativo): usare esclusivamente i comandi Python per ≥1
|
||||
settimana (burn-in `bench run` periodico + smoke), `pwsh` ancora installato.
|
||||
- [ ] **Burn-in verde ≥1 settimana** → `sudo apt-get remove powershell` sull'host
|
||||
Linux.
|
||||
- [ ] Riconfermare i criteri "C done" del §7 dell'analisi sull'host reale.
|
||||
- **Done (codice/docs)**: porting C1–C8 mergeabile, suite verde ≥90%.
|
||||
- **Done (operativo, pendente)**: `pwsh` assente dall'host dopo burn-in verde.
|
||||
|
||||
---
|
||||
|
||||
## Ordine consigliato e parallelizzabilità
|
||||
|
||||
`C1` prima di tutto. `C2`–`C6` indipendenti tra loro (parallelizzabili in
|
||||
PR separate, ognuna sopra C1). `C3` deve precedere la rimozione dei moduli
|
||||
`.psm1`. `C7` e `C8` indipendenti dai porting. `C9` ultimo, dopo che
|
||||
C2–C8 sono merge-ati e il burn-in di validazione è verde.
|
||||
|
||||
## Rischi
|
||||
|
||||
- **WinRM 4× concorrenza instabile** (TODO noto): può far fallire `bench run`
|
||||
su Windows con concorrenza alta — non è regressione del porting. Tarare
|
||||
`--concurrency` e annotare.
|
||||
- **Equivalenza output**: validare A/B (`.ps1` vs Python) finché `pwsh` è
|
||||
ancora installato (prerequisito §6) prima della disinstallazione in C9.
|
||||
@@ -0,0 +1,198 @@
|
||||
# Fase C — Eliminazione dipendenza PowerShell dall'host Linux
|
||||
|
||||
> **Stato**: **codice completato** (C1–C8 implementati + docs C9, 2026-06-07);
|
||||
> resta solo la parte **operativa** di C9 (cutover ≥1 settimana + `apt-get
|
||||
> remove powershell` sull'host reale).
|
||||
> Vedi [overview](ideas-overview.md) e il
|
||||
> [piano di applicazione](idea-3-implementation-plan.md) (checklist operativa).
|
||||
>
|
||||
> **Analisi rifatta il 2026-06-07** sullo stato reale del repo (la versione
|
||||
> precedente proponeva gruppi CLI inesistenti e ometteva metà della
|
||||
> superficie `pwsh`).
|
||||
|
||||
---
|
||||
|
||||
## 1. Obiettivo
|
||||
|
||||
Rendere l'host Linux completamente indipendente da PowerShell Core (`pwsh`).
|
||||
Dopo la Fase B l'orchestrazione ordinaria (job, timer, monitoring, backup,
|
||||
retention, deploy template) è già Python puro. Restano script `.ps1`
|
||||
eseguiti **manualmente** sull'host Linux con `pwsh` per: burn-in di
|
||||
capacità, benchmark di timing, smoke/E2E test, validazione stato host e
|
||||
setup credenziali. Inoltre il job di lint CI usa `shell: pwsh`.
|
||||
|
||||
**C done**: nessuno script in `scripts/*.ps1` viene più eseguito
|
||||
direttamente sull'host Linux, e nessun job CI che gira su un runner Linux
|
||||
richiede `pwsh`; `pwsh` può essere disinstallato dall'host Linux senza
|
||||
impatti operativi. I `.ps1` restano nel repo come riferimento storico e
|
||||
per l'eventuale runner Windows (fallback dual-boot Passo 9).
|
||||
|
||||
---
|
||||
|
||||
## 2. Superficie `pwsh` reale sull'host Linux
|
||||
|
||||
Lo stato CLI attuale (`src/ci_orchestrator/__main__.py`) espone i gruppi:
|
||||
`wait-ready`, `vm`, `build`, `artifacts`, `monitor`, `report`,
|
||||
`retention`, `template`, `job`. **Non esistono** gruppi `bench`,
|
||||
`smoke`, `validate`, `creds` — vanno creati.
|
||||
|
||||
### 2.1 Cosa blocca davvero la disinstallazione di `pwsh`
|
||||
|
||||
| # | Dipendenza | Dove | Tipo |
|
||||
| - | ---------- | ---- | ---- |
|
||||
| 1 | Burn-in di capacità (4 job × N round) | `Test-CapacityBurnIn.ps1` + wrapper `Start-BurnInTest-Linux.ps1` (lanciato via `sudo -u ci-runner -H pwsh …`, vedi PhaseB-user-checklist §B7) | host Linux, manuale |
|
||||
| 2 | Benchmark timing fasi (clone/start/IP/transport/destroy) | `Measure-CIBenchmark.ps1` — **unico consumatore** di `scripts/_Common.psm1` e `scripts/_Transport.psm1` | host Linux, manuale |
|
||||
| 3 | Smoke end-to-end | `Test-Smoke.ps1 -GuestOS Linux` | host Linux, manuale |
|
||||
| 4 | Build E2E reali (validazione pipeline) | `Test-Ns7zipBuild-Linux.ps1`, `Test-NsinnounpBuild-Linux.ps1` | host Linux, manuale |
|
||||
| 5 | Lint CI (PSScriptAnalyzer) | `.gitea/workflows/lint.yml` → `shell: pwsh`, `runs-on: linux-build` | runner Linux, automatico |
|
||||
| 6 | Installazione `pwsh` di default | `setup-host-linux.sh` step 6/6 (opt-out `--skip-pwsh`) | provisioning host |
|
||||
| 7 | Pester (test dei moduli PS) | `tests/_Common.Tests.ps1` | host/CI, manuale |
|
||||
| 8 | Setup credenziali guest | `Set-CIGuestCredential.ps1` (equivalente Linux mancante in CLI) | host Linux, manuale |
|
||||
|
||||
### 2.2 Script Windows-only (NON bloccano `pwsh` su Linux)
|
||||
|
||||
Hardcodano `F:\`, richiedono `#Requires -RunAsAdministrator` o usano il
|
||||
Windows Credential Manager: **non girano sull'host Linux**, quindi non
|
||||
vanno portati per l'obiettivo di Fase C. Restano per il runner Windows.
|
||||
|
||||
- `Validate-HostState.ps1` — logica Credential Manager/ACL NTFS, path `F:\`.
|
||||
Serve però un **equivalente Linux** (`validate host`), non un porting 1:1.
|
||||
- `Set-CIGuestCredential.ps1` — scrive nel vault SYSTEM via scheduled task.
|
||||
Su Linux le credenziali stanno in `keyrings.alt.file.PlaintextKeyring`
|
||||
(B3, `secret-tool` scartato). Serve un `creds set` Linux-nativo.
|
||||
- `Test-CIGuestWinRM.ps1` — diagnostica WinRM, `RunAsAdministrator`.
|
||||
- `Start-BurnInTest.ps1`, `Test-NsinnounpBuild.ps1`, `Test-E2E-Section3.3.ps1`
|
||||
— wrapper Windows (path `F:\`, template WinBuild).
|
||||
|
||||
### 2.3 Già fuori scope (confermato)
|
||||
|
||||
- `Register-CIScheduledTasks.ps1` — Windows Scheduler.
|
||||
- `Invoke-RetentionPolicy.ps1` / `Backup-CITemplate.ps1` — portati in B5
|
||||
(`retention run` / `template backup`).
|
||||
- Shim in `scripts/` (3 righe → Python) — restano.
|
||||
- Script guest (`Install-CIToolchain-*`, `Prepare-*`, `Deploy-*`,
|
||||
`template/guest-setup/windows/*`) — girano dentro le VM, non sull'host.
|
||||
|
||||
---
|
||||
|
||||
## 3. Architettura prevista (nuovi gruppi CLI)
|
||||
|
||||
```
|
||||
python -m ci_orchestrator bench run --guest-os windows|linux
|
||||
--concurrency N # default 4
|
||||
--rounds N # default 10
|
||||
--build-command "..."
|
||||
--json-out PATH
|
||||
python -m ci_orchestrator bench measure --guest-os windows|linux
|
||||
--iterations N
|
||||
--json-out PATH # append a benchmark.jsonl
|
||||
|
||||
python -m ci_orchestrator smoke run --guest-os windows|linux
|
||||
[--build-command "..."] # default: marker no-op
|
||||
|
||||
python -m ci_orchestrator validate host # vmrun, snapshot template, keyring,
|
||||
# permessi dir, act-runner systemd attivo
|
||||
python -m ci_orchestrator validate guest --host IP [--winrm|--ssh] # diag transport
|
||||
|
||||
python -m ci_orchestrator creds set --target BuildVMGuest --user ... [--password-stdin]
|
||||
```
|
||||
|
||||
- **`bench run`** unifica `Test-CapacityBurnIn.ps1` + `Start-BurnInTest*.ps1`:
|
||||
lancia N job concorrenti (riusando `commands/job.py` in-process o come
|
||||
subprocess del CLI Python — niente `Start-Job`/`pwsh`), per R round,
|
||||
e asserisce dopo ogni round: tutti exit 0, nessuna clone-dir orfana in
|
||||
`CI_BUILD_VMS`, nessun `vm-start.lock` stale. Emette report JSON.
|
||||
- **`bench measure`** porta `Measure-CIBenchmark.ps1` (fasi
|
||||
clone/start/IP/transport/destroy) → consente di **retirare**
|
||||
`_Common.psm1` e `_Transport.psm1` (nessun altro consumatore).
|
||||
- **`smoke run`** porta `Test-Smoke.ps1` + i `Test-*Build-Linux.ps1`
|
||||
(build E2E reali come preset opzionali del build-command).
|
||||
- **`validate host`** è un nuovo equivalente Linux (non un porting di
|
||||
`Validate-HostState.ps1`): verifica `vmrun`, presenza snapshot template,
|
||||
leggibilità keyring file-based, permessi `/var/lib/ci/...`, unit
|
||||
`act-runner`/`ci-*` attive.
|
||||
- **`creds set`** sostituisce `Set-CIGuestCredential.ps1` su Linux usando
|
||||
direttamente il backend `keyring` letto da `ci_orchestrator`.
|
||||
|
||||
**Vincolo di design**: i nuovi comandi devono importare
|
||||
`backends.load_backend` e i transport esistenti, **mai**
|
||||
`WorkstationVmrunBackend` diretto (rotta ESXi Fase D aperta).
|
||||
|
||||
---
|
||||
|
||||
## 4. Nodo aperto — lint PSScriptAnalyzer (dipendenza #5)
|
||||
|
||||
Gli script `.ps1` restano nel repo (guest + fallback Windows), quindi il
|
||||
lint PSSA ha ancora valore. Ma `lint.yml` lo esegue con `shell: pwsh` su
|
||||
`runs-on: linux-build`, il che reintroduce `pwsh` sul runner Linux.
|
||||
**Decisione applicata (C7, 2026-06-07): opzione 1.** Il job `pssa` gira ora su
|
||||
`runs-on: windows-build`, `shell: powershell` (PS 5.1), con guardia
|
||||
`if: github.event_name == 'workflow_dispatch'` così non si accoda su push/PR se
|
||||
il runner Windows non è registrato. Il job `python` resta su `linux-build`.
|
||||
Nessun job su runner Linux usa più `pwsh`.
|
||||
|
||||
Tre opzioni considerate:
|
||||
|
||||
1. **(Consigliata)** Spostare il job `pssa` su un runner Windows
|
||||
(`runs-on: windows-build`, `shell: powershell` PS 5.1) — coerente col
|
||||
fatto che i `.ps1` target sono Windows-native. Richiede un runner
|
||||
Windows registrato (dual-boot Passo 9) o eseguirlo solo on-demand.
|
||||
2. Mantenere `pwsh` **solo nel container/runner di lint**, non sull'host
|
||||
orchestratore: l'obiettivo "host Linux senza pwsh" resta soddisfatto se
|
||||
il job gira in un runner isolato. Pragmatica ma ambigua sul "C done".
|
||||
3. Rimuovere il job PSSA da CI e affidare il lint PS a un hook locale
|
||||
manuale. Riduce copertura.
|
||||
|
||||
---
|
||||
|
||||
## 5. Aggiornamenti collaterali
|
||||
|
||||
- `setup-host-linux.sh`: invertire il default dello step 6 (no install
|
||||
`pwsh`; flag `--with-pwsh` per opt-in) o rimuovere lo step.
|
||||
- `.gitea/workflows/lint.yml`: applicare la decisione del §4.
|
||||
- `tests/_Common.Tests.ps1`: rimuovere insieme ai moduli `.psm1` quando
|
||||
`bench measure` li retira (Passo C3); la copertura passa a pytest.
|
||||
- `CLAUDE.md`: la sezione "PowerShell 5.1 Constraints" resta valida per gli
|
||||
script guest Windows; aggiungere nota che l'host Linux non usa più `pwsh`.
|
||||
- `docs/RUNBOOK.md`: aggiornare procedure burn-in / smoke / validazione ai
|
||||
nuovi comandi Python.
|
||||
- `plans/archived/2026-06-07/PhaseB-user-checklist.md` §B7 (archiviato):
|
||||
nota che il comando `pwsh Test-CapacityBurnIn.ps1` è superato da
|
||||
`bench run`.
|
||||
- `TODO.md`: chiudere la voce shim `$IsWindows` bug (f22b142) — sparisce
|
||||
con la rimozione di `Start-Job`/`pwsh` nel burn-in.
|
||||
|
||||
---
|
||||
|
||||
## 6. Prerequisiti
|
||||
|
||||
- Fase B completata e stabile (≥1 settimana, Passo 8 ✅).
|
||||
- `pwsh` ancora installato durante il porting (per confronto output A/B).
|
||||
- Nessuna dipendenza da `pwsh` nei timer systemd (già soddisfatta da B5).
|
||||
|
||||
---
|
||||
|
||||
## 7. Criteri di "C done"
|
||||
|
||||
- `bench run --concurrency 4 --rounds 10` (Win + Linux) produce esito
|
||||
equivalente a `Test-CapacityBurnIn.ps1` (stesse asserzioni di cleanup).
|
||||
- `bench measure` riproduce le fasi di `Measure-CIBenchmark.ps1`; i moduli
|
||||
`_Common.psm1`/`_Transport.psm1` e `tests/_Common.Tests.ps1` rimossi.
|
||||
- `smoke run --guest-os linux` e `--guest-os windows` passano.
|
||||
- `validate host` esce 0 su host configurato correttamente.
|
||||
- `creds set` sostituisce `Set-CIGuestCredential.ps1` sull'host Linux.
|
||||
- Decisione §4 applicata: nessun job CI su runner Linux richiede `pwsh`.
|
||||
- `setup-host-linux.sh` non installa più `pwsh` di default.
|
||||
- `pwsh --version` può essere assente sull'host Linux senza che alcun
|
||||
test, timer, job CI o procedura di RUNBOOK fallisca.
|
||||
- Suite pytest ≥90% coverage, verde.
|
||||
|
||||
---
|
||||
|
||||
## 8. Rollback
|
||||
|
||||
I `.ps1` originali restano nel repo e funzionano su host Windows o su
|
||||
Linux con `pwsh` reinstallato (`setup-host-linux.sh --with-pwsh`). Il
|
||||
rollback è reinstallare `pwsh` e usare gli script esistenti. Nessuna
|
||||
distruzione di dati: il porting aggiunge comandi, non rimuove i `.ps1`
|
||||
finché il burn-in di validazione non è verde.
|
||||
@@ -1,9 +1,10 @@
|
||||
# Fase C — Supporto ESXi per le Build VM (futuro)
|
||||
# Fase D — Supporto ESXi per le Build VM (futuro)
|
||||
|
||||
> **Stato**: **piano differito**, da rivalutare dopo che Fase A
|
||||
> ([rewrite Python](idea-1-python-rewrite.md)) e Fase B
|
||||
> ([host Linux](idea-2-linux-host.md)) sono stabili in produzione.
|
||||
> Vedi [overview](ideas-overview.md).
|
||||
> ([rewrite Python](idea-1-python-rewrite.md)), Fase B
|
||||
> ([host Linux](idea-2-linux-host.md)) e Fase C
|
||||
> ([eliminazione PowerShell](idea-3-powershell-removal.md)) sono stabili
|
||||
> in produzione. Vedi [overview](ideas-overview.md).
|
||||
>
|
||||
> Questo documento esiste per:
|
||||
> 1. Garantire che il design di Fase A abbia gli hook corretti
|
||||
+16
-9
@@ -3,11 +3,12 @@
|
||||
Decisione presa: implementare **prima Idea 1 (Python) + Idea 2 (host Linux)**
|
||||
in sequenza, e **solo dopo** valutare Idea 3 (ESXi).
|
||||
|
||||
| Fase | Cosa | File di dettaglio |
|
||||
| ---- | ---------------------------------------------------------------------------------- | ---------------------------------------------------- |
|
||||
| A | Rewrite in Python sull'host Windows attuale (codebase cross-platform-ready) | [idea-1-python-rewrite.md](idea-1-python-rewrite.md) |
|
||||
| B | Migrazione host: Windows 11 + Workstation Pro → Linux Mint + Workstation Pro Linux | [idea-2-linux-host.md](idea-2-linux-host.md) |
|
||||
| C | (Futuro, condizionato a A+B stabili) Aggiunta backend ESXi | [idea-3-esxi-support.md](idea-3-esxi-support.md) |
|
||||
| Fase | Cosa | File di dettaglio |
|
||||
| ---- | ---------------------------------------------------------------------------------- | -------------------------------------------------------------- |
|
||||
| A | ✅ Rewrite in Python sull'host Windows attuale (codebase cross-platform-ready) | [idea-1-python-rewrite.md](archived/2026-05-23/idea-1-python-rewrite.md) |
|
||||
| B | ✅ Migrazione host: Windows 11 + Workstation Pro → Linux Mint + Workstation Pro Linux | [idea-2-linux-host.md](archived/2026-05-23/idea-2-linux-host.md) |
|
||||
| C | (Futuro, dopo B stabile) Eliminazione dipendenza `pwsh` dall'host Linux | [idea-3-powershell-removal.md](idea-3-powershell-removal.md) |
|
||||
| D | (Futuro, condizionato a C stabili) Aggiunta backend ESXi | [idea-4-esxi-support.md](idea-4-esxi-support.md) |
|
||||
|
||||
---
|
||||
|
||||
@@ -49,7 +50,8 @@ in sequenza, e **solo dopo** valutare Idea 3 (ESXi).
|
||||
| ---- | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
|
||||
| A | Python 3.11+ su Windows; ambiente venv in `F:\CI\python\venv\` | Python, pytest, pypsrp/paramiko |
|
||||
| B | Hardware Linux Mint operativo; VMware Workstation Pro Linux installata; storage analogo a `F:\CI\` | systemd, bash, secret-tool, gestione utente di servizio |
|
||||
| C | Hardware ESXi + licenza Essentials o superiore; account vSphere dedicato | pyVmomi, datastore/portgroup management |
|
||||
| C | Fase B stabile ≥1 settimana; `pwsh` ancora installato durante il porting | click, subprocess, pytest |
|
||||
| D | Hardware ESXi + licenza Essentials o superiore; account vSphere dedicato | pyVmomi, datastore/portgroup management |
|
||||
|
||||
---
|
||||
|
||||
@@ -61,7 +63,10 @@ in sequenza, e **solo dopo** valutare Idea 3 (ESXi).
|
||||
- **B done**: act_runner gira come systemd service su Linux Mint; tutti i
|
||||
workflow esistenti (Windows + Linux build) PASS dal nuovo host; storage
|
||||
migrato a `/var/lib/ci/`; capacity burn-in PASS.
|
||||
- **C done** (se attivata): label `windows-build:esxi` operativa con backend
|
||||
- **C done** (se attivata): `pwsh` può essere disinstallato dall'host Linux
|
||||
senza impatti operativi; `bench run`, `validate host` e `smoke run`
|
||||
sostituiscono completamente i relativi `.ps1`; suite pytest verde ≥90%.
|
||||
- **D done** (se attivata): label `windows-build:esxi` operativa con backend
|
||||
pyVmomi; canary su almeno 1 workflow di produzione per ≥1 settimana
|
||||
senza regressioni.
|
||||
|
||||
@@ -74,6 +79,8 @@ in sequenza, e **solo dopo** valutare Idea 3 (ESXi).
|
||||
modulo "side-by-side" non utilizzato.
|
||||
- **Stop in B**: l'host Windows resta primario; eventuale runner Linux
|
||||
secondario può comunque essere utile per workload sperimentali (path
|
||||
"dual-host" descritto in [idea-2-linux-host.md](idea-2-linux-host.md) §7).
|
||||
- **Stop pianificato in C**: nessun impatto — C è opzionale per definizione,
|
||||
"dual-host" descritto in [idea-2-linux-host.md](archived/2026-05-23/idea-2-linux-host.md) §7).
|
||||
- **Stop pianificato in C**: nessun impatto — gli script `.ps1` originali
|
||||
restano funzionanti; `pwsh` resta installato. Il sistema continua a girare.
|
||||
- **Stop pianificato in D**: nessun impatto — D è opzionale per definizione,
|
||||
Workstation Pro su Linux copre già il caso d'uso primario.
|
||||
|
||||
@@ -32,6 +32,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator vm cleanup @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -57,6 +57,13 @@ if ($Credential) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator artifacts collect @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -32,7 +32,14 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator report job @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
|
||||
@@ -32,6 +32,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator job @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -86,6 +86,13 @@ if ($Credential) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator build run @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -1,248 +0,0 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Measures CI infrastructure phase timings: clone, start, IP acquire, WinRM ready, destroy.
|
||||
|
||||
.DESCRIPTION
|
||||
Creates one or more ephemeral VMs from the template, times each phase, destroys
|
||||
them, then prints a summary table. Results are appended to F:\CI\Logs\benchmark.jsonl
|
||||
for long-term trend tracking (compare before/after §3.1 cache, §3.2 7-Zip, etc.).
|
||||
|
||||
Phases measured:
|
||||
clone — vmrun clone (linked clone from snapshot)
|
||||
start — vmrun start (until command returns)
|
||||
ip — vmrun getGuestIPAddress (polling until IP appears)
|
||||
winrm — TCP 5986 reachable (polling)
|
||||
destroy — stop + deleteVM + dir removal
|
||||
|
||||
No actual build is run. Phase 5 build timings come from invoke-ci.jsonl logs
|
||||
produced by Invoke-CIJob.ps1.
|
||||
|
||||
.PARAMETER TemplatePath
|
||||
Full path to the template VM's .vmx file.
|
||||
Default: F:\CI\Templates\WinBuild2025\WinBuild2025.vmx
|
||||
|
||||
.PARAMETER SnapshotName
|
||||
Snapshot name to clone from. Default: BaseClean
|
||||
|
||||
.PARAMETER CloneBaseDir
|
||||
Directory for ephemeral clone folders. Default: F:\CI\BuildVMs
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER Iterations
|
||||
Number of clone→ready→destroy cycles to run. Multiple iterations reduce timing
|
||||
variance from cold-cache effects. Default: 1
|
||||
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to wait for IP and WinRM per iteration. Default: 300
|
||||
|
||||
.PARAMETER OutputDir
|
||||
Directory where benchmark.jsonl is appended. Default: F:\CI\Logs
|
||||
|
||||
.EXAMPLE
|
||||
# Single baseline run
|
||||
.\Measure-CIBenchmark.ps1
|
||||
|
||||
# 3 iterations for average (first may be slow due to cold host disk cache)
|
||||
.\Measure-CIBenchmark.ps1 -Iterations 3
|
||||
|
||||
# Custom snapshot
|
||||
.\Measure-CIBenchmark.ps1 -SnapshotName BaseClean_20260510
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $TemplatePath = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx',
|
||||
[string] $SnapshotName = 'BaseClean',
|
||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
||||
|
||||
[ValidateRange(1, 10)]
|
||||
[int] $Iterations = 1,
|
||||
|
||||
[ValidateRange(60, 600)]
|
||||
[int] $TimeoutSeconds = 300,
|
||||
|
||||
[string] $OutputDir = 'F:\CI\Logs'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
|
||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
||||
Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null
|
||||
|
||||
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
|
||||
throw "Template VMX not found: $TemplatePath"
|
||||
}
|
||||
if (-not (Test-Path $CloneBaseDir)) {
|
||||
New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null
|
||||
}
|
||||
if (-not (Test-Path $OutputDir)) {
|
||||
New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$benchmarkLog = Join-Path $OutputDir 'benchmark.jsonl'
|
||||
$results = [System.Collections.Generic.List[pscustomobject]]::new()
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "=== CI Benchmark template: $(Split-Path $TemplatePath -Leaf) snapshot: $SnapshotName iterations: $Iterations ==="
|
||||
Write-Host ""
|
||||
|
||||
for ($i = 1; $i -le $Iterations; $i++) {
|
||||
Write-Host "--- Iteration $i / $Iterations ---"
|
||||
$runId = "bench-$(Get-Date -Format 'yyyyMMdd-HHmmss')-$i"
|
||||
$cloneVmx = $null
|
||||
$cloneDir = $null
|
||||
|
||||
$t = [ordered]@{
|
||||
clone = $null
|
||||
start = $null
|
||||
ip = $null
|
||||
winrm = $null
|
||||
destroy = $null
|
||||
deltaKB = $null
|
||||
vmIP = $null
|
||||
error = $null
|
||||
}
|
||||
|
||||
try {
|
||||
# ── Phase: clone ──────────────────────────────────────────────────────
|
||||
$cloneName = "Clone_${runId}"
|
||||
$cloneDir = Join-Path $CloneBaseDir $cloneName
|
||||
$cloneVmx = Join-Path $cloneDir "$cloneName.vmx"
|
||||
|
||||
Write-Host "[bench] Cloning..."
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$null = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
|
||||
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) `
|
||||
-ThrowOnError
|
||||
$t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
Write-Host "[bench] clone: $($t.clone)s"
|
||||
|
||||
# Measure linked-clone disk footprint
|
||||
try {
|
||||
$cloneSizeBytes = (Get-ChildItem -Path $cloneDir -Recurse -File -ErrorAction Stop |
|
||||
Measure-Object -Property Length -Sum).Sum
|
||||
$t.deltaKB = [math]::Round($cloneSizeBytes / 1KB, 0)
|
||||
Write-Host "[bench] clone size: $($t.deltaKB) KB"
|
||||
} catch {
|
||||
Write-Warning "[bench] Could not measure clone size: $_"
|
||||
}
|
||||
|
||||
# ── Phase: start ──────────────────────────────────────────────────────
|
||||
Write-Host "[bench] Starting VM..."
|
||||
$sw.Restart()
|
||||
Invoke-Vmrun -VmrunPath $VmrunPath -Operation start `
|
||||
-Arguments @($cloneVmx, 'nogui') -ThrowOnError | Out-Null
|
||||
$t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
Write-Host "[bench] start: $($t.start)s"
|
||||
|
||||
# ── Phase: IP acquire (primary: guestinfo.ci-ip, fallback: getGuestIPAddress) ──
|
||||
Write-Host "[bench] Waiting for guest IP..."
|
||||
$sw.Restart()
|
||||
$vmIP = Get-GuestIPAddress -VmrunPath $VmrunPath -VmxPath $cloneVmx `
|
||||
-TimeoutSeconds $TimeoutSeconds
|
||||
if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" }
|
||||
$t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
$t.vmIP = $vmIP
|
||||
Write-Host "[bench] ip ($vmIP): $($t.ip)s"
|
||||
|
||||
# ── Phase: WinRM ready (poll TCP 5986) ────────────────────────────────
|
||||
Write-Host "[bench] Waiting for WinRM (TCP 5986)..."
|
||||
$sw.Restart()
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$winrmUp = $false
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$winrmUp = Test-NetConnection -ComputerName $vmIP -Port 5986 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue
|
||||
if ($winrmUp) { break }
|
||||
Start-Sleep -Seconds 3
|
||||
}
|
||||
if (-not $winrmUp) { throw "Timed out waiting for WinRM on $vmIP`:5986 after $TimeoutSeconds s" }
|
||||
$t.winrm = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
Write-Host "[bench] winrm: $($t.winrm)s"
|
||||
}
|
||||
catch {
|
||||
$t.error = "$_"
|
||||
Write-Warning "[bench] Iteration $i failed: $_"
|
||||
}
|
||||
finally {
|
||||
# ── Phase: destroy ────────────────────────────────────────────────────
|
||||
if ($cloneVmx -and (Test-Path $cloneVmx -PathType Leaf)) {
|
||||
Write-Host "[bench] Destroying clone..."
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
& (Join-Path $PSScriptRoot 'Remove-BuildVM.ps1') `
|
||||
-VMPath $cloneVmx -VmrunPath $VmrunPath -GracefulTimeoutSeconds 10 `
|
||||
-ErrorAction SilentlyContinue
|
||||
$t.destroy = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||
Write-Host "[bench] destroy: $($t.destroy)s"
|
||||
} elseif ($cloneDir -and (Test-Path $cloneDir)) {
|
||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
$totalBootSec = if ($t.ip -and $t.winrm) {
|
||||
[math]::Round($t.clone + $t.start + $t.ip + $t.winrm, 2)
|
||||
} else { $null }
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
ts = (Get-Date -Format 'o')
|
||||
runId = $runId
|
||||
iteration = $i
|
||||
template = Split-Path $TemplatePath -Leaf
|
||||
snapshot = $SnapshotName
|
||||
cloneSec = $t.clone
|
||||
startSec = $t.start
|
||||
ipSec = $t.ip
|
||||
winrmSec = $t.winrm
|
||||
destroySec = $t.destroy
|
||||
totalBootSec = $totalBootSec
|
||||
deltaKB = $t.deltaKB
|
||||
vmIP = $t.vmIP
|
||||
error = $t.error
|
||||
}
|
||||
$results.Add($result)
|
||||
|
||||
# Append to JSONL log
|
||||
try {
|
||||
$result | ConvertTo-Json -Compress -Depth 3 |
|
||||
Add-Content -Path $benchmarkLog -Encoding UTF8 -ErrorAction Stop
|
||||
} catch {
|
||||
Write-Warning "[bench] Could not write benchmark.jsonl: $_"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
# ── Summary table ─────────────────────────────────────────────────────────────
|
||||
Write-Host "=== Results ==="
|
||||
$results | Format-Table @(
|
||||
@{ L = 'Iter'; E = { $_.iteration } }
|
||||
@{ L = 'Clone(s)'; E = { $_.cloneSec } }
|
||||
@{ L = 'Start(s)'; E = { $_.startSec } }
|
||||
@{ L = 'IP(s)'; E = { $_.ipSec } }
|
||||
@{ L = 'WinRM(s)'; E = { $_.winrmSec } }
|
||||
@{ L = 'Destroy(s)'; E = { $_.destroySec } }
|
||||
@{ L = 'Boot total'; E = { $_.totalBootSec } }
|
||||
@{ L = 'Delta(KB)'; E = { $_.deltaKB } }
|
||||
@{ L = 'IP'; E = { $_.vmIP } }
|
||||
@{ L = 'Error'; E = { $_.error } }
|
||||
) -AutoSize
|
||||
|
||||
if ($Iterations -gt 1) {
|
||||
$ok = @($results | Where-Object { -not $_.error })
|
||||
if ($ok.Count -gt 0) {
|
||||
$avgBoot = [math]::Round(($ok | ForEach-Object { $_.totalBootSec } |
|
||||
Measure-Object -Average).Average, 2)
|
||||
Write-Host "Average boot-to-WinRM ($($ok.Count) successful runs): $avgBoot s"
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Results appended to: $benchmarkLog"
|
||||
|
||||
@@ -31,6 +31,13 @@ $pyArgs = @(
|
||||
)
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator vm new @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -35,6 +35,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator vm remove @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -68,6 +68,7 @@ param(
|
||||
[System.Security.SecureString] $Password,
|
||||
|
||||
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
|
||||
elseif ($null -ne $IsWindows -and $IsWindows -eq $false) { '/opt/ci/venv/bin/python' }
|
||||
else { 'F:\CI\python\venv\Scripts\python.exe' }),
|
||||
|
||||
[switch] $SkipVerify
|
||||
|
||||
@@ -51,18 +51,16 @@ param(
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
|
||||
$burnInParams = @{
|
||||
RepoUrl = 'https://gitea.emulab.it/Simone/burnin-dummy.git'
|
||||
RepoUrl = 'http://10.10.20.11:3100/Simone/burnin-dummy.git'
|
||||
Branch = 'main'
|
||||
TemplatePath = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx'
|
||||
TemplatePath = '/var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx'
|
||||
SnapshotName = 'BaseClean-Linux'
|
||||
CloneBaseDir = 'F:\CI\BuildVMs'
|
||||
ArtifactBaseDir = 'F:\CI\Artifacts'
|
||||
CloneBaseDir = '/var/lib/ci/build-vms/'
|
||||
ArtifactBaseDir = '/var/lib/ci/artifacts/'
|
||||
GuestCredentialTarget = 'BuildVMGuest'
|
||||
GiteaCredentialTarget = 'GiteaPAT'
|
||||
VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
VmrunPath = '/usr/bin/vmrun'
|
||||
GuestOS = 'Linux'
|
||||
BuildCommand = $BuildCommand
|
||||
Parallelism = $Parallelism
|
||||
@@ -70,6 +68,6 @@ $burnInParams = @{
|
||||
RoundTimeoutMinutes = $RoundTimeoutMinutes
|
||||
}
|
||||
|
||||
& "$scriptDir\Test-CapacityBurnIn.ps1" @burnInParams
|
||||
& (Join-Path $PSScriptRoot 'Test-CapacityBurnIn.ps1') @burnInParams
|
||||
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -55,18 +55,16 @@ param(
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
|
||||
$burnInParams = @{
|
||||
RepoUrl = 'https://gitea.emulab.it/Simone/burnin-dummy.git'
|
||||
RepoUrl = 'http://10.10.20.11:3100/Simone/burnin-dummy.git'
|
||||
Branch = 'main'
|
||||
TemplatePath = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx'
|
||||
TemplatePath = '/var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx'
|
||||
SnapshotName = 'BaseClean'
|
||||
CloneBaseDir = 'F:\CI\BuildVMs'
|
||||
ArtifactBaseDir = 'F:\CI\Artifacts'
|
||||
CloneBaseDir = '/var/lib/ci/build-vms/'
|
||||
ArtifactBaseDir = '/var/lib/ci/artifacts/'
|
||||
GuestCredentialTarget = 'BuildVMGuest'
|
||||
GiteaCredentialTarget = 'GiteaPAT'
|
||||
VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
VmrunPath = '/usr/bin/vmrun'
|
||||
GuestOS = 'Auto'
|
||||
BuildCommand = $BuildCommand
|
||||
Parallelism = $Parallelism
|
||||
@@ -74,6 +72,6 @@ $burnInParams = @{
|
||||
RoundTimeoutMinutes = $RoundTimeoutMinutes
|
||||
}
|
||||
|
||||
& "$scriptDir\Test-CapacityBurnIn.ps1" @burnInParams
|
||||
& (Join-Path $PSScriptRoot 'Test-CapacityBurnIn.ps1') @burnInParams
|
||||
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -74,6 +74,7 @@ param(
|
||||
[string[]] $AuthOrder = @('ntlm', 'negotiate'),
|
||||
|
||||
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
|
||||
elseif ($null -ne $IsWindows -and $IsWindows -eq $false) { '/opt/ci/venv/bin/python' }
|
||||
else { 'F:\CI\python\venv\Scripts\python.exe' }),
|
||||
|
||||
[ValidateRange(5, 300)]
|
||||
|
||||
@@ -214,14 +214,16 @@ for ($r = 1; $r -le $Rounds; $r++) {
|
||||
|
||||
$argArray = $argList.ToArray()
|
||||
|
||||
# Start-Job runs a new PS session; call powershell.exe as child process
|
||||
# so exit $exitCode from Invoke-CIJob.ps1 is captured in $LASTEXITCODE.
|
||||
# Start-Job runs a new PS session; call the platform PS executable as child
|
||||
# process so exit $exitCode from Invoke-CIJob.ps1 is captured in $LASTEXITCODE.
|
||||
# $IsWindows is evaluated inside the job worker (correct OS context).
|
||||
# NOTE: do NOT name the parameter $Args — @Args is a PS reserved splatting
|
||||
# alias for the automatic $Args variable (unbound arguments), which would
|
||||
# be empty here since the parameter binding consumed all positional args.
|
||||
$psJob = Start-Job -ScriptBlock {
|
||||
param([string[]]$ChildArgs)
|
||||
$out = & powershell.exe @ChildArgs 2>&1
|
||||
$psExe = if ($IsWindows) { 'powershell.exe' } else { 'pwsh' }
|
||||
$out = & $psExe @ChildArgs 2>&1
|
||||
[PSCustomObject]@{
|
||||
ExitCode = $LASTEXITCODE
|
||||
Output = ($out -join "`n")
|
||||
|
||||
@@ -32,6 +32,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator wait-ready @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -32,6 +32,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator monitor disk @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -32,6 +32,13 @@ for ($i = 0; $i -lt $args.Count; $i++) {
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
if (-not $venvPython) {
|
||||
# $IsWindows exists only on PS 6+; absent on PS 5.1 (always Windows)
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
$venvPython = '/opt/ci/venv/bin/python'
|
||||
} else {
|
||||
$venvPython = 'F:\CI\python\venv\Scripts\python.exe'
|
||||
}
|
||||
}
|
||||
& $venvPython -m ci_orchestrator monitor runner @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -1,421 +0,0 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Shared helper functions for the Local CI/CD System scripts.
|
||||
|
||||
.DESCRIPTION
|
||||
Import this module at the top of any CI script that needs WinRM session
|
||||
options or vmrun wrappers:
|
||||
|
||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
||||
|
||||
Exported functions:
|
||||
New-CISessionOption — PSSessionOption for WinRM over self-signed TLS
|
||||
Resolve-VmrunPath — Validates vmrun.exe path, throws if missing
|
||||
Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output
|
||||
#>
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
function New-CISessionOption {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Returns a PSSessionOption for WinRM HTTPS with self-signed certificate.
|
||||
.DESCRIPTION
|
||||
All CI build VMs use a self-signed TLS cert on WinRM port 5986.
|
||||
This option set skips CA, CN, and revocation checks — appropriate for
|
||||
an isolated lab network where PKI is not present.
|
||||
.OUTPUTS
|
||||
[System.Management.Automation.Remoting.PSSessionOption]
|
||||
#>
|
||||
[OutputType([System.Management.Automation.Remoting.PSSessionOption])]
|
||||
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseShouldProcessForStateChangingFunctions', '',
|
||||
Justification = 'New-CISessionOption creates an in-memory PSSessionOption object; no system state is changed.')]
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
}
|
||||
|
||||
function Resolve-VmrunPath {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Validates that vmrun.exe exists at the specified path and returns it.
|
||||
.DESCRIPTION
|
||||
Throws a descriptive error if the file is not found, rather than letting
|
||||
a later & call fail with a less useful message.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
.OUTPUTS
|
||||
[string] The validated VmrunPath.
|
||||
#>
|
||||
[OutputType([string])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
)
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath."
|
||||
}
|
||||
return $VmrunPath
|
||||
}
|
||||
|
||||
function Invoke-Vmrun {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs a vmrun -T ws <Operation> command and returns output and exit code.
|
||||
.DESCRIPTION
|
||||
Uniform wrapper so all vmrun calls share the same -T ws flag and output
|
||||
capture pattern. Use -ThrowOnError for operations that must succeed
|
||||
(clone, start); omit it for best-effort operations (stop, getState).
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER Operation
|
||||
vmrun sub-command: clone, start, stop, deleteVM, list, getState,
|
||||
listSnapshots, getGuestIPAddress, etc.
|
||||
.PARAMETER Arguments
|
||||
Additional positional arguments after the operation name.
|
||||
.PARAMETER ThrowOnError
|
||||
When set, throws if vmrun exits non-zero.
|
||||
.OUTPUTS
|
||||
[pscustomobject] with:
|
||||
ExitCode [int] — vmrun exit code
|
||||
Output [string[]] — combined stdout/stderr lines
|
||||
#>
|
||||
[OutputType([pscustomobject])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $Operation,
|
||||
[string[]] $Arguments = @(),
|
||||
[switch] $ThrowOnError
|
||||
)
|
||||
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
||||
$output = & $VmrunPath @cmdArgs 2>&1
|
||||
$exit = $LASTEXITCODE
|
||||
|
||||
if ($ThrowOnError -and $exit -ne 0) {
|
||||
throw "vmrun $Operation failed (exit $exit): $($output -join '; ')"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
ExitCode = $exit
|
||||
Output = $output
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-VmrunBounded {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Runs a vmrun -T ws <Operation> command with a hard wall-clock timeout.
|
||||
.DESCRIPTION
|
||||
Uses [System.Diagnostics.Process] directly (not Start-Process -PassThru) because
|
||||
Start-Process in PowerShell 5.1 does not reliably populate Process.ExitCode after
|
||||
WaitForExit(ms) when output is redirected. ReadToEndAsync() is used for stdout and
|
||||
stderr to prevent deadlock if output buffers fill up before the process exits.
|
||||
|
||||
Use for lifecycle operations: start (180s), stop (60s), deleteVM (90s).
|
||||
Keep Invoke-Vmrun for fast queries: list, readVariable, getGuestIPAddress, getState.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER Operation
|
||||
vmrun sub-command (start, stop, deleteVM, clone, …).
|
||||
.PARAMETER Arguments
|
||||
Additional positional arguments after the operation name.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Hard timeout in seconds. Process is killed if not finished within this time.
|
||||
Default: 120.
|
||||
.PARAMETER ThrowOnTimeout
|
||||
When set, throws if the timeout is reached before vmrun exits.
|
||||
.PARAMETER ThrowOnError
|
||||
When set, throws if vmrun exits with a non-zero code.
|
||||
.OUTPUTS
|
||||
[pscustomobject] with:
|
||||
ExitCode [int] — vmrun exit code (-1 if killed due to timeout)
|
||||
Output [string] — combined stdout/stderr text
|
||||
TimedOut [bool] — true if the process was killed for exceeding TimeoutSeconds
|
||||
ElapsedSeconds [double] — wall-clock seconds from start to exit/kill
|
||||
#>
|
||||
[OutputType([pscustomobject])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $Operation,
|
||||
[string[]] $Arguments = @(),
|
||||
[int] $TimeoutSeconds = 120,
|
||||
[switch] $ThrowOnTimeout,
|
||||
[switch] $ThrowOnError
|
||||
)
|
||||
|
||||
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
|
||||
|
||||
# Quote tokens that contain spaces so the argument string is correct when
|
||||
# passed to ProcessStartInfo.Arguments as a single string.
|
||||
$quotedArgs = $cmdArgs | ForEach-Object {
|
||||
if ($_ -match '\s') { '"' + ($_ -replace '"', '""') + '"' } else { $_ }
|
||||
}
|
||||
$argStr = $quotedArgs -join ' '
|
||||
|
||||
$psi = New-Object System.Diagnostics.ProcessStartInfo
|
||||
$psi.FileName = $VmrunPath
|
||||
$psi.Arguments = $argStr
|
||||
$psi.UseShellExecute = $false
|
||||
$psi.CreateNoWindow = $true
|
||||
$psi.RedirectStandardOutput = $true
|
||||
$psi.RedirectStandardError = $true
|
||||
|
||||
$proc = New-Object System.Diagnostics.Process
|
||||
$proc.StartInfo = $psi
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$timedOut = $false
|
||||
|
||||
[void]$proc.Start()
|
||||
|
||||
# Begin async reads before WaitForExit to avoid deadlock when output buffers fill.
|
||||
$stdoutTask = $proc.StandardOutput.ReadToEndAsync()
|
||||
$stderrTask = $proc.StandardError.ReadToEndAsync()
|
||||
|
||||
$exited = $proc.WaitForExit($TimeoutSeconds * 1000)
|
||||
if (-not $exited) {
|
||||
$timedOut = $true
|
||||
# Kill the entire process tree (taskkill /T) to release any inherited pipe
|
||||
# handles held by child processes (e.g. cmd.exe spawning ping.exe in tests,
|
||||
# or vmrun spawning helper subprocesses). Plain proc.Kill() only kills the
|
||||
# direct process; child processes keep the write end of stdout open.
|
||||
& taskkill /F /T /PID $proc.Id 2>&1 | Out-Null
|
||||
}
|
||||
$sw.Stop()
|
||||
|
||||
$outputText = ''
|
||||
if (-not $timedOut) {
|
||||
# Process exited normally: drain stdout/stderr (up to 5s safety net).
|
||||
[void][System.Threading.Tasks.Task]::WhenAll($stdoutTask, $stderrTask).Wait(5000)
|
||||
$stdoutText = ''
|
||||
$stderrText = ''
|
||||
if ($stdoutTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stdoutText = $stdoutTask.Result
|
||||
}
|
||||
if ($stderrTask.Status -eq [System.Threading.Tasks.TaskStatus]::RanToCompletion) {
|
||||
$stderrText = $stderrTask.Result
|
||||
}
|
||||
$parts = @($stdoutText, $stderrText) | Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
|
||||
$outputText = ($parts | ForEach-Object { $_.TrimEnd() }) -join "`n"
|
||||
}
|
||||
# When timed-out: process was killed; output is irrelevant and potentially
|
||||
# incomplete. Skip drain entirely — no point blocking for a killed process.
|
||||
|
||||
$exitCode = if ($timedOut) { -1 } else { $proc.ExitCode }
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
ExitCode = $exitCode
|
||||
Output = $outputText
|
||||
TimedOut = $timedOut
|
||||
ElapsedSeconds = [math]::Round($sw.Elapsed.TotalSeconds, 1)
|
||||
}
|
||||
|
||||
if ($timedOut -and $ThrowOnTimeout) {
|
||||
throw "vmrun $Operation timed out after ${TimeoutSeconds}s"
|
||||
}
|
||||
if (-not $timedOut -and $exitCode -ne 0 -and $ThrowOnError) {
|
||||
throw "vmrun $Operation failed (exit $exitCode): $outputText"
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
function Get-GuestIPAddress {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Polls a running VMware guest VM until its IP address is available, then returns it.
|
||||
.DESCRIPTION
|
||||
Primary method: reads the 'ci-ip' guestVar written by ci-report-ip.service (Linux)
|
||||
or the equivalent guest-side script via vmware-rpctool. This uses the VMware VMCI
|
||||
channel and does not require TCP/IP connectivity from the host.
|
||||
|
||||
Fallback: vmrun getGuestIPAddress (reliable on Windows guests; may be slow on Linux).
|
||||
|
||||
Returns the detected IP as a plain string, or an empty string if the timeout expires.
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
.PARAMETER VmxPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to poll before giving up. Default: 120.
|
||||
.OUTPUTS
|
||||
[string] Detected IPv4 address, or empty string on timeout.
|
||||
#>
|
||||
[OutputType([string])]
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $VmrunPath,
|
||||
[Parameter(Mandatory)] [string] $VmxPath,
|
||||
[int] $TimeoutSeconds = 120
|
||||
)
|
||||
|
||||
Write-Host "[Get-GuestIPAddress] Polling for VM IP (max ${TimeoutSeconds}s)..."
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$attempt = 0
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$attempt++
|
||||
|
||||
# Primary: guestinfo.ci-ip written by ci-report-ip.service via vmware-rpctool.
|
||||
# NOTE: vmrun readVariable guestVar reads WITHOUT the 'guestinfo.' prefix.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut = (& $VmrunPath -T ws readVariable $VmxPath guestVar 'ci-ip' 2>&1).Trim()
|
||||
$ipRc = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($attempt -le 3 -or ($attempt % 10 -eq 0)) {
|
||||
Write-Host "[Get-GuestIPAddress] [attempt $attempt] guestVar ci-ip: rc=$ipRc out='$ipOut'"
|
||||
}
|
||||
if ($ipRc -eq 0 -and $ipOut -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via guestinfo: $ipOut"
|
||||
return $ipOut
|
||||
}
|
||||
|
||||
# Fallback: vmrun getGuestIPAddress (VMware Tools / open-vm-tools required).
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$ipOut2 = (& $VmrunPath -T ws getGuestIPAddress $VmxPath 2>&1).Trim()
|
||||
$ipRc2 = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
if ($ipRc2 -eq 0 -and $ipOut2 -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') {
|
||||
Write-Host "[Get-GuestIPAddress] IP via getGuestIPAddress: $ipOut2"
|
||||
return $ipOut2
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function Get-GuestDiagnostics {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Best-effort collection of guest diagnostics after a build failure.
|
||||
Never throws — always safe to call from a catch/finally block.
|
||||
|
||||
.PARAMETER IPAddress
|
||||
IP address of the guest VM.
|
||||
|
||||
.PARAMETER GuestOS
|
||||
'Windows' or 'Linux'.
|
||||
|
||||
.PARAMETER Credential
|
||||
PSCredential for WinRM (Windows guests). Ignored for Linux.
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Path to SSH private key (Linux guests). Ignored for Windows.
|
||||
|
||||
.PARAMETER SshUser
|
||||
SSH username (Linux guests). Default: ci_build.
|
||||
|
||||
.PARAMETER DestinationDir
|
||||
Host directory where diagnostics files are written.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $IPAddress,
|
||||
[Parameter(Mandatory)] [ValidateSet('Windows','Linux')]
|
||||
[string] $GuestOS,
|
||||
[pscredential] $Credential,
|
||||
[string] $SshKeyPath,
|
||||
[string] $SshUser = 'ci_build',
|
||||
[Parameter(Mandatory)] [string] $DestinationDir
|
||||
)
|
||||
|
||||
Write-Host "[Diagnostics] Collecting guest diagnostics from $IPAddress ($GuestOS) -> $DestinationDir"
|
||||
try {
|
||||
if (-not (Test-Path $DestinationDir)) {
|
||||
New-Item -ItemType Directory -Path $DestinationDir -Force | Out-Null
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Could not create destination dir: $_"
|
||||
return
|
||||
}
|
||||
|
||||
if ($GuestOS -eq 'Windows') {
|
||||
if (-not $Credential) {
|
||||
Write-Warning "[Diagnostics] No Credential supplied for Windows guest — skipping."
|
||||
return
|
||||
}
|
||||
try {
|
||||
$sessionOption = New-CISessionOption
|
||||
$session = New-PSSession -ComputerName $IPAddress `
|
||||
-Credential $Credential `
|
||||
-SessionOption $sessionOption `
|
||||
-ErrorAction Stop
|
||||
try {
|
||||
# Collect: event log errors (last 50), running processes, disk usage
|
||||
$diagScript = {
|
||||
$out = [System.Text.StringBuilder]::new()
|
||||
[void]$out.AppendLine("=== System Event Log (last 50 errors) ===")
|
||||
Get-EventLog -LogName System -EntryType Error -Newest 50 -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.TimeGenerated) $($_.Source): $($_.Message)") }
|
||||
[void]$out.AppendLine("`n=== Running Processes ===")
|
||||
Get-Process -ErrorAction SilentlyContinue |
|
||||
Sort-Object CPU -Descending |
|
||||
Select-Object -First 30 |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name) PID=$($_.Id) CPU=$($_.CPU)") }
|
||||
[void]$out.AppendLine("`n=== Disk Free Space ===")
|
||||
Get-PSDrive -PSProvider FileSystem -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [void]$out.AppendLine("$($_.Name): Free=$([math]::Round($_.Free/1GB,1))GB Used=$([math]::Round($_.Used/1GB,1))GB") }
|
||||
$out.ToString()
|
||||
}
|
||||
$diagText = Invoke-Command -Session $session -ScriptBlock $diagScript -ErrorAction Stop
|
||||
$diagText | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Windows diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$buildLog = 'C:\CI\build.log'
|
||||
try {
|
||||
Copy-Item -FromSession $session -Path $buildLog `
|
||||
-Destination (Join-Path $DestinationDir 'build.log') `
|
||||
-ErrorAction Stop
|
||||
Write-Host "[Diagnostics] Build log copied."
|
||||
} catch {
|
||||
Write-Host "[Diagnostics] Build log not found in guest (skipping)."
|
||||
}
|
||||
} finally {
|
||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Windows guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
else {
|
||||
# Linux
|
||||
if (-not $SshKeyPath) {
|
||||
Write-Warning "[Diagnostics] No SshKeyPath supplied for Linux guest — skipping."
|
||||
return
|
||||
}
|
||||
$sshOpts = @('-i', $SshKeyPath, '-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-o', 'ConnectTimeout=10', '-o', 'BatchMode=yes')
|
||||
try {
|
||||
# Collect: journalctl errors, ps, df
|
||||
$diagCmd = "journalctl -p err -n 50 --no-pager 2>/dev/null; echo '=== ps ==='; ps aux --sort=-%cpu | head -30; echo '=== df ==='; df -h"
|
||||
$savedEap = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$diagOutput = & ssh.exe @sshOpts "${SshUser}@${IPAddress}" $diagCmd 2>&1
|
||||
$ErrorActionPreference = $savedEap
|
||||
$diagOutput | Set-Content -Path (Join-Path $DestinationDir 'guest-diag.txt') -Encoding UTF8
|
||||
Write-Host "[Diagnostics] Linux diagnostics written."
|
||||
|
||||
# Best-effort: copy CI build log
|
||||
$scpSrc = "${SshUser}@${IPAddress}:/opt/ci/build/ci-build.log"
|
||||
$scpDst = Join-Path $DestinationDir 'build.log'
|
||||
$ErrorActionPreference = 'Continue'
|
||||
& scp.exe @sshOpts $scpSrc $scpDst 2>&1 | Out-Null
|
||||
$ErrorActionPreference = $savedEap
|
||||
if (Test-Path $scpDst) { Write-Host "[Diagnostics] Build log copied." }
|
||||
else { Write-Host "[Diagnostics] Build log not found in guest (skipping)." }
|
||||
} catch {
|
||||
Write-Warning "[Diagnostics] Linux guest diagnostics failed (non-fatal): $_"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun, Invoke-VmrunBounded, Get-GuestIPAddress, Get-GuestDiagnostics
|
||||
@@ -1,201 +0,0 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
SSH/SCP transport helpers for Linux build VMs.
|
||||
|
||||
.DESCRIPTION
|
||||
SSH-only transport layer for the Linux branch of the CI/CD system.
|
||||
The existing WinRM transport (used by Windows VMs) is NOT modified.
|
||||
|
||||
Exported functions:
|
||||
Invoke-SshCommand — run a command in a Linux guest via ssh.exe
|
||||
Copy-SshItem — copy files to/from a Linux guest via scp.exe
|
||||
Test-SshReady — poll until SSH is ready (port 22 open + login succeeds)
|
||||
|
||||
Requirements:
|
||||
- ssh.exe and scp.exe in PATH (default on Windows 11 / Server 2022)
|
||||
- SSH private key at the path provided
|
||||
- Guest user must be set up for key-based auth (no passphrase)
|
||||
#>
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
|
||||
function Invoke-SshCommand {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Run a command in a Linux guest via ssh.exe
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $IP,
|
||||
|
||||
[string] $User = 'ci_build',
|
||||
|
||||
[string] $KeyPath = 'F:\CI\keys\ci_linux',
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string] $Command,
|
||||
|
||||
[int] $ConnectTimeout = 10,
|
||||
|
||||
# Persistent known_hosts file for CI jobs.
|
||||
# When non-empty: StrictHostKeyChecking=accept-new + this file.
|
||||
# When empty (default): permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL)
|
||||
# — used during template provisioning when host key is not yet known.
|
||||
[string] $KnownHostsFile = '',
|
||||
|
||||
# Capture + return stdout/stderr instead of printing to console.
|
||||
[switch] $PassThru,
|
||||
|
||||
# Don't throw on non-zero exit — just return the output.
|
||||
[switch] $AllowFail
|
||||
)
|
||||
|
||||
if ($KnownHostsFile -ne '') {
|
||||
$sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
|
||||
} else {
|
||||
$sshHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
|
||||
}
|
||||
$sshArgs = @(
|
||||
'-i', $KeyPath
|
||||
) + $sshHostKeyOpts + @(
|
||||
'-o', "ConnectTimeout=$ConnectTimeout",
|
||||
'-o', 'BatchMode=yes',
|
||||
"$User@$IP",
|
||||
$Command
|
||||
)
|
||||
|
||||
if ($PassThru) {
|
||||
$output = & ssh @sshArgs 2>&1
|
||||
$exit = $LASTEXITCODE
|
||||
}
|
||||
else {
|
||||
& ssh @sshArgs
|
||||
$exit = $LASTEXITCODE
|
||||
$output = @()
|
||||
}
|
||||
|
||||
if ($exit -ne 0 -and -not $AllowFail) {
|
||||
throw "[SSH] Command failed (exit $exit): $Command"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
ExitCode = [int]$exit
|
||||
Output = [string[]]$output
|
||||
}
|
||||
}
|
||||
|
||||
function Copy-SshItem {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Copy files to/from Linux guest via scp.exe
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $Source,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string] $Destination,
|
||||
|
||||
# Guest IP — used to build user@host prefix when Direction is set.
|
||||
[string] $IP = '',
|
||||
|
||||
[string] $User = 'ci_build',
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[string] $KeyPath,
|
||||
|
||||
[ValidateSet('ToGuest', 'FromGuest', 'Direct')]
|
||||
[string] $Direction = 'Direct',
|
||||
|
||||
# Pass -r to scp (recursive copy).
|
||||
[switch] $Recurse,
|
||||
|
||||
[int] $ConnectTimeout = 10,
|
||||
|
||||
# Persistent known_hosts file for CI jobs.
|
||||
# When non-empty: StrictHostKeyChecking=accept-new + this file.
|
||||
# When empty (default): permissive mode — used during template provisioning.
|
||||
[string] $KnownHostsFile = ''
|
||||
)
|
||||
|
||||
if ($Direction -eq 'ToGuest') {
|
||||
$Destination = "$User@${IP}:$Destination"
|
||||
}
|
||||
elseif ($Direction -eq 'FromGuest') {
|
||||
$Source = "$User@${IP}:$Source"
|
||||
}
|
||||
# Direction = 'Direct': use Source + Destination as-is
|
||||
|
||||
if ($KnownHostsFile -ne '') {
|
||||
$scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=accept-new', '-o', "UserKnownHostsFile=$KnownHostsFile")
|
||||
} else {
|
||||
$scpHostKeyOpts = @('-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=NUL')
|
||||
}
|
||||
$scpArgs = @(
|
||||
'-i', $KeyPath
|
||||
) + $scpHostKeyOpts + @(
|
||||
'-o', "ConnectTimeout=$ConnectTimeout"
|
||||
)
|
||||
if ($Recurse) { $scpArgs += '-r' }
|
||||
$scpArgs += @($Source, $Destination)
|
||||
|
||||
& scp @scpArgs
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "[SCP] Copy failed (exit $LASTEXITCODE): $Source -> $Destination"
|
||||
}
|
||||
}
|
||||
|
||||
function Test-SshReady {
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Poll until SSH port 22 is open and a login command succeeds.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $IP,
|
||||
|
||||
[string] $User = 'ci_build',
|
||||
|
||||
[string] $KeyPath = 'F:\CI\keys\ci_linux',
|
||||
|
||||
[int] $TimeoutSec = 300,
|
||||
|
||||
[int] $PollSec = 10
|
||||
)
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
||||
|
||||
# Phase 1: wait for port 22 to be open
|
||||
$portOpen = $false
|
||||
while (-not $portOpen -and (Get-Date) -lt $deadline) {
|
||||
$portOpen = (Test-NetConnection -ComputerName $IP -Port 22 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue)
|
||||
if (-not $portOpen) {
|
||||
Start-Sleep -Seconds $PollSec
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $portOpen) {
|
||||
throw "[Test-SshReady] Timed out after ${TimeoutSec}s waiting for SSH at $IP"
|
||||
}
|
||||
|
||||
# Phase 2: try ssh echo — success when output contains 'ready'
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$result = Invoke-SshCommand -IP $IP -User $User -KeyPath $KeyPath `
|
||||
-Command 'echo ready' -AllowFail -PassThru
|
||||
if ($result.ExitCode -eq 0 -and ($result.Output -join '') -like '*ready*') {
|
||||
return $true
|
||||
}
|
||||
Start-Sleep -Seconds $PollSec
|
||||
}
|
||||
|
||||
throw "[Test-SshReady] Timed out after ${TimeoutSec}s waiting for SSH at $IP"
|
||||
}
|
||||
|
||||
Export-ModuleMember -Function Invoke-SshCommand, Copy-SshItem, Test-SshReady
|
||||
Executable
+105
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
# install-ci-alias.sh — install a `ci` shell helper that runs the CI
|
||||
# orchestrator as the ci-runner service user against the production venv.
|
||||
#
|
||||
# After install: ci validate host == sudo -u ci-runner \
|
||||
# /opt/ci/venv/bin/python -m ci_orchestrator validate host
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/install-ci-alias.sh # install into the current user's
|
||||
# # ~/.bashrc and ~/.zshrc (if present)
|
||||
# ./scripts/install-ci-alias.sh --system # install system-wide via
|
||||
# # /etc/profile.d/ci-alias.sh (needs sudo)
|
||||
# ./scripts/install-ci-alias.sh --uninstall # remove the installed block
|
||||
#
|
||||
# Idempotent: re-running updates the block in place instead of duplicating it.
|
||||
set -euo pipefail
|
||||
|
||||
PROD_PY="${CI_PROD_PYTHON:-/opt/ci/venv/bin/python}"
|
||||
RUN_USER="${CI_RUN_USER:-ci-runner}"
|
||||
MARK_BEGIN="# >>> ci-orchestrator alias >>>"
|
||||
MARK_END="# <<< ci-orchestrator alias <<<"
|
||||
|
||||
log() { printf '[install-ci-alias] %s\n' "$*"; }
|
||||
die() { printf '[install-ci-alias] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
block() {
|
||||
cat <<EOF
|
||||
${MARK_BEGIN}
|
||||
# Run the CI orchestrator as the ${RUN_USER} service user (production venv).
|
||||
ci() { sudo -u ${RUN_USER} ${PROD_PY} -m ci_orchestrator "\$@"; }
|
||||
# Same, but as the current user (no sudo) — handy for read-only commands.
|
||||
ci-me() { ${PROD_PY} -m ci_orchestrator "\$@"; }
|
||||
${MARK_END}
|
||||
EOF
|
||||
}
|
||||
|
||||
strip_block() {
|
||||
# Remove an existing marked block from $1 (in place). No-op if absent.
|
||||
local file="$1"
|
||||
[ -f "$file" ] || return 0
|
||||
if grep -qF "$MARK_BEGIN" "$file"; then
|
||||
# Delete everything between the markers, inclusive.
|
||||
sed -i "/$(printf '%s' "$MARK_BEGIN" | sed 's/[].[*^$/]/\\&/g')/,/$(printf '%s' "$MARK_END" | sed 's/[].[*^$/]/\\&/g')/d" "$file"
|
||||
# Drop a possible trailing blank line left behind.
|
||||
sed -i -e :a -e '/^\n*$/{$d;N;ba}' "$file" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
install_into() {
|
||||
local file="$1"
|
||||
strip_block "$file"
|
||||
{ printf '\n'; block; } >> "$file"
|
||||
log "updated $file"
|
||||
}
|
||||
|
||||
uninstall_from() {
|
||||
local file="$1"
|
||||
if [ -f "$file" ] && grep -qF "$MARK_BEGIN" "$file"; then
|
||||
strip_block "$file"
|
||||
log "removed block from $file"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local mode="user" action="install"
|
||||
case "${1:-}" in
|
||||
--system) mode="system" ;;
|
||||
--uninstall) action="uninstall" ;;
|
||||
"" ) ;;
|
||||
* ) die "unknown argument: $1 (use --system or --uninstall)" ;;
|
||||
esac
|
||||
|
||||
# Sanity: warn (don't fail) if the production venv is missing.
|
||||
[ -x "$PROD_PY" ] || log "WARNING: $PROD_PY not found — alias installed anyway; fix the venv before use."
|
||||
|
||||
if [ "$mode" = "system" ]; then
|
||||
local target="/etc/profile.d/ci-alias.sh"
|
||||
if [ "$action" = "uninstall" ]; then
|
||||
sudo rm -f "$target" && log "removed $target"
|
||||
else
|
||||
block | sudo tee "$target" >/dev/null
|
||||
sudo chmod 0644 "$target"
|
||||
log "installed $target (applies to all login shells)"
|
||||
fi
|
||||
else
|
||||
local files=()
|
||||
[ -f "$HOME/.bashrc" ] && files+=("$HOME/.bashrc")
|
||||
[ -f "$HOME/.zshrc" ] && files+=("$HOME/.zshrc")
|
||||
# If neither exists, create ~/.bashrc so the alias lands somewhere.
|
||||
[ ${#files[@]} -eq 0 ] && files+=("$HOME/.bashrc")
|
||||
|
||||
for f in "${files[@]}"; do
|
||||
if [ "$action" = "uninstall" ]; then uninstall_from "$f"; else install_into "$f"; fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$action" = "install" ]; then
|
||||
log "done. Open a new shell, or run: source ~/.bashrc"
|
||||
log "try: ci validate host"
|
||||
else
|
||||
log "done. Open a new shell to drop the alias."
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+300
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env bash
|
||||
# setup-host-linux.sh — Bootstrap CI host Linux Mint (Fase B, Step B1)
|
||||
#
|
||||
# Equivalente di Setup-Host.ps1 per l'host Linux.
|
||||
# Eseguire come root: sudo bash setup-host-linux.sh -d /dev/sdf1 [opzioni]
|
||||
#
|
||||
# Cosa fa:
|
||||
# 1. Crea l'utente di servizio ci-runner
|
||||
# 2. Monta la partizione CI su /var/lib/ci via fstab (boot automatico)
|
||||
# Se la partizione ha già una sottocartella ci/, la riorganizza
|
||||
# portando i contenuti alla root della partizione (nessun dato perso)
|
||||
# 3. Crea il layout directory e permessi
|
||||
# 4. Installa Python 3.11+ e crea venv produzione in /opt/ci/venv
|
||||
# 5. (Opzionale) Clona il repo e installa il package nel venv
|
||||
# 6. (Opt-in, --with-pwsh) Installa PowerShell Core
|
||||
# 7. Installa l'alias shell `ci` a livello di sistema (/etc/profile.d/)
|
||||
# L'host Linux NON ha più bisogno di pwsh per il normale funzionamento
|
||||
# (orchestratore Python). Installalo solo se ti serve per script legacy.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Defaults ─────────────────────────────────────────────────────────────────
|
||||
CI_DISK=""
|
||||
CI_ROOT="/var/lib/ci"
|
||||
CI_USER="ci-runner"
|
||||
OPT_CI="/opt/ci"
|
||||
REPO_URL="https://gitea.emulab.it/Simone/local-ci-cd-system.git"
|
||||
|
||||
SKIP_PYTHON=false
|
||||
WITH_PWSH=false
|
||||
SKIP_CLONE=false
|
||||
|
||||
# ── Colori ───────────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
info() { echo -e "${GREEN}==>${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW} ! $*${NC}"; }
|
||||
die() { echo -e "${RED}ERROR: $*${NC}" >&2; exit 1; }
|
||||
|
||||
# ── Uso ──────────────────────────────────────────────────────────────────────
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Uso: sudo bash $(basename "$0") -d <disco> [opzioni]
|
||||
|
||||
Obbligatorio:
|
||||
-d <disco> Partizione CI (es. /dev/sdf1 o /dev/sdf)
|
||||
|
||||
Opzionali:
|
||||
-r <path> CI root (default: /var/lib/ci)
|
||||
-u <utente> Utente di servizio (default: ci-runner)
|
||||
--skip-python Salta installazione Python/venv
|
||||
--skip-clone Salta clone repo + pip install
|
||||
--with-pwsh Installa PowerShell Core (opt-in; NON installato di default).
|
||||
L'host Linux non ha più bisogno di pwsh per il normale
|
||||
funzionamento — usa questo flag solo per script legacy.
|
||||
--skip-pwsh DEPRECATO: no-op. pwsh non è più installato di default.
|
||||
|
||||
Esempi:
|
||||
sudo bash setup-host-linux.sh -d /dev/sdf1
|
||||
sudo bash setup-host-linux.sh -d /dev/sdf1 --with-pwsh
|
||||
sudo bash setup-host-linux.sh -d /dev/sdf1 --skip-clone
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Parse args ───────────────────────────────────────────────────────────────
|
||||
[[ $# -eq 0 ]] && usage
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-d) CI_DISK="$2"; shift 2 ;;
|
||||
-r) CI_ROOT="$2"; shift 2 ;;
|
||||
-u) CI_USER="$2"; shift 2 ;;
|
||||
--skip-python) SKIP_PYTHON=true; shift ;;
|
||||
--with-pwsh) WITH_PWSH=true; shift ;;
|
||||
--skip-pwsh)
|
||||
echo "WARN: --skip-pwsh è deprecato e non ha più effetto: pwsh non è installato di default (usa --with-pwsh per installarlo)." >&2
|
||||
shift ;;
|
||||
--skip-clone) SKIP_CLONE=true; shift ;;
|
||||
-h|--help) usage ;;
|
||||
*) die "Parametro sconosciuto: $1"; ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -z "$CI_DISK" ]] && die "-d <disco> è obbligatorio"
|
||||
[[ $EUID -ne 0 ]] && die "Eseguire come root: sudo bash $0 ..."
|
||||
[[ ! -b "$CI_DISK" ]] && die "$CI_DISK non è un block device"
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════╗"
|
||||
echo "║ CI Host Setup — Linux Mint (Fase B / B1) ║"
|
||||
echo "╚══════════════════════════════════════════════════════╝"
|
||||
echo " Disco CI : $CI_DISK"
|
||||
echo " CI root : $CI_ROOT"
|
||||
echo " Utente : $CI_USER"
|
||||
echo ""
|
||||
|
||||
# ── Step 1: utente di servizio ────────────────────────────────────────────────
|
||||
info "[1/7] Utente di servizio $CI_USER"
|
||||
if id "$CI_USER" &>/dev/null; then
|
||||
warn "Utente $CI_USER esiste già — skip"
|
||||
else
|
||||
useradd -r -m -s /bin/bash "$CI_USER"
|
||||
echo " Creato utente $CI_USER"
|
||||
fi
|
||||
|
||||
# ── Step 2: mount partizione CI ───────────────────────────────────────────────
|
||||
info "[2/7] Mount $CI_DISK → $CI_ROOT"
|
||||
|
||||
if mountpoint -q "$CI_ROOT" 2>/dev/null; then
|
||||
warn "$CI_ROOT già montato — skip riorganizzazione e fstab"
|
||||
else
|
||||
# Recupera UUID e filesystem
|
||||
UUID=$(blkid -s UUID -o value "$CI_DISK" 2>/dev/null)
|
||||
FS_TYPE=$(blkid -s TYPE -o value "$CI_DISK" 2>/dev/null)
|
||||
[[ -z "$UUID" ]] && die "Impossibile determinare UUID di $CI_DISK"
|
||||
[[ -z "$FS_TYPE" ]] && die "Impossibile determinare filesystem di $CI_DISK"
|
||||
echo " UUID=$UUID TYPE=$FS_TYPE"
|
||||
|
||||
# Mount temporaneo per eventuale riorganizzazione
|
||||
TMP_MNT=$(mktemp -d /mnt/ci-setup-XXXXXX)
|
||||
mount "$CI_DISK" "$TMP_MNT"
|
||||
|
||||
if [[ -d "$TMP_MNT/ci" ]]; then
|
||||
echo " Trovata sottocartella ci/ — riorganizzazione in corso..."
|
||||
# Sposta tutto il contenuto di ci/ alla root della partizione
|
||||
shopt -s dotglob nullglob
|
||||
for item in "$TMP_MNT/ci/"*; do
|
||||
name=$(basename "$item")
|
||||
if [[ -e "$TMP_MNT/$name" ]]; then
|
||||
warn " Conflitto: $name già esiste alla root — skip spostamento"
|
||||
else
|
||||
mv "$item" "$TMP_MNT/"
|
||||
echo " Spostato: ci/$name → /"
|
||||
fi
|
||||
done
|
||||
shopt -u dotglob nullglob
|
||||
if [[ -z "$(ls -A "$TMP_MNT/ci" 2>/dev/null)" ]]; then
|
||||
rmdir "$TMP_MNT/ci"
|
||||
echo " Rimossa cartella ci/ vuota"
|
||||
else
|
||||
warn " ci/ non è vuota dopo lo spostamento — controlla manualmente"
|
||||
fi
|
||||
else
|
||||
echo " Nessuna sottocartella ci/ trovata — nessuna riorganizzazione necessaria"
|
||||
fi
|
||||
|
||||
umount "$TMP_MNT"
|
||||
rmdir "$TMP_MNT"
|
||||
|
||||
# Mount point e fstab
|
||||
mkdir -p "$CI_ROOT"
|
||||
if grep -q "UUID=$UUID" /etc/fstab; then
|
||||
warn "Entry fstab per UUID=$UUID già presente — skip"
|
||||
else
|
||||
echo "UUID=$UUID $CI_ROOT $FS_TYPE defaults,nofail 0 2" >> /etc/fstab
|
||||
echo " Aggiunta entry fstab"
|
||||
fi
|
||||
|
||||
mount -a
|
||||
mountpoint -q "$CI_ROOT" || die "mount -a fallito per $CI_ROOT"
|
||||
echo " Montato OK: $(df -h "$CI_ROOT" | tail -1)"
|
||||
fi
|
||||
|
||||
# ── Step 3: layout directory e permessi ───────────────────────────────────────
|
||||
info "[3/7] Layout directory e permessi"
|
||||
|
||||
# Installa acl se mancante
|
||||
if ! command -v setfacl &>/dev/null; then
|
||||
echo " Installazione pacchetto acl..."
|
||||
apt-get install -y acl -qq
|
||||
fi
|
||||
|
||||
mkdir -p \
|
||||
"$CI_ROOT"/{build-vms,artifacts,templates,keys,logs,runner} \
|
||||
"$OPT_CI" \
|
||||
/etc/ci/keys
|
||||
|
||||
chown -R "$CI_USER:$CI_USER" "$CI_ROOT" "$OPT_CI"
|
||||
chmod 750 "$CI_ROOT" "$OPT_CI"
|
||||
for d in build-vms artifacts templates keys logs runner; do
|
||||
chmod 750 "$CI_ROOT/$d"
|
||||
done
|
||||
chmod 700 /etc/ci/keys
|
||||
|
||||
# ACL di default su build-vms (ci-runner eredita rwX sui nuovi file)
|
||||
setfacl -d -m "u:$CI_USER:rwX" "$CI_ROOT/build-vms"
|
||||
echo " Layout OK — ACL default su build-vms applicata"
|
||||
|
||||
# ── Step 4: Python 3.11+ e venv ──────────────────────────────────────────────
|
||||
if [[ "$SKIP_PYTHON" == false ]]; then
|
||||
info "[4/7] Python 3.11+ e venv produzione"
|
||||
|
||||
# Cerca la prima versione di Python >= 3.11 già installata
|
||||
PYTHON_BIN=""
|
||||
for cmd in python3 python3.13 python3.12 python3.11; do
|
||||
if command -v "$cmd" &>/dev/null; then
|
||||
ver=$("$cmd" -c 'import sys; v=sys.version_info; print(v.major*100+v.minor)' 2>/dev/null)
|
||||
if [[ "${ver:-0}" -ge 311 ]]; then
|
||||
PYTHON_BIN="$cmd"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -n "$PYTHON_BIN" ]]; then
|
||||
echo " Python >= 3.11 già presente: $($PYTHON_BIN --version) ($PYTHON_BIN)"
|
||||
apt-get install -y "python3-venv" git -qq 2>/dev/null || true
|
||||
else
|
||||
echo " Nessun Python >= 3.11 trovato — installazione python3.11..."
|
||||
apt-get install -y python3.11 python3.11-venv git -qq
|
||||
PYTHON_BIN="python3.11"
|
||||
fi
|
||||
|
||||
if [[ -d "$OPT_CI/venv" ]]; then
|
||||
warn "venv $OPT_CI/venv già esistente — skip"
|
||||
else
|
||||
"$PYTHON_BIN" -m venv "$OPT_CI/venv"
|
||||
chown -R "$CI_USER:$CI_USER" "$OPT_CI/venv"
|
||||
echo " venv creato in $OPT_CI/venv ($($PYTHON_BIN --version))"
|
||||
fi
|
||||
else
|
||||
info "[4/7] Python/venv — SKIPPED"
|
||||
fi
|
||||
|
||||
# ── Step 5: Clone repo e pip install ─────────────────────────────────────────
|
||||
if [[ "$SKIP_CLONE" == false ]]; then
|
||||
info "[5/7] Clone repo e pip install"
|
||||
REPO_DIR="$OPT_CI/local-ci-cd-system"
|
||||
if [[ -d "$REPO_DIR/.git" ]]; then
|
||||
warn "Repo già presente in $REPO_DIR — skip clone (eseguire git pull manualmente)"
|
||||
else
|
||||
echo " Clone repo..."
|
||||
sudo -u "$CI_USER" git clone "$REPO_URL" "$REPO_DIR"
|
||||
fi
|
||||
echo " pip install..."
|
||||
sudo -u "$CI_USER" "$OPT_CI/venv/bin/pip" install -q -e "$REPO_DIR"
|
||||
verify_out=$(sudo -u "$CI_USER" "$OPT_CI/venv/bin/python" -m ci_orchestrator --help 2>&1) \
|
||||
|| die "ci_orchestrator non importabile"
|
||||
echo " Verifica OK: ${verify_out%%$'\n'*}"
|
||||
else
|
||||
info "[5/7] Clone repo — SKIPPED"
|
||||
fi
|
||||
|
||||
# ── Step 6: PowerShell Core (opt-in via --with-pwsh) ──────────────────────────
|
||||
if [[ "$WITH_PWSH" == true ]]; then
|
||||
info "[6/7] PowerShell Core"
|
||||
if command -v pwsh &>/dev/null; then
|
||||
warn "pwsh già installato: $(pwsh --version) — skip"
|
||||
else
|
||||
echo " Installazione PowerShell Core..."
|
||||
apt-get install -y wget apt-transport-https software-properties-common -qq
|
||||
# Linux Mint riporta la propria versione in VERSION_ID; serve quella Ubuntu base
|
||||
if [[ -f /etc/upstream-release/lsb-release ]]; then
|
||||
UBUNTU_VER=$(grep DISTRIB_RELEASE /etc/upstream-release/lsb-release | cut -d= -f2)
|
||||
else
|
||||
source /etc/os-release
|
||||
UBUNTU_VER="$VERSION_ID"
|
||||
fi
|
||||
wget -q "https://packages.microsoft.com/config/ubuntu/${UBUNTU_VER}/packages-microsoft-prod.deb" \
|
||||
-O /tmp/ms-prod.deb
|
||||
dpkg -i /tmp/ms-prod.deb
|
||||
apt-get update -qq 2>&1 || true # ignora errori di repo terzi (es. chiavi GPG scadute)
|
||||
apt-get install -y powershell -qq
|
||||
rm /tmp/ms-prod.deb
|
||||
echo " Installato: $(pwsh --version)"
|
||||
fi
|
||||
else
|
||||
info "[6/7] PowerShell Core — SKIPPED (opt-in con --with-pwsh)"
|
||||
fi
|
||||
|
||||
# ── Step 7: alias shell `ci` (system-wide) ────────────────────────────────────
|
||||
info "[7/7] Alias shell 'ci'"
|
||||
ALIAS_SCRIPT="$OPT_CI/local-ci-cd-system/scripts/install-ci-alias.sh"
|
||||
if [[ -x "$ALIAS_SCRIPT" ]]; then
|
||||
# Installa /etc/profile.d/ci-alias.sh per tutti gli utenti login.
|
||||
# Passa il venv/utente correnti così l'alias punta a questa installazione.
|
||||
CI_PROD_PYTHON="$OPT_CI/venv/bin/python" CI_RUN_USER="$CI_USER" \
|
||||
bash "$ALIAS_SCRIPT" --system
|
||||
echo " Alias 'ci' installato — apri una nuova shell e prova: ci validate host"
|
||||
else
|
||||
warn "install-ci-alias.sh non trovato in $ALIAS_SCRIPT — skip (clona il repo, Step 5)"
|
||||
fi
|
||||
|
||||
# ── Riepilogo ─────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════╗"
|
||||
echo "║ Setup completato ║"
|
||||
echo "╚══════════════════════════════════════════════════════╝"
|
||||
echo ""
|
||||
echo " CI_ROOT : $CI_ROOT ($(df -h "$CI_ROOT" | awk 'NR==2{print $4}') liberi)"
|
||||
echo " CI_USER : $CI_USER"
|
||||
echo " venv : $OPT_CI/venv"
|
||||
echo ""
|
||||
echo "Passi successivi (manuali):"
|
||||
echo " B2 — Validare template VM in $CI_ROOT/templates/"
|
||||
echo " find $CI_ROOT/templates -name '*.vmx' -exec vmrun -T ws listSnapshots {} \\;"
|
||||
echo " B3 — Copiare chiavi SSH in /etc/ci/keys/ e configurare keyring"
|
||||
echo " B4 — Installare act_runner come systemd service"
|
||||
echo " B5 — Installare timer systemd da deploy/systemd/"
|
||||
echo ""
|
||||
echo "Vedi plans/PhaseB-user-checklist.md per i dettagli."
|
||||
@@ -13,11 +13,17 @@ import click
|
||||
|
||||
from ci_orchestrator import __version__
|
||||
from ci_orchestrator.commands.artifacts import artifacts
|
||||
from ci_orchestrator.commands.bench import bench
|
||||
from ci_orchestrator.commands.build import build
|
||||
from ci_orchestrator.commands.creds import creds
|
||||
from ci_orchestrator.commands.job import job
|
||||
from ci_orchestrator.commands.monitor import monitor
|
||||
from ci_orchestrator.commands.report import report
|
||||
from ci_orchestrator.commands.vm import vm
|
||||
from ci_orchestrator.commands.retention import retention
|
||||
from ci_orchestrator.commands.smoke import smoke
|
||||
from ci_orchestrator.commands.template import template
|
||||
from ci_orchestrator.commands.validate import validate
|
||||
from ci_orchestrator.commands.vm import vm, vmgui
|
||||
|
||||
# Re-export so legacy A1 tests that patched ``cli_module.<name>`` keep working.
|
||||
from ci_orchestrator.commands.wait import ( # noqa: F401 (intentional re-exports)
|
||||
@@ -41,7 +47,14 @@ cli.add_command(build)
|
||||
cli.add_command(artifacts)
|
||||
cli.add_command(monitor)
|
||||
cli.add_command(report)
|
||||
cli.add_command(retention)
|
||||
cli.add_command(template)
|
||||
cli.add_command(job)
|
||||
cli.add_command(bench)
|
||||
cli.add_command(smoke)
|
||||
cli.add_command(validate)
|
||||
cli.add_command(creds)
|
||||
cli.add_command(vmgui)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
|
||||
@@ -0,0 +1,704 @@
|
||||
"""``bench`` sub-commands.
|
||||
|
||||
Phase C ports the manual PowerShell benchmark/burn-in tooling to Python so
|
||||
the Linux host no longer needs ``pwsh``:
|
||||
|
||||
* ``bench run`` — concurrency burn-in (replaces ``Test-CapacityBurnIn.ps1``
|
||||
+ ``Start-BurnInTest*.ps1``).
|
||||
* ``bench measure`` — phase timing (replaces ``Measure-CIBenchmark.ps1``).
|
||||
|
||||
Design rule: import ``backends.load_backend`` and the existing transports
|
||||
only — never ``WorkstationVmrunBackend`` directly (keeps the Phase D ESXi
|
||||
path open). No ``Start-Job``/``pwsh``/``$IsWindows``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import Config, load_config
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# Clone directories left behind by a job are named ``Clone_<job_id>_...``
|
||||
# (see commands/job.py:_make_clone_name and commands/vm.py:vm_new).
|
||||
_CLONE_PREFIX = "Clone_"
|
||||
# A vm-start.lock older than this (seconds) is considered stale leftover.
|
||||
_STALE_LOCK_SECONDS = 5 * 60
|
||||
# Linux guests probe SSH/22; everything else probes WinRM/5986.
|
||||
_SSH_PORT = 22
|
||||
_WINRM_PORT = 5986
|
||||
|
||||
|
||||
def _now() -> float:
|
||||
return time.monotonic()
|
||||
|
||||
|
||||
def _timestamp() -> str:
|
||||
return datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("bench")
|
||||
def bench() -> None:
|
||||
"""Capacity burn-in and phase-timing benchmarks."""
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════ bench run
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobResult:
|
||||
"""Outcome of a single burn-in job slot."""
|
||||
|
||||
job_id: str
|
||||
slot: int
|
||||
exit_code: int
|
||||
status: str
|
||||
elapsed_sec: float
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RoundResult:
|
||||
"""Outcome of one burn-in round (all slots + cleanup assertions)."""
|
||||
|
||||
round: int
|
||||
passed: int
|
||||
failed: int
|
||||
status: str
|
||||
elapsed_sec: float
|
||||
orphans: list[str] = field(default_factory=list)
|
||||
stale_lock: bool = False
|
||||
jobs: list[JobResult] = field(default_factory=list)
|
||||
|
||||
|
||||
def _run_one_job(
|
||||
*,
|
||||
job_id: str,
|
||||
guest_os: str,
|
||||
build_command: str,
|
||||
clone_base_dir: Path,
|
||||
) -> tuple[int, str]:
|
||||
"""Run a single job end to end, returning ``(exit_code, error_message)``.
|
||||
|
||||
Invokes the in-process ``commands.job.job`` command (the same code path
|
||||
production uses) rather than re-spawning ``python -m ci_orchestrator`` or
|
||||
a PowerShell child — closes the ``Start-Job``/``$IsWindows`` shim bug.
|
||||
|
||||
Isolated in its own module-level function so the burn-in harness can be
|
||||
unit tested by monkeypatching it (no real VMs needed).
|
||||
"""
|
||||
from click.testing import CliRunner # local import: drives the job command
|
||||
|
||||
from ci_orchestrator.commands.job import job
|
||||
|
||||
args = [
|
||||
"--job-id",
|
||||
job_id,
|
||||
"--repo-url",
|
||||
"",
|
||||
"--branch",
|
||||
"main",
|
||||
"--guest-os",
|
||||
guest_os,
|
||||
"--clone-base-dir",
|
||||
str(clone_base_dir),
|
||||
"--skip-artifact",
|
||||
]
|
||||
if build_command:
|
||||
args += ["--build-command", build_command]
|
||||
result = CliRunner().invoke(job, args, catch_exceptions=True)
|
||||
if result.exit_code == 0:
|
||||
return 0, ""
|
||||
tail = result.output.strip().splitlines()
|
||||
return result.exit_code, tail[-1] if tail else ""
|
||||
|
||||
|
||||
def _find_orphans(clone_base: Path, job_ids: list[str]) -> list[Path]:
|
||||
"""Return clone dirs in ``clone_base`` belonging to the given jobs."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
prefixes = tuple(f"{_CLONE_PREFIX}{jid}_" for jid in job_ids)
|
||||
return [
|
||||
entry
|
||||
for entry in clone_base.iterdir()
|
||||
if entry.is_dir() and entry.name.startswith(prefixes)
|
||||
]
|
||||
|
||||
|
||||
def _stale_lock(lock_file: Path, max_age_seconds: int = _STALE_LOCK_SECONDS) -> bool:
|
||||
"""True if ``lock_file`` exists and is older than ``max_age_seconds``."""
|
||||
if not lock_file.is_file():
|
||||
return False
|
||||
try:
|
||||
mtime = lock_file.stat().st_mtime
|
||||
except OSError:
|
||||
return False
|
||||
return (time.time() - mtime) > max_age_seconds
|
||||
|
||||
|
||||
def _run_round(
|
||||
*,
|
||||
round_no: int,
|
||||
concurrency: int,
|
||||
guest_os: str,
|
||||
build_command: str,
|
||||
clone_base_dir: Path,
|
||||
lock_file: Path,
|
||||
) -> RoundResult:
|
||||
"""Launch ``concurrency`` concurrent jobs and assert cleanup invariants."""
|
||||
click.echo(f"\n[bench run] Round {round_no}: launching {concurrency} job(s)...")
|
||||
started = _now()
|
||||
ts = _timestamp()
|
||||
slots = {
|
||||
slot: f"burnin-r{round_no}-j{slot}-{ts}"
|
||||
for slot in range(1, concurrency + 1)
|
||||
}
|
||||
|
||||
jobs: list[JobResult] = []
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = {
|
||||
pool.submit(
|
||||
_run_one_job,
|
||||
job_id=job_id,
|
||||
guest_os=guest_os,
|
||||
build_command=build_command,
|
||||
clone_base_dir=clone_base_dir,
|
||||
): (slot, job_id)
|
||||
for slot, job_id in slots.items()
|
||||
}
|
||||
for fut in as_completed(futures):
|
||||
slot, job_id = futures[fut]
|
||||
try:
|
||||
exit_code, error = fut.result()
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
exit_code, error = 1, str(exc)
|
||||
status = "PASS" if exit_code == 0 else "FAIL"
|
||||
jobs.append(
|
||||
JobResult(
|
||||
job_id=job_id,
|
||||
slot=slot,
|
||||
exit_code=exit_code,
|
||||
status=status,
|
||||
elapsed_sec=round(_now() - started, 2),
|
||||
error=error,
|
||||
)
|
||||
)
|
||||
|
||||
jobs.sort(key=lambda j: j.slot)
|
||||
passed = sum(1 for j in jobs if j.status == "PASS")
|
||||
failed = len(jobs) - passed
|
||||
|
||||
# ── Assertion: no orphaned clone directories ─────────────────────────
|
||||
orphans = _find_orphans(clone_base_dir, list(slots.values()))
|
||||
if orphans:
|
||||
failed += 1
|
||||
|
||||
# ── Assertion: no stale vm-start.lock ────────────────────────────────
|
||||
stale = _stale_lock(lock_file)
|
||||
if stale:
|
||||
failed += 1
|
||||
|
||||
status = "PASS" if failed == 0 else "FAIL"
|
||||
result = RoundResult(
|
||||
round=round_no,
|
||||
passed=passed,
|
||||
failed=failed,
|
||||
status=status,
|
||||
elapsed_sec=round(_now() - started, 2),
|
||||
orphans=[str(p) for p in orphans],
|
||||
stale_lock=stale,
|
||||
jobs=jobs,
|
||||
)
|
||||
|
||||
for j in jobs:
|
||||
line = f" slot {j.slot} {j.job_id:<32} {j.status:<5} exit={j.exit_code}"
|
||||
if j.error:
|
||||
line += f" ({j.error})"
|
||||
click.echo(line)
|
||||
click.echo(
|
||||
f" clone cleanup : {'OK' if not orphans else f'FAIL ({len(orphans)} orphan)'}"
|
||||
)
|
||||
for p in result.orphans:
|
||||
click.echo(f" orphan: {p}")
|
||||
click.echo(f" lock cleanup : {'OK' if not stale else 'FAIL (stale lock)'}")
|
||||
click.echo(
|
||||
f" [Round {round_no}] {status} "
|
||||
f"({passed} PASS, {failed} FAIL, {result.elapsed_sec}s)"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _write_run_report(report: dict[str, object], json_out: Path) -> None:
|
||||
json_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_out.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
@bench.command("run")
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="linux",
|
||||
show_default=True,
|
||||
help="Guest OS family of the build VMs under test.",
|
||||
)
|
||||
@click.option(
|
||||
"--concurrency",
|
||||
type=click.IntRange(1, 32),
|
||||
default=4,
|
||||
show_default=True,
|
||||
help="Number of concurrent jobs per round.",
|
||||
)
|
||||
@click.option(
|
||||
"--rounds",
|
||||
type=click.IntRange(1, 1000),
|
||||
default=10,
|
||||
show_default=True,
|
||||
help="Number of sequential rounds to run.",
|
||||
)
|
||||
@click.option(
|
||||
"--build-command",
|
||||
"build_command",
|
||||
default="",
|
||||
help="Build command each job runs (default: a no-op sleep/ping).",
|
||||
)
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"clone_base_dir",
|
||||
default=None,
|
||||
help="Override clone base dir (defaults to config.paths.build_vms).",
|
||||
)
|
||||
@click.option(
|
||||
"--json-out",
|
||||
"json_out",
|
||||
default=None,
|
||||
help="Path for the JSON report (default: CI_ARTIFACTS/bench/<ts>.json).",
|
||||
)
|
||||
def bench_run(
|
||||
guest_os: str,
|
||||
concurrency: int,
|
||||
rounds: int,
|
||||
build_command: str,
|
||||
clone_base_dir: str | None,
|
||||
json_out: str | None,
|
||||
) -> None:
|
||||
"""Run a concurrency burn-in and assert cleanup invariants per round.
|
||||
|
||||
Replaces ``Test-CapacityBurnIn.ps1`` + ``Start-BurnInTest*.ps1``. For
|
||||
each of ``--rounds`` rounds it launches ``--concurrency`` concurrent
|
||||
jobs (in-process, via :func:`concurrent.futures`), then asserts: every
|
||||
job exited 0, no orphaned clone dir remains in the clone base dir, and
|
||||
no ``vm-start.lock`` older than 5 minutes is left behind.
|
||||
"""
|
||||
guest_os = guest_os.lower()
|
||||
config = load_config()
|
||||
base = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||
lock_file = base / "vm-start.lock"
|
||||
|
||||
started_at = datetime.now(tz=UTC)
|
||||
out_path = (
|
||||
Path(json_out)
|
||||
if json_out
|
||||
else config.paths.artifacts / "bench" / f"{_timestamp()}.json"
|
||||
)
|
||||
|
||||
click.echo("=" * 60)
|
||||
click.echo("[bench run] capacity burn-in")
|
||||
click.echo(f" guest-os : {guest_os}")
|
||||
click.echo(f" concurrency : {concurrency}")
|
||||
click.echo(f" rounds : {rounds}")
|
||||
click.echo(f" clone base : {base}")
|
||||
click.echo(f" build cmd : {build_command or '(default no-op)'}")
|
||||
click.echo("=" * 60)
|
||||
|
||||
round_results: list[RoundResult] = []
|
||||
for r in range(1, rounds + 1):
|
||||
round_results.append(
|
||||
_run_round(
|
||||
round_no=r,
|
||||
concurrency=concurrency,
|
||||
guest_os=guest_os,
|
||||
build_command=build_command,
|
||||
clone_base_dir=base,
|
||||
lock_file=lock_file,
|
||||
)
|
||||
)
|
||||
|
||||
total_pass = sum(rr.passed for rr in round_results)
|
||||
total_fail = sum(rr.failed for rr in round_results)
|
||||
rounds_failed = sum(1 for rr in round_results if rr.status == "FAIL")
|
||||
overall = "PASS" if total_fail == 0 else "FAIL"
|
||||
|
||||
# ── Summary table ────────────────────────────────────────────────────
|
||||
click.echo("\n" + "=" * 60)
|
||||
click.echo("[bench run] SUMMARY")
|
||||
click.echo(f" {'Round':<7}{'Pass':<6}{'Fail':<6}{'Elapsed':<10}Status")
|
||||
click.echo(f" {'-' * 5:<7}{'-' * 4:<6}{'-' * 4:<6}{'-' * 8:<10}{'-' * 6}")
|
||||
for rr in round_results:
|
||||
click.echo(
|
||||
f" {rr.round:<7}{rr.passed:<6}{rr.failed:<6}"
|
||||
f"{str(rr.elapsed_sec) + 's':<10}{rr.status}"
|
||||
)
|
||||
click.echo("")
|
||||
click.echo(f" Total jobs : {rounds * concurrency}")
|
||||
click.echo(f" Passed : {total_pass}")
|
||||
click.echo(f" Failed : {total_fail}")
|
||||
click.echo(f" Rounds OK : {rounds - rounds_failed} / {rounds}")
|
||||
click.echo(f"\n OVERALL: {overall}")
|
||||
click.echo("=" * 60)
|
||||
|
||||
report: dict[str, object] = {
|
||||
"ts": started_at.isoformat(),
|
||||
"guestOS": guest_os,
|
||||
"concurrency": concurrency,
|
||||
"rounds": rounds,
|
||||
"buildCommand": build_command,
|
||||
"totalJobs": rounds * concurrency,
|
||||
"totalPass": total_pass,
|
||||
"totalFail": total_fail,
|
||||
"roundsFailed": rounds_failed,
|
||||
"overall": overall,
|
||||
"results": [asdict(rr) for rr in round_results],
|
||||
}
|
||||
try:
|
||||
_write_run_report(report, out_path)
|
||||
click.echo(f"[bench run] report written: {out_path}")
|
||||
except OSError as exc:
|
||||
click.echo(f"[bench run] could not write report: {exc}", err=True)
|
||||
|
||||
if total_fail > 0:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════ bench measure
|
||||
|
||||
|
||||
@dataclass
|
||||
class PhaseTiming:
|
||||
"""Per-iteration phase timings, mirroring ``Measure-CIBenchmark.ps1``.
|
||||
|
||||
Field names of the serialised record (see :meth:`to_record`) are kept
|
||||
identical to the PowerShell version for trend-log continuity.
|
||||
"""
|
||||
|
||||
iteration: int
|
||||
run_id: str
|
||||
template: str
|
||||
snapshot: str
|
||||
guest_os: str
|
||||
clone_sec: float | None = None
|
||||
start_sec: float | None = None
|
||||
ip_sec: float | None = None
|
||||
ready_sec: float | None = None
|
||||
destroy_sec: float | None = None
|
||||
delta_kb: int | None = None
|
||||
vm_ip: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def total_boot_sec(self) -> float | None:
|
||||
if self.ip_sec is None or self.ready_sec is None:
|
||||
return None
|
||||
return round(
|
||||
(self.clone_sec or 0.0)
|
||||
+ (self.start_sec or 0.0)
|
||||
+ self.ip_sec
|
||||
+ self.ready_sec,
|
||||
2,
|
||||
)
|
||||
|
||||
def to_record(self, ts: str) -> dict[str, object]:
|
||||
"""Build a JSON record with the legacy ``Measure-CIBenchmark`` keys."""
|
||||
return {
|
||||
"ts": ts,
|
||||
"runId": self.run_id,
|
||||
"iteration": self.iteration,
|
||||
"template": self.template,
|
||||
"snapshot": self.snapshot,
|
||||
"guestOS": self.guest_os,
|
||||
"cloneSec": self.clone_sec,
|
||||
"startSec": self.start_sec,
|
||||
"ipSec": self.ip_sec,
|
||||
"readySec": self.ready_sec,
|
||||
"destroySec": self.destroy_sec,
|
||||
"totalBootSec": self.total_boot_sec,
|
||||
"deltaKB": self.delta_kb,
|
||||
"vmIP": self.vm_ip,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
def _dir_size_kb(path: Path) -> int | None:
|
||||
"""Total size of files under ``path`` in KiB, or None on error."""
|
||||
try:
|
||||
total = sum(p.stat().st_size for p in path.rglob("*") if p.is_file())
|
||||
except OSError:
|
||||
return None
|
||||
return round(total / 1024)
|
||||
|
||||
|
||||
def _tcp_open(host: str, port: int, timeout: float = 3.0) -> bool:
|
||||
"""Return True if a TCP connection to ``host:port`` succeeds."""
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _wait_port(host: str, port: int, deadline: float, poll: float = 3.0) -> bool:
|
||||
while _now() < deadline:
|
||||
if _tcp_open(host, port):
|
||||
return True
|
||||
time.sleep(poll)
|
||||
return False
|
||||
|
||||
|
||||
def _measure_iteration(
|
||||
*,
|
||||
backend: VmBackend,
|
||||
iteration: int,
|
||||
template: Path,
|
||||
snapshot: str,
|
||||
guest_os: str,
|
||||
clone_base_dir: Path,
|
||||
transport_port: int,
|
||||
timeout_seconds: float,
|
||||
) -> PhaseTiming:
|
||||
"""Clone → start → IP → transport-ready → destroy, timing each phase."""
|
||||
run_id = f"bench-{_timestamp()}-{iteration}"
|
||||
clone_name = f"{_CLONE_PREFIX}{run_id}"
|
||||
clone_dir = clone_base_dir / clone_name
|
||||
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||
timing = PhaseTiming(
|
||||
iteration=iteration,
|
||||
run_id=run_id,
|
||||
template=template.name,
|
||||
snapshot=snapshot,
|
||||
guest_os=guest_os,
|
||||
)
|
||||
handle: VmHandle | None = None
|
||||
try:
|
||||
# ── clone ────────────────────────────────────────────────────────
|
||||
click.echo(f"[bench measure] iteration {iteration}: cloning...")
|
||||
t0 = _now()
|
||||
handle = backend.clone_linked(
|
||||
template=str(template),
|
||||
snapshot=snapshot,
|
||||
name=clone_name,
|
||||
destination=str(clone_vmx),
|
||||
)
|
||||
timing.clone_sec = round(_now() - t0, 2)
|
||||
timing.delta_kb = _dir_size_kb(clone_dir)
|
||||
|
||||
# ── start ────────────────────────────────────────────────────────
|
||||
click.echo("[bench measure] starting...")
|
||||
t0 = _now()
|
||||
backend.start(handle, headless=True)
|
||||
timing.start_sec = round(_now() - t0, 2)
|
||||
|
||||
# ── IP acquire ────────────────────────────────────────────────────
|
||||
click.echo("[bench measure] waiting for guest IP...")
|
||||
t0 = _now()
|
||||
deadline = t0 + timeout_seconds
|
||||
vm_ip: str | None = None
|
||||
while _now() < deadline:
|
||||
vm_ip = backend.get_ip(handle, timeout=min(5.0, deadline - _now()))
|
||||
if vm_ip:
|
||||
break
|
||||
if not vm_ip:
|
||||
raise TimeoutError(f"no guest IP after {timeout_seconds}s")
|
||||
timing.ip_sec = round(_now() - t0, 2)
|
||||
timing.vm_ip = vm_ip
|
||||
|
||||
# ── transport ready ───────────────────────────────────────────────
|
||||
click.echo(f"[bench measure] waiting for port {transport_port}...")
|
||||
t0 = _now()
|
||||
if not _wait_port(vm_ip, transport_port, _now() + timeout_seconds):
|
||||
raise TimeoutError(
|
||||
f"port {transport_port} on {vm_ip} not open after {timeout_seconds}s"
|
||||
)
|
||||
timing.ready_sec = round(_now() - t0, 2)
|
||||
except (BackendError, TimeoutError, OSError) as exc:
|
||||
timing.error = str(exc)
|
||||
click.echo(f"[bench measure] iteration {iteration} failed: {exc}", err=True)
|
||||
finally:
|
||||
# ── destroy ───────────────────────────────────────────────────────
|
||||
t0 = _now()
|
||||
if handle is not None:
|
||||
with contextlib.suppress(BackendError):
|
||||
backend.stop(handle, hard=True)
|
||||
with contextlib.suppress(BackendError):
|
||||
backend.delete(handle)
|
||||
if clone_dir.exists():
|
||||
shutil.rmtree(clone_dir, ignore_errors=True)
|
||||
timing.destroy_sec = round(_now() - t0, 2)
|
||||
return timing
|
||||
|
||||
|
||||
def _resolve_template(config: Config, guest_os: str) -> tuple[Path, str]:
|
||||
"""Pick a default template VMX + snapshot for ``guest_os``."""
|
||||
if guest_os == "linux":
|
||||
return (
|
||||
config.paths.templates / "LinuxBuild2404" / "LinuxBuild2404.vmx",
|
||||
"BaseClean-Linux",
|
||||
)
|
||||
return (
|
||||
config.paths.templates / "WinBuild2025" / "WinBuild2025.vmx",
|
||||
"BaseClean",
|
||||
)
|
||||
|
||||
|
||||
@bench.command("measure")
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="linux",
|
||||
show_default=True,
|
||||
help="Guest OS family of the VM to benchmark.",
|
||||
)
|
||||
@click.option(
|
||||
"--iterations",
|
||||
type=click.IntRange(1, 100),
|
||||
default=1,
|
||||
show_default=True,
|
||||
help="Number of ephemeral VMs to time.",
|
||||
)
|
||||
@click.option(
|
||||
"--template-path",
|
||||
"template_path",
|
||||
default=None,
|
||||
help="Template VMX path (defaults to the per-guest-os template).",
|
||||
)
|
||||
@click.option(
|
||||
"--snapshot-name",
|
||||
"snapshot_name",
|
||||
default=None,
|
||||
help="Snapshot to clone from (defaults to BaseClean / BaseClean-Linux).",
|
||||
)
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"clone_base_dir",
|
||||
default=None,
|
||||
help="Override clone base dir (defaults to config.paths.build_vms).",
|
||||
)
|
||||
@click.option(
|
||||
"--timeout-seconds",
|
||||
"timeout_seconds",
|
||||
type=click.FloatRange(30.0, 1200.0),
|
||||
default=300.0,
|
||||
show_default=True,
|
||||
help="Per-iteration timeout for IP and transport readiness.",
|
||||
)
|
||||
@click.option(
|
||||
"--json-out",
|
||||
"json_out",
|
||||
default=None,
|
||||
help="Path to append JSONL records (default: CI_ARTIFACTS/benchmark.jsonl).",
|
||||
)
|
||||
def bench_measure(
|
||||
guest_os: str,
|
||||
iterations: int,
|
||||
template_path: str | None,
|
||||
snapshot_name: str | None,
|
||||
clone_base_dir: str | None,
|
||||
timeout_seconds: float,
|
||||
json_out: str | None,
|
||||
) -> None:
|
||||
"""Measure clone/start/IP/transport/destroy phase timings.
|
||||
|
||||
Replaces ``Measure-CIBenchmark.ps1``. Records are appended to
|
||||
``benchmark.jsonl`` with the same field names so historical trend data
|
||||
stays comparable.
|
||||
"""
|
||||
guest_os = guest_os.lower()
|
||||
config = load_config()
|
||||
|
||||
default_tpl, default_snap = _resolve_template(config, guest_os)
|
||||
template = Path(template_path) if template_path else default_tpl
|
||||
snapshot = snapshot_name or default_snap
|
||||
base = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||
transport_port = _SSH_PORT if guest_os == "linux" else _WINRM_PORT
|
||||
out_path = Path(json_out) if json_out else config.paths.artifacts / "benchmark.jsonl"
|
||||
|
||||
if not template.is_file():
|
||||
raise click.ClickException(f"Template VMX not found: {template}")
|
||||
|
||||
try:
|
||||
backend = load_backend(config)
|
||||
except BackendNotAvailable as exc:
|
||||
raise click.ClickException(f"backend unavailable: {exc}") from exc
|
||||
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
click.echo("=" * 60)
|
||||
click.echo("[bench measure] phase timing")
|
||||
click.echo(f" template : {template.name}")
|
||||
click.echo(f" snapshot : {snapshot}")
|
||||
click.echo(f" guest-os : {guest_os} (port {transport_port})")
|
||||
click.echo(f" iterations : {iterations}")
|
||||
click.echo("=" * 60)
|
||||
|
||||
timings: list[PhaseTiming] = []
|
||||
for i in range(1, iterations + 1):
|
||||
timing = _measure_iteration(
|
||||
backend=backend,
|
||||
iteration=i,
|
||||
template=template,
|
||||
snapshot=snapshot,
|
||||
guest_os=guest_os,
|
||||
clone_base_dir=base,
|
||||
transport_port=transport_port,
|
||||
timeout_seconds=timeout_seconds,
|
||||
)
|
||||
timings.append(timing)
|
||||
record = timing.to_record(datetime.now(tz=UTC).isoformat())
|
||||
try:
|
||||
with out_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record) + "\n")
|
||||
except OSError as exc:
|
||||
click.echo(f"[bench measure] could not append jsonl: {exc}", err=True)
|
||||
|
||||
# ── Summary table ────────────────────────────────────────────────────
|
||||
click.echo("\n[bench measure] Results")
|
||||
header = f" {'Iter':<5}{'Clone':<8}{'Start':<8}{'IP':<8}{'Ready':<8}{'Destroy':<9}{'Boot':<8}IP"
|
||||
click.echo(header)
|
||||
for t in timings:
|
||||
click.echo(
|
||||
f" {t.iteration:<5}"
|
||||
f"{t.clone_sec!s:<8}{t.start_sec!s:<8}{t.ip_sec!s:<8}"
|
||||
f"{t.ready_sec!s:<8}{t.destroy_sec!s:<9}"
|
||||
f"{t.total_boot_sec!s:<8}{t.vm_ip or '-'}"
|
||||
)
|
||||
if t.error:
|
||||
click.echo(f" error: {t.error}")
|
||||
|
||||
successful = [t for t in timings if t.error is None and t.total_boot_sec is not None]
|
||||
if len(successful) > 1:
|
||||
boots = [t.total_boot_sec for t in successful if t.total_boot_sec is not None]
|
||||
avg = round(sum(boots) / len(boots), 2)
|
||||
click.echo(f"\n Average boot-to-ready ({len(successful)} ok): {avg}s")
|
||||
click.echo(f"\n[bench measure] records appended to: {out_path}")
|
||||
|
||||
|
||||
__all__ = ["bench"]
|
||||
@@ -142,6 +142,7 @@ def _linux_build(
|
||||
artifact_source: str,
|
||||
extra_env: dict[str, str],
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool = False,
|
||||
) -> None:
|
||||
click.echo("[build run] Linux build mode (SSH transport)")
|
||||
|
||||
@@ -151,6 +152,24 @@ def _linux_build(
|
||||
key_path=key_path,
|
||||
known_hosts=known_hosts,
|
||||
) as t:
|
||||
# Clock-skew guard. Ephemeral clones can boot with a wrong clock
|
||||
# (e.g. RTC kept in local time → guest reads it as UTC and runs hours
|
||||
# ahead). Enable NTP and BLOCK until the clock is actually synced, so
|
||||
# the (possibly backward) correction step happens HERE — before the
|
||||
# sources are stamped below. If the step landed *after* the touch,
|
||||
# make would again see source mtimes in the future and warn about
|
||||
# clock skew. Do NOT run `hwclock -s`: on a clone whose RTC is in
|
||||
# local time it pushes the clock further ahead, the very skew we are
|
||||
# removing. Best-effort (check=False); the touch after checkout is the
|
||||
# backstop when NTP is unreachable and the clock simply stays put.
|
||||
t.run(
|
||||
"sudo timedatectl set-ntp true 2>/dev/null || true; "
|
||||
'for _ in $(seq 1 60); do '
|
||||
'[ "$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" '
|
||||
"= yes ] && break; sleep 1; done",
|
||||
check=False,
|
||||
)
|
||||
|
||||
t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
|
||||
|
||||
if host_source_dir:
|
||||
@@ -190,17 +209,33 @@ def _linux_build(
|
||||
"Linux build requires either --host-source-dir or --clone-url."
|
||||
)
|
||||
|
||||
# Normalise source mtimes to the (now-synced) guest clock so make
|
||||
# never sees a file dated in the future and skips the skew warning
|
||||
# plus the spurious rebuild it triggers.
|
||||
t.run(
|
||||
f"find {_sh_quote(workdir)} -exec touch -d {_sh_quote('now')} {{}} +",
|
||||
check=False,
|
||||
)
|
||||
|
||||
env_prefix = "".join(
|
||||
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||
)
|
||||
cmd = build_command or "make"
|
||||
# Optionally run the build under a headless X server (xvfb-run) so GUI
|
||||
# toolkits (e.g. GTK4/PyGObject) have a $DISPLAY. The exported env vars
|
||||
# precede xvfb-run so they propagate into the virtual-display session;
|
||||
# -a auto-selects a free server number. The whole `cd && build` is
|
||||
# wrapped via `sh -c` so xvfb-run drives a single command.
|
||||
inner = f"cd {_sh_quote(workdir)} && {cmd}"
|
||||
run_part = f"xvfb-run -a sh -c {_sh_quote(inner)}" if use_xvfb else inner
|
||||
# PYTHONUNBUFFERED so the guest python flushes promptly and the
|
||||
# streamed output is timely rather than emitted in one block.
|
||||
full_cmd = (
|
||||
f"{env_prefix}export PYTHONUNBUFFERED=1; "
|
||||
f"cd {_sh_quote(workdir)} && {cmd}"
|
||||
full_cmd = f"{env_prefix}export PYTHONUNBUFFERED=1; {run_part}"
|
||||
xvfb_note = " [xvfb-run]" if use_xvfb else ""
|
||||
click.echo(
|
||||
f"[build run] running build{xvfb_note} (env vars redacted): "
|
||||
f"cd {workdir} && {cmd}"
|
||||
)
|
||||
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
|
||||
click.echo("[build run] ----- build output (live) -----")
|
||||
result = t.run_streaming(full_cmd, check=False)
|
||||
click.echo("[build run] ----- end build output -----")
|
||||
@@ -482,6 +517,13 @@ def _windows_build(
|
||||
@click.option("--configuration", default="Release", show_default=True)
|
||||
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--xvfb",
|
||||
"use_xvfb",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Linux only: run the build under xvfb-run (headless X for GUI toolkits).",
|
||||
)
|
||||
@click.option(
|
||||
"--extra-env",
|
||||
"extra_env",
|
||||
@@ -510,6 +552,7 @@ def build_run(
|
||||
configuration: str,
|
||||
skip_artifact: bool,
|
||||
use_shared_cache: bool,
|
||||
use_xvfb: bool,
|
||||
extra_env: tuple[str, ...],
|
||||
) -> None:
|
||||
"""Run a build inside a guest VM, optionally packaging the result.
|
||||
@@ -563,10 +606,13 @@ def build_run(
|
||||
artifact_source=guest_artifact_source,
|
||||
extra_env=extra,
|
||||
skip_artifact=skip_artifact,
|
||||
use_xvfb=use_xvfb,
|
||||
)
|
||||
except (TransportError, TransportCommandError) as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
else:
|
||||
if use_xvfb:
|
||||
click.echo("[build run] --xvfb ignored for Windows guest.")
|
||||
workdir = guest_work_dir or "C:\\CI\\build"
|
||||
output_dir = guest_output_dir or "C:\\CI\\output"
|
||||
target = credential_target or config.guest_cred_target
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""``creds`` sub-commands.
|
||||
|
||||
Phase C replaces ``Set-CIGuestCredential.ps1`` on the Linux host with a
|
||||
keyring-native writer (``keyrings.alt.file.PlaintextKeyring``), matching the
|
||||
read path in :mod:`ci_orchestrator.credentials`.
|
||||
|
||||
* ``creds set`` — store a guest username/password under a target. Passwords
|
||||
are never passed on the command line.
|
||||
|
||||
Write scheme (must mirror :meth:`KeyringCredentialStore.get`):
|
||||
|
||||
* ``keyring.set_password("{target}:meta", "username", <user>)`` — so the
|
||||
reader can recover the username for file backends where
|
||||
``get_credential(target, None)`` is a no-op.
|
||||
* ``keyring.set_password(target, <user>, <password>)`` — the password,
|
||||
keyed by the username under the target service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import click
|
||||
|
||||
|
||||
def _store_credential(target: str, user: str, password: str) -> None:
|
||||
"""Write the two keyring entries the credential reader expects.
|
||||
|
||||
Raises :class:`RuntimeError` if ``keyring`` is not installed.
|
||||
"""
|
||||
try:
|
||||
import keyring
|
||||
except ImportError as exc: # pragma: no cover - dep declared in pyproject
|
||||
raise RuntimeError(
|
||||
"keyring is not installed; run `pip install keyring`."
|
||||
) from exc
|
||||
|
||||
# 1. Username under the "{target}:meta" service, key "username".
|
||||
keyring.set_password(f"{target}:meta", "username", user)
|
||||
# 2. Password under the target service, keyed by the username.
|
||||
keyring.set_password(target, user, password)
|
||||
|
||||
|
||||
@click.group("creds")
|
||||
def creds() -> None:
|
||||
"""Manage guest credentials in the keyring."""
|
||||
|
||||
|
||||
@creds.command("set")
|
||||
@click.option(
|
||||
"--target",
|
||||
default="BuildVMGuest",
|
||||
show_default=True,
|
||||
help="Credential target name (matches config.guest_cred_target).",
|
||||
)
|
||||
@click.option("--user", "user", required=True, help="Guest username to store.")
|
||||
@click.option(
|
||||
"--password-stdin",
|
||||
"password_stdin",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Read the password from stdin instead of prompting.",
|
||||
)
|
||||
def creds_set(target: str, user: str, password_stdin: bool) -> None:
|
||||
"""Store guest credentials so ci_orchestrator can read them back."""
|
||||
if password_stdin:
|
||||
# Read the whole of stdin and strip a single trailing newline only,
|
||||
# so passwords containing internal whitespace survive intact.
|
||||
password = sys.stdin.readline().rstrip("\n").rstrip("\r")
|
||||
else:
|
||||
password = click.prompt(
|
||||
"Guest password",
|
||||
hide_input=True,
|
||||
confirmation_prompt=True,
|
||||
)
|
||||
if not password:
|
||||
raise click.UsageError("password must not be empty.")
|
||||
|
||||
try:
|
||||
_store_credential(target, user, password)
|
||||
except RuntimeError as exc:
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
|
||||
click.echo(f"Stored credential for target {target!r} (user={user}).")
|
||||
|
||||
|
||||
__all__ = ["creds"]
|
||||
@@ -19,6 +19,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
@@ -42,6 +43,7 @@ from ci_orchestrator.commands.build import _linux_build, _windows_build
|
||||
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
|
||||
from ci_orchestrator.config import Config, load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.ip_pool import IpSlotPool
|
||||
from ci_orchestrator.transport.errors import TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
@@ -188,6 +190,28 @@ def _probe_transport(
|
||||
return False
|
||||
|
||||
|
||||
def _inject_guestinfo_ip(
|
||||
vmx_path: Path, ip: str, netmask: str, gateway: str
|
||||
) -> None:
|
||||
"""Write guestinfo.ip-assignment/netmask/gateway into the cloned VMX.
|
||||
|
||||
Removes any pre-existing lines for these keys before appending fresh
|
||||
values, so re-runs are idempotent. Called after clone and before start.
|
||||
"""
|
||||
_KEYS = {"guestinfo.ip-assignment", "guestinfo.netmask", "guestinfo.gateway"}
|
||||
lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
lines = [
|
||||
ln for ln in lines
|
||||
if ln.split("=")[0].strip().lower() not in _KEYS
|
||||
]
|
||||
lines.append(f'guestinfo.ip-assignment = "{ip}"')
|
||||
if netmask:
|
||||
lines.append(f'guestinfo.netmask = "{netmask}"')
|
||||
if gateway:
|
||||
lines.append(f'guestinfo.gateway = "{gateway}"')
|
||||
vmx_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _make_clone_name(job_id: str) -> str:
|
||||
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||
# uuid4 suffix avoids collisions when several clones share a JobId.
|
||||
@@ -242,6 +266,9 @@ def _destroy_clone(
|
||||
backend: VmBackend | None,
|
||||
handle: VmHandle | None,
|
||||
clone_dir: Path | None,
|
||||
*,
|
||||
pool: IpSlotPool | None = None,
|
||||
pool_ip: str | None = None,
|
||||
) -> None:
|
||||
"""Final cleanup. Tolerates every error — must never raise."""
|
||||
if backend is not None and handle is not None:
|
||||
@@ -265,6 +292,12 @@ def _destroy_clone(
|
||||
click.echo(
|
||||
f"[job] could not fully remove: {clone_dir}", err=True
|
||||
)
|
||||
if pool is not None and pool_ip is not None:
|
||||
try:
|
||||
pool.release(pool_ip)
|
||||
click.echo(f"[job] IP slot released: {pool_ip}")
|
||||
except Exception as exc:
|
||||
click.echo(f"[job] IP slot release failed (ignored): {exc}", err=True)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────── command
|
||||
@@ -294,6 +327,13 @@ def _destroy_clone(
|
||||
)
|
||||
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||
@click.option(
|
||||
"--xvfb",
|
||||
"use_xvfb",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Linux only: run the build under xvfb-run (headless X for GUI toolkits).",
|
||||
)
|
||||
@click.option(
|
||||
"--gitea-credential-target",
|
||||
"gitea_credential_target",
|
||||
@@ -362,6 +402,7 @@ def job(
|
||||
use_git_clone: bool,
|
||||
use_shared_cache: bool,
|
||||
skip_artifact: bool,
|
||||
use_xvfb: bool,
|
||||
gitea_credential_target: str,
|
||||
template_path: str | None,
|
||||
snapshot_name: str,
|
||||
@@ -435,10 +476,18 @@ def job(
|
||||
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||
|
||||
handle: VmHandle | None = None
|
||||
slot_pool: IpSlotPool | None = None
|
||||
pool_ip: str | None = None
|
||||
started = datetime.now(tz=UTC)
|
||||
phase_timings: list[tuple[str, float]] = []
|
||||
phase_start = _now()
|
||||
|
||||
# SIGTERM (sent by act_runner on workflow cancellation) must trigger cleanup.
|
||||
def _sigterm(_sig: int, _frame: object) -> None:
|
||||
raise KeyboardInterrupt()
|
||||
|
||||
signal.signal(signal.SIGTERM, _sigterm)
|
||||
|
||||
click.echo("=" * 60)
|
||||
click.echo(f"[job] Job : {job_id}")
|
||||
click.echo(f"[job] Repository : {repo_url}")
|
||||
@@ -474,6 +523,26 @@ def job(
|
||||
)
|
||||
_apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)
|
||||
|
||||
# Resolve guest OS before IP-pool guard — Linux doesn't read guestinfo IP.
|
||||
if guest_os.lower() == "auto":
|
||||
guest_os = _read_guest_os_from_vmx(clone_vmx)
|
||||
click.echo(f"[job] auto-detected guest OS: {guest_os}")
|
||||
else:
|
||||
guest_os = guest_os.lower()
|
||||
|
||||
if config.ip_pool is not None and guest_os != "linux":
|
||||
slot_pool = IpSlotPool(
|
||||
addresses=config.ip_pool.addresses,
|
||||
netmask=config.ip_pool.netmask,
|
||||
gateway=config.ip_pool.gateway,
|
||||
lease_file=config.ip_pool.lease_file,
|
||||
)
|
||||
pool_ip = slot_pool.acquire(clone_name)
|
||||
_inject_guestinfo_ip(
|
||||
clone_vmx, pool_ip, config.ip_pool.netmask, config.ip_pool.gateway
|
||||
)
|
||||
click.echo(f"[job] pre-assigned guest IP: {pool_ip}")
|
||||
|
||||
phase_timings.append(("clone", _now() - phase_start))
|
||||
phase_start = _now()
|
||||
|
||||
@@ -484,20 +553,18 @@ def job(
|
||||
except BackendError as exc:
|
||||
raise click.ClickException(f"vmrun start failed: {exc}") from exc
|
||||
|
||||
# Resolve guest OS now that the clone exists.
|
||||
if guest_os.lower() == "auto":
|
||||
guest_os = _read_guest_os_from_vmx(clone_vmx)
|
||||
click.echo(f"[job] auto-detected guest OS: {guest_os}")
|
||||
else:
|
||||
guest_os = guest_os.lower()
|
||||
|
||||
deadline = _now() + ready_timeout
|
||||
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||
raise click.ClickException("timeout waiting for VM to be running.")
|
||||
ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||
if not ip_address:
|
||||
raise click.ClickException("timeout waiting for guest IP.")
|
||||
click.echo(f"[job] guest IP: {ip_address}")
|
||||
if pool_ip is not None:
|
||||
ip_address: str = pool_ip
|
||||
click.echo(f"[job] guest IP: {ip_address} (pre-assigned, skipping poll)")
|
||||
else:
|
||||
_ip = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||
if not _ip:
|
||||
raise click.ClickException("timeout waiting for guest IP.")
|
||||
ip_address = _ip
|
||||
click.echo(f"[job] guest IP: {ip_address}")
|
||||
phase_timings.append(("start", _now() - phase_start))
|
||||
phase_start = _now()
|
||||
|
||||
@@ -556,6 +623,7 @@ def job(
|
||||
artifact_source=guest_artifact_source,
|
||||
extra_env=extra_env,
|
||||
skip_artifact=skip_artifact,
|
||||
use_xvfb=use_xvfb,
|
||||
)
|
||||
else:
|
||||
_windows_build(
|
||||
@@ -622,17 +690,17 @@ def job(
|
||||
click.echo("=" * 60)
|
||||
except KeyboardInterrupt:
|
||||
click.echo("\n[job] interrupted by user; cleaning up...", err=True)
|
||||
_destroy_clone(backend, handle, clone_dir)
|
||||
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
|
||||
sys.exit(130)
|
||||
except click.ClickException:
|
||||
_destroy_clone(backend, handle, clone_dir)
|
||||
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
|
||||
raise
|
||||
except Exception as exc:
|
||||
click.echo(f"\n[job] FAILURE: {exc}", err=True)
|
||||
_destroy_clone(backend, handle, clone_dir)
|
||||
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
|
||||
raise click.ClickException(str(exc)) from exc
|
||||
else:
|
||||
_destroy_clone(backend, handle, clone_dir)
|
||||
_destroy_clone(backend, handle, clone_dir, pool=slot_pool, pool_ip=pool_ip)
|
||||
|
||||
|
||||
__all__ = ["job"]
|
||||
|
||||
@@ -42,14 +42,14 @@ def _read_events(jsonl_path: Path) -> list[_Event]:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
data = obj.get("data") if isinstance(obj.get("data"), dict) else {}
|
||||
_raw = obj.get("data")
|
||||
data: dict[str, object] = _raw if isinstance(_raw, dict) else {}
|
||||
elapsed: int | None = None
|
||||
if isinstance(data, dict):
|
||||
raw_elapsed = data.get("elapsedSec")
|
||||
if isinstance(raw_elapsed, int | float):
|
||||
elapsed = int(raw_elapsed)
|
||||
raw_elapsed = data.get("elapsedSec")
|
||||
if isinstance(raw_elapsed, int | float):
|
||||
elapsed = int(raw_elapsed)
|
||||
err: str | None = None
|
||||
if isinstance(data, dict) and isinstance(data.get("error"), str):
|
||||
if isinstance(data.get("error"), str):
|
||||
err = str(data["error"])
|
||||
out.append(
|
||||
_Event(
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""``retention`` sub-commands.
|
||||
|
||||
Ports ``scripts/Invoke-RetentionPolicy.ps1`` to Python.
|
||||
|
||||
* ``retention run`` — purge old artifact and log directories based on age;
|
||||
switches to aggressive retention when disk free space is low.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
|
||||
@click.group()
|
||||
def retention() -> None:
|
||||
"""Maintenance: artifact and log retention."""
|
||||
|
||||
|
||||
@retention.command("run")
|
||||
@click.option(
|
||||
"--artifact-dir",
|
||||
default=None,
|
||||
help="Artifact directory (default: CI_ARTIFACTS from config/env).",
|
||||
)
|
||||
@click.option(
|
||||
"--log-dir",
|
||||
default=None,
|
||||
help="Log directory (default: <CI_ROOT>/logs).",
|
||||
)
|
||||
@click.option(
|
||||
"--retention-days",
|
||||
default=30,
|
||||
show_default=True,
|
||||
help="Normal retention threshold in days.",
|
||||
)
|
||||
@click.option(
|
||||
"--aggressive-retention-days",
|
||||
default=7,
|
||||
show_default=True,
|
||||
help="Retention threshold when disk free space is below --min-free-gb.",
|
||||
)
|
||||
@click.option(
|
||||
"--min-free-gb",
|
||||
default=50,
|
||||
show_default=True,
|
||||
help="Switch to aggressive retention when free space drops below this value (GB).",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
is_flag=True,
|
||||
help="Dry run — log what would be deleted without actually deleting.",
|
||||
)
|
||||
def retention_run(
|
||||
artifact_dir: str | None,
|
||||
log_dir: str | None,
|
||||
retention_days: int,
|
||||
aggressive_retention_days: int,
|
||||
min_free_gb: int,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Purge old artifact and log directories based on retention policy."""
|
||||
config = load_config()
|
||||
|
||||
art_path = Path(artifact_dir) if artifact_dir else config.paths.artifacts
|
||||
log_path = Path(log_dir) if log_dir else config.paths.root / "logs"
|
||||
|
||||
check_path = art_path if art_path.exists() else config.paths.root
|
||||
usage = shutil.disk_usage(str(check_path))
|
||||
free_gb = usage.free / (1024**3)
|
||||
|
||||
aggressive = free_gb < min_free_gb
|
||||
effective_days = aggressive_retention_days if aggressive else retention_days
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(days=effective_days)
|
||||
|
||||
click.echo(f"[RetentionPolicy] Free space: {free_gb:.1f} GB")
|
||||
if aggressive:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] WARNING: Free space below {min_free_gb} GB — "
|
||||
f"aggressive retention: {aggressive_retention_days}d",
|
||||
err=True,
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] Normal retention: {effective_days}d "
|
||||
f"(cutoff: {cutoff.strftime('%Y-%m-%d')})"
|
||||
)
|
||||
|
||||
_purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if)
|
||||
_purge_old_dirs(log_path, "log", cutoff, what_if=what_if)
|
||||
|
||||
usage_after = shutil.disk_usage(str(check_path))
|
||||
free_gb_after = usage_after.free / (1024**3)
|
||||
delta = free_gb_after - free_gb
|
||||
click.echo(
|
||||
f"[RetentionPolicy] Done. Free space: {free_gb_after:.1f} GB (delta: +{delta:.1f} GB)"
|
||||
)
|
||||
|
||||
|
||||
def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) -> None:
|
||||
if not base.exists():
|
||||
click.echo(f"[RetentionPolicy] {label} dir not found: {base} — skipping.")
|
||||
return
|
||||
|
||||
old = [
|
||||
d
|
||||
for d in base.iterdir()
|
||||
if d.is_dir()
|
||||
and datetime.fromtimestamp(d.stat().st_mtime, tz=UTC) < cutoff
|
||||
]
|
||||
|
||||
if not old:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] {label}: nothing to purge "
|
||||
f"(cutoff: {cutoff.strftime('%Y-%m-%d')})."
|
||||
)
|
||||
return
|
||||
|
||||
for d in old:
|
||||
age_days = int(
|
||||
(datetime.now(tz=UTC) - datetime.fromtimestamp(d.stat().st_mtime, tz=UTC)).days
|
||||
)
|
||||
if what_if:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] WhatIf: would purge {label} ({age_days}d): {d.name}"
|
||||
)
|
||||
else:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}")
|
||||
@@ -0,0 +1,351 @@
|
||||
"""``smoke`` sub-commands.
|
||||
|
||||
Phase C4 ports the manual smoke / E2E build tooling to Python so the Linux
|
||||
host no longer needs ``pwsh``:
|
||||
|
||||
* ``smoke run`` — end-to-end no-op (or real build) job, replacing
|
||||
``Test-Smoke.ps1`` and ``Test-*Build-Linux.ps1``.
|
||||
|
||||
The command runs **one** full end-to-end job through the in-process
|
||||
``job`` orchestrator (reusing ``commands/job.py`` — no re-spawn of
|
||||
``python -m ci_orchestrator``) and then asserts the three smoke
|
||||
invariants of ``Test-Smoke.ps1``:
|
||||
|
||||
1. the pipeline exited 0,
|
||||
2. the per-job artifact directory was created, and
|
||||
3. a ``job``/``success`` event is present in the ``invoke-ci.jsonl``
|
||||
event log (the same JSONL format ``report job`` consumes).
|
||||
|
||||
Because the in-process ``job`` command does not itself emit the JSONL
|
||||
event log (that was the job of the retired ``Invoke-CIJob.ps1``), this
|
||||
command writes the ``job``/``start`` and ``job``/``success`` events
|
||||
around the orchestration, exactly as the PowerShell wrapper did.
|
||||
|
||||
Design rule: import ``backends.load_backend`` and the existing transports
|
||||
only — never ``WorkstationVmrunBackend`` directly (keeps the Phase D ESXi
|
||||
path open).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.commands.job import job as job_command
|
||||
from ci_orchestrator.config import Config, load_config
|
||||
|
||||
# ─────────────────────────────────────────────────────── OS presets
|
||||
|
||||
# Per-guest defaults mirroring ``Test-Smoke.ps1``: a trivial build that
|
||||
# writes a marker file into the standard output directory, plus the
|
||||
# matching template / snapshot / artifact-source for the OS family.
|
||||
_LINUX_TEMPLATE = "/var/lib/ci/templates/LinuxBuild2404/LinuxBuild2404.vmx"
|
||||
_WINDOWS_TEMPLATE = r"F:\CI\Templates\WinBuild2025\WinBuild2025.vmx"
|
||||
|
||||
_NOOP_BUILD_LINUX = "mkdir -p /opt/ci/output && echo smoke-ok > /opt/ci/output/smoke.txt"
|
||||
_NOOP_BUILD_WINDOWS = (
|
||||
"New-Item -ItemType Directory -Path C:\\CI\\output -Force | Out-Null; "
|
||||
'Set-Content C:\\CI\\output\\smoke.txt "smoke-ok"'
|
||||
)
|
||||
|
||||
# Real Linux build E2E presets (ports of ``Test-*Build-Linux.ps1``).
|
||||
# Exposed via ``--preset`` so the no-op default is never overridden by a
|
||||
# hard-coded ``F:\`` Windows path. Each preset supplies a build command
|
||||
# and the in-guest artifact source directory.
|
||||
_PRESETS: dict[str, tuple[str, str]] = {
|
||||
"ns7zip": (
|
||||
"python3 build_plugin.py --host linux --7zip-version 26.01 "
|
||||
"--configs x64-unicode --verbose",
|
||||
"plugins",
|
||||
),
|
||||
"nsinnounp": (
|
||||
"python3 build_plugin.py --final --dist-dir dist",
|
||||
"dist",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _default_template(guest_os: str) -> str:
|
||||
return _LINUX_TEMPLATE if guest_os == "linux" else _WINDOWS_TEMPLATE
|
||||
|
||||
|
||||
def _default_snapshot(guest_os: str) -> str:
|
||||
return "BaseClean-Linux" if guest_os == "linux" else "BaseClean"
|
||||
|
||||
|
||||
def _noop_build_command(guest_os: str) -> tuple[str, str]:
|
||||
"""Return ``(build_command, guest_artifact_source)`` for the marker job."""
|
||||
if guest_os == "linux":
|
||||
return _NOOP_BUILD_LINUX, "/opt/ci/output"
|
||||
return _NOOP_BUILD_WINDOWS, "C:\\CI\\output"
|
||||
|
||||
|
||||
def _utcnow_iso() -> str:
|
||||
return datetime.now(tz=UTC).isoformat()
|
||||
|
||||
|
||||
def _write_event(
|
||||
jsonl_path: Path,
|
||||
*,
|
||||
job_id: str,
|
||||
phase: str,
|
||||
status: str,
|
||||
elapsed_sec: float | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""Append one event to ``invoke-ci.jsonl`` in ``report job`` format."""
|
||||
data: dict[str, object] = {}
|
||||
if elapsed_sec is not None:
|
||||
data["elapsedSec"] = int(elapsed_sec)
|
||||
if error is not None:
|
||||
data["error"] = error
|
||||
record: dict[str, object] = {
|
||||
"jobId": job_id,
|
||||
"phase": phase,
|
||||
"status": status,
|
||||
"ts": _utcnow_iso(),
|
||||
"data": data,
|
||||
}
|
||||
jsonl_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with jsonl_path.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(record) + "\n")
|
||||
|
||||
|
||||
def _has_job_success(jsonl_path: Path) -> bool:
|
||||
"""True iff the JSONL log contains a ``job``/``success`` event."""
|
||||
try:
|
||||
text = jsonl_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return False
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
if obj.get("phase") == "job" and obj.get("status") == "success":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────── command
|
||||
|
||||
|
||||
@click.group("smoke")
|
||||
def smoke() -> None:
|
||||
"""End-to-end smoke and build validation."""
|
||||
|
||||
|
||||
@smoke.command("run")
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default="linux",
|
||||
show_default=True,
|
||||
help="Guest OS family to smoke-test.",
|
||||
)
|
||||
@click.option(
|
||||
"--build-command",
|
||||
"build_command",
|
||||
default="",
|
||||
help="Build command to run (default: a marker no-op job).",
|
||||
)
|
||||
@click.option(
|
||||
"--preset",
|
||||
type=click.Choice(sorted(_PRESETS), case_sensitive=False),
|
||||
default=None,
|
||||
help="Real Linux build E2E preset (ns7zip / nsinnounp). Linux only.",
|
||||
)
|
||||
@click.option(
|
||||
"--repo-url",
|
||||
"repo_url",
|
||||
default="https://example.invalid/smoke-test.git",
|
||||
show_default=True,
|
||||
help="Repository to build (default: a placeholder; the no-op marker "
|
||||
"job never depends on the clone).",
|
||||
)
|
||||
@click.option("--branch", default="main", show_default=True)
|
||||
@click.option(
|
||||
"--job-id",
|
||||
"job_id",
|
||||
default=None,
|
||||
help="Job id (default: smoke-<timestamp>).",
|
||||
)
|
||||
@click.option(
|
||||
"--template-path",
|
||||
"template_path",
|
||||
default=None,
|
||||
help="Template VMX path (default: per-guest-os standard template).",
|
||||
)
|
||||
@click.option(
|
||||
"--snapshot-name",
|
||||
"snapshot_name",
|
||||
default=None,
|
||||
help="Snapshot to clone from (default: per-guest-os BaseClean[-Linux]).",
|
||||
)
|
||||
@click.option(
|
||||
"--ssh-key-path",
|
||||
"ssh_key_path",
|
||||
default=None,
|
||||
help="SSH private key for the guest (Linux).",
|
||||
)
|
||||
@click.option(
|
||||
"--log-dir",
|
||||
"log_dir",
|
||||
default=None,
|
||||
help="Base directory for invoke-ci.jsonl logs (default: CI_ARTIFACTS).",
|
||||
)
|
||||
def smoke_run(
|
||||
guest_os: str,
|
||||
build_command: str,
|
||||
preset: str | None,
|
||||
repo_url: str,
|
||||
branch: str,
|
||||
job_id: str | None,
|
||||
template_path: str | None,
|
||||
snapshot_name: str | None,
|
||||
ssh_key_path: str | None,
|
||||
log_dir: str | None,
|
||||
) -> None:
|
||||
"""Run one end-to-end job and assert success + artifact + event."""
|
||||
guest_os = guest_os.lower()
|
||||
|
||||
if preset is not None:
|
||||
preset = preset.lower()
|
||||
if guest_os != "linux":
|
||||
raise click.UsageError("--preset is only supported with --guest-os linux.")
|
||||
if build_command:
|
||||
raise click.UsageError("--preset and --build-command are mutually exclusive.")
|
||||
preset_cmd, artifact_source = _PRESETS[preset]
|
||||
effective_build_command = preset_cmd
|
||||
elif build_command:
|
||||
effective_build_command = build_command
|
||||
# A custom command writes to the standard output dir for the OS.
|
||||
_, artifact_source = _noop_build_command(guest_os)
|
||||
else:
|
||||
effective_build_command, artifact_source = _noop_build_command(guest_os)
|
||||
|
||||
config: Config = load_config()
|
||||
resolved_job_id = job_id or f"smoke-{datetime.now(tz=UTC).strftime('%Y%m%d-%H%M%S')}"
|
||||
resolved_template = template_path or _default_template(guest_os)
|
||||
resolved_snapshot = snapshot_name or _default_snapshot(guest_os)
|
||||
|
||||
artifact_base: Path = config.paths.artifacts
|
||||
# Logs live alongside (not inside) the artifacts tree so that writing the
|
||||
# event log never accidentally creates the per-job artifact directory and
|
||||
# masks a build that produced no artifacts (mirrors F:\CI\Logs vs
|
||||
# F:\CI\Artifacts in Test-Smoke.ps1).
|
||||
log_base = Path(log_dir) if log_dir else config.paths.root / "logs"
|
||||
job_artifact_dir = artifact_base / resolved_job_id
|
||||
jsonl_path = log_base / resolved_job_id / "invoke-ci.jsonl"
|
||||
|
||||
click.echo("=" * 60)
|
||||
click.echo(f"[smoke] Guest OS : {guest_os}")
|
||||
click.echo(f"[smoke] Template : {resolved_template}")
|
||||
click.echo(f"[smoke] Snapshot : {resolved_snapshot}")
|
||||
click.echo(f"[smoke] Job id : {resolved_job_id}")
|
||||
if preset is not None:
|
||||
click.echo(f"[smoke] Preset : {preset}")
|
||||
click.echo(f"[smoke] Artifacts : {job_artifact_dir}")
|
||||
click.echo("=" * 60)
|
||||
|
||||
_write_event(
|
||||
jsonl_path, job_id=resolved_job_id, phase="job", status="start"
|
||||
)
|
||||
|
||||
start = time.monotonic()
|
||||
failed = False
|
||||
try:
|
||||
job_command.callback( # type: ignore[misc]
|
||||
job_id=resolved_job_id,
|
||||
repo_url=repo_url,
|
||||
branch=branch,
|
||||
commit="",
|
||||
configuration="Release",
|
||||
build_command=effective_build_command,
|
||||
guest_artifact_source=artifact_source,
|
||||
submodules=True,
|
||||
use_git_clone=True,
|
||||
use_shared_cache=False,
|
||||
skip_artifact=False,
|
||||
use_xvfb=False,
|
||||
gitea_credential_target="GiteaPAT",
|
||||
template_path=resolved_template,
|
||||
snapshot_name=resolved_snapshot,
|
||||
clone_base_dir=None,
|
||||
artifact_base_dir=str(artifact_base),
|
||||
guest_os=guest_os,
|
||||
guest_credential_target=None,
|
||||
ssh_key_path=ssh_key_path,
|
||||
ssh_user="ci_build",
|
||||
ssh_known_hosts_file=None,
|
||||
extra_env_json="",
|
||||
guest_cpu=0,
|
||||
guest_memory_mb=0,
|
||||
ready_timeout=600.0,
|
||||
poll_interval=5.0,
|
||||
vmrun_path=None,
|
||||
)
|
||||
except click.ClickException as exc:
|
||||
failed = True
|
||||
_write_event(
|
||||
jsonl_path,
|
||||
job_id=resolved_job_id,
|
||||
phase="job",
|
||||
status="failure",
|
||||
elapsed_sec=time.monotonic() - start,
|
||||
error=exc.format_message(),
|
||||
)
|
||||
click.echo(f"[smoke] FAIL: pipeline raised: {exc.format_message()}", err=True)
|
||||
|
||||
if not failed:
|
||||
_write_event(
|
||||
jsonl_path,
|
||||
job_id=resolved_job_id,
|
||||
phase="job",
|
||||
status="success",
|
||||
elapsed_sec=time.monotonic() - start,
|
||||
)
|
||||
|
||||
# ── Assert the three smoke invariants ───────────────────────────────
|
||||
click.echo("\n[smoke] Verifying results...")
|
||||
checks_ok = True
|
||||
|
||||
if failed:
|
||||
checks_ok = False
|
||||
else:
|
||||
click.echo("[smoke] OK: pipeline exit 0")
|
||||
|
||||
if job_artifact_dir.is_dir():
|
||||
click.echo(f"[smoke] OK: artifact dir exists: {job_artifact_dir}")
|
||||
else:
|
||||
click.echo(
|
||||
f"[smoke] FAIL: artifact dir not found: {job_artifact_dir}", err=True
|
||||
)
|
||||
checks_ok = False
|
||||
|
||||
if _has_job_success(jsonl_path):
|
||||
click.echo("[smoke] OK: JSONL contains job/success event")
|
||||
else:
|
||||
click.echo(
|
||||
f"[smoke] FAIL: JSONL missing job/success event in: {jsonl_path}",
|
||||
err=True,
|
||||
)
|
||||
checks_ok = False
|
||||
|
||||
click.echo("")
|
||||
if checks_ok:
|
||||
click.echo(f"[smoke] PASSED — {guest_os} smoke test completed successfully.")
|
||||
return
|
||||
raise click.ClickException("smoke test FAILED — one or more checks did not pass.")
|
||||
|
||||
|
||||
__all__ = ["smoke"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,285 @@
|
||||
"""``validate`` sub-commands.
|
||||
|
||||
Phase C adds Linux-native host/guest validation, replacing host-side use of
|
||||
``Validate-HostState.ps1`` and the diagnostic part of ``Test-CIGuestWinRM.ps1``:
|
||||
|
||||
* ``validate host`` — vmrun present, template snapshots present, keyring
|
||||
readable, ``/var/lib/ci`` permissions, systemd units active.
|
||||
* ``validate guest`` — connect to a guest over WinRM/SSH and run a trivial
|
||||
command, reporting the explicit failure cause.
|
||||
|
||||
The host checks are a Linux-native equivalent of ``Validate-HostState.ps1``
|
||||
(not a 1:1 port): no Windows drive paths, no Windows Credential Manager, no
|
||||
NTFS ACLs. The guest check is the diagnostic core of ``Test-CIGuestWinRM.ps1``:
|
||||
unlike the ``is_ready`` probe (which swallows errors and returns a bool),
|
||||
this reports the explicit failure cause.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import Config, load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
|
||||
# Each template directory and the snapshot a clone is taken from. Mirrors the
|
||||
# "VM templates and snapshots" table in CLAUDE.md.
|
||||
_TEMPLATE_SNAPSHOTS: tuple[tuple[str, str], ...] = (
|
||||
("WinBuild2025", "BaseClean"),
|
||||
("WinBuild2022", "BaseClean"),
|
||||
("LinuxBuild2404", "BaseClean-Linux"),
|
||||
)
|
||||
|
||||
# systemd units expected to be active on a configured Linux host.
|
||||
_EXPECTED_UNITS: tuple[str, ...] = ("act-runner.service",)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CheckResult:
|
||||
"""Outcome of a single host check."""
|
||||
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str
|
||||
|
||||
|
||||
@click.group("validate")
|
||||
def validate() -> None:
|
||||
"""Host and guest readiness checks."""
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────── validate host
|
||||
|
||||
def _check_vmrun(config: Config) -> CheckResult:
|
||||
"""vmrun is present and executable (via the backend factory)."""
|
||||
try:
|
||||
backend = load_backend(config)
|
||||
except BackendNotAvailable as exc:
|
||||
return CheckResult("vmrun", False, str(exc))
|
||||
except BackendError as exc: # pragma: no cover - defensive
|
||||
return CheckResult("vmrun", False, str(exc))
|
||||
vmrun_path = getattr(backend, "vmrun_path", None)
|
||||
if vmrun_path:
|
||||
return CheckResult("vmrun", True, f"found at {vmrun_path}")
|
||||
return CheckResult("vmrun", True, "backend loaded")
|
||||
|
||||
|
||||
def _check_snapshots(config: Config) -> CheckResult:
|
||||
"""Each known template VMX exists and exposes its expected snapshot."""
|
||||
try:
|
||||
backend = load_backend(config)
|
||||
except BackendError as exc:
|
||||
return CheckResult("snapshots", False, f"backend unavailable: {exc}")
|
||||
|
||||
templates_dir = config.paths.templates
|
||||
problems: list[str] = []
|
||||
found: list[str] = []
|
||||
for template_name, snapshot in _TEMPLATE_SNAPSHOTS:
|
||||
vmx = templates_dir / template_name / f"{template_name}.vmx"
|
||||
if not vmx.is_file():
|
||||
# A host may run only a subset of templates; skip absent ones
|
||||
# rather than failing on them.
|
||||
continue
|
||||
try:
|
||||
snapshots = backend.list_snapshots(VmHandle(identifier=str(vmx)))
|
||||
except BackendError as exc:
|
||||
problems.append(f"{template_name}: listSnapshots failed ({exc})")
|
||||
continue
|
||||
if snapshot in snapshots:
|
||||
found.append(f"{template_name}->{snapshot}")
|
||||
else:
|
||||
problems.append(
|
||||
f"{template_name}: snapshot {snapshot!r} absent "
|
||||
f"(have: {', '.join(snapshots) or 'none'})"
|
||||
)
|
||||
|
||||
if problems:
|
||||
return CheckResult("snapshots", False, "; ".join(problems))
|
||||
if not found:
|
||||
return CheckResult(
|
||||
"snapshots", False, f"no known template VMX found under {templates_dir}"
|
||||
)
|
||||
return CheckResult("snapshots", True, ", ".join(found))
|
||||
|
||||
|
||||
def _check_keyring(config: Config) -> CheckResult:
|
||||
"""The file-based keyring vault is readable for the guest cred target."""
|
||||
target = config.guest_cred_target
|
||||
try:
|
||||
cred = KeyringCredentialStore().get(target)
|
||||
except KeyError:
|
||||
return CheckResult(
|
||||
"keyring", False, f"credential {target!r} not found in keyring"
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
return CheckResult("keyring", False, str(exc))
|
||||
return CheckResult("keyring", True, f"{target!r} readable (user={cred.username})")
|
||||
|
||||
|
||||
def _check_permissions(config: Config) -> CheckResult:
|
||||
"""``$CI_ROOT`` and key sub-dirs exist and are readable+writable."""
|
||||
root = config.paths.root
|
||||
to_check = [root, config.paths.build_vms, config.paths.artifacts]
|
||||
problems: list[str] = []
|
||||
for path in to_check:
|
||||
if not path.exists():
|
||||
problems.append(f"{path}: missing")
|
||||
continue
|
||||
if not os.access(path, os.R_OK | os.W_OK):
|
||||
problems.append(f"{path}: not read/write for current user")
|
||||
if problems:
|
||||
return CheckResult("permissions", False, "; ".join(problems))
|
||||
return CheckResult("permissions", True, f"{root} read/write")
|
||||
|
||||
|
||||
def _systemctl_is_active(unit: str) -> str | None:
|
||||
"""Return ``systemctl is-active`` state, or ``None`` if unavailable."""
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[systemctl, "is-active", unit],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
return result.stdout.strip().lower() or None
|
||||
|
||||
|
||||
def _check_units() -> CheckResult:
|
||||
"""Expected systemd units report ``active``."""
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return CheckResult(
|
||||
"systemd", False, "systemctl not found (not a systemd host?)"
|
||||
)
|
||||
inactive: list[str] = []
|
||||
active: list[str] = []
|
||||
for unit in _EXPECTED_UNITS:
|
||||
state = _systemctl_is_active(unit)
|
||||
if state == "active":
|
||||
active.append(unit)
|
||||
else:
|
||||
inactive.append(f"{unit}={state or 'unknown'}")
|
||||
if inactive:
|
||||
return CheckResult("systemd", False, "; ".join(inactive))
|
||||
return CheckResult("systemd", True, ", ".join(active))
|
||||
|
||||
|
||||
def run_host_checks(config: Config) -> list[CheckResult]:
|
||||
"""Run every host check and return the per-check results."""
|
||||
return [
|
||||
_check_vmrun(config),
|
||||
_check_snapshots(config),
|
||||
_check_keyring(config),
|
||||
_check_permissions(config),
|
||||
_check_units(),
|
||||
]
|
||||
|
||||
|
||||
@validate.command("host")
|
||||
def validate_host() -> None:
|
||||
"""Check host prerequisites; exit non-zero on any failure."""
|
||||
config = load_config()
|
||||
results = run_host_checks(config)
|
||||
failed = [r for r in results if not r.ok]
|
||||
for result in results:
|
||||
marker = "OK " if result.ok else "FAIL"
|
||||
click.echo(f"[{marker}] {result.name}: {result.detail}")
|
||||
if failed:
|
||||
click.echo(f"\n{len(failed)} check(s) failed.")
|
||||
raise SystemExit(1)
|
||||
click.echo("\nAll host checks passed.")
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────── validate guest
|
||||
|
||||
def _resolve_use_winrm(use_winrm: bool | None, guest_os: str | None) -> bool:
|
||||
"""Decide the transport: explicit flag wins, else infer from guest OS."""
|
||||
if use_winrm is not None:
|
||||
return use_winrm
|
||||
if guest_os is not None:
|
||||
return guest_os.lower() != "linux"
|
||||
raise click.UsageError(
|
||||
"specify the transport: pass --winrm/--ssh or --guest-os."
|
||||
)
|
||||
|
||||
|
||||
def _probe_winrm(host: str, config: Config) -> CheckResult:
|
||||
"""Connect over WinRM and run a trivial command, reporting the cause."""
|
||||
try:
|
||||
cred = KeyringCredentialStore().get(config.guest_cred_target)
|
||||
except (KeyError, RuntimeError) as exc:
|
||||
return CheckResult("winrm", False, f"credential lookup failed: {exc}")
|
||||
transport = WinRmTransport(host, cred.username, cred.password)
|
||||
try:
|
||||
result = transport.run("$true | Out-Null", check=True)
|
||||
except TransportConnectError as exc:
|
||||
return CheckResult("winrm", False, f"connect/auth failed: {exc}")
|
||||
except TransportCommandError as exc:
|
||||
return CheckResult("winrm", False, f"command failed: {exc}")
|
||||
finally:
|
||||
transport.close()
|
||||
return CheckResult("winrm", True, f"connected; exit={result.returncode}")
|
||||
|
||||
|
||||
def _probe_ssh(host: str, config: Config) -> CheckResult:
|
||||
"""Connect over SSH and run a trivial command, reporting the cause."""
|
||||
key_path = str(config.ssh_key_path) if config.ssh_key_path else None
|
||||
transport = SshTransport(host, key_path=key_path)
|
||||
try:
|
||||
result = transport.run("true", check=True, timeout=10.0)
|
||||
except TransportConnectError as exc:
|
||||
return CheckResult("ssh", False, f"connect/auth failed: {exc}")
|
||||
except TransportCommandError as exc:
|
||||
return CheckResult("ssh", False, f"command failed: {exc}")
|
||||
finally:
|
||||
transport.close()
|
||||
return CheckResult("ssh", True, f"connected; exit={result.returncode}")
|
||||
|
||||
|
||||
@validate.command("guest")
|
||||
@click.option("--host", "host", required=True, help="Guest IP or hostname.")
|
||||
@click.option(
|
||||
"--winrm/--ssh",
|
||||
"use_winrm",
|
||||
default=None,
|
||||
help="Force transport (default: inferred from --guest-os).",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||
default=None,
|
||||
help="Guest OS family (used to pick transport when --winrm/--ssh omitted).",
|
||||
)
|
||||
def validate_guest(
|
||||
host: str,
|
||||
use_winrm: bool | None,
|
||||
guest_os: str | None,
|
||||
) -> None:
|
||||
"""Probe a guest's transport and report the explicit failure cause."""
|
||||
config = load_config()
|
||||
winrm = _resolve_use_winrm(use_winrm, guest_os)
|
||||
result = _probe_winrm(host, config) if winrm else _probe_ssh(host, config)
|
||||
marker = "OK " if result.ok else "FAIL"
|
||||
click.echo(f"[{marker}] {result.name} {host}: {result.detail}")
|
||||
if not result.ok:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
__all__ = ["validate"]
|
||||
@@ -12,7 +12,10 @@ instead of scanning the local filesystem.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -194,9 +197,9 @@ def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
try:
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
@@ -398,6 +401,13 @@ def vm_cleanup(
|
||||
show_default=True,
|
||||
help="Informational; reserved for backend hints in Phase C.",
|
||||
)
|
||||
@click.option(
|
||||
"--start",
|
||||
"start_vm",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Start the clone headless immediately after creation.",
|
||||
)
|
||||
def vm_new(
|
||||
template: str,
|
||||
snapshot: str,
|
||||
@@ -405,6 +415,7 @@ def vm_new(
|
||||
job_id: str,
|
||||
vmrun_path: str | None,
|
||||
guest_os: str,
|
||||
start_vm: bool,
|
||||
) -> None:
|
||||
"""Create a linked clone of the template VM for an ephemeral CI build.
|
||||
|
||||
@@ -431,10 +442,10 @@ def vm_new(
|
||||
clone_dir = base / clone_name
|
||||
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||
|
||||
click.echo("[vm new] creating linked clone...")
|
||||
click.echo(f" template : {template}")
|
||||
click.echo(f" snapshot : {snapshot}")
|
||||
click.echo(f" clone vmx: {clone_vmx}")
|
||||
click.echo("[vm new] creating linked clone...", err=True)
|
||||
click.echo(f" template : {template}", err=True)
|
||||
click.echo(f" snapshot : {snapshot}", err=True)
|
||||
click.echo(f" clone vmx: {clone_vmx}", err=True)
|
||||
|
||||
try:
|
||||
backend = _make_backend(vmrun_path)
|
||||
@@ -461,9 +472,208 @@ def vm_new(
|
||||
)
|
||||
|
||||
elapsed = time.monotonic() - start
|
||||
click.echo(f"[vm new] clone created in {elapsed:.1f}s")
|
||||
click.echo(f"[vm new] clone created in {elapsed:.1f}s", err=True)
|
||||
|
||||
if start_vm:
|
||||
click.echo("[vm new] starting VM headless...", err=True)
|
||||
try:
|
||||
backend.start(handle, headless=True)
|
||||
except BackendError as exc:
|
||||
raise click.ClickException(f"vmrun start failed: {exc}") from exc
|
||||
click.echo("[vm new] VM started.", err=True)
|
||||
|
||||
# Final stdout line: the identifier itself, so PS callers can capture it.
|
||||
click.echo(handle.identifier)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm"]
|
||||
# ────────────────────────────────────────────────────────────────────────── open
|
||||
|
||||
|
||||
def _x_display_reachable(env: dict[str, str]) -> tuple[bool, str]:
|
||||
"""Probe whether the X display in ``env`` is usable.
|
||||
|
||||
Uses ``xdpyinfo`` when available (it exits non-zero and prints the reason
|
||||
if the display cannot be opened). Returns ``(ok, detail)``; if the probe
|
||||
tool is missing we optimistically return ``(True, "")`` so the launch is
|
||||
still attempted.
|
||||
"""
|
||||
probe = shutil.which("xdpyinfo")
|
||||
if probe is None:
|
||||
return True, ""
|
||||
try:
|
||||
res = subprocess.run( # fixed argv, no shell
|
||||
[probe],
|
||||
env=env,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return False, str(exc)
|
||||
if res.returncode == 0:
|
||||
return True, ""
|
||||
return False, (res.stderr or b"").decode(errors="replace").strip()
|
||||
|
||||
|
||||
def _launch_gui(argv: list[str], env: dict[str, str], log_path: Path) -> subprocess.Popen[bytes]:
|
||||
"""Spawn the VMware Workstation GUI detached, logging stdio to ``log_path``.
|
||||
|
||||
Isolated so tests can monkeypatch it without spawning a real process. The
|
||||
GUI must outlive the CLI, so the child runs in its own session; its output
|
||||
goes to a log file (not /dev/null) so failures are diagnosable.
|
||||
"""
|
||||
log_fh = log_path.open("wb")
|
||||
return subprocess.Popen( # argv is a fixed list, never a shell string
|
||||
argv,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=log_fh,
|
||||
stderr=subprocess.STDOUT,
|
||||
start_new_session=True,
|
||||
)
|
||||
|
||||
|
||||
@click.command("vmgui")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
default=None,
|
||||
help="Path to a VMX to open. Omit to launch the GUI with no VM loaded.",
|
||||
)
|
||||
@click.option(
|
||||
"--vmware-path",
|
||||
"vmware_path",
|
||||
default="vmware",
|
||||
show_default=True,
|
||||
help="Path to the VMware Workstation GUI binary.",
|
||||
)
|
||||
@click.option(
|
||||
"--power-on",
|
||||
"power_on",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Power the VM on when it is opened (vmware -x).",
|
||||
)
|
||||
@click.option(
|
||||
"--fullscreen",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Power on and enter full-screen mode (vmware -X). Implies --power-on.",
|
||||
)
|
||||
@click.option(
|
||||
"--new-window",
|
||||
"new_window",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Open in a new window instead of a tab (vmware -n).",
|
||||
)
|
||||
@click.option(
|
||||
"--display",
|
||||
"display",
|
||||
default=None,
|
||||
help="X display to use (default: inherit $DISPLAY). e.g. ':0'.",
|
||||
)
|
||||
def vmgui(
|
||||
vmx: str | None,
|
||||
vmware_path: str,
|
||||
power_on: bool,
|
||||
fullscreen: bool,
|
||||
new_window: bool,
|
||||
display: str | None,
|
||||
) -> None:
|
||||
"""Open the VMware Workstation GUI, optionally on a VMX (run as ci-runner).
|
||||
|
||||
Launches the interactive GUI — unlike the headless ``vmrun`` path the rest
|
||||
of the orchestrator uses. With ``--vmx`` the GUI opens that VM; without it
|
||||
the GUI starts bare (no VM loaded). The GUI needs an X display: when invoked
|
||||
through ``sudo -u ci-runner`` the display is usually stripped, so authorise
|
||||
the service user first, e.g. ``xhost +SI:localuser:ci-runner``, and pass
|
||||
``--display :0`` (or export ``DISPLAY``).
|
||||
"""
|
||||
vmx_path: Path | None = None
|
||||
if vmx is not None:
|
||||
vmx_path = Path(vmx)
|
||||
if not vmx_path.is_file():
|
||||
raise click.ClickException(f"VMX file not found: {vmx}")
|
||||
|
||||
gui = shutil.which(vmware_path) or vmware_path
|
||||
|
||||
env = dict(os.environ)
|
||||
if display:
|
||||
env["DISPLAY"] = display
|
||||
disp = env.get("DISPLAY", "")
|
||||
if not disp:
|
||||
raise click.ClickException(
|
||||
"no X display set ($DISPLAY empty and --display omitted). Pass "
|
||||
"--display :0 (and authorise this user on the desktop session: "
|
||||
"run `xhost +SI:localuser:ci-runner` as the logged-in user)."
|
||||
)
|
||||
|
||||
# An inherited XAUTHORITY pointing at another user's cookie (e.g. the
|
||||
# desktop user's ~/.Xauthority) is unreadable by ci-runner and blocks the
|
||||
# connection with "Authorization required". Drop it so the launch relies on
|
||||
# host-based auth (xhost) instead.
|
||||
xauth = env.get("XAUTHORITY")
|
||||
if xauth and not os.access(xauth, os.R_OK):
|
||||
env.pop("XAUTHORITY", None)
|
||||
click.echo(
|
||||
f"[vmgui] dropping unreadable XAUTHORITY ({xauth}); relying on xhost.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
# Pre-flight: fail loudly with the cause instead of silently spawning a
|
||||
# GUI that immediately dies on "cannot open display".
|
||||
ok, detail = _x_display_reachable(env)
|
||||
if not ok:
|
||||
raise click.ClickException(
|
||||
f"cannot open X display {disp!r}: {detail or 'unknown error'}. "
|
||||
"From the desktop session run: xhost +SI:localuser:ci-runner"
|
||||
)
|
||||
|
||||
argv = [gui]
|
||||
if new_window:
|
||||
argv.append("-n")
|
||||
if vmx_path is not None:
|
||||
# Power-on flags only make sense with a VM to open.
|
||||
if fullscreen:
|
||||
argv.append("-X")
|
||||
elif power_on:
|
||||
argv.append("-x")
|
||||
argv.append(str(vmx_path))
|
||||
elif power_on or fullscreen:
|
||||
click.echo(
|
||||
"[vmgui] note: --power-on/--fullscreen ignored without --vmx.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
ts = datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S")
|
||||
log_path = Path(tempfile.gettempdir()) / f"vmgui-{os.getpid()}-{ts}.log"
|
||||
|
||||
click.echo(f"[vmgui] launching VMware GUI: {' '.join(argv)}")
|
||||
try:
|
||||
proc = _launch_gui(argv, env, log_path)
|
||||
except (OSError, ValueError) as exc:
|
||||
raise click.ClickException(f"failed to launch VMware GUI: {exc}") from exc
|
||||
|
||||
# Brief grace: if the GUI dies right away, surface its log instead of
|
||||
# claiming success.
|
||||
time.sleep(1.5)
|
||||
rc = proc.poll()
|
||||
if rc is not None and rc != 0:
|
||||
try:
|
||||
tail = log_path.read_text(errors="replace").strip().splitlines()[-5:]
|
||||
except OSError:
|
||||
tail = []
|
||||
detail = "\n ".join(tail) if tail else "(no output captured)"
|
||||
raise click.ClickException(
|
||||
f"VMware GUI exited immediately (code {rc}). Log {log_path}:\n {detail}"
|
||||
)
|
||||
|
||||
click.echo(
|
||||
f"[vmgui] VMware GUI started (pid {proc.pid}); display={disp}; log={log_path}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm", "vmgui"]
|
||||
|
||||
@@ -22,6 +22,9 @@ from typing import Any
|
||||
|
||||
_ENV_KEYS = ("CI_ROOT", "CI_TEMPLATES", "CI_BUILD_VMS", "CI_ARTIFACTS", "CI_KEYS")
|
||||
|
||||
_IP_POOL_LEASE_DEFAULT_LINUX = "/var/lib/ci/ip-pool.json"
|
||||
_IP_POOL_LEASE_DEFAULT_WIN = r"F:\CI\ip-pool.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Paths:
|
||||
@@ -34,6 +37,26 @@ class Paths:
|
||||
keys: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IpPoolConfig:
|
||||
"""Pre-assigned IP pool for zero-polling VM ready detection.
|
||||
|
||||
When present, the orchestrator injects ``guestinfo.ip-assignment`` into
|
||||
the cloned VMX before starting the VM so the guest can apply a static IP
|
||||
at boot (see ``template/guest-setup/``).
|
||||
"""
|
||||
|
||||
addresses: list[str]
|
||||
netmask: str = "255.255.255.0"
|
||||
gateway: str = ""
|
||||
lease_file: Path = field(
|
||||
default_factory=lambda: Path(
|
||||
_IP_POOL_LEASE_DEFAULT_WIN if os.name == "nt"
|
||||
else _IP_POOL_LEASE_DEFAULT_LINUX
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendConfig:
|
||||
"""Backend selector. ``type`` is currently always 'workstation'."""
|
||||
@@ -51,6 +74,7 @@ class Config:
|
||||
vmrun_path: str | None = None
|
||||
ssh_key_path: Path | None = None
|
||||
guest_cred_target: str = "BuildVMGuest"
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
|
||||
|
||||
def _platform_defaults() -> Paths:
|
||||
@@ -133,6 +157,7 @@ def load_config(
|
||||
vmrun_path: str | None = env.get("CI_VMRUN_PATH")
|
||||
ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None
|
||||
guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest")
|
||||
ip_pool: IpPoolConfig | None = None
|
||||
|
||||
if toml_path and toml_path.is_file():
|
||||
data = _load_toml(toml_path)
|
||||
@@ -148,6 +173,20 @@ def load_config(
|
||||
if ssh_key_path is None and data.get("ssh_key_path"):
|
||||
ssh_key_path = Path(data["ssh_key_path"])
|
||||
guest_cred_target = data.get("guest_cred_target", guest_cred_target)
|
||||
pool_section = data.get("ip_pool", {})
|
||||
if pool_section and pool_section.get("addresses"):
|
||||
addrs = pool_section["addresses"]
|
||||
if isinstance(addrs, list) and addrs:
|
||||
_lease_default = (
|
||||
_IP_POOL_LEASE_DEFAULT_WIN if os.name == "nt"
|
||||
else _IP_POOL_LEASE_DEFAULT_LINUX
|
||||
)
|
||||
ip_pool = IpPoolConfig(
|
||||
addresses=[str(a) for a in addrs],
|
||||
netmask=str(pool_section.get("netmask", "255.255.255.0")),
|
||||
gateway=str(pool_section.get("gateway", "")),
|
||||
lease_file=Path(pool_section.get("lease_file") or _lease_default),
|
||||
)
|
||||
|
||||
paths = _apply_env(paths, env)
|
||||
|
||||
@@ -157,7 +196,8 @@ def load_config(
|
||||
vmrun_path=vmrun_path,
|
||||
ssh_key_path=ssh_key_path,
|
||||
guest_cred_target=guest_cred_target,
|
||||
ip_pool=ip_pool,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BackendConfig", "Config", "Paths", "load_config"]
|
||||
__all__ = ["BackendConfig", "Config", "IpPoolConfig", "Paths", "load_config"]
|
||||
|
||||
@@ -45,15 +45,27 @@ class KeyringCredentialStore:
|
||||
"keyring is not installed; run `pip install keyring`."
|
||||
) from exc
|
||||
|
||||
# 1. Native lookup — works on Windows Credential Manager and backends
|
||||
# that implement get_credential(service, username=None).
|
||||
cred = keyring.get_credential(target, None)
|
||||
if cred is None:
|
||||
# Fallback: some backends only support get_password and need the
|
||||
# username to be the same as the target.
|
||||
password = keyring.get_password(self._service, target)
|
||||
if password is None:
|
||||
raise KeyError(f"Credential '{target}' not found in keyring.")
|
||||
if cred is not None:
|
||||
return Credential(username=cred.username, password=cred.password)
|
||||
|
||||
# 2. Two-entry scheme for file backends (PlaintextKeyring, etc.) where
|
||||
# get_credential(target, None) is a no-op. The username is stored
|
||||
# separately under service="{target}:meta", key="username".
|
||||
username = keyring.get_password(f"{target}:meta", "username")
|
||||
if username is not None:
|
||||
password = keyring.get_password(target, username)
|
||||
if password is not None:
|
||||
return Credential(username=username, password=password)
|
||||
|
||||
# 3. Legacy fallback: password stored under service=self._service.
|
||||
password = keyring.get_password(self._service, target)
|
||||
if password is not None:
|
||||
return Credential(username=target, password=password)
|
||||
return Credential(username=cred.username, password=cred.password)
|
||||
|
||||
raise KeyError(f"Credential '{target}' not found in keyring.")
|
||||
|
||||
|
||||
__all__ = ["Credential", "CredentialStore", "KeyringCredentialStore"]
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""IP slot pool — pre-assigns static IPs to build VMs via VMX guestinfo.
|
||||
|
||||
When ``[ip_pool]`` is configured, the orchestrator allocates one IP from the
|
||||
pool before starting each VM, injects it into the cloned VMX as::
|
||||
|
||||
guestinfo.ip-assignment = "192.168.79.201"
|
||||
guestinfo.netmask = "255.255.255.0"
|
||||
guestinfo.gateway = "192.168.79.2"
|
||||
|
||||
and releases it on VM destroy. The guest applies the static IP at boot:
|
||||
|
||||
* **Linux**: ``/usr/local/bin/ci-report-ip.sh`` (updated version in
|
||||
``template/guest-setup/linux/``) reads ``guestinfo.ip-assignment`` and
|
||||
configures a static NIC address before ``network-online.target`` is reached.
|
||||
|
||||
* **Windows**: the ``CI-StaticIp`` Task Scheduler task
|
||||
(``template/guest-setup/windows/ci-static-ip.ps1``) runs at system startup
|
||||
as SYSTEM and applies the static IP before WinRM starts.
|
||||
|
||||
This eliminates the DHCP / VMware-Tools IP-detect polling loop (20-180 s
|
||||
variance observed in B7 benchmarks). After the VM reaches *running* state the
|
||||
host knows the IP immediately and goes straight to the transport probe.
|
||||
|
||||
Lease file — JSON dict mapping each pool IP to its current owner string, or
|
||||
``null`` if free::
|
||||
|
||||
{
|
||||
"192.168.79.201": "Clone_job123_20260524_abc123",
|
||||
"192.168.79.202": null,
|
||||
"192.168.79.203": null,
|
||||
"192.168.79.204": null
|
||||
}
|
||||
|
||||
File locking: ``fcntl.flock`` (POSIX/Linux); no-op on Windows (low risk —
|
||||
pool size matches concurrency, failures are detectable via transport probe).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class IpSlotPool:
|
||||
"""Process-safe IP slot allocator backed by a JSON lease file."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
addresses: list[str],
|
||||
netmask: str,
|
||||
gateway: str,
|
||||
lease_file: Path,
|
||||
) -> None:
|
||||
if not addresses:
|
||||
raise ValueError("IpSlotPool requires at least one address.")
|
||||
self._addresses = list(addresses)
|
||||
self.netmask = netmask
|
||||
self.gateway = gateway
|
||||
self._lease_file = lease_file
|
||||
self._lock_file = lease_file.with_suffix(".lock")
|
||||
|
||||
# ── public API ─────────────────────────────────────────────────────────
|
||||
|
||||
def acquire(self, owner: str, timeout: float = 60.0) -> str:
|
||||
"""Allocate a free IP slot. Blocks up to *timeout* seconds.
|
||||
|
||||
Raises :exc:`RuntimeError` if the pool stays exhausted past *timeout*.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
with _file_lock(self._lock_file):
|
||||
leases = self._read_leases()
|
||||
for ip in self._addresses:
|
||||
if leases.get(ip) is None:
|
||||
leases[ip] = owner
|
||||
self._write_leases(leases)
|
||||
return ip
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError(
|
||||
f"IP pool exhausted after {timeout:.0f}s; "
|
||||
f"addresses={self._addresses}"
|
||||
)
|
||||
time.sleep(2.0)
|
||||
|
||||
def release(self, ip: str) -> None:
|
||||
"""Return *ip* to the pool (no-op if not in pool)."""
|
||||
with _file_lock(self._lock_file):
|
||||
leases = self._read_leases()
|
||||
if ip in leases:
|
||||
leases[ip] = None
|
||||
self._write_leases(leases)
|
||||
|
||||
# ── internals ──────────────────────────────────────────────────────────
|
||||
|
||||
def _read_leases(self) -> dict[str, str | None]:
|
||||
result: dict[str, str | None] = {ip: None for ip in self._addresses}
|
||||
if not self._lease_file.exists():
|
||||
return result
|
||||
try:
|
||||
raw: object = json.loads(self._lease_file.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return result
|
||||
if not isinstance(raw, dict):
|
||||
return result
|
||||
for ip in self._addresses:
|
||||
val = raw.get(ip)
|
||||
result[ip] = val if isinstance(val, str) else None
|
||||
return result
|
||||
|
||||
def _write_leases(self, leases: dict[str, str | None]) -> None:
|
||||
self._lease_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = self._lease_file.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(leases, indent=2) + "\n", encoding="utf-8")
|
||||
tmp.replace(self._lease_file)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _file_lock(path: Path) -> Generator[None, None, None]:
|
||||
"""Exclusive advisory lock on *path* via ``fcntl.flock`` (POSIX only)."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as fd:
|
||||
try:
|
||||
import fcntl
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
import fcntl
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
__all__ = ["IpSlotPool"]
|
||||
@@ -639,11 +639,29 @@ try {
|
||||
New-NetFirewallRule -DisplayName 'WinRM-HTTPS' -Direction Inbound -Protocol TCP -LocalPort 5986 -Action Allow -Profile Any -RemoteAddress '192.168.79.0/24' | Out-Null
|
||||
L 'WinRM HTTPS listener configured (port 5986, restricted to 192.168.79.0/24)'
|
||||
} catch { L "WinRM HTTPS failed: `$(`$_.Exception.Message)" }
|
||||
winrm set winrm/config/winrs '@{MaxProcessesPerShell="25"}' | Out-Null
|
||||
winrm set winrm/config/winrs '@{MaxProcessesPerShell="100"}' | Out-Null
|
||||
Set-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB 2048 -Force 3>`$null
|
||||
Set-Item WSMan:\localhost\Shell\IdleTimeOut 7200000 -Force 3>`$null
|
||||
# Plugin-level quota: pypsrp (Linux→Win) uses Microsoft.PowerShell endpoint,
|
||||
# which has its own MaxMemoryPerShellMB independent of the Shell quota above.
|
||||
# Without this, parallel native processes (e.g. 3x cl.exe in MSBuild fan-out)
|
||||
# fail with MSB6003 / Win32 1816 'Not enough quota'.
|
||||
Set-Item 'WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB' 2048 -Force 3>`$null
|
||||
Set-Service -Name WinRM -StartupType Automatic 3>`$null
|
||||
L 'WinRM shell limits: MaxMemory=2048MB IdleTimeout=2h MaxProcesses=25; StartType=Automatic'
|
||||
Restart-Service -Name WinRM -Force 3>`$null
|
||||
L 'WinRM limits: Shell+Plugin MaxMemory=2048MB, IdleTimeout=2h, MaxProcesses=100; StartType=Automatic'
|
||||
|
||||
# ── Non-interactive desktop heap (csrss SharedSection) ───────────────────
|
||||
# WinRM sessions run in Session 0 non-interactive window station; default
|
||||
# desktop heap is 768 KB. CreateProcess fails with ERROR_NOT_ENOUGH_QUOTA
|
||||
# (Win32 1816 → MSB6003) when multiple compiler processes spawn in parallel.
|
||||
# Bump the third SharedSection field to 4096 KB. Change is read by csrss only
|
||||
# at boot — the post-install shutdown at the end of this script applies it.
|
||||
`$subsysKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems'
|
||||
`$subsysCur = (Get-ItemProperty -Path `$subsysKey -Name Windows).Windows
|
||||
`$subsysNew = `$subsysCur -replace 'SharedSection=(\d+),(\d+),\d+', 'SharedSection=`$1,`$2,4096'
|
||||
Set-ItemProperty -Path `$subsysKey -Name Windows -Value `$subsysNew
|
||||
L "Desktop heap (non-interactive) SharedSection 3rd field set to 4096 KB (takes effect after reboot)"
|
||||
|
||||
# ── Server Manager / WAC popup / first-logon UX ──────────────────────────
|
||||
# Server Manager auto-start off — machine-wide + current Administrator + Default
|
||||
|
||||
@@ -639,11 +639,29 @@ try {
|
||||
New-NetFirewallRule -DisplayName 'WinRM-HTTPS' -Direction Inbound -Protocol TCP -LocalPort 5986 -Action Allow -Profile Any -RemoteAddress '192.168.79.0/24' | Out-Null
|
||||
L 'WinRM HTTPS listener configured (port 5986, restricted to 192.168.79.0/24)'
|
||||
} catch { L "WinRM HTTPS failed: `$(`$_.Exception.Message)" }
|
||||
winrm set winrm/config/winrs '@{MaxProcessesPerShell="25"}' | Out-Null
|
||||
winrm set winrm/config/winrs '@{MaxProcessesPerShell="100"}' | Out-Null
|
||||
Set-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB 2048 -Force 3>`$null
|
||||
Set-Item WSMan:\localhost\Shell\IdleTimeOut 7200000 -Force 3>`$null
|
||||
# Plugin-level quota: pypsrp (Linux→Win) uses Microsoft.PowerShell endpoint,
|
||||
# which has its own MaxMemoryPerShellMB independent of the Shell quota above.
|
||||
# Without this, parallel native processes (e.g. 3x cl.exe in MSBuild fan-out)
|
||||
# fail with MSB6003 / Win32 1816 'Not enough quota'.
|
||||
Set-Item 'WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB' 2048 -Force 3>`$null
|
||||
Set-Service -Name WinRM -StartupType Automatic 3>`$null
|
||||
L 'WinRM shell limits: MaxMemory=2048MB IdleTimeout=2h MaxProcesses=25; StartType=Automatic'
|
||||
Restart-Service -Name WinRM -Force 3>`$null
|
||||
L 'WinRM limits: Shell+Plugin MaxMemory=2048MB, IdleTimeout=2h, MaxProcesses=100; StartType=Automatic'
|
||||
|
||||
# ── Non-interactive desktop heap (csrss SharedSection) ───────────────────
|
||||
# WinRM sessions run in Session 0 non-interactive window station; default
|
||||
# desktop heap is 768 KB. CreateProcess fails with ERROR_NOT_ENOUGH_QUOTA
|
||||
# (Win32 1816 → MSB6003) when multiple compiler processes spawn in parallel.
|
||||
# Bump the third SharedSection field to 4096 KB. Change is read by csrss only
|
||||
# at boot — the post-install shutdown at the end of this script applies it.
|
||||
`$subsysKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems'
|
||||
`$subsysCur = (Get-ItemProperty -Path `$subsysKey -Name Windows).Windows
|
||||
`$subsysNew = `$subsysCur -replace 'SharedSection=(\d+),(\d+),\d+', 'SharedSection=`$1,`$2,4096'
|
||||
Set-ItemProperty -Path `$subsysKey -Name Windows -Value `$subsysNew
|
||||
L "Desktop heap (non-interactive) SharedSection 3rd field set to 4096 KB (takes effect after reboot)"
|
||||
|
||||
# ── Server Manager / WAC popup / first-logon UX ──────────────────────────
|
||||
# Server Manager auto-start off — machine-wide + current Administrator + Default
|
||||
|
||||
@@ -51,12 +51,15 @@ assert_step() {
|
||||
# --- Step 1: apt update + upgrade ---
|
||||
echo ""
|
||||
echo "=== Step 1: apt update + upgrade ==="
|
||||
# apt-get update always runs — package index must be fresh to locate packages.
|
||||
# --skip-update only skips the (slow) full upgrade.
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq
|
||||
assert_step "apt update completed" true
|
||||
if [ "${SKIP_UPDATE:-0}" = "0" ]; then
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq
|
||||
assert_step "apt update + upgrade completed" true
|
||||
assert_step "apt upgrade completed" true
|
||||
else
|
||||
echo " [SKIP] --skip-update flag set"
|
||||
echo " [SKIP] --skip-update flag set — upgrade skipped (index still refreshed)"
|
||||
fi
|
||||
|
||||
# --- Step 2: build essentials ---
|
||||
@@ -88,6 +91,14 @@ assert_step "git available" command -v git
|
||||
assert_step "7z available" command -v 7z
|
||||
assert_step "curl available" command -v curl
|
||||
|
||||
# --- Step 4a: GTK/GObject + headless display (PyGObject GUI tests) ---
|
||||
echo ""
|
||||
echo "=== Step 4a: GTK4 / GObject introspection + Xvfb ==="
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
|
||||
python3-gi gir1.2-gtk-4.0 xvfb
|
||||
assert_step "python3-gi importable" python3 -c "import gi"
|
||||
assert_step "xvfb available" command -v Xvfb
|
||||
|
||||
# --- Step 4b: .NET SDK (optional) ---
|
||||
echo ""
|
||||
echo "=== Step 4b: .NET SDK (optional) ==="
|
||||
@@ -170,40 +181,99 @@ sudo netplan generate
|
||||
assert_step "99-ci-dhcp-allnics.yaml exists" test -f /etc/netplan/99-ci-dhcp-allnics.yaml
|
||||
|
||||
# --- Step 7b: CI IP reporter via VMware guestinfo ---
|
||||
# The host reads the guest IP via `vmrun readVariable guestVar ci-ip`.
|
||||
# IMPORTANT: vmware-rpctool sets 'guestinfo.ci-ip' but vmrun reads it as just
|
||||
# 'ci-ip' under the guestVar namespace (the 'guestinfo.' prefix is implicit).
|
||||
# This is the official VMware VMCI channel — works even when TCP/IP is not yet
|
||||
# fully initialised on the host. Used as primary; getGuestIPAddress is fallback.
|
||||
# Supports two modes:
|
||||
# Static (ip_pool): reads guestinfo.ip-assignment injected by the host
|
||||
# orchestrator, applies static NIC config, reports back.
|
||||
# DHCP fallback: waits for DHCP address, reports via guestinfo.ci-ip.
|
||||
# Content embedded inline — this script runs inside the VM where the host-side
|
||||
# template/guest-setup/linux/ directory is not present.
|
||||
echo ""
|
||||
echo "=== Step 7b: CI IP reporter ==="
|
||||
sudo tee /usr/local/bin/ci-report-ip.sh > /dev/null << 'EOSCRIPT'
|
||||
echo "=== Step 7b: CI IP reporter (with static IP support) ==="
|
||||
sudo tee /usr/local/bin/ci-report-ip.sh > /dev/null << 'EOREPORTIP'
|
||||
#!/bin/bash
|
||||
# Writes the VM's primary IPv4 to VMware guestinfo.ci-ip on every boot.
|
||||
# Called by ci-report-ip.service.
|
||||
# Retries for up to 60s to tolerate open-vm-tools re-init after UUID change
|
||||
# CI VM IP reporter — called by ci-report-ip.service at boot.
|
||||
#
|
||||
# Supports two modes:
|
||||
# Static (ip_pool): reads guestinfo.ip-assignment injected by the host
|
||||
# orchestrator, applies static NIC config, reports back.
|
||||
# DHCP fallback: waits for DHCP address, reports via guestinfo.ci-ip.
|
||||
#
|
||||
# The VMCI channel (vmware-rpctool) does not require TCP/IP to be initialised —
|
||||
# it communicates via the VMware virtual hardware interface. This means the
|
||||
# static IP can be configured before network-online.target, so by the time SSH
|
||||
# starts listening the NIC already has the correct address.
|
||||
|
||||
RPCTOOL=/usr/bin/vmware-rpctool
|
||||
[ -x "$RPCTOOL" ] || exit 0
|
||||
|
||||
# ── Check for pre-assigned IP from orchestrator ───────────────────────────────
|
||||
assigned_ip=$("$RPCTOOL" "info-get guestinfo.ip-assignment" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
if [[ "$assigned_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
netmask=$("$RPCTOOL" "info-get guestinfo.netmask" 2>/dev/null | tr -d '[:space:]')
|
||||
gateway=$("$RPCTOOL" "info-get guestinfo.gateway" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
# Resolve primary NIC (first non-loopback).
|
||||
iface=$(ip -o link show | awk -F': ' '$2 != "lo" {print $2; exit}')
|
||||
if [ -z "$iface" ]; then
|
||||
echo "ci-report-ip: no non-loopback interface found" | systemd-cat -t ci-report-ip
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compute prefix length from netmask (default /24).
|
||||
prefix=24
|
||||
if [ -n "$netmask" ]; then
|
||||
prefix=$(python3 -c \
|
||||
"import ipaddress; print(ipaddress.IPv4Network('0.0.0.0/$netmask',strict=False).prefixlen)" \
|
||||
2>/dev/null) || prefix=24
|
||||
fi
|
||||
|
||||
# Flush any existing config (DHCP or stale static) and apply.
|
||||
ip addr flush dev "$iface" 2>/dev/null || true
|
||||
ip addr add "${assigned_ip}/${prefix}" dev "$iface"
|
||||
ip link set "$iface" up
|
||||
|
||||
if [[ "$gateway" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
ip route add default via "$gateway" dev "$iface" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Report IP back to host via guestinfo.ci-ip.
|
||||
"$RPCTOOL" "info-set guestinfo.ci-ip $assigned_ip" 2>/dev/null
|
||||
echo "ci-report-ip: applied static IP $assigned_ip/$prefix via guestinfo" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Fallback: DHCP — wait for IP and report ───────────────────────────────────
|
||||
# Retries for up to 60 s to tolerate open-vm-tools re-init after UUID change
|
||||
# (linked clones get a new UUID — vmware-rpctool may fail briefly on first boot).
|
||||
for i in $(seq 1 30); do
|
||||
IP=$(ip -4 route get 1.0.0.0 2>/dev/null \
|
||||
| awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}' \
|
||||
| head -1)
|
||||
if [ -n "$IP" ]; then
|
||||
if /usr/bin/vmware-rpctool "info-set guestinfo.ci-ip $IP" 2>/dev/null; then
|
||||
echo "ci-report-ip: set guestinfo.ci-ip=$IP (attempt $i)" | systemd-cat -t ci-report-ip
|
||||
if "$RPCTOOL" "info-set guestinfo.ci-ip $IP" 2>/dev/null; then
|
||||
echo "ci-report-ip: set guestinfo.ci-ip=$IP via DHCP (attempt $i)" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "ci-report-ip: failed to set guestinfo.ci-ip after 30 attempts" | systemd-cat -t ci-report-ip
|
||||
|
||||
echo "ci-report-ip: failed to set guestinfo.ci-ip after 30 attempts" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 1
|
||||
EOSCRIPT
|
||||
EOREPORTIP
|
||||
sudo chmod +x /usr/local/bin/ci-report-ip.sh
|
||||
|
||||
sudo tee /etc/systemd/system/ci-report-ip.service > /dev/null << 'EOSERVICE'
|
||||
[Unit]
|
||||
Description=Report CI VM IP to VMware host via guestinfo
|
||||
After=open-vm-tools.service network.target
|
||||
# Run after open-vm-tools so vmware-rpctool is available.
|
||||
# Do NOT add network-online.target — we want to run BEFORE the network stack
|
||||
# settles so static IP is applied before SSH/other services start.
|
||||
After=open-vm-tools.service
|
||||
Wants=open-vm-tools.service
|
||||
|
||||
[Service]
|
||||
|
||||
@@ -148,6 +148,11 @@ param(
|
||||
# Skip disk cleanup step (cleanmgr + DISM + cache clear) — speeds up re-runs during dev/debug
|
||||
[switch] $SkipCleanup,
|
||||
|
||||
# Run disk cleanup only — skip all provisioning steps and jump straight to the
|
||||
# cleanup section. Intended for use by Prepare-WinBuild2022.ps1 after the Windows
|
||||
# Update loop completes, so cleanup always runs on the fully-updated system.
|
||||
[switch] $CleanupOnly,
|
||||
|
||||
# Administrator password — used only to write DefaultPassword for autologin.
|
||||
# Optional: if empty, DefaultPassword registry value is left as-is (Deploy may have set it).
|
||||
[string] $AdminPassword = '',
|
||||
@@ -231,6 +236,57 @@ function Assert-Hash {
|
||||
Write-Host " [OK] SHA256 verified: $Label" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Invoke-DiskCleanup {
|
||||
Write-Step "Cleaning up disk"
|
||||
|
||||
# cleanmgr.exe requires an interactive window station — hangs in WinRM Session 0.
|
||||
# Replicate the relevant categories with direct PowerShell cleanup instead.
|
||||
Write-Host "Clearing memory dumps..."
|
||||
Remove-Item 'C:\Windows\Minidump\*' -Force -Recurse -ErrorAction SilentlyContinue
|
||||
Remove-Item 'C:\Windows\memory.dmp' -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Clearing Windows Error Reporting files..."
|
||||
Remove-Item 'C:\ProgramData\Microsoft\Windows\WER\*' -Force -Recurse -ErrorAction SilentlyContinue
|
||||
Write-Host "Clearing Recycle Bin..."
|
||||
Clear-RecycleBin -DriveLetter C -Force -ErrorAction SilentlyContinue
|
||||
|
||||
Write-Host "Stopping WU-adjacent services before cache clear..."
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc', 'bits', 'dosvc', 'TrustedInstaller') {
|
||||
Stop-Service -Name $svc -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
$sdDownload = 'C:\Windows\SoftwareDistribution\Download'
|
||||
Remove-Item $sdDownload -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $sdDownload -Force | Out-Null
|
||||
Write-Host "Windows Update download cache cleared."
|
||||
|
||||
Remove-Item 'C:\Windows\Logs\CBS\*' -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item 'C:\Windows\Temp\*' -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item 'C:\Windows\Prefetch\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
Write-Host "Running DISM component store cleanup (StartComponentCleanup)..."
|
||||
$dism = Start-Process -FilePath 'dism.exe' `
|
||||
-ArgumentList '/Online','/Cleanup-Image','/StartComponentCleanup','/ResetBase' `
|
||||
-Wait -PassThru
|
||||
Write-Host "DISM exited (code $($dism.ExitCode))."
|
||||
Write-Host "Disk cleanup complete."
|
||||
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc') {
|
||||
Set-Service -Name $svc -StartupType Disabled -ErrorAction SilentlyContinue
|
||||
}
|
||||
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Services\wuauserv' `
|
||||
-Name Start -Value 4 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
|
||||
$sdFiles = Get-ChildItem 'C:\Windows\SoftwareDistribution\Download' -Recurse -File -Force -ErrorAction SilentlyContinue
|
||||
$sdBytes = [long]0
|
||||
if ($sdFiles) { foreach ($f in $sdFiles) { $sdBytes += $f.Length } }
|
||||
$sdMB = [math]::Round($sdBytes / 1MB, 1)
|
||||
Write-Host " SoftwareDistribution\Download: $(@($sdFiles).Count) file(s), ${sdMB} MB remaining (WaaSMedicSvc best-effort, WU is disabled)."
|
||||
|
||||
Assert-Step 'Cleanup' 'wuauserv StartType Disabled after cache clear' {
|
||||
(Get-Service wuauserv -ErrorAction Stop).StartType -eq 'Disabled'
|
||||
}
|
||||
}
|
||||
|
||||
# ── Pinned SHA256 hashes for downloaded installers (§1.3) ─────────────────────
|
||||
# Fill in the hash values below. Get them by:
|
||||
# (Invoke-WebRequest '<URL>' -UseBasicParsing -OutFile 'tmp.bin'; (Get-FileHash 'tmp.bin' -Algorithm SHA256).Hash)
|
||||
@@ -272,6 +328,18 @@ $script:Hashes = @{
|
||||
# vcpkg: cloned from GitHub via TLS; pin integrity with $VcpkgRef (commit SHA)
|
||||
}
|
||||
|
||||
# ── CleanupOnly early-exit ────────────────────────────────────────────────────
|
||||
if ($CleanupOnly) {
|
||||
if ($SkipCleanup) {
|
||||
Write-Host "[Setup] -CleanupOnly + -SkipCleanup: nothing to do." -ForegroundColor Yellow
|
||||
exit 0
|
||||
}
|
||||
Write-Host "[Setup] CleanupOnly mode — running disk cleanup on fully-updated system." -ForegroundColor Cyan
|
||||
Invoke-DiskCleanup
|
||||
Write-Host "[Setup] CleanupOnly complete." -ForegroundColor Green
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Pre-flight: remove temp files from any previous partial run ───────────────
|
||||
# These files are normally deleted at the end of each step that creates them.
|
||||
# A previous interrupted run may have left them behind — remove before starting.
|
||||
@@ -318,8 +386,10 @@ Assert-Step 'Defender' 'GPO DisableAntiSpyware=1 (set by Deploy-WinBuild2022)' {
|
||||
Write-Step "WinRM state (validation only — configured at deploy time)"
|
||||
|
||||
# Deploy-WinBuild2022.ps1 post-install.ps1 ran Enable-PSRemoting, set Basic auth over HTTPS,
|
||||
# AllowUnencrypted=false (HTTPS/5986 only), TrustedHosts=*, MaxMemory=2048MB, IdleTimeout=2h,
|
||||
# MaxProcesses=25, StartupType=Automatic. Validated here before the build user and toolchain steps.
|
||||
# AllowUnencrypted=false (HTTPS/5986 only), TrustedHosts=*, Shell+Plugin MaxMemory=2048MB,
|
||||
# IdleTimeout=2h, MaxProcesses=100, StartupType=Automatic, and bumped the non-interactive
|
||||
# desktop heap (SharedSection 3rd field) to 4096 KB. Validated here before the build user
|
||||
# and toolchain steps.
|
||||
|
||||
Assert-Step 'WinRM' 'service Running' {
|
||||
(Get-Service WinRM -ErrorAction Stop).Status -eq 'Running'
|
||||
@@ -333,9 +403,28 @@ Assert-Step 'WinRM' 'AllowUnencrypted=false (HTTPS-only)' {
|
||||
Assert-Step 'WinRM' 'Auth/Basic=true' {
|
||||
(Get-Item WSMan:\localhost\Service\Auth\Basic).Value -eq 'true'
|
||||
}
|
||||
Assert-Step 'WinRM' 'MaxMemoryPerShellMB >= 2048' {
|
||||
Assert-Step 'WinRM' 'Shell MaxMemoryPerShellMB >= 2048' {
|
||||
[int](Get-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB).Value -ge 2048
|
||||
}
|
||||
Assert-Step 'WinRM' 'Plugin Microsoft.PowerShell MaxMemoryPerShellMB >= 2048' {
|
||||
[int](Get-Item 'WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB').Value -ge 2048
|
||||
}
|
||||
if ([int](Get-Item WSMan:\localhost\Shell\MaxProcessesPerShell -ErrorAction SilentlyContinue).Value -lt 100) {
|
||||
Set-Item WSMan:\localhost\Shell\MaxProcessesPerShell 100 -Force
|
||||
}
|
||||
Assert-Step 'WinRM' 'MaxProcessesPerShell >= 100' {
|
||||
[int](Get-Item WSMan:\localhost\Shell\MaxProcessesPerShell).Value -ge 100
|
||||
}
|
||||
$_subsysKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems'
|
||||
$_subsysCur = (Get-ItemProperty $_subsysKey -Name Windows).Windows
|
||||
if (-not ($_subsysCur -match 'SharedSection=\d+,\d+,(\d+)') -or [int]$Matches[1] -lt 4096) {
|
||||
$subsysNew = $_subsysCur -replace 'SharedSection=(\d+),(\d+),\d+', 'SharedSection=$1,$2,4096'
|
||||
Set-ItemProperty $_subsysKey -Name Windows -Value $subsysNew
|
||||
}
|
||||
Assert-Step 'DesktopHeap' 'SharedSection non-interactive >= 4096 KB' {
|
||||
$w = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems' -Name Windows).Windows
|
||||
if ($w -match 'SharedSection=\d+,\d+,(\d+)') { [int]$Matches[1] -ge 4096 } else { $false }
|
||||
}
|
||||
|
||||
# ── Step 3b: Windows KMS Activation ─────────────────────────────────────────
|
||||
# Must run after network is confirmed (Step 3) and before Windows Update (Step 6)
|
||||
@@ -1479,11 +1568,15 @@ if (Test-Path $vcpkgExe) {
|
||||
$cloneArgs += '--depth'
|
||||
$cloneArgs += '1'
|
||||
}
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
& $gitBin.Source @cloneArgs 2>&1 | Write-Host
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($LASTEXITCODE -ne 0) { throw "vcpkg git clone failed (exit $LASTEXITCODE)." }
|
||||
if ($VcpkgRef -ne '') {
|
||||
Write-Host "Checking out vcpkg ref: $VcpkgRef"
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
& $gitBin.Source -C $vcpkgDir checkout $VcpkgRef 2>&1 | Write-Host
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($LASTEXITCODE -ne 0) { throw "vcpkg checkout '$VcpkgRef' failed (exit $LASTEXITCODE)." }
|
||||
}
|
||||
Write-Host "Bootstrapping vcpkg (-disableMetrics)..."
|
||||
@@ -1515,114 +1608,9 @@ Write-Host "Tier-2 Toolchain installation complete." -ForegroundColor Green
|
||||
# ── Cleanup ───────────────────────────────────────────────────────────────────
|
||||
if ($SkipCleanup) {
|
||||
Write-Host "`n[Setup] Skipping disk cleanup (-SkipCleanup switch set)." -ForegroundColor Yellow
|
||||
} else {
|
||||
Invoke-DiskCleanup
|
||||
}
|
||||
else {
|
||||
Write-Step "Cleaning up disk"
|
||||
|
||||
# 1. Windows Disk Cleanup (cleanmgr) — run with /sagerun:1 using preset flags
|
||||
# First register the flags via /sageset:1 in the registry (silent, no UI)
|
||||
$cleanKey = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\VolumeCaches'
|
||||
$sageset = 1
|
||||
$cleanCategories = @(
|
||||
'Active Setup Temp Folders',
|
||||
'BranchCache',
|
||||
'Downloaded Program Files',
|
||||
'Internet Cache Files',
|
||||
'Memory Dump Files',
|
||||
'Old ChkDsk Files',
|
||||
'Previous Installations',
|
||||
'Recycle Bin',
|
||||
'Service Pack Cleanup',
|
||||
'Setup Log Files',
|
||||
'System error memory dump files',
|
||||
'System error minidump files',
|
||||
'Temporary Files',
|
||||
'Temporary Setup Files',
|
||||
'Thumbnail Cache',
|
||||
'Update Cleanup',
|
||||
'Upgrade Discarded Files',
|
||||
'Windows Defender',
|
||||
'Windows Error Reporting Archive Files',
|
||||
'Windows Error Reporting Files',
|
||||
'Windows Error Reporting Queue Files',
|
||||
'Windows Error Reporting Temp Files',
|
||||
'Windows ESD installation files',
|
||||
'Windows Upgrade Log Files'
|
||||
)
|
||||
foreach ($cat in $cleanCategories) {
|
||||
$keyPath = "$cleanKey\$cat"
|
||||
if (Test-Path $keyPath) {
|
||||
Set-ItemProperty -Path $keyPath -Name "StateFlags$('{0:D4}' -f $sageset)" -Value 2 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
Write-Host "Running cleanmgr /sagerun:$sageset (may take a few minutes)..."
|
||||
$proc = Start-Process -FilePath 'cleanmgr.exe' -ArgumentList "/sagerun:$sageset" -Wait -PassThru
|
||||
Write-Host "cleanmgr exited (code $($proc.ExitCode))."
|
||||
|
||||
# 2. Clear Windows Update cache
|
||||
# Stop all services that might hold file locks inside SoftwareDistribution:
|
||||
# wuauserv — Windows Update (already Disabled from Step 6b, stop is idempotent)
|
||||
# UsoSvc — Update Orchestrator (already Disabled from Step 6b)
|
||||
# bits — Background Intelligent Transfer Service (downloads WU payloads)
|
||||
# dosvc — Delivery Optimization (peer-to-peer WU cache)
|
||||
# TrustedInstaller — Windows Modules Installer (unpacks .cab/.msu update packages)
|
||||
Write-Host "Stopping WU-adjacent services before cache clear..."
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc', 'bits', 'dosvc', 'TrustedInstaller') {
|
||||
Stop-Service -Name $svc -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# Delete and recreate the Download folder — more reliable than 'Remove-Item *'
|
||||
# because locked files inside subdirectories silently fail with wildcard delete.
|
||||
$sdDownload = 'C:\Windows\SoftwareDistribution\Download'
|
||||
Remove-Item $sdDownload -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $sdDownload -Force | Out-Null
|
||||
Write-Host "Windows Update download cache cleared."
|
||||
|
||||
# 3. Clear CBS / component store logs
|
||||
Remove-Item 'C:\Windows\Logs\CBS\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 4. Clear TEMP folders
|
||||
Remove-Item 'C:\Windows\Temp\*' -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 5. Clear Prefetch
|
||||
Remove-Item 'C:\Windows\Prefetch\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 6. Run DISM component store cleanup (removes superseded update components)
|
||||
Write-Host "Running DISM component store cleanup (StartComponentCleanup)..."
|
||||
$dism = Start-Process -FilePath 'dism.exe' `
|
||||
-ArgumentList '/Online','/Cleanup-Image','/StartComponentCleanup','/ResetBase' `
|
||||
-Wait -PassThru
|
||||
Write-Host "DISM exited (code $($dism.ExitCode))."
|
||||
|
||||
Write-Host "Disk cleanup complete."
|
||||
|
||||
# DISM StartComponentCleanup / WaaSMedicSvc may reset service StartType during cleanup.
|
||||
# Re-enforce Disabled on both WU services after DISM completes.
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc') {
|
||||
Set-Service -Name $svc -StartupType Disabled -ErrorAction SilentlyContinue
|
||||
}
|
||||
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Services\wuauserv' `
|
||||
-Name Start -Value 4 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# Validation — leftover artifacts from this run should be gone
|
||||
# Note: do NOT assert SoftwareDistribution\Download empty.
|
||||
# WaaSMedicSvc (Windows Update Medic Service) is tamper-protected — cannot be stopped —
|
||||
# and actively restores WU files immediately after deletion. Files reappearing here is
|
||||
# normal and harmless: WU is permanently disabled by Step 6b (service Disabled + GPO keys).
|
||||
# Report remaining size as informational only.
|
||||
$sdFiles = Get-ChildItem 'C:\Windows\SoftwareDistribution\Download' -Recurse -File -Force -ErrorAction SilentlyContinue
|
||||
# Avoid Measure-Object .Sum: under Set-StrictMode -Version Latest, PS 5.1 treats the
|
||||
# nullable Sum property as absent when the pipe is empty, throwing "property not found".
|
||||
$sdBytes = [long]0
|
||||
if ($sdFiles) { foreach ($f in $sdFiles) { $sdBytes += $f.Length } }
|
||||
$sdMB = [math]::Round($sdBytes / 1MB, 1)
|
||||
Write-Host " SoftwareDistribution\Download: $(@($sdFiles).Count) file(s), ${sdMB} MB remaining (WaaSMedicSvc best-effort, WU is disabled)."
|
||||
|
||||
Assert-Step 'Cleanup' 'wuauserv StartType Disabled after cache clear' {
|
||||
(Get-Service wuauserv -ErrorAction Stop).StartType -eq 'Disabled'
|
||||
}
|
||||
} # end if (-not $SkipCleanup)
|
||||
|
||||
# ── Pre-Final re-affirmation ──────────────────────────────────────────────────
|
||||
# Steps 7-10 (installs, cleanup, DISM) can take hours. During that window background
|
||||
|
||||
@@ -148,6 +148,11 @@ param(
|
||||
# Skip disk cleanup step (cleanmgr + DISM + cache clear) — speeds up re-runs during dev/debug
|
||||
[switch] $SkipCleanup,
|
||||
|
||||
# Run disk cleanup only — skip all provisioning steps and jump straight to the
|
||||
# cleanup section. Intended for use by Prepare-WinBuild2025.ps1 after the Windows
|
||||
# Update loop completes, so cleanup always runs on the fully-updated system.
|
||||
[switch] $CleanupOnly,
|
||||
|
||||
# Administrator password — used only to write DefaultPassword for autologin.
|
||||
# Optional: if empty, DefaultPassword registry value is left as-is (Deploy may have set it).
|
||||
[string] $AdminPassword = '',
|
||||
@@ -231,6 +236,67 @@ function Assert-Hash {
|
||||
Write-Host " [OK] SHA256 verified: $Label" -ForegroundColor Green
|
||||
}
|
||||
|
||||
function Invoke-DiskCleanup {
|
||||
Write-Step "Cleaning up disk"
|
||||
|
||||
# cleanmgr.exe requires an interactive window station — hangs in WinRM Session 0.
|
||||
# Replicate the relevant categories with direct PowerShell cleanup instead.
|
||||
Write-Host "Clearing memory dumps..."
|
||||
Remove-Item 'C:\Windows\Minidump\*' -Force -Recurse -ErrorAction SilentlyContinue
|
||||
Remove-Item 'C:\Windows\memory.dmp' -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "Clearing Windows Error Reporting files..."
|
||||
Remove-Item 'C:\ProgramData\Microsoft\Windows\WER\*' -Force -Recurse -ErrorAction SilentlyContinue
|
||||
Write-Host "Clearing Recycle Bin..."
|
||||
Clear-RecycleBin -DriveLetter C -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# Clear Windows Update cache
|
||||
Write-Host "Stopping WU-adjacent services before cache clear..."
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc', 'bits', 'dosvc', 'TrustedInstaller') {
|
||||
Stop-Service -Name $svc -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
$sdDownload = 'C:\Windows\SoftwareDistribution\Download'
|
||||
Remove-Item $sdDownload -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $sdDownload -Force | Out-Null
|
||||
Write-Host "Windows Update download cache cleared."
|
||||
|
||||
# 3. Clear CBS / component store logs
|
||||
Remove-Item 'C:\Windows\Logs\CBS\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 4. Clear TEMP folders
|
||||
Remove-Item 'C:\Windows\Temp\*' -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 5. Clear Prefetch
|
||||
Remove-Item 'C:\Windows\Prefetch\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 6. Run DISM component store cleanup (removes superseded update components)
|
||||
Write-Host "Running DISM component store cleanup (StartComponentCleanup)..."
|
||||
$dism = Start-Process -FilePath 'dism.exe' `
|
||||
-ArgumentList '/Online','/Cleanup-Image','/StartComponentCleanup','/ResetBase' `
|
||||
-Wait -PassThru
|
||||
Write-Host "DISM exited (code $($dism.ExitCode))."
|
||||
|
||||
Write-Host "Disk cleanup complete."
|
||||
|
||||
# DISM / WaaSMedicSvc may reset WU service StartType — re-enforce Disabled.
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc') {
|
||||
Set-Service -Name $svc -StartupType Disabled -ErrorAction SilentlyContinue
|
||||
}
|
||||
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Services\wuauserv' `
|
||||
-Name Start -Value 4 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# Report remaining WU cache size (WaaSMedicSvc restores files — informational only).
|
||||
$sdFiles = Get-ChildItem 'C:\Windows\SoftwareDistribution\Download' -Recurse -File -Force -ErrorAction SilentlyContinue
|
||||
$sdBytes = [long]0
|
||||
if ($sdFiles) { foreach ($f in $sdFiles) { $sdBytes += $f.Length } }
|
||||
$sdMB = [math]::Round($sdBytes / 1MB, 1)
|
||||
Write-Host " SoftwareDistribution\Download: $(@($sdFiles).Count) file(s), ${sdMB} MB remaining (WaaSMedicSvc best-effort, WU is disabled)."
|
||||
|
||||
Assert-Step 'Cleanup' 'wuauserv StartType Disabled after cache clear' {
|
||||
(Get-Service wuauserv -ErrorAction Stop).StartType -eq 'Disabled'
|
||||
}
|
||||
}
|
||||
|
||||
# ── Pinned SHA256 hashes for downloaded installers (§1.3) ─────────────────────
|
||||
# Fill in the hash values below. Get them by:
|
||||
# (Invoke-WebRequest '<URL>' -UseBasicParsing -OutFile 'tmp.bin'; (Get-FileHash 'tmp.bin' -Algorithm SHA256).Hash)
|
||||
@@ -272,6 +338,20 @@ $script:Hashes = @{
|
||||
# vcpkg: cloned from GitHub via TLS; pin integrity with $VcpkgRef (commit SHA)
|
||||
}
|
||||
|
||||
# ── CleanupOnly early-exit ────────────────────────────────────────────────────
|
||||
# Called by Prepare-WinBuild2025.ps1 after the Windows Update loop completes.
|
||||
# Skips all provisioning steps; runs only disk cleanup then exits.
|
||||
if ($CleanupOnly) {
|
||||
if ($SkipCleanup) {
|
||||
Write-Host "[Setup] -CleanupOnly + -SkipCleanup: nothing to do." -ForegroundColor Yellow
|
||||
exit 0
|
||||
}
|
||||
Write-Host "[Setup] CleanupOnly mode — running disk cleanup on fully-updated system." -ForegroundColor Cyan
|
||||
Invoke-DiskCleanup
|
||||
Write-Host "[Setup] CleanupOnly complete." -ForegroundColor Green
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Pre-flight: remove temp files from any previous partial run ───────────────
|
||||
# These files are normally deleted at the end of each step that creates them.
|
||||
# A previous interrupted run may have left them behind — remove before starting.
|
||||
@@ -318,8 +398,10 @@ Assert-Step 'Defender' 'GPO DisableAntiSpyware=1 (set by Deploy-WinBuild2025)' {
|
||||
Write-Step "WinRM state (validation only — configured at deploy time)"
|
||||
|
||||
# Deploy-WinBuild2025.ps1 post-install.ps1 ran Enable-PSRemoting, set Basic auth over HTTPS,
|
||||
# AllowUnencrypted=false (HTTPS/5986 only), TrustedHosts=*, MaxMemory=2048MB, IdleTimeout=2h,
|
||||
# MaxProcesses=25, StartupType=Automatic. Validated here before the build user and toolchain steps.
|
||||
# AllowUnencrypted=false (HTTPS/5986 only), TrustedHosts=*, Shell+Plugin MaxMemory=2048MB,
|
||||
# IdleTimeout=2h, MaxProcesses=100, StartupType=Automatic, and bumped the non-interactive
|
||||
# desktop heap (SharedSection 3rd field) to 4096 KB. Validated here before the build user
|
||||
# and toolchain steps.
|
||||
|
||||
Assert-Step 'WinRM' 'service Running' {
|
||||
(Get-Service WinRM -ErrorAction Stop).Status -eq 'Running'
|
||||
@@ -333,9 +415,28 @@ Assert-Step 'WinRM' 'AllowUnencrypted=false (HTTPS-only)' {
|
||||
Assert-Step 'WinRM' 'Auth/Basic=true' {
|
||||
(Get-Item WSMan:\localhost\Service\Auth\Basic).Value -eq 'true'
|
||||
}
|
||||
Assert-Step 'WinRM' 'MaxMemoryPerShellMB >= 2048' {
|
||||
Assert-Step 'WinRM' 'Shell MaxMemoryPerShellMB >= 2048' {
|
||||
[int](Get-Item WSMan:\localhost\Shell\MaxMemoryPerShellMB).Value -ge 2048
|
||||
}
|
||||
Assert-Step 'WinRM' 'Plugin Microsoft.PowerShell MaxMemoryPerShellMB >= 2048' {
|
||||
[int](Get-Item 'WSMan:\localhost\Plugin\Microsoft.PowerShell\Quotas\MaxMemoryPerShellMB').Value -ge 2048
|
||||
}
|
||||
if ([int](Get-Item WSMan:\localhost\Shell\MaxProcessesPerShell -ErrorAction SilentlyContinue).Value -lt 100) {
|
||||
Set-Item WSMan:\localhost\Shell\MaxProcessesPerShell 100 -Force
|
||||
}
|
||||
Assert-Step 'WinRM' 'MaxProcessesPerShell >= 100' {
|
||||
[int](Get-Item WSMan:\localhost\Shell\MaxProcessesPerShell).Value -ge 100
|
||||
}
|
||||
$_subsysKey = 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems'
|
||||
$_subsysCur = (Get-ItemProperty $_subsysKey -Name Windows).Windows
|
||||
if (-not ($_subsysCur -match 'SharedSection=\d+,\d+,(\d+)') -or [int]$Matches[1] -lt 4096) {
|
||||
$subsysNew = $_subsysCur -replace 'SharedSection=(\d+),(\d+),\d+', 'SharedSection=$1,$2,4096'
|
||||
Set-ItemProperty $_subsysKey -Name Windows -Value $subsysNew
|
||||
}
|
||||
Assert-Step 'DesktopHeap' 'SharedSection non-interactive >= 4096 KB' {
|
||||
$w = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\SubSystems' -Name Windows).Windows
|
||||
if ($w -match 'SharedSection=\d+,\d+,(\d+)') { [int]$Matches[1] -ge 4096 } else { $false }
|
||||
}
|
||||
|
||||
# ── Step 3b: Windows KMS Activation ─────────────────────────────────────────
|
||||
# Must run after network is confirmed (Step 3) and before Windows Update (Step 6)
|
||||
@@ -1479,11 +1580,15 @@ if (Test-Path $vcpkgExe) {
|
||||
$cloneArgs += '--depth'
|
||||
$cloneArgs += '1'
|
||||
}
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
& $gitBin.Source @cloneArgs 2>&1 | Write-Host
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($LASTEXITCODE -ne 0) { throw "vcpkg git clone failed (exit $LASTEXITCODE)." }
|
||||
if ($VcpkgRef -ne '') {
|
||||
Write-Host "Checking out vcpkg ref: $VcpkgRef"
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
& $gitBin.Source -C $vcpkgDir checkout $VcpkgRef 2>&1 | Write-Host
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($LASTEXITCODE -ne 0) { throw "vcpkg checkout '$VcpkgRef' failed (exit $LASTEXITCODE)." }
|
||||
}
|
||||
Write-Host "Bootstrapping vcpkg (-disableMetrics)..."
|
||||
@@ -1515,114 +1620,9 @@ Write-Host "Tier-2 Toolchain installation complete." -ForegroundColor Green
|
||||
# ── Cleanup ───────────────────────────────────────────────────────────────────
|
||||
if ($SkipCleanup) {
|
||||
Write-Host "`n[Setup] Skipping disk cleanup (-SkipCleanup switch set)." -ForegroundColor Yellow
|
||||
} else {
|
||||
Invoke-DiskCleanup
|
||||
}
|
||||
else {
|
||||
Write-Step "Cleaning up disk"
|
||||
|
||||
# 1. Windows Disk Cleanup (cleanmgr) — run with /sagerun:1 using preset flags
|
||||
# First register the flags via /sageset:1 in the registry (silent, no UI)
|
||||
$cleanKey = 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\VolumeCaches'
|
||||
$sageset = 1
|
||||
$cleanCategories = @(
|
||||
'Active Setup Temp Folders',
|
||||
'BranchCache',
|
||||
'Downloaded Program Files',
|
||||
'Internet Cache Files',
|
||||
'Memory Dump Files',
|
||||
'Old ChkDsk Files',
|
||||
'Previous Installations',
|
||||
'Recycle Bin',
|
||||
'Service Pack Cleanup',
|
||||
'Setup Log Files',
|
||||
'System error memory dump files',
|
||||
'System error minidump files',
|
||||
'Temporary Files',
|
||||
'Temporary Setup Files',
|
||||
'Thumbnail Cache',
|
||||
'Update Cleanup',
|
||||
'Upgrade Discarded Files',
|
||||
'Windows Defender',
|
||||
'Windows Error Reporting Archive Files',
|
||||
'Windows Error Reporting Files',
|
||||
'Windows Error Reporting Queue Files',
|
||||
'Windows Error Reporting Temp Files',
|
||||
'Windows ESD installation files',
|
||||
'Windows Upgrade Log Files'
|
||||
)
|
||||
foreach ($cat in $cleanCategories) {
|
||||
$keyPath = "$cleanKey\$cat"
|
||||
if (Test-Path $keyPath) {
|
||||
Set-ItemProperty -Path $keyPath -Name "StateFlags$('{0:D4}' -f $sageset)" -Value 2 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
Write-Host "Running cleanmgr /sagerun:$sageset (may take a few minutes)..."
|
||||
$proc = Start-Process -FilePath 'cleanmgr.exe' -ArgumentList "/sagerun:$sageset" -Wait -PassThru
|
||||
Write-Host "cleanmgr exited (code $($proc.ExitCode))."
|
||||
|
||||
# 2. Clear Windows Update cache
|
||||
# Stop all services that might hold file locks inside SoftwareDistribution:
|
||||
# wuauserv — Windows Update (already Disabled from Step 6b, stop is idempotent)
|
||||
# UsoSvc — Update Orchestrator (already Disabled from Step 6b)
|
||||
# bits — Background Intelligent Transfer Service (downloads WU payloads)
|
||||
# dosvc — Delivery Optimization (peer-to-peer WU cache)
|
||||
# TrustedInstaller — Windows Modules Installer (unpacks .cab/.msu update packages)
|
||||
Write-Host "Stopping WU-adjacent services before cache clear..."
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc', 'bits', 'dosvc', 'TrustedInstaller') {
|
||||
Stop-Service -Name $svc -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# Delete and recreate the Download folder — more reliable than 'Remove-Item *'
|
||||
# because locked files inside subdirectories silently fail with wildcard delete.
|
||||
$sdDownload = 'C:\Windows\SoftwareDistribution\Download'
|
||||
Remove-Item $sdDownload -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $sdDownload -Force | Out-Null
|
||||
Write-Host "Windows Update download cache cleared."
|
||||
|
||||
# 3. Clear CBS / component store logs
|
||||
Remove-Item 'C:\Windows\Logs\CBS\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 4. Clear TEMP folders
|
||||
Remove-Item 'C:\Windows\Temp\*' -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 5. Clear Prefetch
|
||||
Remove-Item 'C:\Windows\Prefetch\*' -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# 6. Run DISM component store cleanup (removes superseded update components)
|
||||
Write-Host "Running DISM component store cleanup (StartComponentCleanup)..."
|
||||
$dism = Start-Process -FilePath 'dism.exe' `
|
||||
-ArgumentList '/Online','/Cleanup-Image','/StartComponentCleanup','/ResetBase' `
|
||||
-Wait -PassThru
|
||||
Write-Host "DISM exited (code $($dism.ExitCode))."
|
||||
|
||||
Write-Host "Disk cleanup complete."
|
||||
|
||||
# DISM StartComponentCleanup / WaaSMedicSvc may reset service StartType during cleanup.
|
||||
# Re-enforce Disabled on both WU services after DISM completes.
|
||||
foreach ($svc in 'wuauserv', 'UsoSvc') {
|
||||
Set-Service -Name $svc -StartupType Disabled -ErrorAction SilentlyContinue
|
||||
}
|
||||
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Services\wuauserv' `
|
||||
-Name Start -Value 4 -Type DWord -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# Validation — leftover artifacts from this run should be gone
|
||||
# Note: do NOT assert SoftwareDistribution\Download empty.
|
||||
# WaaSMedicSvc (Windows Update Medic Service) is tamper-protected — cannot be stopped —
|
||||
# and actively restores WU files immediately after deletion. Files reappearing here is
|
||||
# normal and harmless: WU is permanently disabled by Step 6b (service Disabled + GPO keys).
|
||||
# Report remaining size as informational only.
|
||||
$sdFiles = Get-ChildItem 'C:\Windows\SoftwareDistribution\Download' -Recurse -File -Force -ErrorAction SilentlyContinue
|
||||
# Avoid Measure-Object .Sum: under Set-StrictMode -Version Latest, PS 5.1 treats the
|
||||
# nullable Sum property as absent when the pipe is empty, throwing "property not found".
|
||||
$sdBytes = [long]0
|
||||
if ($sdFiles) { foreach ($f in $sdFiles) { $sdBytes += $f.Length } }
|
||||
$sdMB = [math]::Round($sdBytes / 1MB, 1)
|
||||
Write-Host " SoftwareDistribution\Download: $(@($sdFiles).Count) file(s), ${sdMB} MB remaining (WaaSMedicSvc best-effort, WU is disabled)."
|
||||
|
||||
Assert-Step 'Cleanup' 'wuauserv StartType Disabled after cache clear' {
|
||||
(Get-Service wuauserv -ErrorAction Stop).StartType -eq 'Disabled'
|
||||
}
|
||||
} # end if (-not $SkipCleanup)
|
||||
|
||||
# ── Pre-Final re-affirmation ──────────────────────────────────────────────────
|
||||
# Steps 7-10 (installs, cleanup, DISM) can take hours. During that window background
|
||||
|
||||
@@ -163,7 +163,25 @@ param(
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (IWR, etc.)
|
||||
|
||||
# On Linux, PSWSMan must be imported to register the native WSMan client library.
|
||||
# Install once: sudo pwsh -c "Install-Module PSWSMan -Force; Install-WSMan"
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
Import-Module PSWSMan -ErrorAction Stop
|
||||
}
|
||||
|
||||
function Test-TcpPort {
|
||||
param([string]$ComputerName, [int]$Port, [int]$TimeoutMs = 3000)
|
||||
try {
|
||||
$tcp = [System.Net.Sockets.TcpClient]::new()
|
||||
$ar = $tcp.BeginConnect($ComputerName, $Port, $null, $null)
|
||||
$ok = $ar.AsyncWaitHandle.WaitOne($TimeoutMs, $false)
|
||||
$connected = $ok -and $tcp.Connected
|
||||
$tcp.Close()
|
||||
return $connected
|
||||
} catch { return $false }
|
||||
}
|
||||
|
||||
# ── Locate vmrun.exe ──────────────────────────────────────────────────────────
|
||||
$vmrunCandidates = @(
|
||||
@@ -247,8 +265,7 @@ Write-Host "[Prepare] Waiting for WinRM on $VMIPAddress`:5986 (timeout ${winrmTi
|
||||
$winrmDeadline = [datetime]::UtcNow.AddSeconds($winrmTimeoutSec)
|
||||
$winrmReady = $false
|
||||
while ([datetime]::UtcNow -lt $winrmDeadline) {
|
||||
if (Test-NetConnection -ComputerName $VMIPAddress -Port 5986 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue -ErrorAction SilentlyContinue) {
|
||||
if (Test-TcpPort -ComputerName $VMIPAddress -Port 5986 -TimeoutMs 5000) {
|
||||
$winrmReady = $true; break
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
@@ -275,31 +292,38 @@ else {
|
||||
$credential = New-Object System.Management.Automation.PSCredential($AdminUsername, $secPwd)
|
||||
}
|
||||
|
||||
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck
|
||||
|
||||
# ── Configure HOST-side WinRM client (TrustedHosts for HTTPS Basic auth) ──────
|
||||
# HTTPS/5986 does not require AllowUnencrypted on the host side. Only TrustedHosts
|
||||
# needs appending so PS accepts the non-domain VM. Restored in the finally block.
|
||||
# Windows only: TrustedHosts must include the VM IP so PS accepts the non-domain
|
||||
# VM over HTTPS. On Linux, New-PSSession uses OMI and -SkipCACheck/-SkipCNCheck
|
||||
# in sessionOptions are sufficient — TrustedHosts does not apply.
|
||||
$prevTrustedHosts = $null
|
||||
try {
|
||||
$prevTrustedHosts = (Get-Item WSMan:\localhost\Client\TrustedHosts -ErrorAction Stop).Value
|
||||
# Add VM IP to TrustedHosts if not already present
|
||||
if ($prevTrustedHosts -ne '*' -and $prevTrustedHosts -notlike "*$VMIPAddress*") {
|
||||
$newTrusted = if ($prevTrustedHosts -eq '') { $VMIPAddress } else { "$prevTrustedHosts,$VMIPAddress" }
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $newTrusted -Force -ErrorAction Stop
|
||||
$isLinuxHost = ($null -ne $IsWindows -and $IsWindows -eq $false)
|
||||
if (-not $isLinuxHost) {
|
||||
try {
|
||||
$prevTrustedHosts = (Get-Item WSMan:\localhost\Client\TrustedHosts -ErrorAction Stop).Value
|
||||
# Add VM IP to TrustedHosts if not already present
|
||||
if ($prevTrustedHosts -ne '*' -and $prevTrustedHosts -notlike "*$VMIPAddress*") {
|
||||
$newTrusted = if ($prevTrustedHosts -eq '') { $VMIPAddress } else { "$prevTrustedHosts,$VMIPAddress" }
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $newTrusted -Force -ErrorAction Stop
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts configured."
|
||||
Assert-Step 'HostWinRM' "TrustedHosts includes $VMIPAddress (or '*')" {
|
||||
$th = (Get-Item WSMan:\localhost\Client\TrustedHosts).Value
|
||||
$th -eq '*' -or $th -like "*$VMIPAddress*"
|
||||
}
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts configured."
|
||||
Assert-Step 'HostWinRM' "TrustedHosts includes $VMIPAddress (or '*')" {
|
||||
$th = (Get-Item WSMan:\localhost\Client\TrustedHosts).Value
|
||||
$th -eq '*' -or $th -like "*$VMIPAddress*"
|
||||
catch {
|
||||
Write-Warning "[Prepare] Could not configure host WinRM TrustedHosts (need elevation?)."
|
||||
Write-Warning "Run once in an elevated PowerShell on the HOST:"
|
||||
Write-Warning " Set-Item WSMan:\localhost\Client\TrustedHosts -Value '*' -Force"
|
||||
Write-Warning "Then re-run this script."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Warning "[Prepare] Could not configure host WinRM TrustedHosts (need elevation?)."
|
||||
Write-Warning "Run once in an elevated PowerShell on the HOST:"
|
||||
Write-Warning " Set-Item WSMan:\localhost\Client\TrustedHosts -Value '*' -Force"
|
||||
Write-Warning "Then re-run this script."
|
||||
exit 1
|
||||
else {
|
||||
Write-Host "[Prepare] Linux host — TrustedHosts not applicable, skipping."
|
||||
}
|
||||
|
||||
|
||||
@@ -497,6 +521,29 @@ try {
|
||||
-Authentication Basic -ErrorAction Stop
|
||||
}
|
||||
|
||||
# ── Install CI-StaticIp scheduled task (ip_pool guestinfo static IP) ────────
|
||||
$staticIpScriptPath = Join-Path $scriptDir 'guest-setup\windows\ci-static-ip.ps1'
|
||||
if (Test-Path $staticIpScriptPath -PathType Leaf) {
|
||||
Write-Host "[Prepare] Installing CI-StaticIp scheduled task on guest..."
|
||||
$staticIpContent = Get-Content -LiteralPath $staticIpScriptPath -Raw
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param([string]$ScriptContent)
|
||||
Set-Content -LiteralPath 'C:\CI\ci-static-ip.ps1' -Value $ScriptContent `
|
||||
-Encoding UTF8 -Force
|
||||
$action = New-ScheduledTaskAction `
|
||||
-Execute 'powershell.exe' `
|
||||
-Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1'
|
||||
$trigger = New-ScheduledTaskTrigger -AtStartup
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
|
||||
-LogonType ServiceAccount -RunLevel Highest
|
||||
Register-ScheduledTask -TaskName 'CI-StaticIp' -Action $action `
|
||||
-Trigger $trigger -Principal $principal -Force | Out-Null
|
||||
} -ArgumentList $staticIpContent
|
||||
Write-Host "[Prepare] CI-StaticIp task registered." -ForegroundColor Green
|
||||
} else {
|
||||
Write-Warning "[Prepare] guest-setup\windows\ci-static-ip.ps1 not found alongside this script — skipping CI-StaticIp task. ip_pool feature will not work for this template."
|
||||
}
|
||||
|
||||
# ── Post-setup remote validation ──────────────────────────────────────────
|
||||
Write-Host "[Prepare] Running post-setup validation against guest..."
|
||||
$guestState = Invoke-Command -Session $session -ScriptBlock {
|
||||
@@ -505,15 +552,17 @@ try {
|
||||
$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + ';' +
|
||||
[System.Environment]::GetEnvironmentVariable('PATH','User')
|
||||
[PSCustomObject]@{
|
||||
WinRMRunning = (Get-Service WinRM -ErrorAction SilentlyContinue).Status -eq 'Running'
|
||||
UserExists = [bool](Get-LocalUser -Name $BuildUser -ErrorAction SilentlyContinue)
|
||||
UserIsAdmin = [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUser)
|
||||
UACDisabled = (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' -Name EnableLUA -ErrorAction SilentlyContinue).EnableLUA -eq 0
|
||||
FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)
|
||||
DotNetVersion = (Get-Command dotnet -ErrorAction SilentlyContinue) | ForEach-Object { (& $_.Source --version 2>&1) }
|
||||
PythonExists = Test-Path 'C:\Python\python.exe' -PathType Leaf
|
||||
MSBuildExists = Test-Path $msbuildPath -PathType Leaf
|
||||
CIDirsPresent = (Test-Path 'C:\CI\build') -and (Test-Path 'C:\CI\output') -and (Test-Path 'C:\CI\scripts')
|
||||
WinRMRunning = (Get-Service WinRM -ErrorAction SilentlyContinue).Status -eq 'Running'
|
||||
UserExists = [bool](Get-LocalUser -Name $BuildUser -ErrorAction SilentlyContinue)
|
||||
UserIsAdmin = [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUser)
|
||||
UACDisabled = (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' -Name EnableLUA -ErrorAction SilentlyContinue).EnableLUA -eq 0
|
||||
FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)
|
||||
DotNetVersion = (Get-Command dotnet -ErrorAction SilentlyContinue) | ForEach-Object { (& $_.Source --version 2>&1) }
|
||||
PythonExists = Test-Path 'C:\Python\python.exe' -PathType Leaf
|
||||
MSBuildExists = Test-Path $msbuildPath -PathType Leaf
|
||||
CIDirsPresent = (Test-Path 'C:\CI\build') -and (Test-Path 'C:\CI\output') -and (Test-Path 'C:\CI\scripts')
|
||||
StaticIpTask = [bool](Get-ScheduledTask -TaskName 'CI-StaticIp' -ErrorAction SilentlyContinue)
|
||||
StaticIpScript = Test-Path 'C:\CI\ci-static-ip.ps1' -PathType Leaf
|
||||
}
|
||||
} -ArgumentList $BuildUsername
|
||||
|
||||
@@ -526,6 +575,8 @@ try {
|
||||
Assert-Step 'PostSetup' 'guest Python installed' { $guestState.PythonExists }
|
||||
Assert-Step 'PostSetup' 'guest MSBuild present' { $guestState.MSBuildExists }
|
||||
Assert-Step 'PostSetup' 'guest C:\CI\{build,output,scripts} present' { $guestState.CIDirsPresent }
|
||||
Assert-Step 'PostSetup' 'guest CI-StaticIp task registered' { $guestState.StaticIpTask }
|
||||
Assert-Step 'PostSetup' 'guest C:\CI\ci-static-ip.ps1 present' { $guestState.StaticIpScript }
|
||||
|
||||
Write-Host "[Prepare] All post-setup validations passed (.NET $($guestState.DotNetVersion))." -ForegroundColor Green
|
||||
|
||||
@@ -717,11 +768,11 @@ try {
|
||||
finally {
|
||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||
|
||||
# Restore host TrustedHosts to its original value
|
||||
if ($null -ne $prevTrustedHosts) {
|
||||
# Restore host TrustedHosts to its original value (Windows only)
|
||||
if (-not $isLinuxHost -and $null -ne $prevTrustedHosts) {
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $prevTrustedHosts -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -163,7 +163,25 @@ param(
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (IWR, etc.)
|
||||
|
||||
# On Linux, PSWSMan must be imported to register the native WSMan client library.
|
||||
# Install once: sudo pwsh -c "Install-Module PSWSMan -Force; Install-WSMan"
|
||||
if ($null -ne $IsWindows -and $IsWindows -eq $false) {
|
||||
Import-Module PSWSMan -ErrorAction Stop
|
||||
}
|
||||
|
||||
function Test-TcpPort {
|
||||
param([string]$ComputerName, [int]$Port, [int]$TimeoutMs = 3000)
|
||||
try {
|
||||
$tcp = [System.Net.Sockets.TcpClient]::new()
|
||||
$ar = $tcp.BeginConnect($ComputerName, $Port, $null, $null)
|
||||
$ok = $ar.AsyncWaitHandle.WaitOne($TimeoutMs, $false)
|
||||
$connected = $ok -and $tcp.Connected
|
||||
$tcp.Close()
|
||||
return $connected
|
||||
} catch { return $false }
|
||||
}
|
||||
|
||||
# ── Locate vmrun.exe ──────────────────────────────────────────────────────────
|
||||
$vmrunCandidates = @(
|
||||
@@ -247,8 +265,7 @@ Write-Host "[Prepare] Waiting for WinRM on $VMIPAddress`:5986 (timeout ${winrmTi
|
||||
$winrmDeadline = [datetime]::UtcNow.AddSeconds($winrmTimeoutSec)
|
||||
$winrmReady = $false
|
||||
while ([datetime]::UtcNow -lt $winrmDeadline) {
|
||||
if (Test-NetConnection -ComputerName $VMIPAddress -Port 5986 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue -ErrorAction SilentlyContinue) {
|
||||
if (Test-TcpPort -ComputerName $VMIPAddress -Port 5986 -TimeoutMs 5000) {
|
||||
$winrmReady = $true; break
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
@@ -275,31 +292,38 @@ else {
|
||||
$credential = New-Object System.Management.Automation.PSCredential($AdminUsername, $secPwd)
|
||||
}
|
||||
|
||||
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck
|
||||
|
||||
# ── Configure HOST-side WinRM client (TrustedHosts for HTTPS Basic auth) ──────
|
||||
# HTTPS/5986 does not require AllowUnencrypted on the host side. Only TrustedHosts
|
||||
# needs appending so PS accepts the non-domain VM. Restored in the finally block.
|
||||
# Windows only: TrustedHosts must include the VM IP so PS accepts the non-domain
|
||||
# VM over HTTPS. On Linux, New-PSSession uses OMI and -SkipCACheck/-SkipCNCheck
|
||||
# in sessionOptions are sufficient — TrustedHosts does not apply.
|
||||
$prevTrustedHosts = $null
|
||||
try {
|
||||
$prevTrustedHosts = (Get-Item WSMan:\localhost\Client\TrustedHosts -ErrorAction Stop).Value
|
||||
# Add VM IP to TrustedHosts if not already present
|
||||
if ($prevTrustedHosts -ne '*' -and $prevTrustedHosts -notlike "*$VMIPAddress*") {
|
||||
$newTrusted = if ($prevTrustedHosts -eq '') { $VMIPAddress } else { "$prevTrustedHosts,$VMIPAddress" }
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $newTrusted -Force -ErrorAction Stop
|
||||
$isLinuxHost = ($null -ne $IsWindows -and $IsWindows -eq $false)
|
||||
if (-not $isLinuxHost) {
|
||||
try {
|
||||
$prevTrustedHosts = (Get-Item WSMan:\localhost\Client\TrustedHosts -ErrorAction Stop).Value
|
||||
# Add VM IP to TrustedHosts if not already present
|
||||
if ($prevTrustedHosts -ne '*' -and $prevTrustedHosts -notlike "*$VMIPAddress*") {
|
||||
$newTrusted = if ($prevTrustedHosts -eq '') { $VMIPAddress } else { "$prevTrustedHosts,$VMIPAddress" }
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $newTrusted -Force -ErrorAction Stop
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts configured."
|
||||
Assert-Step 'HostWinRM' "TrustedHosts includes $VMIPAddress (or '*')" {
|
||||
$th = (Get-Item WSMan:\localhost\Client\TrustedHosts).Value
|
||||
$th -eq '*' -or $th -like "*$VMIPAddress*"
|
||||
}
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts configured."
|
||||
Assert-Step 'HostWinRM' "TrustedHosts includes $VMIPAddress (or '*')" {
|
||||
$th = (Get-Item WSMan:\localhost\Client\TrustedHosts).Value
|
||||
$th -eq '*' -or $th -like "*$VMIPAddress*"
|
||||
catch {
|
||||
Write-Warning "[Prepare] Could not configure host WinRM TrustedHosts (need elevation?)."
|
||||
Write-Warning "Run once in an elevated PowerShell on the HOST:"
|
||||
Write-Warning " Set-Item WSMan:\localhost\Client\TrustedHosts -Value '*' -Force"
|
||||
Write-Warning "Then re-run this script."
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Warning "[Prepare] Could not configure host WinRM TrustedHosts (need elevation?)."
|
||||
Write-Warning "Run once in an elevated PowerShell on the HOST:"
|
||||
Write-Warning " Set-Item WSMan:\localhost\Client\TrustedHosts -Value '*' -Force"
|
||||
Write-Warning "Then re-run this script."
|
||||
exit 1
|
||||
else {
|
||||
Write-Host "[Prepare] Linux host — TrustedHosts not applicable, skipping."
|
||||
}
|
||||
|
||||
|
||||
@@ -497,6 +521,33 @@ try {
|
||||
-Authentication Basic -ErrorAction Stop
|
||||
}
|
||||
|
||||
# ── Install CI-StaticIp scheduled task (ip_pool guestinfo static IP) ────────
|
||||
# Reads ci-static-ip.ps1 from alongside this script and installs it into the
|
||||
# guest as C:\CI\ci-static-ip.ps1, then registers the CI-StaticIp task
|
||||
# (At Startup, SYSTEM) so the guest applies a pre-assigned static IP from
|
||||
# guestinfo.ip-assignment before WinRM starts on each clone boot.
|
||||
$staticIpScriptPath = Join-Path $scriptDir 'guest-setup\windows\ci-static-ip.ps1'
|
||||
if (Test-Path $staticIpScriptPath -PathType Leaf) {
|
||||
Write-Host "[Prepare] Installing CI-StaticIp scheduled task on guest..."
|
||||
$staticIpContent = Get-Content -LiteralPath $staticIpScriptPath -Raw
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param([string]$ScriptContent)
|
||||
Set-Content -LiteralPath 'C:\CI\ci-static-ip.ps1' -Value $ScriptContent `
|
||||
-Encoding UTF8 -Force
|
||||
$action = New-ScheduledTaskAction `
|
||||
-Execute 'powershell.exe' `
|
||||
-Argument '-NoProfile -NonInteractive -ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1'
|
||||
$trigger = New-ScheduledTaskTrigger -AtStartup
|
||||
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
|
||||
-LogonType ServiceAccount -RunLevel Highest
|
||||
Register-ScheduledTask -TaskName 'CI-StaticIp' -Action $action `
|
||||
-Trigger $trigger -Principal $principal -Force | Out-Null
|
||||
} -ArgumentList $staticIpContent
|
||||
Write-Host "[Prepare] CI-StaticIp task registered." -ForegroundColor Green
|
||||
} else {
|
||||
Write-Warning "[Prepare] guest-setup\windows\ci-static-ip.ps1 not found alongside this script — skipping CI-StaticIp task. ip_pool feature will not work for this template."
|
||||
}
|
||||
|
||||
# ── Post-setup remote validation ──────────────────────────────────────────
|
||||
Write-Host "[Prepare] Running post-setup validation against guest..."
|
||||
$guestState = Invoke-Command -Session $session -ScriptBlock {
|
||||
@@ -505,15 +556,17 @@ try {
|
||||
$env:PATH = [System.Environment]::GetEnvironmentVariable('PATH','Machine') + ';' +
|
||||
[System.Environment]::GetEnvironmentVariable('PATH','User')
|
||||
[PSCustomObject]@{
|
||||
WinRMRunning = (Get-Service WinRM -ErrorAction SilentlyContinue).Status -eq 'Running'
|
||||
UserExists = [bool](Get-LocalUser -Name $BuildUser -ErrorAction SilentlyContinue)
|
||||
UserIsAdmin = [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUser)
|
||||
UACDisabled = (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' -Name EnableLUA -ErrorAction SilentlyContinue).EnableLUA -eq 0
|
||||
FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)
|
||||
DotNetVersion = (Get-Command dotnet -ErrorAction SilentlyContinue) | ForEach-Object { (& $_.Source --version 2>&1) }
|
||||
PythonExists = Test-Path 'C:\Python\python.exe' -PathType Leaf
|
||||
MSBuildExists = Test-Path $msbuildPath -PathType Leaf
|
||||
CIDirsPresent = (Test-Path 'C:\CI\build') -and (Test-Path 'C:\CI\output') -and (Test-Path 'C:\CI\scripts')
|
||||
WinRMRunning = (Get-Service WinRM -ErrorAction SilentlyContinue).Status -eq 'Running'
|
||||
UserExists = [bool](Get-LocalUser -Name $BuildUser -ErrorAction SilentlyContinue)
|
||||
UserIsAdmin = [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUser)
|
||||
UACDisabled = (Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System' -Name EnableLUA -ErrorAction SilentlyContinue).EnableLUA -eq 0
|
||||
FirewallOff = -not (Get-NetFirewallProfile | Where-Object Enabled -eq $true)
|
||||
DotNetVersion = (Get-Command dotnet -ErrorAction SilentlyContinue) | ForEach-Object { (& $_.Source --version 2>&1) }
|
||||
PythonExists = Test-Path 'C:\Python\python.exe' -PathType Leaf
|
||||
MSBuildExists = Test-Path $msbuildPath -PathType Leaf
|
||||
CIDirsPresent = (Test-Path 'C:\CI\build') -and (Test-Path 'C:\CI\output') -and (Test-Path 'C:\CI\scripts')
|
||||
StaticIpTask = [bool](Get-ScheduledTask -TaskName 'CI-StaticIp' -ErrorAction SilentlyContinue)
|
||||
StaticIpScript = Test-Path 'C:\CI\ci-static-ip.ps1' -PathType Leaf
|
||||
}
|
||||
} -ArgumentList $BuildUsername
|
||||
|
||||
@@ -526,6 +579,8 @@ try {
|
||||
Assert-Step 'PostSetup' 'guest Python installed' { $guestState.PythonExists }
|
||||
Assert-Step 'PostSetup' 'guest MSBuild present' { $guestState.MSBuildExists }
|
||||
Assert-Step 'PostSetup' 'guest C:\CI\{build,output,scripts} present' { $guestState.CIDirsPresent }
|
||||
Assert-Step 'PostSetup' 'guest CI-StaticIp task registered' { $guestState.StaticIpTask }
|
||||
Assert-Step 'PostSetup' 'guest C:\CI\ci-static-ip.ps1 present' { $guestState.StaticIpScript }
|
||||
|
||||
Write-Host "[Prepare] All post-setup validations passed (.NET $($guestState.DotNetVersion))." -ForegroundColor Green
|
||||
|
||||
@@ -717,11 +772,11 @@ try {
|
||||
finally {
|
||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||
|
||||
# Restore host TrustedHosts to its original value
|
||||
if ($null -ne $prevTrustedHosts) {
|
||||
# Restore host TrustedHosts to its original value (Windows only)
|
||||
if (-not $isLinuxHost -and $null -ne $prevTrustedHosts) {
|
||||
Set-Item WSMan:\localhost\Client\TrustedHosts -Value $prevTrustedHosts -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||
}
|
||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Report CI VM IP to VMware host via guestinfo
|
||||
# Run after open-vm-tools so vmware-rpctool is available.
|
||||
# Do NOT add network-online.target — we want to run BEFORE the network stack
|
||||
# settles so static IP is applied before SSH/other services start.
|
||||
After=open-vm-tools.service
|
||||
Wants=open-vm-tools.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/ci-report-ip.sh
|
||||
RemainAfterExit=yes
|
||||
TimeoutStartSec=120
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
# CI VM IP reporter — called by ci-report-ip.service at boot.
|
||||
#
|
||||
# Phase C extension: reads guestinfo.ip-assignment from the VMX (written by the
|
||||
# host orchestrator before vmrun start). If present, applies a static IP to
|
||||
# the primary NIC and reports it back immediately via guestinfo.ci-ip, skipping
|
||||
# DHCP entirely. If absent, falls back to the original DHCP-detect behaviour.
|
||||
#
|
||||
# The VMCI channel (vmware-rpctool) does not require TCP/IP to be initialised —
|
||||
# it communicates via the VMware virtual hardware interface. This means the
|
||||
# static IP can be configured before network-online.target, so by the time SSH
|
||||
# starts listening the NIC already has the correct address.
|
||||
#
|
||||
# Install:
|
||||
# sudo cp ci-report-ip.sh /usr/local/bin/ci-report-ip.sh
|
||||
# sudo chmod +x /usr/local/bin/ci-report-ip.sh
|
||||
# sudo cp ci-report-ip.service /etc/systemd/system/ci-report-ip.service
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl enable ci-report-ip.service
|
||||
|
||||
RPCTOOL=/usr/bin/vmware-rpctool
|
||||
[ -x "$RPCTOOL" ] || exit 0
|
||||
|
||||
# ── Check for pre-assigned IP from orchestrator ───────────────────────────────
|
||||
assigned_ip=$("$RPCTOOL" "info-get guestinfo.ip-assignment" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
if [[ "$assigned_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
netmask=$("$RPCTOOL" "info-get guestinfo.netmask" 2>/dev/null | tr -d '[:space:]')
|
||||
gateway=$("$RPCTOOL" "info-get guestinfo.gateway" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
# Resolve primary NIC (first non-loopback).
|
||||
iface=$(ip -o link show | awk -F': ' '$2 != "lo" {print $2; exit}')
|
||||
if [ -z "$iface" ]; then
|
||||
echo "ci-report-ip: no non-loopback interface found" | systemd-cat -t ci-report-ip
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compute prefix length from netmask (default /24).
|
||||
prefix=24
|
||||
if [ -n "$netmask" ]; then
|
||||
prefix=$(python3 -c \
|
||||
"import ipaddress; print(ipaddress.IPv4Network('0.0.0.0/$netmask',strict=False).prefixlen)" \
|
||||
2>/dev/null) || prefix=24
|
||||
fi
|
||||
|
||||
# Flush any existing config (DHCP or stale static) and apply.
|
||||
ip addr flush dev "$iface" 2>/dev/null || true
|
||||
ip addr add "${assigned_ip}/${prefix}" dev "$iface"
|
||||
ip link set "$iface" up
|
||||
|
||||
if [[ "$gateway" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
ip route add default via "$gateway" dev "$iface" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Report IP back to host via guestinfo.ci-ip.
|
||||
"$RPCTOOL" "info-set guestinfo.ci-ip $assigned_ip" 2>/dev/null
|
||||
echo "ci-report-ip: applied static IP $assigned_ip/$prefix via guestinfo" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Fallback: DHCP — wait for IP and report ───────────────────────────────────
|
||||
# Retries for up to 60 s to tolerate open-vm-tools re-init after UUID change
|
||||
# (linked clones get a new UUID — vmware-rpctool may fail briefly on first boot).
|
||||
for i in $(seq 1 30); do
|
||||
IP=$(ip -4 route get 1.0.0.0 2>/dev/null \
|
||||
| awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}' \
|
||||
| head -1)
|
||||
if [ -n "$IP" ]; then
|
||||
if "$RPCTOOL" "info-set guestinfo.ci-ip $IP" 2>/dev/null; then
|
||||
echo "ci-report-ip: set guestinfo.ci-ip=$IP via DHCP (attempt $i)" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "ci-report-ip: failed to set guestinfo.ci-ip after 30 attempts" \
|
||||
| systemd-cat -t ci-report-ip
|
||||
exit 1
|
||||
@@ -0,0 +1,115 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Applies a pre-assigned static IP from VMware guestinfo at boot.
|
||||
|
||||
.DESCRIPTION
|
||||
Called by the CI-StaticIp Task Scheduler task at system startup (SYSTEM
|
||||
account). Reads guestinfo.ip-assignment written by the host orchestrator
|
||||
before vmrun start. If present, disables DHCP and applies the static IP
|
||||
before WinRM has a chance to start on a DHCP address, eliminating the
|
||||
20-180 s VMware-Tools IP-detect polling loop.
|
||||
|
||||
If guestinfo.ip-assignment is absent (template used standalone or
|
||||
ip_pool not configured), exits 0 immediately — DHCP runs normally.
|
||||
|
||||
Install into the template VM:
|
||||
1. Copy this script to C:\CI\ci-static-ip.ps1
|
||||
2. Create a Task Scheduler task:
|
||||
Name: CI-StaticIp
|
||||
Trigger: At System Startup
|
||||
Action: powershell.exe -NoProfile -NonInteractive
|
||||
-ExecutionPolicy Bypass -File C:\CI\ci-static-ip.ps1
|
||||
Run As: SYSTEM
|
||||
Run with highest privileges: Yes
|
||||
3. Take the BaseClean snapshot from powered-OFF state.
|
||||
|
||||
The VMCI channel (vmware-rpctool) does not require TCP/IP — it works
|
||||
via the VMware virtual hardware interface, so guestinfo is readable
|
||||
before the NIC is configured.
|
||||
#>
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# vmware-rpctool.exe was removed in recent VMware Tools versions; use vmtoolsd --cmd instead.
|
||||
$vmtoolsd = 'C:\Program Files\VMware\VMware Tools\vmtoolsd.exe'
|
||||
if (-not (Test-Path $vmtoolsd -PathType Leaf)) { exit 0 }
|
||||
|
||||
function Read-GuestInfo {
|
||||
param([string]$Key)
|
||||
# vmtoolsd writes to a file handle but not to a PS pipeline — use cmd.exe redirection.
|
||||
$tmp = [System.IO.Path]::GetTempFileName()
|
||||
try {
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
cmd /c "`"$vmtoolsd`" --cmd `"info-get guestinfo.$Key`" > `"$tmp`" 2>&1"
|
||||
$ErrorActionPreference = $savedEap
|
||||
$text = (Get-Content $tmp -ErrorAction SilentlyContinue | Out-String).Trim()
|
||||
if ($text -and $text -notmatch 'No value found|Error') {
|
||||
return $text
|
||||
}
|
||||
} finally {
|
||||
Remove-Item $tmp -ErrorAction SilentlyContinue
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
$ip = Read-GuestInfo 'ip-assignment'
|
||||
$netmask = Read-GuestInfo 'netmask'
|
||||
$gateway = Read-GuestInfo 'gateway'
|
||||
|
||||
# No pre-assigned IP — let DHCP run normally.
|
||||
if (-not ($ip -match '^\d{1,3}(\.\d{1,3}){3}$')) { exit 0 }
|
||||
|
||||
# Convert dotted-decimal netmask to prefix length (default /24).
|
||||
function Get-PrefixLength {
|
||||
param([string]$Mask)
|
||||
if (-not ($Mask -match '^\d{1,3}(\.\d{1,3}){3}$')) { return 24 }
|
||||
$bits = 0
|
||||
foreach ($octet in $Mask.Split('.')) {
|
||||
$b = [int]$octet
|
||||
while ($b -gt 0) { $bits += ($b -band 1); $b = $b -shr 1 }
|
||||
}
|
||||
return $bits
|
||||
}
|
||||
$prefix = Get-PrefixLength $netmask
|
||||
|
||||
# Target the first Up adapter (skip loopback / Teredo / Bluetooth).
|
||||
$adapter = Get-NetAdapter |
|
||||
Where-Object { $_.Status -eq 'Up' -and $_.InterfaceDescription -notmatch 'Loopback|Bluetooth|Teredo' } |
|
||||
Select-Object -First 1
|
||||
if (-not $adapter) {
|
||||
Write-EventLog -LogName Application -Source 'ci-static-ip' -EventId 1 `
|
||||
-EntryType Error -Message 'No Up adapter found.' -ErrorAction SilentlyContinue
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Disable DHCP and remove any existing IPv4 addresses and routes.
|
||||
$adapter | Set-NetIPInterface -Dhcp Disabled -ErrorAction SilentlyContinue
|
||||
Get-NetRoute -InterfaceAlias $adapter.Name -AddressFamily IPv4 -ErrorAction SilentlyContinue |
|
||||
Remove-NetRoute -Confirm:$false -ErrorAction SilentlyContinue
|
||||
$adapter | Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue |
|
||||
Remove-NetIPAddress -Confirm:$false -ErrorAction SilentlyContinue
|
||||
|
||||
# Apply static IP (no DefaultGateway here — add route separately to avoid conflicts).
|
||||
New-NetIPAddress -InterfaceAlias $adapter.Name -IPAddress $ip `
|
||||
-PrefixLength $prefix -ErrorAction Stop | Out-Null
|
||||
|
||||
# Add default route if gateway supplied.
|
||||
if ($gateway -match '^\d{1,3}(\.\d{1,3}){3}$') {
|
||||
New-NetRoute -InterfaceAlias $adapter.Name -DestinationPrefix '0.0.0.0/0' `
|
||||
-NextHop $gateway -ErrorAction SilentlyContinue | Out-Null
|
||||
}
|
||||
|
||||
# Use the gateway as DNS (fallback: Google DNS).
|
||||
$dns = @()
|
||||
if ($gateway -match '^\d{1,3}(\.\d{1,3}){3}$') { $dns += $gateway }
|
||||
$dns += '8.8.8.8'
|
||||
Set-DnsClientServerAddress -InterfaceAlias $adapter.Name `
|
||||
-ServerAddresses $dns -ErrorAction SilentlyContinue
|
||||
|
||||
# Report IP back via guestinfo.ci-ip so the host also has it.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
& $vmtoolsd --cmd "info-set guestinfo.ci-ip $ip" 2>&1 | Out-Null
|
||||
$ErrorActionPreference = $savedEap
|
||||
|
||||
exit 0
|
||||
@@ -1,181 +0,0 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Pester v5 tests for scripts/_Common.psm1
|
||||
#>
|
||||
|
||||
BeforeAll {
|
||||
Import-Module (Join-Path $PSScriptRoot '..\scripts\_Common.psm1') -Force
|
||||
|
||||
# Fake vmrun: accepts any args, exits with %FAKE_VMRUN_EXIT% (default 0)
|
||||
$script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-$PID.cmd"
|
||||
Set-Content $script:FakeVmrun -Value @'
|
||||
@echo off
|
||||
echo fake-vmrun-output
|
||||
if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% )
|
||||
'@
|
||||
}
|
||||
|
||||
AfterAll {
|
||||
Remove-Item $script:FakeVmrun -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
Describe 'New-CISessionOption' {
|
||||
It 'returns a PSSessionOption' {
|
||||
$opt = New-CISessionOption
|
||||
$opt | Should -BeOfType [System.Management.Automation.Remoting.PSSessionOption]
|
||||
}
|
||||
|
||||
It 'has SkipCACheck = true' {
|
||||
(New-CISessionOption).SkipCACheck | Should -Be $true
|
||||
}
|
||||
|
||||
It 'has SkipCNCheck = true' {
|
||||
(New-CISessionOption).SkipCNCheck | Should -Be $true
|
||||
}
|
||||
|
||||
It 'has SkipRevocationCheck = true' {
|
||||
(New-CISessionOption).SkipRevocationCheck | Should -Be $true
|
||||
}
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
Describe 'Resolve-VmrunPath' {
|
||||
It 'returns the path when the file exists' {
|
||||
$tmp = [System.IO.Path]::GetTempFileName()
|
||||
try {
|
||||
Resolve-VmrunPath -VmrunPath $tmp | Should -Be $tmp
|
||||
} finally {
|
||||
Remove-Item $tmp -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
It 'throws with a descriptive message when file is missing' {
|
||||
{ Resolve-VmrunPath -VmrunPath 'C:\DoesNotExist\vmrun.exe' } |
|
||||
Should -Throw -ExpectedMessage '*vmrun.exe not found*'
|
||||
}
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
Describe 'Invoke-Vmrun' {
|
||||
It 'returns ExitCode 0 and Output when command succeeds' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$r = Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'list'
|
||||
$r.ExitCode | Should -Be 0
|
||||
"$($r.Output)" | Should -Match 'fake-vmrun-output'
|
||||
}
|
||||
|
||||
It 'returns non-zero ExitCode without throwing by default' {
|
||||
$env:FAKE_VMRUN_EXIT = '1'
|
||||
$r = Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'clone'
|
||||
$r.ExitCode | Should -Be 1
|
||||
}
|
||||
|
||||
It 'throws when -ThrowOnError and ExitCode is non-zero' {
|
||||
$env:FAKE_VMRUN_EXIT = '1'
|
||||
{ Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'clone' -ThrowOnError } |
|
||||
Should -Throw -ExpectedMessage '*clone failed*'
|
||||
}
|
||||
|
||||
It 'does not throw when -ThrowOnError and ExitCode is 0' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
{ Invoke-Vmrun -VmrunPath $script:FakeVmrun -Operation 'start' -ThrowOnError } |
|
||||
Should -Not -Throw
|
||||
}
|
||||
|
||||
AfterEach {
|
||||
$env:FAKE_VMRUN_EXIT = $null
|
||||
}
|
||||
}
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
Describe 'Invoke-VmrunBounded' {
|
||||
BeforeAll {
|
||||
# Fake vmrun that sleeps for $FAKE_VMRUN_SLEEP seconds then exits with
|
||||
# $FAKE_VMRUN_EXIT. Both default to 0 if unset.
|
||||
$script:SlowFakeVmrun = Join-Path $env:TEMP "fake-vmrun-slow-$PID.cmd"
|
||||
Set-Content $script:SlowFakeVmrun -Value @'
|
||||
@echo off
|
||||
if "%FAKE_VMRUN_SLEEP%"=="" goto :nosleep
|
||||
ping -n %FAKE_VMRUN_SLEEP% 127.0.0.1 >nul
|
||||
:nosleep
|
||||
echo fake-vmrun-bounded-output
|
||||
if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 0 ) else ( exit /b %FAKE_VMRUN_EXIT% )
|
||||
'@
|
||||
}
|
||||
|
||||
AfterAll {
|
||||
Remove-Item $script:SlowFakeVmrun -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
AfterEach {
|
||||
$env:FAKE_VMRUN_EXIT = $null
|
||||
$env:FAKE_VMRUN_SLEEP = $null
|
||||
}
|
||||
|
||||
It 'returns ExitCode 0, TimedOut false and Output when command succeeds quickly' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -Arguments @('some.vmx', 'nogui') -TimeoutSeconds 30
|
||||
$r.ExitCode | Should -Be 0
|
||||
$r.TimedOut | Should -Be $false
|
||||
"$($r.Output)" | Should -Match 'fake-vmrun-bounded-output'
|
||||
$r.ElapsedSeconds | Should -BeGreaterOrEqual 0
|
||||
}
|
||||
|
||||
It 'returns non-zero ExitCode without throwing by default' {
|
||||
$env:FAKE_VMRUN_EXIT = '3'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 30
|
||||
$r.ExitCode | Should -Be 3
|
||||
$r.TimedOut | Should -Be $false
|
||||
}
|
||||
|
||||
It 'throws when -ThrowOnError and ExitCode is non-zero' {
|
||||
$env:FAKE_VMRUN_EXIT = '2'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'stop' -TimeoutSeconds 30 -ThrowOnError } |
|
||||
Should -Throw -ExpectedMessage '*stop failed*'
|
||||
}
|
||||
|
||||
It 'does not throw when -ThrowOnError and ExitCode is 0' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 30 -ThrowOnError } |
|
||||
Should -Not -Throw
|
||||
}
|
||||
|
||||
It 'times out and returns TimedOut = true when process exceeds TimeoutSeconds' {
|
||||
# FAKE_VMRUN_SLEEP uses ping -n N which waits roughly (N-1) seconds.
|
||||
# Set sleep >> timeout so the process is guaranteed to still be running.
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2
|
||||
$r.TimedOut | Should -Be $true
|
||||
$r.ExitCode | Should -Be -1
|
||||
$r.ElapsedSeconds | Should -BeLessOrEqual 10 # killed well before 60s
|
||||
}
|
||||
|
||||
It 'throws when -ThrowOnTimeout and process times out' {
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'deleteVM' -TimeoutSeconds 2 -ThrowOnTimeout } |
|
||||
Should -Throw -ExpectedMessage '*timed out*'
|
||||
}
|
||||
|
||||
It 'does not throw on timeout when neither flag is set' {
|
||||
$env:FAKE_VMRUN_SLEEP = '60'
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
{ Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'start' -TimeoutSeconds 2 } |
|
||||
Should -Not -Throw
|
||||
}
|
||||
|
||||
It 'ElapsedSeconds is populated and plausible' {
|
||||
$env:FAKE_VMRUN_EXIT = '0'
|
||||
$env:FAKE_VMRUN_SLEEP = '0'
|
||||
$r = Invoke-VmrunBounded -VmrunPath $script:SlowFakeVmrun -Operation 'list' -TimeoutSeconds 30
|
||||
$r.ElapsedSeconds | Should -BeGreaterOrEqual 0
|
||||
$r.ElapsedSeconds | Should -BeLessThan 30
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
"""Tests for backends/protocol.py.
|
||||
|
||||
Covers VmState, VmHandle, and the Protocol stub bodies (the ``...``
|
||||
ellipsis in each method), which exist as documentation-level defaults
|
||||
and are callable on the Protocol class itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from ci_orchestrator.backends.protocol import VmBackend, VmHandle, VmState
|
||||
|
||||
|
||||
def test_vm_state_values() -> None:
|
||||
assert VmState.POWERED_OFF == "powered_off"
|
||||
assert VmState.POWERED_ON == "powered_on"
|
||||
assert VmState.SUSPENDED == "suspended"
|
||||
assert VmState.UNKNOWN == "unknown"
|
||||
|
||||
|
||||
def test_vm_handle_fields() -> None:
|
||||
h = VmHandle(identifier="x.vmx", name="job-1")
|
||||
assert h.identifier == "x.vmx"
|
||||
assert h.name == "job-1"
|
||||
|
||||
|
||||
def test_vm_handle_name_defaults_to_none() -> None:
|
||||
h = VmHandle(identifier="x.vmx")
|
||||
assert h.name is None
|
||||
|
||||
|
||||
def test_vm_handle_is_frozen() -> None:
|
||||
h = VmHandle("x.vmx")
|
||||
with pytest.raises((AttributeError, TypeError)):
|
||||
h.identifier = "y.vmx" # type: ignore[misc]
|
||||
|
||||
|
||||
# ── Protocol stub bodies (lines 53, 57, 61, 65, 69, 73, 81) ──────────────────
|
||||
# The Protocol stubs are callable as default implementations that return None.
|
||||
# Exercising them satisfies coverage without asserting behaviour (they are
|
||||
# documentation stubs, not functional implementations).
|
||||
|
||||
def test_protocol_stubs_are_callable() -> None:
|
||||
# Protocol.__init__ blocks VmBackend() in Python 3.12; bypass via __new__.
|
||||
b: VmBackend = object.__new__(VmBackend)
|
||||
h = VmHandle("x.vmx")
|
||||
assert b.clone_linked("tpl.vmx", "snap", "name") is None # line 53
|
||||
assert b.start(h) is None # line 57
|
||||
assert b.stop(h) is None # line 61
|
||||
assert b.delete(h) is None # line 65
|
||||
assert b.get_ip(h) is None # line 69
|
||||
assert b.list_snapshots(h) is None # line 73
|
||||
assert b.is_running(h) is None # line 81
|
||||
@@ -322,3 +322,234 @@ def test_collect_linux_no_files_raises(
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "no files were transferred" in result.output
|
||||
|
||||
|
||||
# ── NEW: coverage gap tests ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
from ci_orchestrator.transport.errors import TransportError # noqa: E402
|
||||
|
||||
|
||||
def test_collect_windows_include_logs_fetch_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 187 (191): log fetch succeeds → file deposited, no error message."""
|
||||
_patch(monkeypatch)
|
||||
payload = _zip_payload(tmp_path)
|
||||
|
||||
class _LogFetchOk(_win_transport_with_zip(payload)): # type: ignore[misc]
|
||||
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||
super().__init__(*a, **kw)
|
||||
# The log-listing command returns one log file path.
|
||||
self.run_overrides["C:\\CI\\build"] = _FakeResult(
|
||||
stdout="C:\\CI\\build\\build.log\r\n"
|
||||
)
|
||||
|
||||
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||
self.fetched.append((remote_path, str(local_path)))
|
||||
# Write something for the log file fetch too.
|
||||
Path(local_path).write_bytes(payload if local_path.endswith(".zip") else b"log content")
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "WinRmTransport", _LogFetchOk)
|
||||
out_dir = tmp_path / "out"
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.7",
|
||||
"--guest-artifact-path", "C:\\CI\\output",
|
||||
"--host-artifact-dir", str(out_dir),
|
||||
"--include-logs",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
# fetch was called for both the zip and the log
|
||||
fetched_remotes = [r for r, _ in _FakeTransport.instances[0].fetched]
|
||||
assert "C:\\CI\\build\\build.log" in fetched_remotes
|
||||
assert "fetching log" in result.output
|
||||
|
||||
|
||||
def test_collect_windows_include_logs_fetch_fails(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Lines 192-193: log fetch raises TransportError → error message printed, continues."""
|
||||
_patch(monkeypatch)
|
||||
payload = _zip_payload(tmp_path)
|
||||
|
||||
class _LogFetchFail(_win_transport_with_zip(payload)): # type: ignore[misc]
|
||||
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||
super().__init__(*a, **kw)
|
||||
self.run_overrides["C:\\CI\\build"] = _FakeResult(
|
||||
stdout="C:\\CI\\build\\fail.log\r\n"
|
||||
)
|
||||
|
||||
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||
if remote_path.endswith(".log"):
|
||||
raise TransportError("log fetch boom")
|
||||
# Normal zip fetch
|
||||
self.fetched.append((remote_path, str(local_path)))
|
||||
Path(local_path).write_bytes(payload)
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "WinRmTransport", _LogFetchFail)
|
||||
out_dir = tmp_path / "out"
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.7",
|
||||
"--guest-artifact-path", "C:\\CI\\output",
|
||||
"--host-artifact-dir", str(out_dir),
|
||||
"--include-logs",
|
||||
],
|
||||
)
|
||||
# Should succeed overall (error logged but not fatal)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "log fetch failed" in result.output
|
||||
|
||||
|
||||
def test_collect_windows_include_logs_no_artifacts(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 199: include_logs=True, no artifacts extracted → ClickException."""
|
||||
_patch(monkeypatch)
|
||||
|
||||
# Transport that returns a real-looking zip but empty after extraction,
|
||||
# and returns log listing.
|
||||
empty_zip = tmp_path / "empty.zip"
|
||||
import zipfile as _zf
|
||||
with _zf.ZipFile(empty_zip, "w"):
|
||||
pass
|
||||
empty_zip_bytes = empty_zip.read_bytes()
|
||||
|
||||
class _NoArtifacts(_FakeTransport):
|
||||
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||
super().__init__(*a, **kw)
|
||||
# no log lines returned
|
||||
self.run_overrides["C:\\CI\\build"] = _FakeResult(stdout="")
|
||||
|
||||
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||
self.fetched.append((remote_path, str(local_path)))
|
||||
Path(local_path).write_bytes(empty_zip_bytes)
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "WinRmTransport", _NoArtifacts)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.7",
|
||||
"--guest-artifact-path", "C:\\CI\\output",
|
||||
"--host-artifact-dir", str(tmp_path / "out"),
|
||||
"--include-logs",
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "no artifacts after collect" in result.output
|
||||
|
||||
|
||||
def test_collect_linux_key_from_config(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 261: --guest-os linux with no --ssh-key-path but config.ssh_key_path set."""
|
||||
_patch(monkeypatch)
|
||||
|
||||
# Build a real tarball for the fake transport to return.
|
||||
payload_dir = tmp_path / "payload"
|
||||
payload_dir.mkdir()
|
||||
(payload_dir / "out.txt").write_text("hi", encoding="utf-8")
|
||||
tar_local = tmp_path / "src.tar.gz"
|
||||
with tarfile.open(tar_local, "w:gz") as tf:
|
||||
for f in payload_dir.iterdir():
|
||||
tf.add(f, arcname=f.name)
|
||||
payload_bytes = tar_local.read_bytes()
|
||||
|
||||
captured_key: list[Any] = []
|
||||
|
||||
class _LinuxCapture(_FakeTransport):
|
||||
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||
super().__init__(*a, **kw)
|
||||
captured_key.append(kw.get("key_path"))
|
||||
|
||||
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||
self.fetched.append((remote_path, str(local_path)))
|
||||
Path(local_path).write_bytes(payload_bytes)
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "SshTransport", _LinuxCapture)
|
||||
|
||||
# Patch load_config to return a config that has ssh_key_path set.
|
||||
from ci_orchestrator.config import BackendConfig, Config, Paths
|
||||
|
||||
fake_key = tmp_path / "ci_linux"
|
||||
_test_paths = Paths(
|
||||
root=tmp_path, templates=tmp_path, build_vms=tmp_path,
|
||||
artifacts=tmp_path, keys=tmp_path,
|
||||
)
|
||||
_test_cfg = Config(paths=_test_paths, backend=BackendConfig(), ssh_key_path=fake_key)
|
||||
monkeypatch.setattr(artifacts_module, "load_config", lambda: _test_cfg)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.5",
|
||||
"--guest-os", "linux",
|
||||
"--guest-artifact-path", "/opt/ci/output",
|
||||
"--host-artifact-dir", str(tmp_path / "out"),
|
||||
# no --ssh-key-path → should come from config
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert captured_key and captured_key[0] == str(fake_key)
|
||||
|
||||
|
||||
def test_collect_transport_error_raises_click_exception(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 280: TransportError during collect → ClickException."""
|
||||
_patch(monkeypatch)
|
||||
|
||||
class _Boom(_FakeTransport):
|
||||
def __enter__(self) -> _Boom:
|
||||
raise TransportError("connect failed")
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "WinRmTransport", _Boom)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.7",
|
||||
"--guest-artifact-path", "C:\\CI\\output",
|
||||
"--host-artifact-dir", str(tmp_path / "out"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "connect failed" in result.output
|
||||
|
||||
|
||||
def test_collect_windows_log_listing_skips_blank_lines(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 187 (continue): listing stdout has blank lines → skipped silently."""
|
||||
_patch(monkeypatch)
|
||||
payload = _zip_payload(tmp_path)
|
||||
|
||||
class _BlankLines(_win_transport_with_zip(payload)): # type: ignore[misc]
|
||||
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||
super().__init__(*a, **kw)
|
||||
# Return listing with leading/trailing blank lines only — no real log paths.
|
||||
self.run_overrides["C:\\CI\\build"] = _FakeResult(stdout="\r\n \r\n")
|
||||
|
||||
monkeypatch.setattr(artifacts_module, "WinRmTransport", _BlankLines)
|
||||
out_dir = tmp_path / "out"
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"artifacts", "collect",
|
||||
"--ip-address", "10.0.0.7",
|
||||
"--guest-artifact-path", "C:\\CI\\output",
|
||||
"--host-artifact-dir", str(out_dir),
|
||||
"--include-logs",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
# No "fetching log" output — all listing lines were blank and skipped.
|
||||
assert "fetching log" not in result.output
|
||||
|
||||
@@ -0,0 +1,487 @@
|
||||
"""Tests for the ``bench`` sub-commands (Phase C2 burn-in + C3 measure).
|
||||
|
||||
The C1 ``--help`` smoke tests are kept; behavioural tests mock the backend
|
||||
and the per-job runner so no real VMs are spun up.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.bench as bench_module
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.backends.errors import BackendNotAvailable, BackendOperationFailed
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.commands.bench import (
|
||||
JobResult,
|
||||
PhaseTiming,
|
||||
_find_orphans,
|
||||
_stale_lock,
|
||||
bench,
|
||||
)
|
||||
|
||||
# ── C1 smoke tests (kept) ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_bench_help_lists_subcommands() -> None:
|
||||
result = CliRunner().invoke(cli, ["bench", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "run" in result.output
|
||||
assert "measure" in result.output
|
||||
|
||||
|
||||
def test_bench_run_help() -> None:
|
||||
result = CliRunner().invoke(cli, ["bench", "run", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--concurrency" in result.output
|
||||
assert "--rounds" in result.output
|
||||
|
||||
|
||||
def test_bench_measure_help() -> None:
|
||||
result = CliRunner().invoke(cli, ["bench", "measure", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--iterations" in result.output
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_find_orphans_matches_job_prefix(tmp_path: Path) -> None:
|
||||
(tmp_path / "Clone_burnin-r1-j1-x_20260101_aaa").mkdir()
|
||||
(tmp_path / "Clone_other-job_20260101_bbb").mkdir()
|
||||
(tmp_path / "not-a-clone").mkdir()
|
||||
orphans = _find_orphans(tmp_path, ["burnin-r1-j1-x"])
|
||||
assert len(orphans) == 1
|
||||
assert orphans[0].name.startswith("Clone_burnin-r1-j1-x_")
|
||||
|
||||
|
||||
def test_find_orphans_missing_base_dir(tmp_path: Path) -> None:
|
||||
assert _find_orphans(tmp_path / "ghost", ["j1"]) == []
|
||||
|
||||
|
||||
def test_stale_lock_detects_old_file(tmp_path: Path) -> None:
|
||||
import os
|
||||
|
||||
lock = tmp_path / "vm-start.lock"
|
||||
lock.write_text("x")
|
||||
old = lock.stat().st_mtime - (10 * 60)
|
||||
os.utime(lock, (old, old))
|
||||
assert _stale_lock(lock) is True
|
||||
|
||||
|
||||
def test_stale_lock_fresh_file_is_ok(tmp_path: Path) -> None:
|
||||
lock = tmp_path / "vm-start.lock"
|
||||
lock.write_text("x")
|
||||
assert _stale_lock(lock) is False
|
||||
|
||||
|
||||
def test_stale_lock_missing_file(tmp_path: Path) -> None:
|
||||
assert _stale_lock(tmp_path / "absent.lock") is False
|
||||
|
||||
|
||||
# ── bench run ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _ci_paths(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
"""Point load_config at temp build-vms/artifacts dirs."""
|
||||
env = {
|
||||
"CI_BUILD_VMS": str(tmp_path / "build-vms"),
|
||||
"CI_ARTIFACTS": str(tmp_path / "artifacts"),
|
||||
}
|
||||
(tmp_path / "build-vms").mkdir()
|
||||
(tmp_path / "artifacts").mkdir()
|
||||
real = bench_module.load_config
|
||||
monkeypatch.setattr(bench_module, "load_config", lambda: real(env=env))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_bench_run_all_pass(monkeypatch: pytest.MonkeyPatch, _ci_paths: Path) -> None:
|
||||
calls: list[str] = []
|
||||
|
||||
def _fake_job(*, job_id: str, **_kw: Any) -> tuple[int, str]:
|
||||
calls.append(job_id)
|
||||
return 0, ""
|
||||
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", _fake_job)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "run", "--concurrency", "3", "--rounds", "2"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "OVERALL: PASS" in result.output
|
||||
assert len(calls) == 6 # 3 jobs * 2 rounds
|
||||
# Report file written under artifacts/bench.
|
||||
reports = list((_ci_paths / "artifacts" / "bench").glob("*.json"))
|
||||
assert len(reports) == 1
|
||||
data = json.loads(reports[0].read_text())
|
||||
assert data["overall"] == "PASS"
|
||||
assert data["totalJobs"] == 6
|
||||
assert len(data["results"]) == 2
|
||||
|
||||
|
||||
def test_bench_run_failing_job_marks_round_fail(
|
||||
monkeypatch: pytest.MonkeyPatch, _ci_paths: Path
|
||||
) -> None:
|
||||
def _fake_job(*, job_id: str, slot: int = 0, **_kw: Any) -> tuple[int, str]:
|
||||
return 1, "clone failed: boom"
|
||||
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", _fake_job)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "run", "--concurrency", "1", "--rounds", "1"]
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "OVERALL: FAIL" in result.output
|
||||
assert "clone failed: boom" in result.output
|
||||
|
||||
|
||||
def test_bench_run_orphan_dir_fails_round(
|
||||
monkeypatch: pytest.MonkeyPatch, _ci_paths: Path
|
||||
) -> None:
|
||||
base = _ci_paths / "build-vms"
|
||||
|
||||
def _fake_job(*, job_id: str, **_kw: Any) -> tuple[int, str]:
|
||||
# Leave an orphan clone dir behind for this job.
|
||||
(base / f"Clone_{job_id}_20260101_orphan").mkdir()
|
||||
return 0, ""
|
||||
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", _fake_job)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "run", "--concurrency", "1", "--rounds", "1"]
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "orphan" in result.output
|
||||
assert "OVERALL: FAIL" in result.output
|
||||
|
||||
|
||||
def test_bench_run_stale_lock_fails_round(
|
||||
monkeypatch: pytest.MonkeyPatch, _ci_paths: Path
|
||||
) -> None:
|
||||
base = _ci_paths / "build-vms"
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", lambda **_kw: (0, ""))
|
||||
# _stale_lock reports a stale lock regardless of file age.
|
||||
monkeypatch.setattr(bench_module, "_stale_lock", lambda _p: True)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["bench", "run", "--concurrency", "1", "--rounds", "1", "--clone-base-dir", str(base)],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "stale lock" in result.output
|
||||
|
||||
|
||||
def test_bench_run_custom_json_out(
|
||||
monkeypatch: pytest.MonkeyPatch, _ci_paths: Path
|
||||
) -> None:
|
||||
out = _ci_paths / "custom" / "report.json"
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", lambda **_kw: (0, ""))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["bench", "run", "--concurrency", "1", "--rounds", "1", "--json-out", str(out)],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert out.is_file()
|
||||
|
||||
|
||||
def test_run_one_job_invokes_job_command(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""_run_one_job drives the real job command and maps exit codes."""
|
||||
import ci_orchestrator.commands.job as job_module
|
||||
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
class _FakeResult:
|
||||
exit_code = 0
|
||||
output = "ok\n"
|
||||
|
||||
class _FakeRunner:
|
||||
def invoke(self, _cmd: Any, args: list[str], **_kw: Any) -> _FakeResult:
|
||||
captured["args"] = args
|
||||
return _FakeResult()
|
||||
|
||||
monkeypatch.setattr("click.testing.CliRunner", lambda *a, **k: _FakeRunner())
|
||||
del job_module # imported for symmetry; not otherwise needed
|
||||
code, err = bench_module._run_one_job(
|
||||
job_id="j1",
|
||||
guest_os="linux",
|
||||
build_command="echo hi",
|
||||
clone_base_dir=tmp_path,
|
||||
)
|
||||
assert code == 0
|
||||
assert err == ""
|
||||
assert "--job-id" in captured["args"]
|
||||
assert "--build-command" in captured["args"]
|
||||
|
||||
|
||||
def test_run_one_job_failure_returns_tail(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
class _FakeResult:
|
||||
exit_code = 1
|
||||
output = "line one\nError: clone failed\n"
|
||||
|
||||
class _FakeRunner:
|
||||
def invoke(self, *_a: Any, **_k: Any) -> _FakeResult:
|
||||
return _FakeResult()
|
||||
|
||||
monkeypatch.setattr("click.testing.CliRunner", lambda *a, **k: _FakeRunner())
|
||||
code, err = bench_module._run_one_job(
|
||||
job_id="j1", guest_os="linux", build_command="", clone_base_dir=tmp_path
|
||||
)
|
||||
assert code == 1
|
||||
assert err == "Error: clone failed"
|
||||
|
||||
|
||||
# ── bench measure ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeBackend:
|
||||
def __init__(self, *, ip: str | None = "10.0.0.5", fail: str = "") -> None:
|
||||
self._ip = ip
|
||||
self._fail = fail
|
||||
self.calls: list[str] = []
|
||||
|
||||
def clone_linked(
|
||||
self, template: str, snapshot: str, name: str, destination: str | None = None
|
||||
) -> VmHandle:
|
||||
self.calls.append("clone")
|
||||
if destination:
|
||||
d = Path(destination)
|
||||
d.parent.mkdir(parents=True, exist_ok=True)
|
||||
d.write_text('guestOS = "ubuntu-64"', encoding="utf-8")
|
||||
if self._fail == "clone":
|
||||
raise BackendOperationFailed("clone", 1, "no")
|
||||
return VmHandle(identifier=destination or name, name=name)
|
||||
|
||||
def start(self, handle: VmHandle, headless: bool = True) -> None:
|
||||
self.calls.append("start")
|
||||
|
||||
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||
self.calls.append("get_ip")
|
||||
return self._ip
|
||||
|
||||
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||
self.calls.append("stop")
|
||||
|
||||
def delete(self, handle: VmHandle) -> None:
|
||||
self.calls.append("delete")
|
||||
|
||||
|
||||
def _make_template(tmp_path: Path) -> Path:
|
||||
tpl = tmp_path / "tpl" / "LinuxBuild2404.vmx"
|
||||
tpl.parent.mkdir(parents=True)
|
||||
tpl.write_text('guestOS = "ubuntu-64"', encoding="utf-8")
|
||||
return tpl
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _measure_paths(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
|
||||
env = {
|
||||
"CI_BUILD_VMS": str(tmp_path / "build-vms"),
|
||||
"CI_ARTIFACTS": str(tmp_path / "artifacts"),
|
||||
}
|
||||
real = bench_module.load_config
|
||||
monkeypatch.setattr(bench_module, "load_config", lambda: real(env=env))
|
||||
return tmp_path
|
||||
|
||||
|
||||
def test_bench_measure_happy_path(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
backend = _FakeBackend(ip="10.0.0.5")
|
||||
monkeypatch.setattr(bench_module, "load_backend", lambda _c: backend)
|
||||
# transport ready immediately
|
||||
monkeypatch.setattr(bench_module, "_tcp_open", lambda *a, **k: True)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"bench",
|
||||
"measure",
|
||||
"--guest-os",
|
||||
"linux",
|
||||
"--iterations",
|
||||
"2",
|
||||
"--template-path",
|
||||
str(tpl),
|
||||
"--timeout-seconds",
|
||||
"30",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "clone" in backend.calls
|
||||
assert "delete" in backend.calls
|
||||
jsonl = _measure_paths / "artifacts" / "benchmark.jsonl"
|
||||
assert jsonl.is_file()
|
||||
records = [json.loads(line) for line in jsonl.read_text().splitlines()]
|
||||
assert len(records) == 2
|
||||
# Legacy field names preserved.
|
||||
assert set(records[0]) >= {
|
||||
"ts", "runId", "iteration", "template", "snapshot", "guestOS",
|
||||
"cloneSec", "startSec", "ipSec", "readySec", "destroySec",
|
||||
"totalBootSec", "deltaKB", "vmIP", "error",
|
||||
}
|
||||
assert records[0]["vmIP"] == "10.0.0.5"
|
||||
assert records[0]["error"] is None
|
||||
assert "Average boot-to-ready" in result.output
|
||||
|
||||
|
||||
def test_bench_measure_ip_timeout_records_error(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
backend = _FakeBackend(ip=None) # never returns an IP
|
||||
monkeypatch.setattr(bench_module, "load_backend", lambda _c: backend)
|
||||
# Make the IP-wait loop terminate fast: a monotonic clock that jumps
|
||||
# 100s per call, so any deadline is exceeded within a couple of polls.
|
||||
counter = {"t": 0.0}
|
||||
|
||||
def _fast_now() -> float:
|
||||
counter["t"] += 100.0
|
||||
return counter["t"]
|
||||
|
||||
monkeypatch.setattr(bench_module, "_now", _fast_now)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["bench", "measure", "--template-path", str(tpl), "--timeout-seconds", "30"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
jsonl = _measure_paths / "artifacts" / "benchmark.jsonl"
|
||||
rec = json.loads(jsonl.read_text().splitlines()[0])
|
||||
assert rec["error"] is not None
|
||||
assert rec["readySec"] is None
|
||||
# destroy still ran (finally block).
|
||||
assert "delete" in backend.calls
|
||||
|
||||
|
||||
def test_bench_measure_template_missing(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "measure", "--template-path", str(_measure_paths / "nope.vmx")]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Template VMX not found" in result.output
|
||||
|
||||
|
||||
def test_bench_measure_backend_unavailable(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
|
||||
def _raise(_c: Any) -> Any:
|
||||
raise BackendNotAvailable("no vmrun")
|
||||
|
||||
monkeypatch.setattr(bench_module, "load_backend", _raise)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "measure", "--template-path", str(tpl)]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "backend unavailable" in result.output
|
||||
|
||||
|
||||
def test_bench_measure_clone_failure_records_error(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
backend = _FakeBackend(fail="clone")
|
||||
monkeypatch.setattr(bench_module, "load_backend", lambda _c: backend)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "measure", "--template-path", str(tpl), "--timeout-seconds", "30"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
jsonl = _measure_paths / "artifacts" / "benchmark.jsonl"
|
||||
rec = json.loads(jsonl.read_text().splitlines()[0])
|
||||
assert rec["error"] is not None
|
||||
assert rec["cloneSec"] is None
|
||||
|
||||
|
||||
def test_bench_measure_transport_port_timeout(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
backend = _FakeBackend(ip="10.0.0.9")
|
||||
monkeypatch.setattr(bench_module, "load_backend", lambda _c: backend)
|
||||
# IP is acquired immediately, but the transport port never opens.
|
||||
monkeypatch.setattr(bench_module, "_wait_port", lambda *a, **k: False)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "measure", "--template-path", str(tpl), "--timeout-seconds", "30"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
jsonl = _measure_paths / "artifacts" / "benchmark.jsonl"
|
||||
rec = json.loads(jsonl.read_text().splitlines()[0])
|
||||
assert rec["readySec"] is None
|
||||
assert "port" in (rec["error"] or "")
|
||||
|
||||
|
||||
def test_bench_measure_destroy_suppresses_backend_errors(
|
||||
monkeypatch: pytest.MonkeyPatch, _measure_paths: Path
|
||||
) -> None:
|
||||
tpl = _make_template(_measure_paths)
|
||||
|
||||
class _FlakyBackend(_FakeBackend):
|
||||
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||
self.calls.append("stop")
|
||||
raise BackendOperationFailed("stop", 1, "no")
|
||||
|
||||
def delete(self, handle: VmHandle) -> None:
|
||||
self.calls.append("delete")
|
||||
raise BackendOperationFailed("delete", 1, "no")
|
||||
|
||||
backend = _FlakyBackend(ip="10.0.0.5")
|
||||
monkeypatch.setattr(bench_module, "load_backend", lambda _c: backend)
|
||||
monkeypatch.setattr(bench_module, "_tcp_open", lambda *a, **k: True)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "measure", "--template-path", str(tpl), "--timeout-seconds", "30"]
|
||||
)
|
||||
# stop/delete raising must not crash the command.
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "stop" in backend.calls
|
||||
assert "delete" in backend.calls
|
||||
|
||||
|
||||
def test_bench_run_report_write_failure_is_tolerated(
|
||||
monkeypatch: pytest.MonkeyPatch, _ci_paths: Path
|
||||
) -> None:
|
||||
monkeypatch.setattr(bench_module, "_run_one_job", lambda **_kw: (0, ""))
|
||||
|
||||
def _boom(_report: Any, _path: Any) -> None:
|
||||
raise OSError("disk full")
|
||||
|
||||
monkeypatch.setattr(bench_module, "_write_run_report", _boom)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["bench", "run", "--concurrency", "1", "--rounds", "1"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "could not write report" in result.output
|
||||
|
||||
|
||||
def test_bench_measure_default_template_windows(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""--guest-os windows resolves the WinBuild default template + snapshot."""
|
||||
from ci_orchestrator.config import load_config as real_load
|
||||
|
||||
cfg = real_load(env={"CI_ROOT": str(tmp_path)})
|
||||
tpl, snap = bench_module._resolve_template(cfg, "windows")
|
||||
assert tpl.name == "WinBuild2025.vmx"
|
||||
assert snap == "BaseClean"
|
||||
tpl_l, snap_l = bench_module._resolve_template(cfg, "linux")
|
||||
assert tpl_l.name == "LinuxBuild2404.vmx"
|
||||
assert snap_l == "BaseClean-Linux"
|
||||
|
||||
|
||||
# ── dataclass behaviour ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_phase_timing_total_boot_none_when_incomplete() -> None:
|
||||
t = PhaseTiming(iteration=1, run_id="r", template="t", snapshot="s", guest_os="linux")
|
||||
assert t.total_boot_sec is None
|
||||
t.clone_sec, t.start_sec, t.ip_sec, t.ready_sec = 1.0, 2.0, 3.0, 4.0
|
||||
assert t.total_boot_sec == 10.0
|
||||
|
||||
|
||||
def test_job_result_dataclass_defaults() -> None:
|
||||
jr = JobResult(job_id="j", slot=1, exit_code=0, status="PASS", elapsed_sec=1.0)
|
||||
assert jr.error == ""
|
||||
assert bench is not None
|
||||
@@ -10,8 +10,9 @@ from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.build as build_module
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.config import BackendConfig, Config, Paths
|
||||
from ci_orchestrator.credentials import Credential
|
||||
from ci_orchestrator.transport.errors import TransportCommandError
|
||||
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
@@ -211,6 +212,83 @@ def test_build_run_linux_clone_url_happy_path(
|
||||
assert any("/work/build" in c for c in cmds)
|
||||
|
||||
|
||||
def test_build_run_linux_xvfb_wraps_command(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""--xvfb wraps the Linux build in xvfb-run; env exports stay outside."""
|
||||
_patch(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build", "run",
|
||||
"--ip-address", "10.0.0.5",
|
||||
"--guest-os", "linux",
|
||||
"--clone-url", "https://example/repo.git",
|
||||
"--build-command", "make all",
|
||||
"--guest-linux-work-dir", "/work",
|
||||
"--extra-env", "FOO=bar",
|
||||
"--xvfb",
|
||||
"--ssh-key-path", str(tmp_path / "key"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||
# Build wrapped under xvfb-run via sh -c; env export precedes it.
|
||||
assert any(
|
||||
"xvfb-run -a sh -c 'cd /work && make all'" in c
|
||||
and "export FOO=bar" in c
|
||||
for c in cmds
|
||||
)
|
||||
|
||||
|
||||
def test_build_run_linux_no_xvfb_runs_plain(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Without --xvfb the build runs directly (no xvfb-run wrapper)."""
|
||||
_patch(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build", "run",
|
||||
"--ip-address", "10.0.0.5",
|
||||
"--guest-os", "linux",
|
||||
"--clone-url", "https://example/repo.git",
|
||||
"--build-command", "make all",
|
||||
"--guest-linux-work-dir", "/work",
|
||||
"--ssh-key-path", str(tmp_path / "key"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||
assert any("cd /work && make all" in c for c in cmds)
|
||||
assert not any("xvfb-run" in c for c in cmds)
|
||||
|
||||
|
||||
def test_build_run_xvfb_ignored_on_windows(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""--xvfb is a no-op (with a notice) for Windows guests."""
|
||||
_patch(monkeypatch)
|
||||
src = tmp_path / "src"
|
||||
src.mkdir()
|
||||
(src / "f.txt").write_text("x")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build", "run",
|
||||
"--ip-address", "10.0.0.9",
|
||||
"--guest-os", "windows",
|
||||
"--host-source-dir", str(src),
|
||||
"--build-command", "echo hi",
|
||||
"--xvfb",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--xvfb ignored for Windows guest" in result.output
|
||||
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||
assert not any("xvfb-run" in c for c in cmds)
|
||||
|
||||
|
||||
def test_build_run_linux_host_source_dir(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
@@ -370,3 +448,393 @@ def test_build_run_windows_host_source_dir(
|
||||
# Staging copies the raw source into the collect dir (no zip-in-zip).
|
||||
assert any("Copy-Item" in c and "'dist'" in c for c in cmds)
|
||||
assert not any("Compress-Archive" in c and "'dist'" in c for c in cmds)
|
||||
|
||||
|
||||
# ── _parse_extra_env unit tests ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_parse_extra_env_empty_string_skipped() -> None:
|
||||
"""Line 62: empty strings in the iterable are silently skipped (continue branch)."""
|
||||
result = build_module._parse_extra_env(["", "FOO=bar", ""])
|
||||
assert result == {"FOO": "bar"}
|
||||
|
||||
|
||||
def test_parse_extra_env_no_equals_raises() -> None:
|
||||
"""Line 63-66: value with no '=' must raise BadParameter."""
|
||||
import click
|
||||
|
||||
with pytest.raises(click.BadParameter, match="KEY=VALUE"):
|
||||
build_module._parse_extra_env(["BADVALUE"])
|
||||
|
||||
|
||||
def test_parse_extra_env_empty_key_raises() -> None:
|
||||
"""Line 69: '=value' with empty key must raise BadParameter."""
|
||||
import click
|
||||
|
||||
with pytest.raises(click.BadParameter, match="key cannot be empty"):
|
||||
build_module._parse_extra_env(["=value"])
|
||||
|
||||
|
||||
# ── _zip_dir skips subdirectories ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_zip_dir_skips_subdirectories(tmp_path: Path) -> None:
|
||||
"""Line 80->79: _zip_dir iterates rglob('*'); directories are skipped (is_file() == False)."""
|
||||
import zipfile
|
||||
|
||||
src = tmp_path / "src"
|
||||
src.mkdir()
|
||||
(src / "file.txt").write_text("hello", encoding="utf-8")
|
||||
subdir = src / "subdir"
|
||||
subdir.mkdir()
|
||||
(subdir / "nested.txt").write_text("world", encoding="utf-8")
|
||||
|
||||
archive = tmp_path / "out.zip"
|
||||
build_module._zip_dir(src, archive)
|
||||
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
names = zf.namelist()
|
||||
# Only files should appear; the bare subdirectory entry must not be present.
|
||||
assert "file.txt" in names
|
||||
assert str(Path("subdir") / "nested.txt") in names or "subdir/nested.txt" in names
|
||||
# No entry that ends in '/' (a directory entry).
|
||||
assert not any(n.endswith("/") for n in names)
|
||||
|
||||
|
||||
# ── _report_build_output: stdout empty, stderr present ────────────────────
|
||||
|
||||
|
||||
def test_report_build_output_stderr_only(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
"""Lines 108->111, 112-113: when stdout is empty, only stderr branch executes."""
|
||||
result = build_module._report_build_output("", "some error text")
|
||||
captured = capsys.readouterr()
|
||||
assert "some error text" in captured.err
|
||||
assert captured.out.count("[build run]") == 2 # header + footer lines
|
||||
# The tail should reflect the stderr content.
|
||||
assert "some error text" in result
|
||||
|
||||
|
||||
# ── _linux_build: neither host_source_dir nor clone_url ───────────────────
|
||||
|
||||
|
||||
def test_linux_build_no_source_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Line 189: _linux_build with no source raises ClickException."""
|
||||
import click
|
||||
|
||||
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||
with pytest.raises(click.ClickException, match="either --host-source-dir or --clone-url"):
|
||||
build_module._linux_build(
|
||||
ip_address="10.0.0.1",
|
||||
ssh_user="ci_build",
|
||||
key_path=None,
|
||||
known_hosts=None,
|
||||
workdir="/opt/ci/build",
|
||||
output_dir="/opt/ci/output",
|
||||
host_source_dir=None,
|
||||
clone_url=None,
|
||||
clone_branch="main",
|
||||
clone_commit="",
|
||||
clone_submodules=False,
|
||||
build_command="make",
|
||||
artifact_source="dist",
|
||||
extra_env={},
|
||||
skip_artifact=False,
|
||||
)
|
||||
|
||||
|
||||
# ── _linux_build: clone_url + clone_commit triggers fetch+checkout ─────────
|
||||
|
||||
|
||||
def test_linux_build_clone_commit_fetch_checkout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Lines 230-233: clone_commit causes 3 subprocess calls (clone, fetch, checkout) via SSH."""
|
||||
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||
build_module._linux_build(
|
||||
ip_address="10.0.0.1",
|
||||
ssh_user="ci_build",
|
||||
key_path=None,
|
||||
known_hosts=None,
|
||||
workdir="/opt/ci/build",
|
||||
output_dir="/opt/ci/output",
|
||||
host_source_dir=None,
|
||||
clone_url="http://repo/example.git",
|
||||
clone_branch="main",
|
||||
clone_commit="abc123",
|
||||
clone_submodules=False,
|
||||
build_command="true",
|
||||
artifact_source="dist",
|
||||
extra_env={},
|
||||
skip_artifact=True,
|
||||
)
|
||||
instance = _FakeTransport.instances[0]
|
||||
cmds = [c[0] for c in instance.runs]
|
||||
assert any("git clone" in c for c in cmds)
|
||||
assert any("fetch --depth 1 origin" in c and "abc123" in c for c in cmds)
|
||||
assert any("checkout" in c and "abc123" in c for c in cmds)
|
||||
|
||||
|
||||
# ── _windows_build: clone_url + clone_commit triggers fetch+checkout ───────
|
||||
|
||||
|
||||
def test_windows_build_clone_commit_fetch_checkout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Lines 315-321: clone_commit in windows path causes fetch+checkout run."""
|
||||
_patch(monkeypatch)
|
||||
build_module._windows_build(
|
||||
ip_address="10.0.0.7",
|
||||
credential_target="BuildVMGuest",
|
||||
workdir="C:\\CI\\build",
|
||||
output_dir="C:\\CI\\output",
|
||||
host_source_dir=None,
|
||||
clone_url="https://example/repo.git",
|
||||
clone_branch="main",
|
||||
clone_commit="deadbeef",
|
||||
clone_submodules=False,
|
||||
build_command="echo done",
|
||||
artifact_source="dist",
|
||||
configuration="Release",
|
||||
extra_env={},
|
||||
skip_artifact=True,
|
||||
use_shared_cache=False,
|
||||
)
|
||||
instance = _FakeTransport.instances[0]
|
||||
cmds = [c[0] for c in instance.runs]
|
||||
assert any("git clone" in c for c in cmds)
|
||||
assert any("fetch --depth 1 origin" in c and "deadbeef" in c for c in cmds)
|
||||
assert any("checkout" in c and "deadbeef" in c for c in cmds)
|
||||
|
||||
|
||||
# ── _windows_build: no source raises ClickException ────────────────────────
|
||||
|
||||
|
||||
def test_windows_build_no_source_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Line 321: _windows_build with no source raises ClickException."""
|
||||
import click
|
||||
|
||||
_patch(monkeypatch)
|
||||
with pytest.raises(click.ClickException, match="either --host-source-dir or --clone-url"):
|
||||
build_module._windows_build(
|
||||
ip_address="10.0.0.7",
|
||||
credential_target="BuildVMGuest",
|
||||
workdir="C:\\CI\\build",
|
||||
output_dir="C:\\CI\\output",
|
||||
host_source_dir=None,
|
||||
clone_url=None,
|
||||
clone_branch="main",
|
||||
clone_commit="",
|
||||
clone_submodules=False,
|
||||
build_command="echo done",
|
||||
artifact_source="dist",
|
||||
configuration="Release",
|
||||
extra_env={},
|
||||
skip_artifact=False,
|
||||
use_shared_cache=False,
|
||||
)
|
||||
|
||||
|
||||
def test_windows_build_extra_env_semicolon_appended(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Line 330: when extra_env is non-empty, env_setup gets '; ' appended before build command."""
|
||||
_patch(monkeypatch)
|
||||
build_module._windows_build(
|
||||
ip_address="10.0.0.7",
|
||||
credential_target="BuildVMGuest",
|
||||
workdir="C:\\CI\\build",
|
||||
output_dir="C:\\CI\\output",
|
||||
host_source_dir=None,
|
||||
clone_url="https://example/repo.git",
|
||||
clone_branch="main",
|
||||
clone_commit="",
|
||||
clone_submodules=False,
|
||||
build_command="echo done",
|
||||
artifact_source="dist",
|
||||
configuration="Release",
|
||||
extra_env={"MY_VAR": "hello"},
|
||||
skip_artifact=True,
|
||||
use_shared_cache=False,
|
||||
)
|
||||
instance = _FakeTransport.instances[0]
|
||||
cmds = [c[0] for c in instance.runs]
|
||||
# The build command should contain the injected env var followed by '; '
|
||||
assert any("$env:MY_VAR = 'hello'" in c for c in cmds)
|
||||
|
||||
|
||||
def test_windows_build_build_failure_raises(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Lines 365-367: when the build command returns non-zero, ClickException is raised."""
|
||||
import click
|
||||
|
||||
_patch(monkeypatch)
|
||||
|
||||
class _FailsBuild(_FakeTransport):
|
||||
def run(self, script: str, *, check: bool = True) -> _FakeResult:
|
||||
if "echo fail" in script:
|
||||
return _FakeResult(stdout="", stderr="boom", returncode=1)
|
||||
return super().run(script, check=check)
|
||||
|
||||
monkeypatch.setattr(build_module, "WinRmTransport", _FailsBuild)
|
||||
with pytest.raises(click.ClickException, match="build command failed"):
|
||||
build_module._windows_build(
|
||||
ip_address="10.0.0.7",
|
||||
credential_target="BuildVMGuest",
|
||||
workdir="C:\\CI\\build",
|
||||
output_dir="C:\\CI\\output",
|
||||
host_source_dir=None,
|
||||
clone_url="https://example/repo.git",
|
||||
clone_branch="main",
|
||||
clone_commit="",
|
||||
clone_submodules=False,
|
||||
build_command="echo fail",
|
||||
artifact_source="dist",
|
||||
configuration="Release",
|
||||
extra_env={},
|
||||
skip_artifact=False,
|
||||
use_shared_cache=False,
|
||||
)
|
||||
|
||||
|
||||
# ── _linux_build: artifact_source == output_dir → collecting in place ─────
|
||||
|
||||
|
||||
def test_linux_build_artifact_source_equals_output_dir(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Lines 365, 370-371: when artifact_source is the output_dir itself, the
|
||||
'collecting in place' branch runs an existence/non-empty check."""
|
||||
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||
output_dir = "/opt/ci/output"
|
||||
build_module._linux_build(
|
||||
ip_address="10.0.0.1",
|
||||
ssh_user="ci_build",
|
||||
key_path=None,
|
||||
known_hosts=None,
|
||||
workdir="/opt/ci/build",
|
||||
output_dir=output_dir,
|
||||
host_source_dir=None,
|
||||
clone_url="https://example/repo.git",
|
||||
clone_branch="main",
|
||||
clone_commit="",
|
||||
clone_submodules=False,
|
||||
build_command="true",
|
||||
# Use the absolute output dir so srcpath == out
|
||||
artifact_source=output_dir,
|
||||
extra_env={},
|
||||
skip_artifact=False,
|
||||
)
|
||||
instance = _FakeTransport.instances[0]
|
||||
cmds = [c[0] for c in instance.runs]
|
||||
# The 'collecting in place' guard command must appear.
|
||||
assert any("if [ ! -d" in c and output_dir in c for c in cmds)
|
||||
# The wipe-and-copy command must NOT appear.
|
||||
assert not any("rm -rf" in c and "mkdir -p" in c and output_dir in c for c in cmds)
|
||||
|
||||
|
||||
# ── build run CLI: ssh_key_path from config ────────────────────────────────
|
||||
|
||||
|
||||
def _make_config(ssh_key_path: Path | None = None) -> Config:
|
||||
paths = Paths(
|
||||
root=Path("/var/lib/ci"),
|
||||
templates=Path("/var/lib/ci/templates"),
|
||||
build_vms=Path("/var/lib/ci/build-vms"),
|
||||
artifacts=Path("/var/lib/ci/artifacts"),
|
||||
keys=Path("/var/lib/ci/keys"),
|
||||
)
|
||||
return Config(paths=paths, backend=BackendConfig(), ssh_key_path=ssh_key_path)
|
||||
|
||||
|
||||
def test_build_run_linux_ssh_key_from_config(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 548: when --ssh-key-path is omitted but config.ssh_key_path is set, it is used."""
|
||||
_patch(monkeypatch)
|
||||
key = tmp_path / "ci_linux"
|
||||
monkeypatch.setattr(build_module, "load_config", lambda: _make_config(ssh_key_path=key))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build",
|
||||
"run",
|
||||
"--ip-address",
|
||||
"10.0.0.5",
|
||||
"--guest-os",
|
||||
"linux",
|
||||
"--clone-url",
|
||||
"https://example/repo.git",
|
||||
"--build-command",
|
||||
"true",
|
||||
"--skip-artifact",
|
||||
# deliberately NO --ssh-key-path
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
# The transport must have been instantiated with the config key path.
|
||||
instance = _FakeTransport.instances[0]
|
||||
assert str(key) in (instance.kwargs.get("key_path") or instance.args[1] if len(instance.args) > 1 else "") or instance.kwargs.get("key_path") == str(key)
|
||||
|
||||
|
||||
# ── build run CLI: linux TransportError → ClickException ──────────────────
|
||||
|
||||
|
||||
def test_build_run_linux_transport_error(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Line 568: TransportError from _linux_build is caught and re-raised as ClickException."""
|
||||
_patch(monkeypatch)
|
||||
monkeypatch.setattr(build_module, "load_config", lambda: _make_config())
|
||||
|
||||
class _RaisesTransport(_FakeTransport):
|
||||
def __enter__(self) -> _RaisesTransport:
|
||||
raise TransportError("connection refused")
|
||||
|
||||
monkeypatch.setattr(build_module, "SshTransport", _RaisesTransport)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build",
|
||||
"run",
|
||||
"--ip-address",
|
||||
"10.0.0.5",
|
||||
"--guest-os",
|
||||
"linux",
|
||||
"--clone-url",
|
||||
"https://example/repo.git",
|
||||
"--build-command",
|
||||
"true",
|
||||
"--ssh-key-path",
|
||||
str(tmp_path / "key"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "connection refused" in result.output
|
||||
|
||||
|
||||
# ── build run CLI: windows TransportError → ClickException ────────────────
|
||||
|
||||
|
||||
def test_build_run_windows_transport_error(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Lines 591-592: TransportError from _windows_build is caught and re-raised as ClickException."""
|
||||
_patch(monkeypatch)
|
||||
monkeypatch.setattr(build_module, "load_config", lambda: _make_config())
|
||||
|
||||
class _RaisesTransport(_FakeTransport):
|
||||
def __enter__(self) -> _RaisesTransport:
|
||||
raise TransportError("winrm unreachable")
|
||||
|
||||
monkeypatch.setattr(build_module, "WinRmTransport", _RaisesTransport)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"build",
|
||||
"run",
|
||||
"--ip-address",
|
||||
"10.0.0.7",
|
||||
"--guest-os",
|
||||
"windows",
|
||||
"--clone-url",
|
||||
"https://example/repo.git",
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "winrm unreachable" in result.output
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Tests for ``creds set`` (C6).
|
||||
|
||||
CLI registration smoke tests (``--help``) are kept; behavioural tests mock
|
||||
the ``keyring`` module and include a round-trip via ``KeyringCredentialStore``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
|
||||
# ── --help smoke tests (kept) ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_creds_help_lists_set() -> None:
|
||||
result = CliRunner().invoke(cli, ["creds", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "set" in result.output
|
||||
|
||||
|
||||
def test_creds_set_help() -> None:
|
||||
result = CliRunner().invoke(cli, ["creds", "set", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--target" in result.output
|
||||
assert "--user" in result.output
|
||||
|
||||
|
||||
# ── fake keyring ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _install_fake_keyring(monkeypatch: pytest.MonkeyPatch) -> dict[tuple[str, str], str]:
|
||||
"""Install a fake ``keyring`` module with a two-key password store.
|
||||
|
||||
Returns the backing dict so tests can inspect what was written. The fake
|
||||
implements both ``set_password`` (writer) and ``get_password`` /
|
||||
``get_credential`` (reader), so round-trips work end to end.
|
||||
"""
|
||||
store: dict[tuple[str, str], str] = {}
|
||||
fake = types.ModuleType("keyring")
|
||||
|
||||
def set_password(service: str, key: str, password: str) -> None:
|
||||
store[(service, key)] = password
|
||||
|
||||
def get_password(service: str, key: str) -> str | None:
|
||||
return store.get((service, key))
|
||||
|
||||
def get_credential(service: str, _username: str | None) -> Any:
|
||||
# File backends are a no-op here (mirrors PlaintextKeyring); the reader
|
||||
# falls through to the two-entry scheme.
|
||||
return None
|
||||
|
||||
fake.set_password = set_password # type: ignore[attr-defined]
|
||||
fake.get_password = get_password # type: ignore[attr-defined]
|
||||
fake.get_credential = get_credential # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, "keyring", fake)
|
||||
return store
|
||||
|
||||
|
||||
# ── write scheme ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_creds_set_writes_two_entries(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store = _install_fake_keyring(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "BuildVMGuest", "--user", "ci_build",
|
||||
"--password-stdin"],
|
||||
input="s3cret\n",
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
# username under "{target}:meta"
|
||||
assert store[("BuildVMGuest:meta", "username")] == "ci_build"
|
||||
# password under target keyed by username
|
||||
assert store[("BuildVMGuest", "ci_build")] == "s3cret"
|
||||
|
||||
|
||||
def test_creds_set_round_trips_via_store(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Round-trip: write with `creds set`, read back with the reader."""
|
||||
_install_fake_keyring(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "BuildVMGuest", "--user", "WINBUILD\\ci",
|
||||
"--password-stdin"],
|
||||
input="p@ss word!\n",
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
cred = KeyringCredentialStore().get("BuildVMGuest")
|
||||
assert cred.username == "WINBUILD\\ci"
|
||||
assert cred.password == "p@ss word!"
|
||||
|
||||
|
||||
def test_creds_set_default_target(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store = _install_fake_keyring(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--user", "ci_build", "--password-stdin"],
|
||||
input="abc\n",
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert store[("BuildVMGuest:meta", "username")] == "ci_build"
|
||||
|
||||
|
||||
# ── password sourcing ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_creds_set_prompts_when_no_stdin_flag(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
store = _install_fake_keyring(monkeypatch)
|
||||
# click.prompt with confirmation_prompt reads the value twice.
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "T", "--user", "u"],
|
||||
input="hunter2\nhunter2\n",
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert store[("T", "u")] == "hunter2"
|
||||
|
||||
|
||||
def test_creds_set_empty_password_rejected(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_install_fake_keyring(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "T", "--user", "u", "--password-stdin"],
|
||||
input="\n",
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "empty" in result.output.lower()
|
||||
|
||||
|
||||
def test_creds_set_strips_trailing_newline(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
store = _install_fake_keyring(monkeypatch)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "T", "--user", "u", "--password-stdin"],
|
||||
input="secret\r\n",
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert store[("T", "u")] == "secret"
|
||||
|
||||
|
||||
def test_creds_set_never_accepts_password_option() -> None:
|
||||
"""The CLI must NOT expose a --password option."""
|
||||
result = CliRunner().invoke(cli, ["creds", "set", "--help"])
|
||||
assert "--password " not in result.output
|
||||
assert "--password=" not in result.output
|
||||
|
||||
|
||||
def test_creds_set_keyring_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When keyring import fails, surface a clean ClickException."""
|
||||
import ci_orchestrator.commands.creds as creds_module
|
||||
|
||||
def _raise(_t: str, _u: str, _p: str) -> None:
|
||||
raise RuntimeError("keyring is not installed; run `pip install keyring`.")
|
||||
|
||||
monkeypatch.setattr(creds_module, "_store_credential", _raise)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["creds", "set", "--target", "T", "--user", "u", "--password-stdin"],
|
||||
input="x\n",
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "keyring is not installed" in result.output
|
||||
|
||||
|
||||
def test_store_credential_invokes_keyring(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Unit-level: _store_credential writes both entries to the module."""
|
||||
import ci_orchestrator.commands.creds as creds_module
|
||||
|
||||
store = _install_fake_keyring(monkeypatch)
|
||||
creds_module._store_credential("Tgt", "user1", "pw1")
|
||||
assert store[("Tgt:meta", "username")] == "user1"
|
||||
assert store[("Tgt", "user1")] == "pw1"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -659,3 +659,49 @@ def test_runner_pruned_old_entries(
|
||||
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
# ── NEW: coverage gap tests ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_disk_alert_json_includes_message_field(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Lines 186-187: disk below threshold + --json → JSON has 'message' key, exit 1."""
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=5))
|
||||
result = CliRunner().invoke(
|
||||
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
payload = json.loads(result.output.strip())
|
||||
assert payload["status"] == "alert"
|
||||
assert "message" in payload
|
||||
assert "below" in payload["message"]
|
||||
|
||||
|
||||
def test_runner_running_gitea_online_no_webhook(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Lines 398->402: online==0 but no webhook_url → branch skips _post_webhook,
|
||||
continues to return (line 402). Covers the 398->402 branch (webhook_url falsy)."""
|
||||
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 0)
|
||||
|
||||
class _Store:
|
||||
def get(self, _target: str) -> Any:
|
||||
from ci_orchestrator.credentials import Credential
|
||||
return Credential("u", "tok")
|
||||
|
||||
import ci_orchestrator.credentials as creds
|
||||
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"monitor", "runner",
|
||||
"--state-dir", str(tmp_path),
|
||||
"--gitea-url", "https://g/",
|
||||
"--gitea-credential-target", "tgt",
|
||||
# no --webhook-url → webhook_url is ""
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "0 online runners" in result.output
|
||||
|
||||
@@ -210,3 +210,37 @@ def test_report_job_failed_filter_no_results(tmp_path: Path) -> None:
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "No matching jobs" in result.output
|
||||
|
||||
|
||||
# ── NEW: coverage gap tests ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_report_job_detail_no_failure_event(tmp_path: Path) -> None:
|
||||
"""Lines 201->203: detail view where events exist but no failure → no 'Error:' line."""
|
||||
_write_jsonl(tmp_path / "run-ok" / "invoke-ci.jsonl", _success_events("run-ok"))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["report", "job", "--log-dir", str(tmp_path), "--job-id", "run-ok"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Job: run-ok" in result.output
|
||||
# No failure event → no "Error:" line should appear
|
||||
assert "Error:" not in result.output
|
||||
|
||||
|
||||
def test_report_job_list_skips_empty_jsonl(tmp_path: Path) -> None:
|
||||
"""Line 218: list mode with an empty/unreadable JSONL file → that file is skipped."""
|
||||
# Write one valid job
|
||||
_write_jsonl(tmp_path / "run-good" / "invoke-ci.jsonl", _success_events("run-good"))
|
||||
# Write an empty JSONL (no events) — should be silently skipped
|
||||
empty_jsonl = tmp_path / "run-empty" / "invoke-ci.jsonl"
|
||||
empty_jsonl.parent.mkdir(parents=True)
|
||||
empty_jsonl.write_text("", encoding="utf-8")
|
||||
|
||||
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
# Only the valid job appears
|
||||
assert "run-good" in result.output
|
||||
# The empty dir's name must not appear as a row (it was skipped)
|
||||
# We can confirm by checking only one data row exists: "success" appears once
|
||||
assert result.output.count("success") >= 1
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Tests for ``ci_orchestrator retention run``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.retention as ret
|
||||
from ci_orchestrator.__main__ import cli
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _fake_usage(free_gb: float, total_gb: float = 500.0) -> shutil._ntuple_diskusage:
|
||||
gb = 1024**3
|
||||
return shutil._ntuple_diskusage(
|
||||
int(total_gb * gb),
|
||||
int((total_gb - free_gb) * gb),
|
||||
int(free_gb * gb),
|
||||
)
|
||||
|
||||
|
||||
def _make_old_dir(base: Path, name: str, age_days: int = 40) -> Path:
|
||||
d = base / name
|
||||
d.mkdir(parents=True)
|
||||
old_mtime = time.time() - age_days * 86400
|
||||
import os
|
||||
|
||||
os.utime(d, (old_mtime, old_mtime))
|
||||
return d
|
||||
|
||||
|
||||
def _make_recent_dir(base: Path, name: str) -> Path:
|
||||
d = base / name
|
||||
d.mkdir(parents=True)
|
||||
return d
|
||||
|
||||
|
||||
# ── retention run — basic purge ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_purge_old_artifact_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
art = tmp_path / "artifacts"
|
||||
art.mkdir()
|
||||
old = _make_old_dir(art, "job-001")
|
||||
recent = _make_recent_dir(art, "job-002")
|
||||
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not old.exists(), "old dir should have been deleted"
|
||||
assert recent.exists(), "recent dir should survive"
|
||||
assert "Purged artifact" in result.output
|
||||
|
||||
|
||||
def test_nothing_to_purge_message(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
art = tmp_path / "artifacts"
|
||||
art.mkdir()
|
||||
_make_recent_dir(art, "job-001")
|
||||
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "nothing to purge" in result.output
|
||||
|
||||
|
||||
def test_missing_artifact_dir_skipped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"retention",
|
||||
"run",
|
||||
"--artifact-dir",
|
||||
str(tmp_path / "nonexistent"),
|
||||
"--log-dir",
|
||||
str(tmp_path / "logs"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "not found" in result.output
|
||||
|
||||
|
||||
# ── aggressive mode ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_aggressive_mode_triggered(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
art = tmp_path / "artifacts"
|
||||
art.mkdir()
|
||||
_make_old_dir(art, "job-001", age_days=10) # 10 days old, > aggressive threshold (7d)
|
||||
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=5))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"retention",
|
||||
"run",
|
||||
"--artifact-dir",
|
||||
str(art),
|
||||
"--log-dir",
|
||||
str(tmp_path / "logs"),
|
||||
"--min-free-gb",
|
||||
"50",
|
||||
"--aggressive-retention-days",
|
||||
"7",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "aggressive retention" in result.output
|
||||
assert "WARNING" in result.output
|
||||
|
||||
|
||||
def test_aggressive_mode_not_triggered_when_space_ok(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
art = tmp_path / "artifacts"
|
||||
art.mkdir()
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Normal retention" in result.output
|
||||
assert "WARNING" not in result.output
|
||||
|
||||
|
||||
# ── --what-if ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_what_if_does_not_delete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
art = tmp_path / "artifacts"
|
||||
art.mkdir()
|
||||
old = _make_old_dir(art, "job-001")
|
||||
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"retention",
|
||||
"run",
|
||||
"--artifact-dir",
|
||||
str(art),
|
||||
"--log-dir",
|
||||
str(tmp_path / "logs"),
|
||||
"--what-if",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert old.exists(), "--what-if must not delete anything"
|
||||
assert "WhatIf" in result.output
|
||||
|
||||
|
||||
# ── log dir purge ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
log = tmp_path / "logs"
|
||||
log.mkdir()
|
||||
old = _make_old_dir(log, "run-001")
|
||||
|
||||
monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"retention",
|
||||
"run",
|
||||
"--artifact-dir",
|
||||
str(tmp_path / "artifacts"),
|
||||
"--log-dir",
|
||||
str(log),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not old.exists()
|
||||
assert "Purged log" in result.output
|
||||
@@ -0,0 +1,286 @@
|
||||
"""Tests for ``ci_orchestrator smoke`` (Phase C1 scaffolding + C4 behaviour)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.smoke as smoke_module
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.config import BackendConfig, Config, Paths
|
||||
|
||||
# ──────────────────────────────────────────────────────── C1 --help smoke
|
||||
|
||||
|
||||
def test_smoke_help_lists_run() -> None:
|
||||
result = CliRunner().invoke(cli, ["smoke", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "run" in result.output
|
||||
|
||||
|
||||
def test_smoke_run_help() -> None:
|
||||
result = CliRunner().invoke(cli, ["smoke", "run", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--guest-os" in result.output
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _patch_config(monkeypatch: pytest.MonkeyPatch, artifacts: Path) -> None:
|
||||
paths = Paths(
|
||||
root=artifacts.parent,
|
||||
templates=artifacts.parent / "templates",
|
||||
build_vms=artifacts.parent / "build-vms",
|
||||
artifacts=artifacts,
|
||||
keys=artifacts.parent / "keys",
|
||||
)
|
||||
cfg = Config(paths=paths, backend=BackendConfig(), ip_pool=None)
|
||||
monkeypatch.setattr(smoke_module, "load_config", lambda: cfg)
|
||||
|
||||
|
||||
def _patch_job(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
artifacts: Path,
|
||||
make_artifact: bool = True,
|
||||
raises: bool = False,
|
||||
) -> dict[str, list[Any]]:
|
||||
"""Replace the in-process ``job`` command with a recording fake."""
|
||||
calls: dict[str, list[Any]] = {"job": []}
|
||||
|
||||
def _fake_callback(**kwargs: Any) -> None:
|
||||
calls["job"].append(kwargs)
|
||||
if make_artifact:
|
||||
job_dir = artifacts / kwargs["job_id"]
|
||||
job_dir.mkdir(parents=True, exist_ok=True)
|
||||
(job_dir / "smoke.txt").write_text("smoke-ok", encoding="utf-8")
|
||||
if raises:
|
||||
import click
|
||||
|
||||
raise click.ClickException("clone failed: boom")
|
||||
|
||||
# ``job_command`` is a click.Command; patch its callback.
|
||||
monkeypatch.setattr(smoke_module.job_command, "callback", _fake_callback)
|
||||
return calls
|
||||
|
||||
|
||||
def _read_events(jsonl_path: Path) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for line in jsonl_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
out.append(json.loads(line))
|
||||
return out
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────── C4 behaviour
|
||||
|
||||
|
||||
def test_smoke_run_linux_happy_path(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--guest-os", "linux", "--job-id", "smoke-1"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "PASSED" in result.output
|
||||
|
||||
# The job ran once with the Linux no-op marker preset.
|
||||
assert len(calls["job"]) == 1
|
||||
kw = calls["job"][0]
|
||||
assert kw["guest_os"] == "linux"
|
||||
assert kw["job_id"] == "smoke-1"
|
||||
assert "smoke.txt" in kw["build_command"]
|
||||
assert kw["guest_artifact_source"] == "/opt/ci/output"
|
||||
|
||||
# Artifact dir was created and the success event is present.
|
||||
assert (artifacts / "smoke-1").is_dir()
|
||||
events = _read_events(tmp_path / "logs" / "smoke-1" / "invoke-ci.jsonl")
|
||||
assert any(e["phase"] == "job" and e["status"] == "start" for e in events)
|
||||
assert any(e["phase"] == "job" and e["status"] == "success" for e in events)
|
||||
|
||||
|
||||
def test_smoke_run_windows_uses_windows_preset(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--guest-os", "windows", "--job-id", "smoke-win"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
kw = calls["job"][0]
|
||||
assert kw["guest_os"] == "windows"
|
||||
assert kw["guest_artifact_source"] == "C:\\CI\\output"
|
||||
assert "C:\\CI\\output" in kw["build_command"]
|
||||
|
||||
|
||||
def test_smoke_run_failure_when_job_raises(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
_patch_job(monkeypatch, artifacts=artifacts, make_artifact=False, raises=True)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--guest-os", "linux", "--job-id", "smoke-fail"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "FAILED" in result.output
|
||||
# A failure event was logged and there is no success event.
|
||||
events = _read_events(tmp_path / "logs" / "smoke-fail" / "invoke-ci.jsonl")
|
||||
assert any(e["status"] == "failure" for e in events)
|
||||
assert not any(e["status"] == "success" for e in events)
|
||||
|
||||
|
||||
def test_smoke_run_fails_when_artifact_missing(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
# Job "succeeds" but produces no artifact dir → smoke must still fail.
|
||||
_patch_job(monkeypatch, artifacts=artifacts, make_artifact=False)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--guest-os", "linux", "--job-id", "smoke-noart"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "artifact dir not found" in result.output
|
||||
|
||||
|
||||
def test_smoke_run_preset_ns7zip(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["smoke", "run", "--guest-os", "linux", "--preset", "ns7zip", "--job-id", "p1"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
kw = calls["job"][0]
|
||||
assert "build_plugin.py" in kw["build_command"]
|
||||
assert "7zip-version" in kw["build_command"]
|
||||
assert kw["guest_artifact_source"] == "plugins"
|
||||
|
||||
|
||||
def test_smoke_run_preset_nsinnounp(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["smoke", "run", "--preset", "nsinnounp", "--job-id", "p2"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
kw = calls["job"][0]
|
||||
assert kw["build_command"] == "python3 build_plugin.py --final --dist-dir dist"
|
||||
assert kw["guest_artifact_source"] == "dist"
|
||||
|
||||
|
||||
def test_smoke_run_preset_rejected_on_windows(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
_patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--guest-os", "windows", "--preset", "ns7zip"]
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "only supported with --guest-os linux" in result.output
|
||||
|
||||
|
||||
def test_smoke_run_preset_and_build_command_mutually_exclusive(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
_patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"smoke", "run", "--preset", "ns7zip",
|
||||
"--build-command", "echo hi",
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "mutually exclusive" in result.output
|
||||
|
||||
|
||||
def test_smoke_run_custom_build_command(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"smoke", "run", "--guest-os", "linux",
|
||||
"--build-command", "make all",
|
||||
"--job-id", "custom-1",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
kw = calls["job"][0]
|
||||
assert kw["build_command"] == "make all"
|
||||
# Custom command keeps the OS-default artifact source.
|
||||
assert kw["guest_artifact_source"] == "/opt/ci/output"
|
||||
|
||||
|
||||
def test_has_job_success_tolerates_malformed_jsonl(tmp_path: Path) -> None:
|
||||
"""Garbage/missing log lines must not raise and must report no success."""
|
||||
missing = tmp_path / "nope.jsonl"
|
||||
assert smoke_module._has_job_success(missing) is False
|
||||
|
||||
jsonl = tmp_path / "log.jsonl"
|
||||
jsonl.write_text(
|
||||
"\n" # blank line
|
||||
"not json\n" # JSONDecodeError
|
||||
"[1, 2, 3]\n" # valid JSON but not a dict
|
||||
'{"phase": "job", "status": "start"}\n', # dict, no success
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert smoke_module._has_job_success(jsonl) is False
|
||||
|
||||
jsonl.write_text('{"phase": "job", "status": "success"}\n', encoding="utf-8")
|
||||
assert smoke_module._has_job_success(jsonl) is True
|
||||
|
||||
|
||||
def test_smoke_run_default_job_id_and_separate_log_dir(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
artifacts = tmp_path / "artifacts"
|
||||
logs = tmp_path / "logs"
|
||||
_patch_config(monkeypatch, artifacts)
|
||||
calls = _patch_job(monkeypatch, artifacts=artifacts)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli, ["smoke", "run", "--log-dir", str(logs)]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
generated_id = calls["job"][0]["job_id"]
|
||||
assert generated_id.startswith("smoke-")
|
||||
# JSONL written under the override log dir, artifacts under config dir.
|
||||
assert (logs / generated_id / "invoke-ci.jsonl").is_file()
|
||||
assert (artifacts / generated_id).is_dir()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user