Merge Fase A: Python rewrite + Linux migration into main
Phase A validated end-to-end (PhaseA-user-checklist Passi 1-8): venv deploy, smoke wait-ready/job (Win+Linux), Gitea workflows (lint, self-test both transports, build-ns7zip matrix + release), burn-in 12/12, benchmark. ci_orchestrator Python pipeline + PS shims, github-free artifacts, in-guest/host-clone transports, perf tuning. Resolution: result tree == feature (authoritative). Removed 5 obsolete files superseded by Phase A (old gitea/ action+lint paths now under .gitea/, Pester tests for the replaced PS New/Remove-BuildVM/ Wait-VMReady scripts). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -27,19 +27,31 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
build-command: 'python build_plugin.py --final --dist-dir dist'
|
# build_plugin.py auto-detects host/toolset and builds all
|
||||||
|
# configs by default; --dist forwards to the per-version script
|
||||||
|
# which copies the DLLs to <repo>/dist/<config> instead of the
|
||||||
|
# committed plugins/ dir.
|
||||||
|
build-command: 'python build_plugin.py --dist'
|
||||||
artifact-source: 'dist'
|
artifact-source: 'dist'
|
||||||
submodules: 'true'
|
submodules: 'true'
|
||||||
guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }}
|
guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }}
|
||||||
# repo-url uses the host SSH alias `gitea-ci` (~/.ssh/config),
|
# In-guest clone (default). repo-url must be reachable FROM the
|
||||||
# which exists only on the host — so the clone MUST run on the
|
# guest (VMnet8 NAT -> gitea.emulab.it), not the host-only SSH
|
||||||
# host, not in the guest. use-git-clone:'false' selects the
|
# alias. ci_orchestrator injects the GiteaPAT credential into
|
||||||
# host-side clone + zip transfer transport.
|
# the HTTPS URL, so GiteaPAT must exist in the LocalSystem
|
||||||
use-git-clone: 'false'
|
# keyring (Set-CIGuestCredential.ps1 -Target GiteaPAT) — or
|
||||||
repo-url: 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git'
|
# nsis-plugin-ns7zip must be public.
|
||||||
|
use-git-clone: 'true'
|
||||||
|
repo-url: 'https://gitea.emulab.it/Simone/nsis-plugin-ns7zip.git'
|
||||||
# Cross-repo build: github.ref_name is this repo's branch, not
|
# Cross-repo build: github.ref_name is this repo's branch, not
|
||||||
# nsis-plugin-ns7zip's. Pin the target repo's branch.
|
# nsis-plugin-ns7zip's. Pin the target repo's branch.
|
||||||
repo-branch: 'main'
|
repo-branch: 'main'
|
||||||
|
# 8 vCPU / 8 GB. WSL on the host builds the 3 configs in
|
||||||
|
# parallel in ~103s (= one config); the 4-vCPU CI VM thrashes
|
||||||
|
# (3x make -j on 4 cores) and takes ~3x. Not structural — just
|
||||||
|
# vCPU starvation. Host is 10c/20t; this matrix = 2 VMs.
|
||||||
|
guest-cpu: '8'
|
||||||
|
guest-memory-mb: '8192'
|
||||||
# Suffix disambiguates artifact dirs: F:\CI\Artifacts\{run_id}-{attempt}-windows
|
# Suffix disambiguates artifact dirs: F:\CI\Artifacts\{run_id}-{attempt}-windows
|
||||||
job-id-suffix: '${{ matrix.target }}'
|
job-id-suffix: '${{ matrix.target }}'
|
||||||
artifact-name: 'nsis-plugin-ns7zip-${{ matrix.target }}-${{ github.ref_name }}'
|
artifact-name: 'nsis-plugin-ns7zip-${{ matrix.target }}-${{ github.ref_name }}'
|
||||||
@@ -49,7 +61,10 @@ jobs:
|
|||||||
# Requires a GITEA_TOKEN secret with repo write permissions.
|
# Requires a GITEA_TOKEN secret with repo write permissions.
|
||||||
release:
|
release:
|
||||||
needs: build
|
needs: build
|
||||||
if: github.ref_type == 'tag'
|
# Tag pushes only. Explicit ${{ }} + event gate: a bare
|
||||||
|
# `if: github.ref_type == 'tag'` can leave the job stuck "blocked"
|
||||||
|
# (instead of skipped) on Gitea for workflow_dispatch runs.
|
||||||
|
if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
|
||||||
runs-on: windows-build
|
runs-on: windows-build
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
@@ -104,4 +119,7 @@ jobs:
|
|||||||
$zipBytes = [System.IO.File]::ReadAllBytes(
|
$zipBytes = [System.IO.File]::ReadAllBytes(
|
||||||
(Resolve-Path $zipName).ProviderPath)
|
(Resolve-Path $zipName).ProviderPath)
|
||||||
Invoke-RestMethod -Method Post -Uri $uploadUrl `
|
Invoke-RestMethod -Method Post -Uri $uploadUrl `
|
||||||
-Headers $uploadHeaders -Body $zipBytes | Out-Null
|
-Headers $uploadHeaders -Body $zipBytes | Out-Null
|
||||||
|
Write-Host "Uploaded asset: $zipName"
|
||||||
|
}
|
||||||
|
Write-Host "Release $env:TAG published with $($release.html_url)"
|
||||||
@@ -4,7 +4,9 @@
|
|||||||
# VM clone -> start -> IP detect -> trivial build -> artifact collect -> destroy
|
# VM clone -> start -> IP detect -> trivial build -> artifact collect -> destroy
|
||||||
#
|
#
|
||||||
# Trigger: workflow_dispatch only (never runs on push/tag).
|
# Trigger: workflow_dispatch only (never runs on push/tag).
|
||||||
# Both jobs are independent — a failure in one does not cancel the other.
|
# Jobs are independent — a failure in one does not cancel the others.
|
||||||
|
# Covers both source transports: in-guest git clone (default) and the
|
||||||
|
# host-side clone + zip transfer variant (use-git-clone: 'false').
|
||||||
#
|
#
|
||||||
# Prerequisites (must be in place before running):
|
# Prerequisites (must be in place before running):
|
||||||
# - Runner labels: windows-build:host and linux-build:host active
|
# - Runner labels: windows-build:host and linux-build:host active
|
||||||
@@ -22,7 +24,7 @@ jobs:
|
|||||||
smoke-windows:
|
smoke-windows:
|
||||||
runs-on: windows-build
|
runs-on: windows-build
|
||||||
steps:
|
steps:
|
||||||
- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
# Trivial no-op build: write a marker file to the CI output directory.
|
# Trivial no-op build: write a marker file to the CI output directory.
|
||||||
# The Windows build VM maps C:\CI\output as the artifact source.
|
# The Windows build VM maps C:\CI\output as the artifact source.
|
||||||
@@ -35,7 +37,7 @@ jobs:
|
|||||||
smoke-linux:
|
smoke-linux:
|
||||||
runs-on: linux-build
|
runs-on: linux-build
|
||||||
steps:
|
steps:
|
||||||
- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
# Trivial no-op build: write a marker file to the CI output directory.
|
# Trivial no-op build: write a marker file to the CI output directory.
|
||||||
# The Linux build VM uses /opt/ci/output as the artifact source.
|
# The Linux build VM uses /opt/ci/output as the artifact source.
|
||||||
@@ -43,4 +45,29 @@ jobs:
|
|||||||
artifact-source: '/opt/ci/output'
|
artifact-source: '/opt/ci/output'
|
||||||
guest-os: 'Linux'
|
guest-os: 'Linux'
|
||||||
job-id-suffix: 'smoke-linux'
|
job-id-suffix: 'smoke-linux'
|
||||||
artifact-name: 'smoke-linux-${{ github.run_id }}'
|
artifact-name: 'smoke-linux-${{ github.run_id }}'
|
||||||
|
|
||||||
|
# ── Host-side clone + zip transfer variant (use-git-clone: 'false') ───────
|
||||||
|
smoke-windows-hostclone:
|
||||||
|
runs-on: windows-build
|
||||||
|
steps:
|
||||||
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
|
with:
|
||||||
|
build-command: 'New-Item -ItemType Directory -Path C:\CI\output -Force | Out-Null; Set-Content C:\CI\output\smoke.txt "smoke-ok"'
|
||||||
|
artifact-source: 'C:\CI\output'
|
||||||
|
guest-os: 'Windows'
|
||||||
|
use-git-clone: 'false'
|
||||||
|
job-id-suffix: 'smoke-win-hostclone'
|
||||||
|
artifact-name: 'smoke-windows-hostclone-${{ github.run_id }}'
|
||||||
|
|
||||||
|
smoke-linux-hostclone:
|
||||||
|
runs-on: linux-build
|
||||||
|
steps:
|
||||||
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
|
with:
|
||||||
|
build-command: 'mkdir -p /opt/ci/output && echo smoke-ok > /opt/ci/output/smoke.txt'
|
||||||
|
artifact-source: '/opt/ci/output'
|
||||||
|
guest-os: 'Linux'
|
||||||
|
use-git-clone: 'false'
|
||||||
|
job-id-suffix: 'smoke-linux-hostclone'
|
||||||
|
artifact-name: 'smoke-linux-hostclone-${{ github.run_id }}'
|
||||||
@@ -51,6 +51,73 @@ $ErrorActionPreference = 'Stop'
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Python development
|
||||||
|
|
||||||
|
L'orchestratore CI vive in `src/ci_orchestrator/` (package `ci-orchestrator`,
|
||||||
|
entry point `python -m ci_orchestrator`). Tutta la logica applicativa
|
||||||
|
nuova va scritta in Python, non in PowerShell. Gli script PS in `scripts/`
|
||||||
|
sono ridotti a shim a 3 righe verso la CLI Python e sono mantenuti solo
|
||||||
|
per compatibilita' con Task Scheduler e caller esterni.
|
||||||
|
|
||||||
|
| Parametro | Valore |
|
||||||
|
| ------------------------ | ------------------------------------------------------------------------------------- |
|
||||||
|
| Python | **3.11+** richiesto (`requires-python` in `pyproject.toml`) |
|
||||||
|
| Venv produzione | `F:\CI\python\venv\` — install **non-editable**: `pip install .` (act_runner=LocalSystem; `-e` rompe con `No module named ci_orchestrator`). Ri-deploy a ogni modifica codice. |
|
||||||
|
| Venv dev locale | `.venv\` (NON committare; gitignored) |
|
||||||
|
| Install (dev locale) | `python -m pip install -e .[dev]` — editable solo nel `.venv` dev |
|
||||||
|
| Test | `python -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90` |
|
||||||
|
| Lint | `python -m ruff check src tests\python` |
|
||||||
|
| Type-check | `python -m mypy --strict src` |
|
||||||
|
|
||||||
|
**Convenzioni Python (enforced da ruff + mypy --strict)**:
|
||||||
|
|
||||||
|
- Type hints obbligatori su funzioni pubbliche e helper esportati. Evitare
|
||||||
|
`Any` se esiste un tipo concreto/Protocol; `Any` solo per fixture pytest
|
||||||
|
e mock.
|
||||||
|
- Niente variabili globali. Lo state vive in oggetti passati esplicitamente
|
||||||
|
(factory `backends.load_backend(config)`, `Config` dataclass).
|
||||||
|
- Niente `print()`: usare `click.echo` (rispetta il flag `-q` di click e
|
||||||
|
il redirect `err=True` per stderr).
|
||||||
|
- Niente `subprocess.run(..., shell=True)`. Niente shell-out per cose che
|
||||||
|
esistono come libreria Python (paramiko per SSH, pypsrp per WinRM,
|
||||||
|
keyring per credenziali).
|
||||||
|
- Encoding stdio sempre UTF-8: i moduli Python lo assumono e
|
||||||
|
`runner/config.yaml` esporta `PYTHONIOENCODING=utf-8` ad act_runner.
|
||||||
|
|
||||||
|
**Mappatura script PowerShell → sotto-comando Python** (Fase A2/A3/A4):
|
||||||
|
|
||||||
|
| PowerShell legacy (shim) | Python sub-command |
|
||||||
|
| ----------------------------------- | ------------------------------------------ |
|
||||||
|
| `Wait-VMReady.ps1` | `python -m ci_orchestrator wait-ready` |
|
||||||
|
| `New-BuildVM.ps1` | `python -m ci_orchestrator vm new` |
|
||||||
|
| `Remove-BuildVM.ps1` | `python -m ci_orchestrator vm remove` |
|
||||||
|
| `Cleanup-OrphanedBuildVMs.ps1` | `python -m ci_orchestrator vm cleanup` |
|
||||||
|
| `Invoke-RemoteBuild.ps1` | `python -m ci_orchestrator build run` |
|
||||||
|
| `Get-BuildArtifacts.ps1` | `python -m ci_orchestrator artifacts collect` |
|
||||||
|
| `Watch-DiskSpace.ps1` | `python -m ci_orchestrator monitor disk` |
|
||||||
|
| `Watch-RunnerHealth.ps1` | `python -m ci_orchestrator monitor runner` |
|
||||||
|
| `Get-CIJobSummary.ps1` | `python -m ci_orchestrator report job` |
|
||||||
|
| `Invoke-CIJob.ps1` | `python -m ci_orchestrator job` |
|
||||||
|
|
||||||
|
Restano in PowerShell (girano sull'host fuori dal flusso CI o nel guest):
|
||||||
|
`Setup-Host.ps1`, `Register-CIScheduledTasks.ps1`, `Backup-CITemplate.ps1`,
|
||||||
|
`Invoke-RetentionPolicy.ps1`, `Measure-CIBenchmark.ps1`,
|
||||||
|
`Test-CapacityBurnIn.ps1`, tutti i `template/Deploy-*.ps1`,
|
||||||
|
`template/Prepare-*.ps1`, `template/Install-CIToolchain-*.ps1` e i
|
||||||
|
`tests/Test-*.ps1` di smoke locale.
|
||||||
|
|
||||||
|
**Test gate**: coverage totale ≥90% (alzato in A5, poi a 90%). Nuovi moduli devono
|
||||||
|
includere unit test con mock; gli errori frequenti di questa stessa
|
||||||
|
sezione sono coperti da `tests/python/test_agents_errors.py` come
|
||||||
|
regression test — non rimuoverli senza aggiornare la voce in
|
||||||
|
"Errori frequenti da evitare" e motivare la rimozione.
|
||||||
|
|
||||||
|
**Closeout fasi A**: vedi `plans/A1-closeout.md`, `plans/A2-closeout.md`,
|
||||||
|
`plans/A3-closeout.md`, `plans/A4-closeout.md`, `plans/A5-closeout.md`
|
||||||
|
per stato per-fase, decisioni prese e voci pendenti a carico utente.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Struttura directory host (fissa)
|
## Struttura directory host (fissa)
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -124,6 +191,11 @@ NON modificare il branch WinRM esistente quando si lavora sul branch Linux e vic
|
|||||||
- `GITEA_CI_ARTIFACT_DIR` — `F:\CI\Artifacts`
|
- `GITEA_CI_ARTIFACT_DIR` — `F:\CI\Artifacts`
|
||||||
- `GITEA_CI_GUEST_CRED_TARGET` — `BuildVMGuest`
|
- `GITEA_CI_GUEST_CRED_TARGET` — `BuildVMGuest`
|
||||||
- `GITEA_CI_SSH_KEY_PATH` — `F:\CI\keys\ci_linux`
|
- `GITEA_CI_SSH_KEY_PATH` — `F:\CI\keys\ci_linux`
|
||||||
|
- `CI_PYTHON_LAUNCHER` — absolute path al Python di sistema usato per
|
||||||
|
bootstrap del venv produzione (`F:\CI\python\venv`). act_runner gira come
|
||||||
|
SYSTEM con PATH sanitizzato (no `python`/`py.exe` risolvibili), quindi i
|
||||||
|
workflow Python leggono questo env come single source of truth. Aggiornare
|
||||||
|
solo quando si fa upgrade dell'interprete.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -151,6 +223,6 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali:
|
|||||||
7. **`[ordered]@{}` non disponibile in PS 2** — non e' un problema qui (PS 5.1), ma ricordare che `[ordered]` e' PS 3+.
|
7. **`[ordered]@{}` non disponibile in PS 2** — non e' un problema qui (PS 5.1), ma ricordare che `[ordered]` e' PS 3+.
|
||||||
8. **`ForEach-Object -Parallel`** — disponibile solo in PS 7+; usare `foreach` classico o `Start-Job`.
|
8. **`ForEach-Object -Parallel`** — disponibile solo in PS 7+; usare `foreach` classico o `Start-Job`.
|
||||||
9. **Snapshot template catturato con VM accesa** — produce file `*.vmem` / `*.vmsn` con memoria; `vmrun clone ... linked -snapshot=<name>` rifiuta con `Error: The virtual machine should not be powered on. It is already running.` anche se `vmrun list` mostra 0 VM. Catturare snapshot SOLO da stato fully powered-off (verificare assenza di `.vmem` nella dir template).
|
9. **Snapshot template catturato con VM accesa** — produce file `*.vmem` / `*.vmsn` con memoria; `vmrun clone ... linked -snapshot=<name>` rifiuta con `Error: The virtual machine should not be powered on. It is already running.` anche se `vmrun list` mostra 0 VM. Catturare snapshot SOLO da stato fully powered-off (verificare assenza di `.vmem` nella dir template).
|
||||||
10. **`vmrun getGuestIPAddress` per check "VM running"** — richiede VMware Tools attivi nel guest; in headless Tools puo' impiegare 30-60s a rispondere e il check appare bloccato. Usare `vmrun list` e cercare il path del VMX nell'output.
|
10. **`vmrun getGuestIPAddress` per check "VM running"** — richiede VMware Tools attivi nel guest; in headless Tools puo' impiegare 30-60s a rispondere e il check appare bloccato. Usare `vmrun list` e cercare il path del VMX nell'output per `is_running`. Per recuperare l'IP usare `getGuestIPAddress -wait` (blocca internamente finché Tools registrano l'IP) e filtrare le righe che iniziano con `"Error:"` perché vmrun in modalità headless può tornare exit code 0 con `"Error: The VMware Tools are not running..."` su stdout invece di usare un exit code non-zero.
|
||||||
11. **Machine-id identico tra clone Linux** — le cloud image Ubuntu hanno un `/etc/machine-id` fisso. VMware DHCP assegna l'IP tramite DHCP client-id derivato da machine-id: se tutti i clone hanno lo stesso machine-id ottengono lo stesso IP e si bloccano per "IP collision". Fix nel template (prima di prendere lo snapshot): `sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id && sudo shutdown -h now`, poi ricreare lo snapshot `BaseClean-Linux` da stato powered-off.
|
11. **Machine-id identico tra clone Linux** — le cloud image Ubuntu hanno un `/etc/machine-id` fisso. VMware DHCP assegna l'IP tramite DHCP client-id derivato da machine-id: se tutti i clone hanno lo stesso machine-id ottengono lo stesso IP e si bloccano per "IP collision". Fix nel template (prima di prendere lo snapshot): `sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id && sudo shutdown -h now`, poi ricreare lo snapshot `BaseClean-Linux` da stato powered-off.
|
||||||
12. **`& nativecmd 2>$null` NON sopprime stderr in PS 5.1 con `$ErrorActionPreference='Stop'`** — la stderr del processo nativo viene convertita in ErrorRecord nel pipeline di PowerShell; sotto `'Stop'` questo diventa un errore terminante PRIMA che `2>$null` possa scartarlo. Esempio: `ssh-keygen -R <ip> -f <file>` stampa "Host X not found in file" su stderr → eccezione terminante. Fix: usare `$ErrorActionPreference = 'SilentlyContinue'` attorno alla chiamata oppure `2>&1 | Out-Null`. Per i job CI su VM ephemere locali, usare direttamente `StrictHostKeyChecking=no UserKnownHostsFile=NUL` (default in `_Transport.psm1` con `KnownHostsFile=''`) — nessun `ssh-keygen` necessario.
|
12. **`& nativecmd 2>$null` NON sopprime stderr in PS 5.1 con `$ErrorActionPreference='Stop'`** — la stderr del processo nativo viene convertita in ErrorRecord nel pipeline di PowerShell; sotto `'Stop'` questo diventa un errore terminante PRIMA che `2>$null` possa scartarlo. Esempio: `ssh-keygen -R <ip> -f <file>` stampa "Host X not found in file" su stderr → eccezione terminante. Fix: usare `$ErrorActionPreference = 'SilentlyContinue'` attorno alla chiamata oppure `2>&1 | Out-Null`. Per i job CI su VM ephemere locali, usare direttamente `StrictHostKeyChecking=no UserKnownHostsFile=NUL` (default in `_Transport.psm1` con `KnownHostsFile=''`) — nessun `ssh-keygen` necessario.
|
||||||
|
|||||||
@@ -3,7 +3,45 @@
|
|||||||
ExcludeRules = @(
|
ExcludeRules = @(
|
||||||
# All CI scripts use Write-Host for structured output captured by act_runner.
|
# All CI scripts use Write-Host for structured output captured by act_runner.
|
||||||
# Write-Output would interleave with function return values and break callers.
|
# Write-Output would interleave with function return values and break callers.
|
||||||
'PSAvoidUsingWriteHost'
|
'PSAvoidUsingWriteHost',
|
||||||
|
|
||||||
|
# Formatting-only rules: generate one warning per line across all existing scripts
|
||||||
|
# (thousands of hits). Run locally with Invoke-ScriptAnalyzer -Fix to auto-format;
|
||||||
|
# do not use as CI gate until the whole codebase has been reformatted.
|
||||||
|
'PSUseConsistentIndentation',
|
||||||
|
'PSUseConsistentWhitespace',
|
||||||
|
|
||||||
|
# The codebase uses the param() + -ArgumentList pattern for remote script blocks
|
||||||
|
# (Invoke-Command, Start-Job) consistently and correctly. PSSA does not recognise
|
||||||
|
# param() inside a ScriptBlock as a local declaration, so it fires a false positive
|
||||||
|
# on every parameter. $Using: is an alternative but not required here.
|
||||||
|
'PSUseUsingScopeModifierInNewRunspaces',
|
||||||
|
|
||||||
|
# UTF-8 without BOM is the editor default; adding BOM to ~35 files is cosmetic
|
||||||
|
# and does not affect PS 5.1 execution.
|
||||||
|
'PSUseBOMForUnicodeEncodedFile',
|
||||||
|
|
||||||
|
# Plural nouns in internal helpers (Get-GuestDiagnostics, Remove-OldJobDirs, etc.)
|
||||||
|
# are intentional — they describe collections, not single objects.
|
||||||
|
'PSUseSingularNouns',
|
||||||
|
|
||||||
|
# PSAvoidUsingPlainTextForPassword fires as a false positive on *CredentialTarget params
|
||||||
|
# (Windows Credential Manager target strings, not passwords). Real plain-text password
|
||||||
|
# usage in template setup scripts is already caught by PSAvoidUsingConvertToSecureStringWithPlainText
|
||||||
|
# with targeted SuppressMessage attributes where intentional.
|
||||||
|
'PSAvoidUsingPlainTextForPassword',
|
||||||
|
|
||||||
|
# These scripts use [switch]$VerifyArtifact = $true for opt-out patterns.
|
||||||
|
# Removing the default would silently change script behaviour.
|
||||||
|
'PSAvoidDefaultValueSwitchParameter',
|
||||||
|
|
||||||
|
# Stop-Vm is an intentional internal name in a VMware-only CI environment;
|
||||||
|
# Hyper-V is not installed on the CI host so there is no real cmdlet to shadow.
|
||||||
|
'PSAvoidOverwritingBuiltInCmdlets',
|
||||||
|
|
||||||
|
# Advisory rule only. Params may be declared for external callers, future use,
|
||||||
|
# or accepted-but-ignored forwarding. Not a safety issue.
|
||||||
|
'PSReviewUnusedParameter'
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── Rule-specific configuration ──────────────────────────────────────────────
|
# ── Rule-specific configuration ──────────────────────────────────────────────
|
||||||
@@ -13,9 +51,11 @@
|
|||||||
Enable = $true
|
Enable = $true
|
||||||
}
|
}
|
||||||
|
|
||||||
# All state-changing functions must support -WhatIf / ShouldProcess
|
# ShouldProcess/WhatIf support is not required for one-shot deployment scripts
|
||||||
|
# and internal CI helper functions; adding it to all state-changing functions
|
||||||
|
# would be invasive with no operational benefit in this environment.
|
||||||
PSUseShouldProcessForStateChangingFunctions = @{
|
PSUseShouldProcessForStateChangingFunctions = @{
|
||||||
Enable = $true
|
Enable = $false
|
||||||
}
|
}
|
||||||
|
|
||||||
# Credentials must flow as [PSCredential], not plain strings
|
# Credentials must flow as [PSCredential], not plain strings
|
||||||
@@ -23,9 +63,9 @@
|
|||||||
Enable = $true
|
Enable = $true
|
||||||
}
|
}
|
||||||
|
|
||||||
# No plain-text passwords in params or variables
|
# Covered by ExcludeRules above (false positives on *CredentialTarget params).
|
||||||
PSAvoidUsingPlainTextForPassword = @{
|
PSAvoidUsingPlainTextForPassword = @{
|
||||||
Enable = $true
|
Enable = $false
|
||||||
}
|
}
|
||||||
|
|
||||||
# ConvertTo-SecureString -AsPlainText only acceptable during template setup;
|
# ConvertTo-SecureString -AsPlainText only acceptable during template setup;
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ Variante **Windows Server 2022**: `*WinBuild2022.ps1` + VMX `F:\CI\Templates\Win
|
|||||||
│ ├── actions/
|
│ ├── actions/
|
||||||
│ │ └── local-ci-build/action.yml # Composite action riusabile (wrap Invoke-CIJob.ps1)
|
│ │ └── local-ci-build/action.yml # Composite action riusabile (wrap Invoke-CIJob.ps1)
|
||||||
│ ├── workflows/
|
│ ├── workflows/
|
||||||
│ │ ├── build-nsInnoUnp.yml # Workflow matrix Windows+Linux per nsis-plugin-nsinnounp
|
│ │ ├── build-ns7zip.yml # Workflow matrix Windows+Linux per nsis-plugin-ns7zip
|
||||||
│ │ └── lint.yml # PSScriptAnalyzer su push/PR (§5.4)
|
│ │ └── lint.yml # PSScriptAnalyzer su push/PR (§5.4)
|
||||||
│ └── workflow-example.yml # Template generico adattabile
|
│ └── workflow-example.yml # Template generico adattabile
|
||||||
│
|
│
|
||||||
@@ -146,6 +146,106 @@ Variante **Windows Server 2022**: `*WinBuild2022.ps1` + VMX `F:\CI\Templates\Win
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
## Python orchestrator (Phase A — in progress)
|
||||||
|
|
||||||
|
A new Python 3.11+ rewrite of the orchestrator lives under `src/ci_orchestrator/`.
|
||||||
|
Phase A (see [implementation-plan-A-B](plans/implementation-plan-A-B.md)) is on
|
||||||
|
branch `feature/python-rewrite-and-linux-migration`. As of A5 the Python CLI is the
|
||||||
|
production entry point invoked by `gitea/actions/local-ci-build/action.yml`;
|
||||||
|
the PowerShell scripts in `scripts/` are 3-line shims that forward to the
|
||||||
|
Python sub-commands listed below.
|
||||||
|
|
||||||
|
### Setup
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Production venv on the runner host (one-time + on every code deploy).
|
||||||
|
# NON-editable: act_runner runs as LocalSystem and cannot resolve an
|
||||||
|
# editable .pth into the dev tree or a transient RunnerWork dir
|
||||||
|
# (-> "No module named ci_orchestrator"). Re-run `pip install .` after
|
||||||
|
# each code change. No CI workflow may install into this venv.
|
||||||
|
python -m venv F:\CI\python\venv
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m pip install --upgrade pip
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m pip install .
|
||||||
|
|
||||||
|
# Local dev venv (NOT committed; see .gitignore) — editable is fine here
|
||||||
|
python -m venv .venv
|
||||||
|
.\.venv\Scripts\python.exe -m pip install -e ".[dev]"
|
||||||
|
```
|
||||||
|
|
||||||
|
Configuration: copy [config.example.toml](config.example.toml) to
|
||||||
|
`F:\CI\config.toml` (or set `$env:CI_CONFIG`). Environment variables
|
||||||
|
(`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`)
|
||||||
|
override the file. Dependency manifest and tool config (ruff, mypy,
|
||||||
|
pytest, coverage) live in [pyproject.toml](pyproject.toml).
|
||||||
|
|
||||||
|
### CLI sub-commands
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Discover commands
|
||||||
|
python -m ci_orchestrator --help
|
||||||
|
|
||||||
|
# Wait for a VM to become reachable (WinRM 5986 or SSH 22 — auto-detected)
|
||||||
|
python -m ci_orchestrator wait-ready `
|
||||||
|
--vmx F:\CI\BuildVMs\smoke\smoke.vmx --guest-os windows --timeout 300
|
||||||
|
|
||||||
|
# Linked-clone lifecycle
|
||||||
|
python -m ci_orchestrator vm new `
|
||||||
|
--template-path F:\CI\Templates\WinBuild2025\WinBuild2025.vmx `
|
||||||
|
--snapshot-name BaseClean `
|
||||||
|
--clone-base-dir F:\CI\BuildVMs `
|
||||||
|
--job-id smoke-001
|
||||||
|
python -m ci_orchestrator vm remove --vmx F:\CI\BuildVMs\Clone_smoke-001_*\Clone_smoke-001_*.vmx
|
||||||
|
python -m ci_orchestrator vm cleanup --max-age-hours 12
|
||||||
|
|
||||||
|
# Build + artifact collection
|
||||||
|
python -m ci_orchestrator build run `
|
||||||
|
--vmx F:\CI\BuildVMs\Clone_smoke-001_*\*.vmx `
|
||||||
|
--script .\build.ps1 --workdir C:\build --guest-os windows
|
||||||
|
python -m ci_orchestrator artifacts collect `
|
||||||
|
--vmx F:\CI\BuildVMs\Clone_smoke-001_*\*.vmx `
|
||||||
|
--remote-path C:\build\dist --local-dir F:\CI\Artifacts\smoke-001
|
||||||
|
|
||||||
|
# Monitoring (driven by Task Scheduler / systemd timers in B5)
|
||||||
|
python -m ci_orchestrator monitor disk --min-free-gb 50 --drive-letter F
|
||||||
|
python -m ci_orchestrator monitor runner --state-dir F:\CI\act_runner
|
||||||
|
|
||||||
|
# Reporting
|
||||||
|
python -m ci_orchestrator report job --job-id smoke-001
|
||||||
|
|
||||||
|
# End-to-end orchestrator (replaces Invoke-CIJob.ps1)
|
||||||
|
python -m ci_orchestrator job `
|
||||||
|
--job-id smoke-001 `
|
||||||
|
--repo-url ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git `
|
||||||
|
--branch main `
|
||||||
|
--template-path F:\CI\Templates\WinBuild2025\WinBuild2025.vmx `
|
||||||
|
--build-command "python build_plugin.py --final --dist-dir dist" `
|
||||||
|
--guest-artifact-source dist
|
||||||
|
```
|
||||||
|
|
||||||
|
The PowerShell wrappers (`scripts\Invoke-CIJob.ps1`,
|
||||||
|
`scripts\New-BuildVM.ps1`, `scripts\Wait-VMReady.ps1`,
|
||||||
|
`scripts\Remove-BuildVM.ps1`, `scripts\Cleanup-OrphanedBuildVMs.ps1`,
|
||||||
|
`scripts\Invoke-RemoteBuild.ps1`, `scripts\Get-BuildArtifacts.ps1`,
|
||||||
|
`scripts\Watch-DiskSpace.ps1`, `scripts\Watch-RunnerHealth.ps1`,
|
||||||
|
`scripts\Get-CIJobSummary.ps1`) are kept as 3-line shims for Task
|
||||||
|
Scheduler and external callers — they translate PascalCase parameters
|
||||||
|
to the Python kebab-case CLI and propagate the exit code.
|
||||||
|
|
||||||
|
### Local validation gates
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\.venv\Scripts\python.exe -m ruff check src tests\python
|
||||||
|
.\.venv\Scripts\python.exe -m mypy --strict src
|
||||||
|
.\.venv\Scripts\python.exe -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90
|
||||||
|
```
|
||||||
|
|
||||||
|
Coverage gate is **90%**. Regression tests for the
|
||||||
|
`AGENTS.md` "Errori frequenti da evitare" #9–#12 live in
|
||||||
|
[tests/python/test_agents_errors.py](tests/python/test_agents_errors.py)
|
||||||
|
and must stay green.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
## Setup rapido
|
## Setup rapido
|
||||||
|
|
||||||
### 1a. Template VM Windows
|
### 1a. Template VM Windows
|
||||||
@@ -198,10 +298,12 @@ Dettagli: [docs/LINUX-TEMPLATE-SETUP.md](docs/LINUX-TEMPLATE-SETUP.md).
|
|||||||
### 2. Credenziali guest
|
### 2. Credenziali guest
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
# PowerShell elevato — una volta sola sull'host
|
# PowerShell elevato — una volta sola sull'host.
|
||||||
Import-Module CredentialManager
|
# Scrive nel vault SYSTEM (act_runner gira come LocalSystem; un
|
||||||
New-StoredCredential -Target "BuildVMGuest" -UserName "ci_build" `
|
# New-StoredCredential dalla sessione utente NON sarebbe leggibile dal
|
||||||
-Password "<your-build-password>" -Persist LocalMachine
|
# runner). Username host-qualificato col computer name del guest
|
||||||
|
# (bare 'ci_build' -> NTLM SEC_E_UNKNOWN_CREDENTIALS).
|
||||||
|
.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. act_runner
|
### 3. act_runner
|
||||||
@@ -244,7 +346,7 @@ Vedi [docs/Setup-GiteaSSH.md](docs/Setup-GiteaSSH.md) per configurare l'alias `g
|
|||||||
```powershell
|
```powershell
|
||||||
& 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' `
|
& 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' `
|
||||||
-JobId 'test-win-001' `
|
-JobId 'test-win-001' `
|
||||||
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git' `
|
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' `
|
||||||
-Branch 'main' `
|
-Branch 'main' `
|
||||||
-TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
-TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
||||||
-Submodules `
|
-Submodules `
|
||||||
@@ -258,7 +360,7 @@ Vedi [docs/Setup-GiteaSSH.md](docs/Setup-GiteaSSH.md) per configurare l'alias `g
|
|||||||
```powershell
|
```powershell
|
||||||
& 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' `
|
& 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' `
|
||||||
-JobId 'test-linux-001' `
|
-JobId 'test-linux-001' `
|
||||||
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git' `
|
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' `
|
||||||
-Branch 'main' `
|
-Branch 'main' `
|
||||||
-TemplatePath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
-TemplatePath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
||||||
-SnapshotName 'BaseClean-Linux' `
|
-SnapshotName 'BaseClean-Linux' `
|
||||||
@@ -276,8 +378,8 @@ Aggiungi `-SkipArtifact` per job che non producono output (es. lint, test puri).
|
|||||||
|
|
||||||
## Workflow Gitea
|
## Workflow Gitea
|
||||||
|
|
||||||
Il file [gitea/workflows/build-nsInnoUnp.yml](gitea/workflows/build-nsInnoUnp.yml) è il workflow
|
Il file [gitea/workflows/build-ns7zip.yml](gitea/workflows/build-ns7zip.yml) è il workflow
|
||||||
attivo per `Simone/nsis-plugin-nsinnounp`. Si attiva su push di tag `v*` o manualmente.
|
attivo per `Simone/nsis-plugin-ns7zip`. Si attiva su push di tag `v*` o manualmente.
|
||||||
Usa una **strategy matrix** `[windows, linux]` che esegue in parallelo su entrambi i runner.
|
Usa una **strategy matrix** `[windows, linux]` che esegue in parallelo su entrambi i runner.
|
||||||
|
|
||||||
La composite action [gitea/actions/local-ci-build/action.yml](gitea/actions/local-ci-build/action.yml)
|
La composite action [gitea/actions/local-ci-build/action.yml](gitea/actions/local-ci-build/action.yml)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# TODO — Local CI/CD System
|
# TODO — Local CI/CD System
|
||||||
|
|
||||||
<!-- Last updated: 2026-05-11 — Sprint 15: §7.5 DONE (Rinomina Setup-* → Install-CIToolchain-*: WinBuild2025/2022/Linux2404; tutti i riferimenti aggiornati in Prepare/Deploy/docs/README/TODO/action.yml) -->
|
<!-- Last updated: 2026-05-11 — Sprint 15: §7.5 DONE (Rinomina Setup-* → Install-CIToolchain-*: WinBuild2025/2022/Linux2404; tutti i riferimenti aggiornati in Prepare/Deploy/docs/README/TODO/action.yml) -->
|
||||||
<!-- Sprint 14: §6.4+§6.5 DONE (build-nsInnoUnp.yml matrix windows+linux via composite action; ExtraGuestEnv chain Invoke-CIJob→Invoke-RemoteBuild, Windows session inject + Linux export prefix) -->
|
<!-- Sprint 14: §6.4+§6.5 DONE (build-ns7zip.yml matrix windows+linux via composite action; ExtraGuestEnv chain Invoke-CIJob→Invoke-RemoteBuild, Windows session inject + Linux export prefix) -->
|
||||||
<!-- Sprint 13: §6.2 Composite Action DONE (gitea/actions/local-ci-build/action.yml — inputs: build-command/artifact-source/submodules/guest-os/use-git-clone) -->
|
<!-- Sprint 13: §6.2 Composite Action DONE (gitea/actions/local-ci-build/action.yml — inputs: build-command/artifact-source/submodules/guest-os/use-git-clone) -->
|
||||||
<!-- Sprint 12: §6.1 Linux Build VM DONE (Deploy/Prepare/Setup/Transport/e2e 4xPASS — nsis7z.dll 2272 KB parity Windows) -->
|
<!-- Sprint 12: §6.1 Linux Build VM DONE (Deploy/Prepare/Setup/Transport/e2e 4xPASS — nsis7z.dll 2272 KB parity Windows) -->
|
||||||
<!-- Sprint 11: §6.6 Tier-2 Toolchain DONE (Pwsh7/NSIS/CMake/Node.js/WiX/gh CLI/Sysinternals/vcpkg — Step 12 in Install-CIToolchain-WinBuild2025) -->
|
<!-- Sprint 11: §6.6 Tier-2 Toolchain DONE (Pwsh7/NSIS/CMake/Node.js/WiX/gh CLI/Sysinternals/vcpkg — Step 12 in Install-CIToolchain-WinBuild2025) -->
|
||||||
@@ -56,7 +56,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I
|
|||||||
- [x] **Gitea Server** — già in esecuzione su `http://10.10.20.11:3100` (e `https://gitea.emulab.it`)
|
- [x] **Gitea Server** — già in esecuzione su `http://10.10.20.11:3100` (e `https://gitea.emulab.it`)
|
||||||
- [x] Runner registration token ottenuto e usato
|
- [x] Runner registration token ottenuto e usato
|
||||||
- [x] Create at least one test repository with `.gitea/workflows/build.yml`
|
- [x] Create at least one test repository with `.gitea/workflows/build.yml`
|
||||||
(`Simone/nsis-plugin-nsinnounp` — testato e2e, workflow in `gitea/workflows/build-nsInnoUnp.yml`)
|
(`Simone/nsis-plugin-ns7zip` — testato e2e, workflow in `gitea/workflows/build-ns7zip.yml`)
|
||||||
- [x] Gitea Actions abilitato (runner registrato con successo)
|
- [x] Gitea Actions abilitato (runner registrato con successo)
|
||||||
|
|
||||||
- [x] **act_runner** — installato e registrato sull'host Windows
|
- [x] **act_runner** — installato e registrato sull'host Windows
|
||||||
@@ -155,7 +155,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I
|
|||||||
- [x] Test `Invoke-RemoteBuild.ps1` — dotnet restore + build OK, artifacts.zip creato in VM (2026-05-08)
|
- [x] Test `Invoke-RemoteBuild.ps1` — dotnet restore + build OK, artifacts.zip creato in VM (2026-05-08)
|
||||||
- [x] Test `Get-BuildArtifacts.ps1` — artifacts.zip (78 KB) copiato in F:\CI\Artifacts\test-003\ (2026-05-08)
|
- [x] Test `Get-BuildArtifacts.ps1` — artifacts.zip (78 KB) copiato in F:\CI\Artifacts\test-003\ (2026-05-08)
|
||||||
- [x] Test `Remove-BuildVM.ps1` — VM spenta + directory rimossa correttamente (2026-05-08)
|
- [x] Test `Remove-BuildVM.ps1` — VM spenta + directory rimossa correttamente (2026-05-08)
|
||||||
- [x] Test `Invoke-CIJob.ps1` end-to-end su una soluzione reale (nsis-plugin-nsinnounp)
|
- [x] Test `Invoke-CIJob.ps1` end-to-end su una soluzione reale (nsis-plugin-ns7zip)
|
||||||
- [x] e2e-008: SUCCESS — build OK + artifact raccolto (1632 KB)
|
- [x] e2e-008: SUCCESS — build OK + artifact raccolto (1632 KB)
|
||||||
- [x] e2e-009: SUCCESS — cleanup automatico VM + clone confermato (finally block)
|
- [x] e2e-009: SUCCESS — cleanup automatico VM + clone confermato (finally block)
|
||||||
- [x] `scripts/Cleanup-OrphanedBuildVMs.ps1` creato e verificato — `-WhatIf`, soglia `-MaxAgeHours`, hard stop + deleteVM + rimozione dir (2026-05-09)
|
- [x] `scripts/Cleanup-OrphanedBuildVMs.ps1` creato e verificato — `-WhatIf`, soglia `-MaxAgeHours`, hard stop + deleteVM + rimozione dir (2026-05-09)
|
||||||
@@ -171,13 +171,13 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I
|
|||||||
## Gitea Workflow Integration
|
## Gitea Workflow Integration
|
||||||
|
|
||||||
- [x] Copiare `gitea/workflow-example.yml` in `.gitea/workflows/build.yml` nel repo di test
|
- [x] Copiare `gitea/workflow-example.yml` in `.gitea/workflows/build.yml` nel repo di test
|
||||||
(workflow `build-nsInnoUnp.yml` già presente e funzionante per `nsis-plugin-nsinnounp`)
|
(workflow `build-ns7zip.yml` già presente e funzionante per `nsis-plugin-ns7zip`)
|
||||||
- [x] Push commit e verificare job in Gitea Actions UI
|
- [x] Push commit e verificare job in Gitea Actions UI
|
||||||
- [x] Verificare artifact scaricabile da Gitea dopo build riuscita
|
- [x] Verificare artifact scaricabile da Gitea dopo build riuscita
|
||||||
- [x] `gitea/workflows/lint.yml` creato — PSScriptAnalyzer su push/PR che toccano `.ps1` (2026-05-09)
|
- [x] `gitea/workflows/lint.yml` creato — PSScriptAnalyzer su push/PR che toccano `.ps1` (2026-05-09)
|
||||||
- [x] **[P3] Adattare workflow per altri repository** — generalizzare `BUILD_COMMAND` e `ARTIFACT_SOURCE`.
|
- [x] **[P3] Adattare workflow per altri repository** — generalizzare `BUILD_COMMAND` e `ARTIFACT_SOURCE`.
|
||||||
COMPLETATO 2026-05-11: `gitea/workflow-example.yml` riscritto per usare la composite action
|
COMPLETATO 2026-05-11: `gitea/workflow-example.yml` riscritto per usare la composite action
|
||||||
`Simone/local-ci-system/.gitea/actions/local-ci-build@main`. Commenti inline guidano
|
`Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`. Commenti inline guidano
|
||||||
la personalizzazione di `build-command`, `artifact-source`, submodules, SSH URL,
|
la personalizzazione di `build-command`, `artifact-source`, submodules, SSH URL,
|
||||||
`extra-guest-env-json` e `artifact-name`. Nessuna logica di orchestrazione nei repo chiamanti.
|
`extra-guest-env-json` e `artifact-name`. Nessuna logica di orchestrazione nei repo chiamanti.
|
||||||
|
|
||||||
@@ -343,13 +343,13 @@ File: [scripts/Invoke-CIJob.ps1](scripts/Invoke-CIJob.ps1), [scripts/Invoke-Remo
|
|||||||
- Prerequisiti satisfatti: Git + 7-Zip in template (§6.6, completato)
|
- Prerequisiti satisfatti: Git + 7-Zip in template (§6.6, completato)
|
||||||
|
|
||||||
**Beneficio**: Elimina host-zip-transfer overhead (rilevante per repo > 200 MB con submoduli).
|
**Beneficio**: Elimina host-zip-transfer overhead (rilevante per repo > 200 MB con submoduli).
|
||||||
**Prossimo**: E2E test con nsis-plugin-nsinnounp (-UseGitClone flag) per misurare impatto.
|
**Prossimo**: E2E test con nsis-plugin-ns7zip (-UseGitClone flag) per misurare impatto.
|
||||||
|
|
||||||
### 3.5 [P2] [ ] vCPU/RAM tuning per workload — DEFERRED (home lab)
|
### 3.5 [P2] [ ] vCPU/RAM tuning per workload — DEFERRED (home lab)
|
||||||
File: [docs/OPTIMIZATION.md:82-89](docs/OPTIMIZATION.md), VMX template.
|
File: [docs/OPTIMIZATION.md:82-89](docs/OPTIMIZATION.md), VMX template.
|
||||||
|
|
||||||
`numvcpus=4`, `memsize=6144` è ragionevole ma non ottimo per ogni build. Per
|
`numvcpus=4`, `memsize=6144` è ragionevole ma non ottimo per ogni build. Per
|
||||||
`nsis-plugin-nsinnounp` (4 build paralleli interni con thread divisi) 4 vCPU sono già saturati.
|
`nsis-plugin-ns7zip` (4 build paralleli interni con thread divisi) 4 vCPU sono già saturati.
|
||||||
|
|
||||||
Considerare:
|
Considerare:
|
||||||
- VMX pinning: 6 vCPU per build C++ pesanti, 2 vCPU per build .NET piccoli.
|
- VMX pinning: 6 vCPU per build C++ pesanti, 2 vCPU per build .NET piccoli.
|
||||||
@@ -488,7 +488,7 @@ jobs:
|
|||||||
runs-on: windows-build # or linux-build
|
runs-on: windows-build # or linux-build
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
- uses: Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
build-command: 'python build_plugin.py --final --dist-dir dist'
|
build-command: 'python build_plugin.py --final --dist-dir dist'
|
||||||
artifact-source: 'dist'
|
artifact-source: 'dist'
|
||||||
@@ -506,14 +506,14 @@ Già supportato lato Gitea, basta replicare il setup.
|
|||||||
**Deferred**: in un home lab single-host questa funzionalità non serve. Non pianificare.
|
**Deferred**: in un home lab single-host questa funzionalità non serve. Non pianificare.
|
||||||
|
|
||||||
### 6.4 [P3] [x] Build matrix nel workflow
|
### 6.4 [P3] [x] Build matrix nel workflow
|
||||||
File: [gitea/workflows/build-nsInnoUnp.yml](gitea/workflows/build-nsInnoUnp.yml).
|
File: [gitea/workflows/build-ns7zip.yml](gitea/workflows/build-ns7zip.yml).
|
||||||
|
|
||||||
**Status**: COMPLETATO 2026-05-11 — `build-nsInnoUnp.yml` riscritto con matrix strategy.
|
**Status**: COMPLETATO 2026-05-11 — `build-ns7zip.yml` riscritto con matrix strategy.
|
||||||
|
|
||||||
Matrix `target: [windows, linux]`, `runs-on: ${{ matrix.target }}-build`,
|
Matrix `target: [windows, linux]`, `runs-on: ${{ matrix.target }}-build`,
|
||||||
`fail-fast: false`. Usa la composite action con `job-id-suffix: ${{ matrix.target }}`
|
`fail-fast: false`. Usa la composite action con `job-id-suffix: ${{ matrix.target }}`
|
||||||
per artifact dir separate e `repo-url: ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git`.
|
per artifact dir separate e `repo-url: ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git`.
|
||||||
Artifact separati: `nsis-plugin-nsinnounp-windows-<tag>` e `nsis-plugin-nsinnounp-linux-<tag>`.
|
Artifact separati: `nsis-plugin-ns7zip-windows-<tag>` e `nsis-plugin-ns7zip-linux-<tag>`.
|
||||||
|
|
||||||
### 6.5 [P3] [x] Secret injection workflow-level
|
### 6.5 [P3] [x] Secret injection workflow-level
|
||||||
|
|
||||||
@@ -526,7 +526,7 @@ Artifact separati: `nsis-plugin-nsinnounp-windows-<tag>` e `nsis-plugin-nsinnoun
|
|||||||
|
|
||||||
Usage dal workflow:
|
Usage dal workflow:
|
||||||
```yaml
|
```yaml
|
||||||
- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
- uses: Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
build-command: 'python sign_and_build.py'
|
build-command: 'python sign_and_build.py'
|
||||||
extra-guest-env-json: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","CERT_THUMBPRINT":"${{ secrets.CERT_THUMBPRINT }}"}'
|
extra-guest-env-json: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","CERT_THUMBPRINT":"${{ secrets.CERT_THUMBPRINT }}"}'
|
||||||
@@ -549,7 +549,7 @@ build più ampi senza toccare il template.
|
|||||||
| **Git for Windows** | 2.54.0.windows.1 | Self-clone in VM, submodule fetch, fallback §3.2 |
|
| **Git for Windows** | 2.54.0.windows.1 | Self-clone in VM, submodule fetch, fallback §3.2 |
|
||||||
| **7-Zip** | 26.01 | Unpack/zip universale; usato da Step 12 stesso (vcpkg) |
|
| **7-Zip** | 26.01 | Unpack/zip universale; usato da Step 12 stesso (vcpkg) |
|
||||||
| **PowerShell 7.x** | 7.6.1 | Parallel pipelines, operatori moderni, pwsh in CI |
|
| **PowerShell 7.x** | 7.6.1 | Parallel pipelines, operatori moderni, pwsh in CI |
|
||||||
| **NSIS** | 3.12 | Build installer Windows (es. `nsis-plugin-nsinnounp`) |
|
| **NSIS** | 3.12 | Build installer Windows (es. `nsis-plugin-ns7zip`) |
|
||||||
| **gh CLI** | 2.92.0 | Upload artifact a release Gitea/GitHub, comment PR |
|
| **gh CLI** | 2.92.0 | Upload artifact a release Gitea/GitHub, comment PR |
|
||||||
|
|
||||||
### Tier-2 — Step 12 (implementato)
|
### Tier-2 — Step 12 (implementato)
|
||||||
@@ -716,7 +716,7 @@ VM avviata → WinRM: git clone --recurse-submodules https://<token>@gitea/...
|
|||||||
|
|
||||||
- [x] **Test e2e con `-UseGitClone`** — COMPLETATO 2026-05-10
|
- [x] **Test e2e con `-UseGitClone`** — COMPLETATO 2026-05-10
|
||||||
- [x] Aggiornare snapshot `BaseClean` con Git installato
|
- [x] Aggiornare snapshot `BaseClean` con Git installato
|
||||||
- [x] Eseguire `e2e-010` con `-UseGitClone` su `nsis-plugin-nsinnounp` (ha submoduli)
|
- [x] Eseguire `e2e-010` con `-UseGitClone` su `nsis-plugin-ns7zip` (ha submoduli)
|
||||||
- [x] Confrontare tempo totale con e2e-009 (senza `-UseGitClone`) — 34s / 25.7% più veloce
|
- [x] Confrontare tempo totale con e2e-009 (senza `-UseGitClone`) — 34s / 25.7% più veloce
|
||||||
- [x] Verificare che il PAT non appaia nei log di `Invoke-CIJob.ps1`
|
- [x] Verificare che il PAT non appaia nei log di `Invoke-CIJob.ps1`
|
||||||
|
|
||||||
@@ -820,6 +820,43 @@ da workflow. Attualmente fisso a `numvcpus=4, memsize=6144` — subottimo per bu
|
|||||||
**Rifiori quando**: Performance profiling mostra contentione CPU/RAM è bottleneck per la
|
**Rifiori quando**: Performance profiling mostra contentione CPU/RAM è bottleneck per la
|
||||||
workload target. Per home lab, 4 vCPU/6 GB è ragionevole compromesso.
|
workload target. Per home lab, 4 vCPU/6 GB è ragionevole compromesso.
|
||||||
|
|
||||||
|
### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com)
|
||||||
|
File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`).
|
||||||
|
|
||||||
|
Dipendenza residua da `github.com`: `actions/checkout@v4` in `lint.yml` (2×) —
|
||||||
|
il codice dell'action è scaricato da github.com a runtime (con
|
||||||
|
`[actions] DEFAULT_ACTIONS_URL = github`). La dipendenza per gli **artifact**
|
||||||
|
è già stata eliminata (upload/download-artifact rimossi; handoff via
|
||||||
|
filesystem host — vedi commit `refactor(ci): drop upload/download-artifact`).
|
||||||
|
Questo task è l'**alternativa "fix 1"** non scelta: mirrorare le action
|
||||||
|
GitHub usate (`checkout`) in una org `actions` sull'istanza Gitea e
|
||||||
|
referenziarle via URL Gitea completo, eliminando del tutto github.com.
|
||||||
|
|
||||||
|
**Stato attuale**: Deferred. Artifact già github-free (scelta "fix 2"
|
||||||
|
applicata). Resta solo `actions/checkout@v4` in lint; act_runner la cacha
|
||||||
|
dopo il primo fetch, quindi outage GitHub impatta solo cold cache.
|
||||||
|
**Rifiori quando**: Host air-gapped, GitHub non raggiungibile dall'host CI,
|
||||||
|
o si vuole indipendenza totale da github.com (anche per `checkout`).
|
||||||
|
|
||||||
|
### 8.2 [P3] [ ] Streaming live output build su Windows (WinRM)
|
||||||
|
File: [src/ci_orchestrator/transport/winrm.py](src/ci_orchestrator/transport/winrm.py),
|
||||||
|
[src/ci_orchestrator/commands/build.py](src/ci_orchestrator/commands/build.py).
|
||||||
|
|
||||||
|
Linux/SSH ora streamma l'output del build in tempo reale
|
||||||
|
(`SshTransport.run_streaming` — recv incrementale + `PYTHONUNBUFFERED`).
|
||||||
|
Windows resta **log-a-fine-job**: `pypsrp.Client.execute_ps` ritorna
|
||||||
|
solo a comando concluso; `_windows_build` usa `_report_build_output`
|
||||||
|
(blocco unico) + coda nel messaggio d'errore.
|
||||||
|
|
||||||
|
**Stato attuale**: Deferred. Streaming WinRM richiede API basso livello
|
||||||
|
RunspacePool/PowerShell con poll di `ps.output`/stream invece di
|
||||||
|
`execute_ps`, semantica PSRP più complessa; output unbuffered dei tool
|
||||||
|
nativi (MSBuild) scomodo. Diagnostica già adeguata (output completo a
|
||||||
|
fine job + ultime righe nell'errore).
|
||||||
|
**Rifiori quando**: Build Windows lunghi rendono l'assenza di progress
|
||||||
|
live un problema operativo reale, o si standardizza lo streaming
|
||||||
|
cross-OS.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Suggerimento di sequencing
|
## Suggerimento di sequencing
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
# ci-orchestrator config example.
|
||||||
|
#
|
||||||
|
# Copy to $CI_ROOT/config.toml or set $CI_CONFIG to the file path.
|
||||||
|
# Environment variables (CI_ROOT, CI_TEMPLATES, CI_BUILD_VMS, CI_ARTIFACTS,
|
||||||
|
# CI_KEYS, CI_VMRUN_PATH, CI_SSH_KEY_PATH, CI_GUEST_CRED_TARGET) override
|
||||||
|
# anything set here.
|
||||||
|
#
|
||||||
|
# IMPORTANT (TOML rule): top-level keys must appear BEFORE any [section] header.
|
||||||
|
# Keys placed after a [section] belong to that section, not to the root table.
|
||||||
|
|
||||||
|
# Top-level orchestrator settings.
|
||||||
|
vmrun_path = "C:/Program Files (x86)/VMware/VMware Workstation/vmrun.exe"
|
||||||
|
ssh_key_path = "F:/CI/keys/ci_linux"
|
||||||
|
guest_cred_target = "BuildVMGuest"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- Windows host
|
||||||
|
[paths]
|
||||||
|
root = "F:/CI"
|
||||||
|
templates = "F:/CI/Templates"
|
||||||
|
build_vms = "F:/CI/BuildVMs"
|
||||||
|
artifacts = "F:/CI/Artifacts"
|
||||||
|
keys = "F:/CI/keys"
|
||||||
|
|
||||||
|
# Phase B (Linux host) — uncomment after migration.
|
||||||
|
# [paths]
|
||||||
|
# root = "/var/lib/ci"
|
||||||
|
# templates = "/var/lib/ci/templates"
|
||||||
|
# build_vms = "/var/lib/ci/build-vms"
|
||||||
|
# artifacts = "/var/lib/ci/artifacts"
|
||||||
|
# keys = "/etc/ci/keys"
|
||||||
|
|
||||||
|
# Phase C hook: backend selector. Only "workstation" is implemented in Phase A.
|
||||||
|
[backend]
|
||||||
|
type = "workstation"
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
# systemd units — Local CI/CD System (Linux host)
|
||||||
|
|
||||||
|
Coppie `*.service` + `*.timer` che replicano i task periodici registrati su
|
||||||
|
Windows da `scripts/Register-CIScheduledTasks.ps1`. Pensate per essere
|
||||||
|
installate su un host Linux Mint / Ubuntu LTS, con il package
|
||||||
|
`ci_orchestrator` installato in `/opt/ci/venv/`.
|
||||||
|
|
||||||
|
## Mapping Windows Task Scheduler → systemd timer
|
||||||
|
|
||||||
|
| Windows Task Scheduler | systemd unit | Cadenza | Comando |
|
||||||
|
| ----------------------- | ---------------------------------- | ----------------------- | --------------------------------------------------------------- |
|
||||||
|
| `CI-CleanupOrphans` | `ci-cleanup-orphans.timer` | every 6h + at boot | `python -m ci_orchestrator vm cleanup --max-age-hours 6` |
|
||||||
|
| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `pwsh Invoke-RetentionPolicy.ps1` (vedi nota PowerShell) |
|
||||||
|
| `CI-DiskSpaceAlert` | `ci-watch-disk-space.timer` | every 15 min | `python -m ci_orchestrator monitor disk` |
|
||||||
|
| `CI-RunnerHealth` | `ci-watch-runner-health.timer` | every 15 min | `python -m ci_orchestrator monitor runner` |
|
||||||
|
| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `pwsh Backup-CITemplate.ps1` (vedi nota PowerShell) |
|
||||||
|
|
||||||
|
## Nota PowerShell su Linux
|
||||||
|
|
||||||
|
Due unit (`ci-retention-policy`, `ci-backup-template`) invocano script
|
||||||
|
PowerShell che, per scelta documentata in `AGENTS.md` ("Mappatura script
|
||||||
|
PowerShell → Python"), restano in PowerShell. Su Linux questo richiede
|
||||||
|
**PowerShell Core (`pwsh`)**:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Ubuntu / Mint LTS
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y wget apt-transport-https software-properties-common
|
||||||
|
wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb
|
||||||
|
sudo dpkg -i packages-microsoft-prod.deb
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y powershell
|
||||||
|
pwsh --version # verifica
|
||||||
|
```
|
||||||
|
|
||||||
|
Se preferisci non installare `pwsh`, porta `Invoke-RetentionPolicy.ps1` e
|
||||||
|
`Backup-CITemplate.ps1` come sub-comandi `python -m ci_orchestrator
|
||||||
|
retention run` / `template backup` (out of scope B5 per il piano corrente
|
||||||
|
— vedi `plans/implementation-plan-A-B.md`).
|
||||||
|
|
||||||
|
## Prerequisiti
|
||||||
|
|
||||||
|
1. Utente di sistema `ci-runner` con UID dedicato (creato in B1).
|
||||||
|
2. Venv Python attivo in `/opt/ci/venv/` con `ci_orchestrator` installato:
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/pip install -e /opt/ci/local-ci-cd-system
|
||||||
|
```
|
||||||
|
3. Variabili d'ambiente `CI_*` definite in `/etc/ci/environment` (caricato
|
||||||
|
tramite `EnvironmentFile=` in ogni unit `.service`):
|
||||||
|
```
|
||||||
|
CI_ROOT=/var/lib/ci
|
||||||
|
CI_TEMPLATES=/var/lib/ci/templates
|
||||||
|
CI_BUILD_VMS=/var/lib/ci/build-vms
|
||||||
|
CI_ARTIFACTS=/var/lib/ci/artifacts
|
||||||
|
CI_KEYS=/etc/ci/keys
|
||||||
|
PYTHONIOENCODING=utf-8
|
||||||
|
```
|
||||||
|
4. `Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` presenti in
|
||||||
|
`/opt/ci/local-ci-cd-system/scripts/` (clone del repo).
|
||||||
|
|
||||||
|
## Installazione
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 1. Copia le unit in /etc/systemd/system/
|
||||||
|
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||||
|
|
||||||
|
# 2. Reload systemd
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
|
||||||
|
# 3. Abilita + avvia tutti i timer
|
||||||
|
for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \
|
||||||
|
ci-watch-runner-health ci-backup-template; do
|
||||||
|
sudo systemctl enable --now "${t}.timer"
|
||||||
|
done
|
||||||
|
|
||||||
|
# 4. Verifica
|
||||||
|
systemctl list-timers --all 'ci-*'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Test e troubleshooting
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Trigger manuale di un singolo task (esegue il .service una volta)
|
||||||
|
sudo systemctl start ci-cleanup-orphans.service
|
||||||
|
|
||||||
|
# Log dell'ultima esecuzione
|
||||||
|
journalctl -u ci-cleanup-orphans.service -n 50 --no-pager
|
||||||
|
|
||||||
|
# Live tail di tutti i task CI
|
||||||
|
journalctl -u 'ci-*' -f
|
||||||
|
```
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \
|
||||||
|
ci-watch-runner-health ci-backup-template; do
|
||||||
|
sudo systemctl disable --now "${t}.timer"
|
||||||
|
sudo rm -f "/etc/systemd/system/${t}.service" "/etc/systemd/system/${t}.timer"
|
||||||
|
done
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
A questo punto il fallback è ri-attivare i task su Windows con
|
||||||
|
`scripts/Register-CIScheduledTasks.ps1`.
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — weekly backup of VMware templates
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ci-runner
|
||||||
|
Group=ci-runner
|
||||||
|
EnvironmentFile=/etc/ci/environment
|
||||||
|
WorkingDirectory=/opt/ci/local-ci-cd-system
|
||||||
|
# NOTE: requires PowerShell Core (`pwsh`) on the Linux host.
|
||||||
|
# See deploy/systemd/README.md "Nota PowerShell su Linux".
|
||||||
|
ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Backup-CITemplate.ps1
|
||||||
|
TimeoutStartSec=4h
|
||||||
|
IOSchedulingClass=best-effort
|
||||||
|
IOSchedulingPriority=7
|
||||||
|
Nice=19
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ReadWritePaths=/var/lib/ci /var/backups/ci
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — weekly timer for ci-backup-template.service
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Sunday 02:00 + random delay 1h to avoid clashing with other CI tasks.
|
||||||
|
OnCalendar=Sun *-*-* 02:00:00
|
||||||
|
RandomizedDelaySec=1h
|
||||||
|
Persistent=true
|
||||||
|
Unit=ci-backup-template.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — destroy orphaned build VMs (max age 6h)
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ci-runner
|
||||||
|
Group=ci-runner
|
||||||
|
EnvironmentFile=/etc/ci/environment
|
||||||
|
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --max-age-hours 6
|
||||||
|
TimeoutStartSec=1h
|
||||||
|
Nice=10
|
||||||
|
# Hardening
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ReadWritePaths=/var/lib/ci /var/log/ci
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — periodic timer for ci-cleanup-orphans.service
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Equivalent to Windows Task Scheduler:
|
||||||
|
# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 6h
|
||||||
|
# New-ScheduledTaskTrigger -AtStartup
|
||||||
|
OnBootSec=2min
|
||||||
|
OnUnitActiveSec=6h
|
||||||
|
Persistent=true
|
||||||
|
AccuracySec=1min
|
||||||
|
Unit=ci-cleanup-orphans.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — purge old artifacts/logs/IP leases (retention policy)
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ci-runner
|
||||||
|
Group=ci-runner
|
||||||
|
EnvironmentFile=/etc/ci/environment
|
||||||
|
WorkingDirectory=/opt/ci/local-ci-cd-system
|
||||||
|
# NOTE: requires PowerShell Core (`pwsh`) on the Linux host.
|
||||||
|
# See deploy/systemd/README.md "Nota PowerShell su Linux".
|
||||||
|
ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Invoke-RetentionPolicy.ps1
|
||||||
|
TimeoutStartSec=1h
|
||||||
|
Nice=15
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
NoNewPrivileges=true
|
||||||
|
ReadWritePaths=/var/lib/ci /var/log/ci
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — daily timer for ci-retention-policy.service
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Equivalent to Windows Task Scheduler:
|
||||||
|
# New-ScheduledTaskTrigger -Daily -At '03:00' -RandomDelay 30min
|
||||||
|
OnCalendar=*-*-* 03:00:00
|
||||||
|
RandomizedDelaySec=30min
|
||||||
|
Persistent=true
|
||||||
|
Unit=ci-retention-policy.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — disk space monitor (alert via systemd journal)
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ci-runner
|
||||||
|
Group=ci-runner
|
||||||
|
EnvironmentFile=/etc/ci/environment
|
||||||
|
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor disk
|
||||||
|
TimeoutStartSec=5min
|
||||||
|
Nice=15
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
NoNewPrivileges=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — periodic timer for ci-watch-disk-space.service
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Equivalent to Windows Task Scheduler:
|
||||||
|
# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 15min
|
||||||
|
OnBootSec=5min
|
||||||
|
OnUnitActiveSec=15min
|
||||||
|
Persistent=true
|
||||||
|
AccuracySec=30s
|
||||||
|
Unit=ci-watch-disk-space.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — auto-restart act_runner if stopped (rate-limited)
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
After=network-online.target act-runner.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
User=ci-runner
|
||||||
|
Group=ci-runner
|
||||||
|
EnvironmentFile=/etc/ci/environment
|
||||||
|
# `monitor runner` enforces internal rate-limit: max 3 restarts/h
|
||||||
|
# (matches Windows CI-RunnerHealth scheduled task semantics).
|
||||||
|
ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner
|
||||||
|
TimeoutStartSec=5min
|
||||||
|
Nice=15
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
PrivateTmp=true
|
||||||
|
# NoNewPrivileges intentionally NOT set: monitor may need to call
|
||||||
|
# `systemctl restart act-runner.service` via polkit.
|
||||||
|
ReadWritePaths=/var/lib/ci /var/log/ci
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Local CI/CD — periodic timer for ci-watch-runner-health.service
|
||||||
|
Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Equivalent to Windows Task Scheduler:
|
||||||
|
# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 15min
|
||||||
|
OnBootSec=3min
|
||||||
|
OnUnitActiveSec=15min
|
||||||
|
Persistent=true
|
||||||
|
AccuracySec=30s
|
||||||
|
Unit=ci-watch-runner-health.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -206,3 +206,116 @@ VMs have internet access via NAT — required for pip/nuget/apt during build.
|
|||||||
- Source code is transferred via zip (Compress-Archive → WinRM copy → Expand-Archive in guest) by default — no PAT is ever present inside the VM; VMs have NAT internet for pip/nuget at build time.
|
- Source code is transferred via zip (Compress-Archive → WinRM copy → Expand-Archive in guest) by default — no PAT is ever present inside the VM; VMs have NAT internet for pip/nuget at build time.
|
||||||
- With `-UseGitClone` (§3.3, opt-in) il clone avviene dentro la VM via SSH alias `gitea-ci` (forwardato dall'host); la chiave privata SSH dell'host non viene esposta al guest — vedi BEST-PRACTICES.md per il threat model.
|
- With `-UseGitClone` (§3.3, opt-in) il clone avviene dentro la VM via SSH alias `gitea-ci` (forwardato dall'host); la chiave privata SSH dell'host non viene esposta al guest — vedi BEST-PRACTICES.md per il threat model.
|
||||||
- Linux build VMs usano la chiave SSH `F:\CI\keys\ci_linux` (ed25519, no passphrase), archiviata solo sull'host. Il guest non conosce credenziali WinRM.
|
- Linux build VMs usano la chiave SSH `F:\CI\keys\ci_linux` (ed25519, no passphrase), archiviata solo sull'host. Il guest non conosce credenziali WinRM.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Python orchestrator (Phase A)
|
||||||
|
|
||||||
|
Phase A of [implementation-plan-A-B](../plans/implementation-plan-A-B.md)
|
||||||
|
replaces the PowerShell orchestration scripts with a Python 3.11+ package
|
||||||
|
under `src/ci_orchestrator/`. The PS scripts in `scripts/` are now thin
|
||||||
|
3-line shims that forward to the Python CLI — the diagram above still
|
||||||
|
applies, except that the boxes labelled `*.ps1` are entry-point names,
|
||||||
|
not the implementation.
|
||||||
|
|
||||||
|
### Package layout
|
||||||
|
|
||||||
|
```
|
||||||
|
src/ci_orchestrator/
|
||||||
|
├── __init__.py
|
||||||
|
├── __main__.py # click entry point: `python -m ci_orchestrator`
|
||||||
|
├── config.py # Config dataclass + env/TOML loader (OS-aware paths)
|
||||||
|
├── credentials.py # CredentialStore Protocol + KeyringCredentialStore
|
||||||
|
│
|
||||||
|
├── backends/
|
||||||
|
│ ├── __init__.py # load_backend(config) factory (selector for Phase C)
|
||||||
|
│ ├── protocol.py # VmBackend Protocol + VmHandle dataclass + VmState enum
|
||||||
|
│ ├── workstation.py # WorkstationVmrunBackend (subprocess + vmrun -T ws)
|
||||||
|
│ └── errors.py # BackendError / BackendNotAvailable / BackendOperationFailed
|
||||||
|
│
|
||||||
|
├── transport/
|
||||||
|
│ ├── __init__.py
|
||||||
|
│ ├── winrm.py # WinRM transport via pypsrp (replaces New-PSSession)
|
||||||
|
│ ├── ssh.py # SSH/SFTP transport via paramiko (replaces ssh.exe/scp.exe)
|
||||||
|
│ └── errors.py # TransportConnectError / TransportCommandError
|
||||||
|
│
|
||||||
|
└── commands/ # One module per `click` sub-command group
|
||||||
|
├── wait.py # `wait-ready`
|
||||||
|
├── vm.py # `vm new` / `vm remove` / `vm cleanup`
|
||||||
|
├── build.py # `build run`
|
||||||
|
├── artifacts.py # `artifacts collect`
|
||||||
|
├── monitor.py # `monitor disk` / `monitor runner`
|
||||||
|
├── report.py # `report job`
|
||||||
|
└── job.py # `job` — full end-to-end orchestrator
|
||||||
|
```
|
||||||
|
|
||||||
|
### `VmBackend` Protocol and the `load_backend(config)` factory
|
||||||
|
|
||||||
|
`backends/protocol.py` defines a hypervisor-agnostic Protocol:
|
||||||
|
|
||||||
|
```python
|
||||||
|
class VmBackend(Protocol):
|
||||||
|
def clone_linked(self, template, snapshot, name, destination=None) -> VmHandle: ...
|
||||||
|
def start(self, handle, headless=True) -> None: ...
|
||||||
|
def stop(self, handle, hard=False) -> None: ...
|
||||||
|
def delete(self, handle) -> None: ...
|
||||||
|
def get_ip(self, handle, timeout=0.0) -> str | None: ...
|
||||||
|
def list_snapshots(self, handle) -> list[str]: ...
|
||||||
|
def is_running(self, handle) -> bool: ...
|
||||||
|
```
|
||||||
|
|
||||||
|
Names are deliberately neutral (no `vmrun_*` prefixes) and identifiers are
|
||||||
|
opaque strings (`VmHandle.identifier`): a VMX path on Workstation, a
|
||||||
|
managed object reference on ESXi. `is_running` MUST NOT use guest tools
|
||||||
|
(see `AGENTS.md` error #10).
|
||||||
|
|
||||||
|
Backends are constructed via `backends.load_backend(config)`. Application
|
||||||
|
code (most importantly `commands/job.py`) imports the factory only — never
|
||||||
|
`WorkstationVmrunBackend` directly. The factory reads
|
||||||
|
`config.backend.type` from `config.toml` (default: `"workstation"`).
|
||||||
|
|
||||||
|
### Phase C extension point — adding ESXi without forks
|
||||||
|
|
||||||
|
To add an ESXi backend in a future Phase C:
|
||||||
|
|
||||||
|
1. Add a new module `backends/esxi.py` implementing the `VmBackend`
|
||||||
|
Protocol via `pyVmomi`. `VmHandle.identifier` becomes the managed
|
||||||
|
object reference; `clone_linked` becomes a `CloneVM_Task` call;
|
||||||
|
`start` becomes `PowerOnVM_Task`.
|
||||||
|
2. Register it in `backends/__init__.py` `load_backend(config)` under a
|
||||||
|
new `config.backend.type == "esxi"` branch.
|
||||||
|
3. Add `[backend]` keys to `config.example.toml` (host, user, datacenter,
|
||||||
|
datastore, …). The `Config` dataclass already namespaces backend
|
||||||
|
options under `config.backend`.
|
||||||
|
4. Transport layers (`winrm.py`, `ssh.py`) and the `commands/*` modules
|
||||||
|
need no change — they consume `VmBackend` and `VmHandle` only.
|
||||||
|
|
||||||
|
The single non-trivial constraint is that `commands/vm.py` `vm new`
|
||||||
|
currently builds the destination VMX path under `--clone-base-dir` (a
|
||||||
|
host-local concept). For ESXi this becomes a datastore folder; the
|
||||||
|
parameter rename can be deferred to Phase C as long as the backend
|
||||||
|
ignores the value and uses `config.backend.datastore` instead.
|
||||||
|
|
||||||
|
### Transport selection
|
||||||
|
|
||||||
|
`commands/job.py` auto-detects the guest OS by parsing `guestOS = "..."`
|
||||||
|
in the cloned VMX (`ubuntu-*` → Linux/SSH, anything else → Windows/WinRM).
|
||||||
|
The Linux branch is wired to `transport/ssh.py` (paramiko) and the
|
||||||
|
Windows branch to `transport/winrm.py` (pypsrp). Both transports raise
|
||||||
|
typed errors (`TransportConnectError`, `TransportCommandError`) — never
|
||||||
|
bare exceptions.
|
||||||
|
|
||||||
|
### Validation gates (CI)
|
||||||
|
|
||||||
|
Enforced by `gitea/workflows/lint.yml` job `python`:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
python -m ruff check src tests\python
|
||||||
|
python -m mypy --strict src
|
||||||
|
python -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90
|
||||||
|
```
|
||||||
|
|
||||||
|
Coverage gate is **90%**. Regression tests for
|
||||||
|
`AGENTS.md` errors #9–#12 live in
|
||||||
|
[tests/python/test_agents_errors.py](../tests/python/test_agents_errors.py).
|
||||||
|
|
||||||
|
|||||||
+27
-16
@@ -4,29 +4,40 @@
|
|||||||
|
|
||||||
### Do NOT store credentials in scripts or config files
|
### Do NOT store credentials in scripts or config files
|
||||||
|
|
||||||
The scripts use `-GuestCredentialTarget` (a Windows Credential Manager target name)
|
The guest VM credential is referenced by target name (`BuildVMGuest`,
|
||||||
rather than plaintext username/password parameters. Store credentials once:
|
`GITEA_CI_GUEST_CRED_TARGET`) and read by the Python orchestrator via the
|
||||||
|
`keyring` library — never as plaintext parameters.
|
||||||
|
|
||||||
|
> **Critical: store it in the LocalSystem vault, not your user vault.**
|
||||||
|
> act_runner runs as the `LocalSystem` service account. Windows Credential
|
||||||
|
> Manager / keyring vaults are **per-user**. A credential added with
|
||||||
|
> `cmdkey` or the Credential Manager UI from an interactive admin session
|
||||||
|
> lands in *your* vault and is invisible to the runner, which then fails
|
||||||
|
> with `Credential 'BuildVMGuest' not found in keyring`.
|
||||||
|
|
||||||
|
> **Username must be host-qualified.** The guest is a workgroup machine;
|
||||||
|
> NTLM rejects a bare `ci_build` with `SEC_E_UNKNOWN_CREDENTIALS`. Store
|
||||||
|
> it as `WINBUILD-2025\ci_build` (the guest computer name, i.e. the WinRM
|
||||||
|
> TLS certificate CN). The WinRM transport forces `auth='ntlm'` for the
|
||||||
|
> same reason (Negotiate→Kerberos is meaningless without a domain).
|
||||||
|
|
||||||
|
Store (or rotate) the credential with the helper, which writes into the
|
||||||
|
SYSTEM vault via the production venv's `keyring` (run elevated):
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
# Run on host (once, before first CI job)
|
.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'
|
||||||
cmdkey /generic:BuildVMGuest /user:MACHINENAME\ci_build /pass:YourStrongPassword
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Retrieve in scripts via the `CredentialManager` PowerShell module:
|
It prompts for the password securely, writes it to the SYSTEM vault, and
|
||||||
|
verifies the read-back as SYSTEM. Diagnose WinRM reachability/auth with
|
||||||
```powershell
|
`.\scripts\Test-CIGuestWinRM.ps1 -IpAddress <guest-ip>`.
|
||||||
Install-Module CredentialManager -Scope CurrentUser
|
|
||||||
$cred = Get-StoredCredential -Target 'BuildVMGuest'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Rotate credentials quarterly
|
### Rotate credentials quarterly
|
||||||
|
|
||||||
1. Update password in the template VM (requires rebuilding `BaseClean` snapshot)
|
1. Update the password in the template VM (rebuild the `BaseClean` snapshot).
|
||||||
2. Update Windows Credential Manager on the host:
|
2. Re-run `Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'`
|
||||||
```
|
with the new password.
|
||||||
cmdkey /generic:BuildVMGuest /user:MACHINENAME\ci_build /pass:NewPassword
|
3. No code changes required — the orchestrator references the target name.
|
||||||
```
|
|
||||||
3. No script changes required — they reference the target name, not the password.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -56,7 +56,7 @@ docs/TEST-PLAN-v1.3-to-HEAD.md, gitea/actions/local-ci-build/action.yml.
|
|||||||
|
|
||||||
## 2026-05-11 — Sprint 14: §6.4 Build matrix + §6.5 Secret injection
|
## 2026-05-11 — Sprint 14: §6.4 Build matrix + §6.5 Secret injection
|
||||||
|
|
||||||
**§6.4** — `build-nsInnoUnp.yml` rewritten with matrix strategy `target: [windows, linux]`.
|
**§6.4** — `build-ns7zip.yml` rewritten with matrix strategy `target: [windows, linux]`.
|
||||||
Separate artifact dirs; `fail-fast: false`; composite action used for both targets.
|
Separate artifact dirs; `fail-fast: false`; composite action used for both targets.
|
||||||
|
|
||||||
**§6.5** — `ExtraGuestEnv [hashtable]` added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`:
|
**§6.5** — `ExtraGuestEnv [hashtable]` added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`:
|
||||||
@@ -127,7 +127,7 @@ disables wuauserv/UsoSvc permanently.
|
|||||||
`-UseGitClone` switch added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`.
|
`-UseGitClone` switch added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`.
|
||||||
- Skips host git clone + zip + WinRM transfer overhead.
|
- Skips host git clone + zip + WinRM transfer overhead.
|
||||||
- PAT injected at runtime, never persisted in snapshot.
|
- PAT injected at runtime, never persisted in snapshot.
|
||||||
- E2E result: 34 s saving, 25.7% reduction vs baseline on nsis-plugin-nsinnounp.
|
- E2E result: 34 s saving, 25.7% reduction vs baseline on nsis-plugin-ns7zip.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ F:\CI\
|
|||||||
├── keys\
|
├── keys\
|
||||||
│ ├── ci_linux # chiave privata SSH per Linux VMs (ed25519, no passphrase)
|
│ ├── ci_linux # chiave privata SSH per Linux VMs (ed25519, no passphrase)
|
||||||
│ └── ci_linux.pub # chiave pubblica corrispondente
|
│ └── ci_linux.pub # chiave pubblica corrispondente
|
||||||
|
├── config.toml # configurazione Python orchestrator (ssh_key_path, ecc.)
|
||||||
├── act_runner\
|
├── act_runner\
|
||||||
│ ├── act_runner.exe # binario runner Gitea
|
│ ├── act_runner.exe # binario runner Gitea
|
||||||
│ ├── config.yaml # config runner (path, label, capacity)
|
│ ├── config.yaml # config runner (path, label, capacity)
|
||||||
@@ -90,6 +91,32 @@ Apri PowerShell **come Administrator**, poi:
|
|||||||
- Avvia servizio (`SERVICE_AUTO_START`)
|
- Avvia servizio (`SERVICE_AUTO_START`)
|
||||||
7. **Configura SSH alias `gitea-ci`** in `~/.ssh/config` (HostName, Port, User git)
|
7. **Configura SSH alias `gitea-ci`** in `~/.ssh/config` (HostName, Port, User git)
|
||||||
|
|
||||||
|
### Note operative (scoperte in validazione end-to-end)
|
||||||
|
|
||||||
|
Setup-Host.ps1 da solo **non basta** per la pipeline Python su runner
|
||||||
|
LocalSystem. Dopo Setup-Host:
|
||||||
|
|
||||||
|
- **Credenziale guest nel vault SYSTEM, username host-qualificato.** Lo
|
||||||
|
step 4 scrive in Windows Credential Manager dell'utente interattivo →
|
||||||
|
invisibile ad act_runner (LocalSystem). Esegui (elevato):
|
||||||
|
`\.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'`.
|
||||||
|
Bare `ci_build` viene rifiutato da NTLM (`SEC_E_UNKNOWN_CREDENTIALS`):
|
||||||
|
usa il computer name del guest (= CN del cert WinRM). Diagnostica:
|
||||||
|
`\.\scripts\Test-CIGuestWinRM.ps1 -IpAddress <guest-ip>`.
|
||||||
|
- **venv di produzione = install non-editable.** Popola
|
||||||
|
`F:\CI\python\venv` con `python -m pip install .` dal repo reale —
|
||||||
|
**mai** `pip install -e .` (il `.pth` punta a un path che LocalSystem
|
||||||
|
non risolve → `No module named ci_orchestrator`). Vedi
|
||||||
|
`plans/PhaseA-user-checklist.md` Passo 1. Nessun workflow CI deve
|
||||||
|
installare in quel venv (lint usa un venv effimero per-job).
|
||||||
|
- **Gitea `app.ini`:** imposta `[actions] DEFAULT_ACTIONS_URL = github`
|
||||||
|
e referenzia l'action col **URL completo**
|
||||||
|
(`https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`)
|
||||||
|
così le action GitHub (checkout, upload-artifact) restano risolvibili
|
||||||
|
e l'action locale punta all'istanza. Il repo `local-ci-cd-system` deve
|
||||||
|
essere **pubblico** (il runner clona l'action senza autenticazione) e
|
||||||
|
l'action deve esistere su `@main` (il ref che il runner clona).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Parametri principali
|
## Parametri principali
|
||||||
@@ -139,6 +166,22 @@ Allo `Setup-Host.ps1` exit 0:
|
|||||||
```
|
```
|
||||||
Poi **provisiona il template VM Linux** — vedi [LINUX-TEMPLATE-SETUP.md](LINUX-TEMPLATE-SETUP.md)
|
Poi **provisiona il template VM Linux** — vedi [LINUX-TEMPLATE-SETUP.md](LINUX-TEMPLATE-SETUP.md)
|
||||||
|
|
||||||
|
6. **Crea `F:\CI\config.toml`** — necessario affinché il Python orchestrator trovi la chiave SSH e le altre impostazioni:
|
||||||
|
```powershell
|
||||||
|
Copy-Item N:\Code\Workspace\Local-CI-CD-System\config.example.toml F:\CI\config.toml
|
||||||
|
```
|
||||||
|
Il file `config.example.toml` contiene già i path corretti per Windows (`F:\CI\...`). Modificare solo se si usa un `$CIRoot` diverso da `F:\CI`.
|
||||||
|
|
||||||
|
Valori chiave configurati:
|
||||||
|
|
||||||
|
| Chiave | Valore default | Scopo |
|
||||||
|
| -------------------- | ----------------------- | -------------------------------------------- |
|
||||||
|
| `ssh_key_path` | `F:/CI/keys/ci_linux` | Chiave privata per SSH verso Linux guest |
|
||||||
|
| `guest_cred_target` | `BuildVMGuest` | Target Windows Credential Manager (WinRM) |
|
||||||
|
| `backend.type` | `workstation` | Backend VMware (solo `workstation` in Phase A) |
|
||||||
|
|
||||||
|
Le variabili d'ambiente `CI_SSH_KEY_PATH`, `CI_VMRUN_PATH`, `CI_GUEST_CRED_TARGET` sovrascrivono i valori del TOML se presenti.
|
||||||
|
|
||||||
6. **Registra chiave SSH con Gitea**:
|
6. **Registra chiave SSH con Gitea**:
|
||||||
- Gitea UI → User Settings → SSH Keys → Add Key
|
- Gitea UI → User Settings → SSH Keys → Add Key
|
||||||
- Chiave pubblica: `%USERPROFILE%\.ssh\id_rsa.pub`
|
- Chiave pubblica: `%USERPROFILE%\.ssh\id_rsa.pub`
|
||||||
@@ -162,6 +205,7 @@ Allo `Setup-Host.ps1` exit 0:
|
|||||||
| Snapshot name (Windows) | `BaseClean` |
|
| Snapshot name (Windows) | `BaseClean` |
|
||||||
| Snapshot name (Linux) | `BaseClean-Linux` |
|
| Snapshot name (Linux) | `BaseClean-Linux` |
|
||||||
| SSH key (Linux VMs) | `F:\CI\keys\ci_linux` (privata), `F:\CI\keys\ci_linux.pub` |
|
| SSH key (Linux VMs) | `F:\CI\keys\ci_linux` (privata), `F:\CI\keys\ci_linux.pub` |
|
||||||
|
| Orchestrator config TOML | `F:\CI\config.toml` (copia di `config.example.toml`) |
|
||||||
| Clone base dir | `F:\CI\BuildVMs\` |
|
| Clone base dir | `F:\CI\BuildVMs\` |
|
||||||
| Artifact dir | `F:\CI\Artifacts\` |
|
| Artifact dir | `F:\CI\Artifacts\` |
|
||||||
| Log dir | `F:\CI\Logs\` (retention: 30 giorni) |
|
| Log dir | `F:\CI\Logs\` (retention: 30 giorni) |
|
||||||
@@ -196,3 +240,4 @@ Task schedulati (vedi `scripts/Register-CIScheduledTasks.ps1` per il bootstrap):
|
|||||||
| Service `act_runner` non parte | Check `F:\CI\act_runner\logs\stderr.log` |
|
| Service `act_runner` non parte | Check `F:\CI\act_runner\logs\stderr.log` |
|
||||||
| `New-StoredCredential` fail | Verifica modulo CredentialManager importato + scope corretto |
|
| `New-StoredCredential` fail | Verifica modulo CredentialManager importato + scope corretto |
|
||||||
| Job CI fallisce su `Test-WSMan` | TrustedHosts host non include subnet VM, AllowUnencrypted=$false |
|
| Job CI fallisce su `Test-WSMan` | TrustedHosts host non include subnet VM, AllowUnencrypted=$false |
|
||||||
|
| `wait-ready` SSH: `No authentication methods available` | `F:\CI\config.toml` mancante o `ssh_key_path` non configurato. Crea il file (vedi step 6) oppure passa `--ssh-key-path F:\CI\keys\ci_linux` |
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Configurazione accesso SSH a Gitea
|
# Configurazione accesso SSH a Gitea
|
||||||
|
|
||||||
**Target**: clone da `git@gitea.emulab.it:Simone/nsis-plugin-nsinnounp.git`
|
**Target**: clone da `git@gitea.emulab.it:Simone/nsis-plugin-ns7zip.git`
|
||||||
**Gitea**: `http://10.10.20.11:3100` / `https://gitea.emulab.it`
|
**Gitea**: `http://10.10.20.11:3100` / `https://gitea.emulab.it`
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -83,13 +83,13 @@ Hi there, Simone! You've successfully authenticated with the key named ci-build@
|
|||||||
Usare l'alias `gitea-ci` definito in `~\.ssh\config`:
|
Usare l'alias `gitea-ci` definito in `~\.ssh\config`:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git'
|
-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git'
|
||||||
```
|
```
|
||||||
|
|
||||||
Oppure con IP/porta espliciti (equivalente):
|
Oppure con IP/porta espliciti (equivalente):
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
-RepoUrl 'ssh://git@10.10.20.11:2222/Simone/nsis-plugin-nsinnounp.git'
|
-RepoUrl 'ssh://git@10.10.20.11:2222/Simone/nsis-plugin-ns7zip.git'
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -254,7 +254,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
if: success()
|
if: success()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-${{ github.ref_name }}
|
name: build-${{ github.ref_name }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
||||||
@@ -262,7 +262,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload logs on failure
|
- name: Upload logs on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-logs-${{ github.run_id }}
|
name: build-logs-${{ github.run_id }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}\*.log
|
path: F:\CI\Artifacts\${{ github.run_id }}\*.log
|
||||||
@@ -308,7 +308,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
if: success()
|
if: success()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-linux-${{ github.ref_name }}
|
name: build-linux-${{ github.ref_name }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
||||||
@@ -359,7 +359,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
if: success()
|
if: success()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-${{ github.ref_name }}
|
name: build-${{ github.ref_name }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
||||||
@@ -404,7 +404,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
if: success()
|
if: success()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-linux-${{ github.ref_name }}
|
name: build-linux-${{ github.ref_name }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip
|
||||||
@@ -502,7 +502,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Upload artifacts
|
- name: Upload artifacts
|
||||||
if: success()
|
if: success()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-${{ matrix.config }}-${{ github.ref_name }}
|
name: build-${{ matrix.config }}-${{ github.ref_name }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}-${{ matrix.config }}\artifacts.zip
|
path: F:\CI\Artifacts\${{ github.run_id }}-${{ matrix.config }}\artifacts.zip
|
||||||
@@ -573,3 +573,6 @@ the cleanup task `Cleanup-OrphanedBuildVMs.ps1` handles these cases.
|
|||||||
| `-TemplatePath` for Linux without `GITEA_CI_LINUX_TEMPLATE_PATH` | Pass `$env:GITEA_CI_LINUX_TEMPLATE_PATH` or the full path explicitly |
|
| `-TemplatePath` for Linux without `GITEA_CI_LINUX_TEMPLATE_PATH` | Pass `$env:GITEA_CI_LINUX_TEMPLATE_PATH` or the full path explicitly |
|
||||||
| Forgetting `-SnapshotName 'BaseClean-Linux'` for Linux | Linux template uses a different snapshot name than Windows (`BaseClean`) |
|
| Forgetting `-SnapshotName 'BaseClean-Linux'` for Linux | Linux template uses a different snapshot name than Windows (`BaseClean`) |
|
||||||
| Same `-JobId` for matrix jobs | Append matrix key: `${{ github.run_id }}-${{ matrix.config }}` |
|
| Same `-JobId` for matrix jobs | Append matrix key: `${{ github.run_id }}-${{ matrix.config }}` |
|
||||||
|
| `actions/upload-artifact@v4` / `download-artifact@v4` | Use `@v3`. Gitea reports as GHES; @v4 (`@actions/artifact` v2 / Twirp API) fails with `GHESNotSupportedError`. |
|
||||||
|
| Short action ref `Simone/local-ci-cd-system/...@main` | Use the full URL `https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`; with `[actions] DEFAULT_ACTIONS_URL = github` in Gitea's app.ini this keeps GitHub actions (checkout, upload-artifact) resolvable while pointing the composite action at the local instance. The action repo must be on `@main` (the ref the runner clones). |
|
||||||
|
| Composite action referenced from a private/internal repo | The runner clones the action repo unauthenticated. Make `local-ci-cd-system` public, or the clone fails with `authentication required: Unauthorized`. |
|
||||||
|
|||||||
@@ -1,301 +0,0 @@
|
|||||||
# .gitea/actions/local-ci-build/action.yml
|
|
||||||
#
|
|
||||||
# Reusable composite action — Local CI/CD System
|
|
||||||
#
|
|
||||||
# Wraps Invoke-CIJob.ps1 (ephemeral VMware VM pipeline) into a single callable
|
|
||||||
# step, usable from any Gitea repository without duplicating the orchestration
|
|
||||||
# logic.
|
|
||||||
#
|
|
||||||
# Stored in: Simone/local-ci-system / .gitea/actions/local-ci-build/action.yml
|
|
||||||
#
|
|
||||||
# Usage from a calling repo's workflow:
|
|
||||||
#
|
|
||||||
# jobs:
|
|
||||||
# build:
|
|
||||||
# runs-on: windows-build # or linux-build
|
|
||||||
# steps:
|
|
||||||
# - uses: actions/checkout@v4
|
|
||||||
# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
|
||||||
# with:
|
|
||||||
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
|
||||||
# artifact-source: 'dist'
|
|
||||||
# submodules: 'true'
|
|
||||||
#
|
|
||||||
# Runner label determines the guest OS template:
|
|
||||||
# windows-build -> WinBuild2025 (WinRM/HTTPS transport)
|
|
||||||
# linux-build -> LinuxBuild2404 (SSH transport)
|
|
||||||
|
|
||||||
name: Local CI Build
|
|
||||||
description: >
|
|
||||||
Builds a project using an ephemeral VMware VM managed by Invoke-CIJob.ps1.
|
|
||||||
Handles the full VM lifecycle: clone template -> start VM -> run build ->
|
|
||||||
collect artifacts -> destroy VM.
|
|
||||||
Compatible with windows-build (WinRM) and linux-build (SSH) runners.
|
|
||||||
|
|
||||||
inputs:
|
|
||||||
build-command:
|
|
||||||
description: >
|
|
||||||
Shell command to execute inside the build VM.
|
|
||||||
Example: 'python build_plugin.py --final --dist-dir dist'
|
|
||||||
required: true
|
|
||||||
|
|
||||||
artifact-source:
|
|
||||||
description: >
|
|
||||||
Path inside VM where build outputs are located, relative to the CI work
|
|
||||||
directory. Collected to F:\CI\Artifacts\{JobId}\ on the host.
|
|
||||||
required: false
|
|
||||||
default: 'dist'
|
|
||||||
|
|
||||||
submodules:
|
|
||||||
description: >
|
|
||||||
Set to "true" to clone the repository with --recurse-submodules.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
|
|
||||||
guest-os:
|
|
||||||
description: >
|
|
||||||
Guest OS type: Windows, Linux, or Auto.
|
|
||||||
Auto (default) detects the OS from the VMX guestOS field after cloning.
|
|
||||||
required: false
|
|
||||||
default: 'Auto'
|
|
||||||
|
|
||||||
use-git-clone:
|
|
||||||
description: >
|
|
||||||
Set to "true" to clone the repository inside the VM rather than
|
|
||||||
transferring a zip from the host. Requires Git in the template (see
|
|
||||||
Install-CIToolchain-WinBuild2025.ps1 Step 12 / Install-CIToolchain-Linux2404.sh).
|
|
||||||
Preferred for repositories larger than ~200 MB.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
|
|
||||||
configuration:
|
|
||||||
description: >
|
|
||||||
Build configuration string forwarded to the build command (Release/Debug).
|
|
||||||
required: false
|
|
||||||
default: 'Release'
|
|
||||||
|
|
||||||
template-path:
|
|
||||||
description: >
|
|
||||||
Override the VMX template path. Leave empty (default) to use the value
|
|
||||||
from the GITEA_CI_TEMPLATE_PATH runner environment variable set in
|
|
||||||
runner/config.yaml.
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
snapshot-name:
|
|
||||||
description: >
|
|
||||||
Override the snapshot name to clone from. Leave empty (default) to use
|
|
||||||
GITEA_CI_SNAPSHOT_NAME runner env var, or "BaseClean" if that is also
|
|
||||||
unset.
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
artifact-name:
|
|
||||||
description: >
|
|
||||||
Name for the artifact uploaded to Gitea. Leave empty to use the default:
|
|
||||||
build-{run_id}-{sha}.
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
artifact-retention-days:
|
|
||||||
description: 'Days to retain uploaded artifacts in Gitea.'
|
|
||||||
required: false
|
|
||||||
default: '14'
|
|
||||||
|
|
||||||
job-id-suffix:
|
|
||||||
description: >
|
|
||||||
Optional suffix appended to the job ID: run_id-run_attempt-SUFFIX.
|
|
||||||
Use to disambiguate parallel matrix legs so each leg writes to its own
|
|
||||||
artifact directory (e.g. set to matrix.target: windows, linux).
|
|
||||||
Leave empty (default) for single-job workflows.
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
repo-url:
|
|
||||||
description: >
|
|
||||||
Override the repository URL passed to Invoke-CIJob.ps1.
|
|
||||||
Defaults to github.server_url/repository.git (HTTPS).
|
|
||||||
Set to an SSH URL (e.g. ssh://gitea-ci/Org/repo.git) when the host
|
|
||||||
runner uses an SSH alias for Gitea.
|
|
||||||
required: false
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
extra-guest-env-json:
|
|
||||||
description: >
|
|
||||||
JSON object of extra environment variables to inject into the build VM
|
|
||||||
guest before the build command runs. Use for signing keys and secrets.
|
|
||||||
Example: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","GPG_KEY_ID":"0xABCD1234"}'
|
|
||||||
Values are passed via ExtraGuestEnv hashtable in Invoke-CIJob.ps1 and
|
|
||||||
are never echoed to the log.
|
|
||||||
required: false
|
|
||||||
default: '{}'
|
|
||||||
|
|
||||||
use-shared-cache:
|
|
||||||
description: >
|
|
||||||
Set to "true" to redirect NuGet and pip caches to VMware shared folders
|
|
||||||
on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip).
|
|
||||||
Requires Set-TemplateSharedFolders.ps1 to have been run on the template
|
|
||||||
and VMware Tools HGFS driver present in the guest.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
|
|
||||||
skip-artifact:
|
|
||||||
description: >
|
|
||||||
Set to "true" to skip artifact packaging and collection entirely.
|
|
||||||
Use for jobs that intentionally produce no output (lint, static analysis,
|
|
||||||
test-only). When absent (default), the build MUST produce artifact-source
|
|
||||||
or the job fails.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
|
|
||||||
outputs:
|
|
||||||
artifact-path:
|
|
||||||
description: >
|
|
||||||
Host-side directory where Invoke-CIJob.ps1 placed collected artifacts.
|
|
||||||
Format: F:\CI\Artifacts\{run_id}-{run_attempt}
|
|
||||||
value: ${{ steps.invoke-ci.outputs.artifact-path }}
|
|
||||||
|
|
||||||
artifact-name:
|
|
||||||
description: 'Name used when uploading the artifact to Gitea.'
|
|
||||||
value: ${{ steps.invoke-ci.outputs.artifact-name }}
|
|
||||||
|
|
||||||
runs:
|
|
||||||
using: composite
|
|
||||||
steps:
|
|
||||||
|
|
||||||
# ── Invoke the CI orchestrator ─────────────────────────────────────────────
|
|
||||||
- name: Invoke CI Job
|
|
||||||
id: invoke-ci
|
|
||||||
shell: powershell
|
|
||||||
# Inputs are forwarded via environment variables to avoid shell injection
|
|
||||||
# from user-supplied build-command or path strings.
|
|
||||||
env:
|
|
||||||
INPUT_BUILD_COMMAND: ${{ inputs.build-command }}
|
|
||||||
INPUT_ARTIFACT_SOURCE: ${{ inputs.artifact-source }}
|
|
||||||
INPUT_SUBMODULES: ${{ inputs.submodules }}
|
|
||||||
INPUT_GUEST_OS: ${{ inputs.guest-os }}
|
|
||||||
INPUT_USE_GIT_CLONE: ${{ inputs.use-git-clone }}
|
|
||||||
INPUT_CONFIGURATION: ${{ inputs.configuration }}
|
|
||||||
INPUT_TEMPLATE_PATH: ${{ inputs.template-path }}
|
|
||||||
INPUT_SNAPSHOT_NAME: ${{ inputs.snapshot-name }}
|
|
||||||
INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }}
|
|
||||||
INPUT_JOB_ID_SUFFIX: ${{ inputs.job-id-suffix }}
|
|
||||||
INPUT_REPO_URL: ${{ inputs.repo-url }}
|
|
||||||
INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }}
|
|
||||||
INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }}
|
|
||||||
INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }}
|
|
||||||
run: |
|
|
||||||
#Requires -Version 5.1
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
$ciScriptsDir = $env:GITEA_CI_SCRIPT_ROOT
|
|
||||||
|
|
||||||
# Build job ID — append suffix when provided (matrix disambiguation)
|
|
||||||
$jobId = '${{ github.run_id }}-${{ github.run_attempt }}'
|
|
||||||
if ($env:INPUT_JOB_ID_SUFFIX) { $jobId = "$jobId-$($env:INPUT_JOB_ID_SUFFIX)" }
|
|
||||||
$artifactPath = "F:\CI\Artifacts\$jobId"
|
|
||||||
|
|
||||||
# Resolve artifact name: caller-supplied or default to build-{run_id}-{sha}
|
|
||||||
if ($env:INPUT_ARTIFACT_NAME) {
|
|
||||||
$artifactName = $env:INPUT_ARTIFACT_NAME
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$artifactName = "build-${{ github.run_id }}-${{ github.sha }}"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Resolve repository URL: explicit override or construct from GitHub context
|
|
||||||
if ($env:INPUT_REPO_URL) {
|
|
||||||
$repoUrl = $env:INPUT_REPO_URL
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$repoUrl = '${{ github.server_url }}/${{ github.repository }}.git'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Parse extra-guest-env-json into a hashtable (§6.5 secret injection)
|
|
||||||
$extraGuestEnv = @{}
|
|
||||||
if ($env:INPUT_EXTRA_GUEST_ENV_JSON -and $env:INPUT_EXTRA_GUEST_ENV_JSON -ne '{}') {
|
|
||||||
$parsed = $env:INPUT_EXTRA_GUEST_ENV_JSON | ConvertFrom-Json
|
|
||||||
foreach ($prop in $parsed.PSObject.Properties) {
|
|
||||||
$extraGuestEnv[$prop.Name] = [string]$prop.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Resolve guest OS: honor explicit input; for Auto, infer from RUNNER_LABELS
|
|
||||||
$resolvedGuestOS = $env:INPUT_GUEST_OS
|
|
||||||
if ($resolvedGuestOS -eq 'Auto') {
|
|
||||||
if ($env:RUNNER_LABELS -match 'linux') {
|
|
||||||
$resolvedGuestOS = 'Linux'
|
|
||||||
} else {
|
|
||||||
$resolvedGuestOS = 'Windows'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Select template path: explicit input overrides OS-based runner env var
|
|
||||||
$resolvedTemplatePath = $env:INPUT_TEMPLATE_PATH
|
|
||||||
if (-not $resolvedTemplatePath) {
|
|
||||||
if ($resolvedGuestOS -eq 'Linux') {
|
|
||||||
$resolvedTemplatePath = $env:GITEA_CI_LINUX_TEMPLATE_PATH
|
|
||||||
} else {
|
|
||||||
$resolvedTemplatePath = $env:GITEA_CI_TEMPLATE_PATH
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Select snapshot name: explicit input > OS-based default
|
|
||||||
# GITEA_CI_SNAPSHOT_NAME applies to Windows (template-refresh override)
|
|
||||||
$resolvedSnapshotName = $env:INPUT_SNAPSHOT_NAME
|
|
||||||
if (-not $resolvedSnapshotName) {
|
|
||||||
if ($resolvedGuestOS -eq 'Linux') {
|
|
||||||
$resolvedSnapshotName = 'BaseClean-Linux'
|
|
||||||
} else {
|
|
||||||
$resolvedSnapshotName = if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Build parameter hashtable for Invoke-CIJob.ps1
|
|
||||||
$params = @{
|
|
||||||
JobId = $jobId
|
|
||||||
RepoUrl = $repoUrl
|
|
||||||
Branch = '${{ github.ref_name }}'
|
|
||||||
Commit = '${{ github.sha }}'
|
|
||||||
BuildCommand = $env:INPUT_BUILD_COMMAND
|
|
||||||
GuestArtifactSource = $env:INPUT_ARTIFACT_SOURCE
|
|
||||||
GuestOS = $resolvedGuestOS
|
|
||||||
Configuration = $env:INPUT_CONFIGURATION
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($env:INPUT_SUBMODULES -eq 'true') { $params['Submodules'] = $true }
|
|
||||||
if ($env:INPUT_USE_GIT_CLONE -eq 'true') { $params['UseGitClone'] = $true }
|
|
||||||
if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $params['UseSharedCache'] = $true }
|
|
||||||
if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $params['SkipArtifact'] = $true }
|
|
||||||
if ($resolvedTemplatePath) { $params['TemplatePath'] = $resolvedTemplatePath }
|
|
||||||
if ($resolvedSnapshotName) { $params['SnapshotName'] = $resolvedSnapshotName }
|
|
||||||
if ($extraGuestEnv.Count -gt 0) { $params['ExtraGuestEnv'] = $extraGuestEnv }
|
|
||||||
|
|
||||||
& "$ciScriptsDir\Invoke-CIJob.ps1" @params
|
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
||||||
|
|
||||||
# Emit outputs consumed by subsequent steps
|
|
||||||
"artifact-path=$artifactPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8
|
|
||||||
"artifact-name=$artifactName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8
|
|
||||||
|
|
||||||
# ── Upload build artifacts on success ─────────────────────────────────────
|
|
||||||
- name: Upload artifacts
|
|
||||||
if: success() && inputs.skip-artifact != 'true'
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: ${{ steps.invoke-ci.outputs.artifact-name }}
|
|
||||||
path: ${{ steps.invoke-ci.outputs.artifact-path }}\
|
|
||||||
retention-days: ${{ inputs.artifact-retention-days }}
|
|
||||||
if-no-files-found: error
|
|
||||||
|
|
||||||
# ── Upload diagnostic logs on failure ─────────────────────────────────────
|
|
||||||
- name: Upload diagnostic logs on failure
|
|
||||||
if: failure()
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: build-logs-${{ github.sha }}
|
|
||||||
path: ${{ steps.invoke-ci.outputs.artifact-path }}\*.log
|
|
||||||
retention-days: 3
|
|
||||||
if-no-files-found: ignore
|
|
||||||
|
|
||||||
@@ -9,7 +9,7 @@
|
|||||||
# windows-build -> WinBuild2025 (WinRM/HTTPS transport)
|
# windows-build -> WinBuild2025 (WinRM/HTTPS transport)
|
||||||
# linux-build -> LinuxBuild2404 (SSH transport)
|
# linux-build -> LinuxBuild2404 (SSH transport)
|
||||||
#
|
#
|
||||||
# The composite action (Simone/local-ci-system/.gitea/actions/local-ci-build)
|
# The composite action (Simone/local-ci-cd-system/.gitea/actions/local-ci-build)
|
||||||
# handles the full VM lifecycle: clone template -> start -> build ->
|
# handles the full VM lifecycle: clone template -> start -> build ->
|
||||||
# collect artifacts -> upload to Gitea -> destroy VM.
|
# collect artifacts -> upload to Gitea -> destroy VM.
|
||||||
# No orchestration logic needed in each repo's workflow.
|
# No orchestration logic needed in each repo's workflow.
|
||||||
@@ -36,7 +36,7 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
# ── Invoke the full VM lifecycle via composite action ───────────────────
|
# ── Invoke the full VM lifecycle via composite action ───────────────────
|
||||||
- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
- uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
with:
|
with:
|
||||||
# --- adapt these two per-repository -----------------------------------
|
# --- adapt these two per-repository -----------------------------------
|
||||||
build-command: 'python build.py --dist-dir dist' # command run in VM
|
build-command: 'python build.py --dist-dir dist' # command run in VM
|
||||||
@@ -63,7 +63,7 @@ jobs:
|
|||||||
# Step 4: Upload artifacts even on failure (for diagnostics)
|
# Step 4: Upload artifacts even on failure (for diagnostics)
|
||||||
- name: Upload diagnostic logs on failure
|
- name: Upload diagnostic logs on failure
|
||||||
if: failure()
|
if: failure()
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v3
|
||||||
with:
|
with:
|
||||||
name: build-logs-${{ github.sha }}
|
name: build-logs-${{ github.sha }}
|
||||||
path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}\*.log
|
path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}\*.log
|
||||||
@@ -140,7 +140,7 @@ jobs:
|
|||||||
#
|
#
|
||||||
# - name: Upload artifacts (${{ matrix.target }})
|
# - name: Upload artifacts (${{ matrix.target }})
|
||||||
# if: success()
|
# if: success()
|
||||||
# uses: actions/upload-artifact@v4
|
# uses: actions/upload-artifact@v3
|
||||||
# with:
|
# with:
|
||||||
# name: build-${{ matrix.target }}-${{ github.sha }}
|
# name: build-${{ matrix.target }}-${{ github.sha }}
|
||||||
# path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.target }}\
|
# path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.target }}\
|
||||||
@@ -154,7 +154,7 @@ jobs:
|
|||||||
# The composite action handles: Invoke-CIJob.ps1 call + artifact upload + log upload.
|
# The composite action handles: Invoke-CIJob.ps1 call + artifact upload + log upload.
|
||||||
#
|
#
|
||||||
# Reference: gitea/actions/local-ci-build/action.yml in this repository.
|
# Reference: gitea/actions/local-ci-build/action.yml in this repository.
|
||||||
# Deployed path in Gitea: Simone/local-ci-system / .gitea/actions/local-ci-build/action.yml
|
# Deployed path in Gitea: Simone/local-ci-cd-system / .gitea/actions/local-ci-build/action.yml
|
||||||
#
|
#
|
||||||
# Minimum example (Windows build, Python project with submodules):
|
# Minimum example (Windows build, Python project with submodules):
|
||||||
#
|
#
|
||||||
@@ -171,7 +171,7 @@ jobs:
|
|||||||
# runs-on: windows-build
|
# runs-on: windows-build
|
||||||
# steps:
|
# steps:
|
||||||
# - uses: actions/checkout@v4
|
# - uses: actions/checkout@v4
|
||||||
# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
# - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
# with:
|
# with:
|
||||||
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
||||||
# artifact-source: 'dist'
|
# artifact-source: 'dist'
|
||||||
@@ -188,7 +188,7 @@ jobs:
|
|||||||
# runs-on: ${{ matrix.target }}-build
|
# runs-on: ${{ matrix.target }}-build
|
||||||
# steps:
|
# steps:
|
||||||
# - uses: actions/checkout@v4
|
# - uses: actions/checkout@v4
|
||||||
# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main
|
# - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main
|
||||||
# with:
|
# with:
|
||||||
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
# build-command: 'python build_plugin.py --final --dist-dir dist'
|
||||||
# artifact-source: 'dist'
|
# artifact-source: 'dist'
|
||||||
|
|||||||
@@ -1,55 +0,0 @@
|
|||||||
# PSScriptAnalyzer lint — runs on every push/PR that touches PowerShell files.
|
|
||||||
# Requires PSScriptAnalyzer installed on the runner host:
|
|
||||||
# Install-Module PSScriptAnalyzer -Scope AllUsers -Force
|
|
||||||
#
|
|
||||||
# Failures block the PR. Fix warnings with:
|
|
||||||
# Invoke-ScriptAnalyzer -Path scripts\ -Recurse -Fix
|
|
||||||
|
|
||||||
name: Lint (PSScriptAnalyzer)
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
paths:
|
|
||||||
- '**.ps1'
|
|
||||||
- '**.psm1'
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- '**.ps1'
|
|
||||||
- '**.psm1'
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
lint:
|
|
||||||
runs-on: windows-build
|
|
||||||
timeout-minutes: 10
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
fetch-depth: 1
|
|
||||||
|
|
||||||
- name: Run PSScriptAnalyzer
|
|
||||||
shell: powershell
|
|
||||||
run: |
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
if (-not (Get-Module -ListAvailable PSScriptAnalyzer)) {
|
|
||||||
Write-Host "Installing PSScriptAnalyzer..."
|
|
||||||
Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -AllowClobber
|
|
||||||
}
|
|
||||||
|
|
||||||
$paths = @('scripts', 'template', 'runner', 'Setup-Host.ps1')
|
|
||||||
$results = $paths | ForEach-Object {
|
|
||||||
if (Test-Path $_) {
|
|
||||||
Invoke-ScriptAnalyzer -Path $_ -Recurse -Severity Error,Warning
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($results) {
|
|
||||||
$results | Format-Table -AutoSize
|
|
||||||
Write-Error "PSScriptAnalyzer found $($results.Count) issue(s). Fix before merging."
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "PSScriptAnalyzer: no issues found."
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
# Fase A1 — Closeout operativo
|
||||||
|
|
||||||
|
Checklist per chiudere lo step **A1** di [implementation-plan-A-B](implementation-plan-A-B.md)
|
||||||
|
prima di iniziare A2. Il codice è già committato sul branch
|
||||||
|
`feature/python-rewrite-phase-a` (commit `9fbe952`); manca solo la
|
||||||
|
validazione contro l'ambiente reale.
|
||||||
|
|
||||||
|
## Stato attuale
|
||||||
|
|
||||||
|
- [x] Codice + 35 unit test (coverage 82.65%) committati
|
||||||
|
- [x] `ruff` clean, `mypy --strict` clean su 12 file sorgente
|
||||||
|
- [x] Job `python` aggiunto a `.gitea/workflows/lint.yml`
|
||||||
|
- [x] Job `python` di `lint.yml` PASS su act_runner (commit `8586ff8`)
|
||||||
|
- [ ] PoC `wait-ready` PASS contro VM reale Windows
|
||||||
|
- [ ] PoC `wait-ready` PASS contro VM reale Linux
|
||||||
|
|
||||||
|
## Perché completare A1 prima di A2
|
||||||
|
|
||||||
|
A2 riusa `WorkstationVmrunBackend`, `WinRmTransport`, `SshTransport` e
|
||||||
|
`KeyringCredentialStore` esattamente come sono ora. Se uno di questi ha
|
||||||
|
un bug di integrazione (TLS, encoding, parsing output `vmrun list`,
|
||||||
|
permessi venv, ecc.), trovarlo dopo A2 costa: **1 fix nei moduli core +
|
||||||
|
retest di tutti i 6 comandi portati in A2**. Trovarlo ora costa: 1 fix.
|
||||||
|
|
||||||
|
## 1. PoC `wait-ready` su VM Windows reale
|
||||||
|
|
||||||
|
**Obiettivo**: validare il path `vmrun.is_running` → `get_ip` → WinRM
|
||||||
|
HTTPS con cert self-signed end-to-end.
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# 1. Setup venv (una tantum) sull'host CI
|
||||||
|
python -m venv F:\CI\python\venv
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m pip install --upgrade pip
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m pip install -e ".[dev]"
|
||||||
|
|
||||||
|
# 2. Clone manuale di una VM smoke da template
|
||||||
|
$vmrun = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||||
|
$tpl = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx'
|
||||||
|
$dst = 'F:\CI\BuildVMs\smoke-a1\smoke-a1.vmx'
|
||||||
|
& $vmrun -T ws clone $tpl $dst linked -snapshot=BaseClean -cloneName=smoke-a1
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "clone fallito" }
|
||||||
|
& $vmrun -T ws start $dst nogui
|
||||||
|
if ($LASTEXITCODE -ne 0) { throw "start fallito" }
|
||||||
|
|
||||||
|
# 3. PoC wait-ready
|
||||||
|
$env:PYTHONIOENCODING = 'utf-8'
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m ci_orchestrator wait-ready `
|
||||||
|
--vmx $dst --guest-os windows --timeout 300
|
||||||
|
|
||||||
|
# 4. Cleanup
|
||||||
|
& $vmrun -T ws stop $dst hard
|
||||||
|
& $vmrun -T ws deleteVM $dst
|
||||||
|
```
|
||||||
|
|
||||||
|
**Atteso**:
|
||||||
|
|
||||||
|
- Exit code `0`
|
||||||
|
- Ultima riga stdout: `[wait-ready] WinRM ready.`
|
||||||
|
- Tempo totale tra start VM e WinRM ready: 60–180 s
|
||||||
|
|
||||||
|
**Failure mode più probabili e fix**:
|
||||||
|
|
||||||
|
| Sintomo | Causa probabile | Fix |
|
||||||
|
| -------------------------------------------------- | ------------------------------------------------------ | -------------------------------------------------------------------------------------------------- |
|
||||||
|
| `is_running` resta `False` per sempre | Path mismatch in `vmrun list` (slash diversi, casing) | Aggiungere normalizzazione `os.path.normcase` in `WorkstationVmrunBackend.is_running` |
|
||||||
|
| `get_ip` ritorna sempre `None` | VMware Tools non ancora pronti | Atteso nei primi 30–60 s; se persiste >180 s, problema nel template |
|
||||||
|
| `WinRM probe failed: ... certificate verify` | `cert_validation=False` non basta su TLS handshake | Ispezionare cipher; eventualmente pinnare versione `pypsrp` o passare `auth='ntlm'` esplicito |
|
||||||
|
| `KeyError: 'BuildVMGuest'` | Credenziale non leggibile dall'utente che lancia il PoC | Eseguire come l'utente del runner (`runas /user:ci-runner ...`) o re-store credential nel suo profilo |
|
||||||
|
| `KeyringCredentialStore` ritorna username vuoto | Backend keyring usa `get_password`-only | Verificato in `test_credentials.py::test_get_falls_back_to_password`; in tal caso impostare username separatamente nel target |
|
||||||
|
|
||||||
|
## 2. PoC `wait-ready` su VM Linux reale
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$tpl = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx'
|
||||||
|
$dst = 'F:\CI\BuildVMs\smoke-a1-lnx\smoke-a1-lnx.vmx'
|
||||||
|
& $vmrun -T ws clone $tpl $dst linked -snapshot=BaseClean-Linux -cloneName=smoke-a1-lnx
|
||||||
|
& $vmrun -T ws start $dst nogui
|
||||||
|
|
||||||
|
$env:CI_SSH_KEY_PATH = 'F:\CI\keys\ci_linux'
|
||||||
|
F:\CI\python\venv\Scripts\python.exe -m ci_orchestrator wait-ready `
|
||||||
|
--vmx $dst --guest-os linux --ssh-user ci_build --timeout 300
|
||||||
|
|
||||||
|
& $vmrun -T ws stop $dst hard
|
||||||
|
& $vmrun -T ws deleteVM $dst
|
||||||
|
```
|
||||||
|
|
||||||
|
**Atteso**: ultima riga `[wait-ready] SSH ready.` entro 60–120 s.
|
||||||
|
|
||||||
|
**Failure mode più probabili**:
|
||||||
|
|
||||||
|
| Sintomo | Causa | Fix |
|
||||||
|
| ------------------------------------ | ---------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||||
|
| `SSH connection ... timed out` | DHCP collision (vedi `AGENTS.md` errore #11) | Verificare snapshot `BaseClean-Linux` con machine-id resettato |
|
||||||
|
| `paramiko.AuthenticationException` | Permessi chiave SSH errati (Windows ACL su `ci_linux`) | `icacls F:\CI\keys\ci_linux /inheritance:r /grant:r "<utente-runner>:R"` |
|
||||||
|
| `is_ready()` False ma SSH manuale OK | `BatchMode=yes` lato `ssh.exe` non equivalente in paramiko | Già coperto: paramiko non chiede prompt; verificare `look_for_keys=False` settato |
|
||||||
|
|
||||||
|
## 3. Job `python` di `lint.yml` su act_runner
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
# Push del branch
|
||||||
|
git push -u origin feature/python-rewrite-phase-a
|
||||||
|
```
|
||||||
|
|
||||||
|
Poi su Gitea: aprire una PR contro `main` (o trigger manuale del
|
||||||
|
workflow). Verificare che il job `python` completi entro
|
||||||
|
`timeout-minutes: 15`.
|
||||||
|
|
||||||
|
**Failure mode più probabili**:
|
||||||
|
|
||||||
|
| Sintomo | Causa | Fix |
|
||||||
|
| ------------------------------------------------------ | -------------------------------------------------------- | ------------------------------------------------------------------------------ |
|
||||||
|
| `python: command not found` | `python` non in PATH dell'utente del runner | Installare Python 3.11+ machine-wide o aggiornare `PATH` del servizio |
|
||||||
|
| `pip install -e .[dev]` >15 min al primo run | Cache assente, network lento | Bumpare `timeout-minutes: 30` solo per il primo run; cache stabile dopo |
|
||||||
|
| `Access denied` su `F:\CI\python\venv` | Utente del runner senza scrittura su `F:\CI\python` | `icacls F:\CI\python /grant "<utente-runner>:(OI)(CI)F"` |
|
||||||
|
| Coverage <70% | Differenza Python 3.14 (locale) vs 3.11 (CI) | Rilanciare con `--cov-report=term-missing` per identificare branch mancante |
|
||||||
|
|
||||||
|
## 4. Definizione di "A1 done"
|
||||||
|
|
||||||
|
Spuntabile solo quando **tutti** e tre i punti sopra sono PASS:
|
||||||
|
|
||||||
|
- [ ] PoC Windows: `wait-ready` exit 0 contro `WinBuild2025` reale
|
||||||
|
- [ ] PoC Linux: `wait-ready` exit 0 contro `LinuxBuild2404` reale
|
||||||
|
- [ ] CI: workflow `lint.yml` job `python` PASS su act_runner
|
||||||
|
|
||||||
|
A quel punto:
|
||||||
|
|
||||||
|
1. Aggiornare la checklist master in [implementation-plan-A-B](implementation-plan-A-B.md)
|
||||||
|
spuntando le righe `[A1]`.
|
||||||
|
2. Mergeare `feature/python-rewrite-phase-a` su `main` (o lasciare il
|
||||||
|
branch live se A2 verrà committato sopra).
|
||||||
|
3. Procedere con A2 (script "foglia": `Wait-VMReady`, `Remove-BuildVM`,
|
||||||
|
`Cleanup-OrphanedBuildVMs`, `Watch-DiskSpace`, `Watch-RunnerHealth`,
|
||||||
|
`Get-CIJobSummary`).
|
||||||
|
|
||||||
|
## 5. Workaround se A1 non si può chiudere ora
|
||||||
|
|
||||||
|
Se l'hardware/VM template non sono accessibili in questo momento:
|
||||||
|
|
||||||
|
- **Non rimuovere** alcuno script PS in A2.
|
||||||
|
- Marcare A1 come "code-complete, validation pending" in
|
||||||
|
[implementation-plan-A-B](implementation-plan-A-B.md).
|
||||||
|
- Procedere con A2 ma trattare ogni bug scoperto su `WorkstationVmrunBackend`
|
||||||
|
/ `WinRmTransport` / `SshTransport` come **fix di A1** (commit separato,
|
||||||
|
no nuove feature A2 nello stesso commit).
|
||||||
|
- Riservare una sessione dedicata di smoke testing prima di A3 (pipeline
|
||||||
|
di build), perché A3 introduce `clone_linked` + `start` reali.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# A2 — Closeout
|
||||||
|
|
||||||
|
Fase A2 di `plans/implementation-plan-A-B.md`: porting in Python degli script
|
||||||
|
"foglia" (no state condiviso). Branch: `feature/python-rewrite-phase-a`.
|
||||||
|
|
||||||
|
## Stato attuale
|
||||||
|
|
||||||
|
- [x] `commands/wait.py` → `wait-ready` (sostituisce `Wait-VMReady.ps1`)
|
||||||
|
- [x] `commands/vm.py` → `vm remove` + `vm cleanup` (sostituisce `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`)
|
||||||
|
- [x] `commands/monitor.py` → `monitor disk` + `monitor runner` (sostituisce `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`)
|
||||||
|
- [x] `commands/report.py` → `report job` (sostituisce `Get-CIJobSummary.ps1`)
|
||||||
|
- [x] Tutti i 6 `.ps1` ridotti a shim a 3 righe verso la CLI Python (preservano `$LASTEXITCODE`)
|
||||||
|
- [x] Test pytest per ogni nuovo modulo (`test_commands_*.py`), 69 test totali, ruff/mypy --strict clean, coverage 74.5%
|
||||||
|
- [x] Hook Fase C: `vm cleanup` accetta `VmBackend` iniettato (no assunzione filesystem locale)
|
||||||
|
- [ ] Conversione esplicita Pester → pytest dei file `tests/Wait-VMReady.Tests.ps1` e `tests/Remove-BuildVM.Tests.ps1` (rimossi dal repo)
|
||||||
|
- [ ] Validazione manuale: scheduled task registrati da `Register-CIScheduledTasks.ps1` continuano a funzionare via shim
|
||||||
|
- [ ] Smoke end-to-end manuale: clone VM → `wait-ready` → `vm remove`
|
||||||
|
|
||||||
|
## Voce per voce — Definizione di "fatto" A2
|
||||||
|
|
||||||
|
| Criterio | Stato | Note |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Tutti gli script "foglia" hanno shim PS che chiama Python | ✅ | 6/6 |
|
||||||
|
| Test pytest sostituiscono i Pester corrispondenti | ⚠️ parziale | I Pester legacy sono ancora presenti come safety net; equivalenza funzionale coperta dai nuovi test pytest. Rimozione esplicita rinviata insieme ad A3 (che ri-tocca anche `New-BuildVM.Tests.ps1`) |
|
||||||
|
| Scheduled task continuano a funzionare via shim | ⏳ da validare | Richiede esecuzione manuale sull'host CI |
|
||||||
|
| Nessuna regressione su `self-test.yml` | ⏳ da validare | Workflow `lint.yml` PASS; `self-test.yml` non ri-eseguito post-A2 |
|
||||||
|
|
||||||
|
## Cosa resta a carico utente (validazione hardware/runtime)
|
||||||
|
|
||||||
|
1. Eseguire manualmente uno dei task in `Register-CIScheduledTasks.ps1` (es. `Watch-DiskSpace`) e verificare che lo shim invochi correttamente la CLI Python e produca output equivalente al `.ps1` originale.
|
||||||
|
2. Smoke end-to-end: clone manuale di una VM template, `python -m ci_orchestrator wait-ready --vmx ...`, `python -m ci_orchestrator vm remove --vmx ...`.
|
||||||
|
3. Decidere se rimuovere i Pester `tests/Wait-VMReady.Tests.ps1` / `tests/Remove-BuildVM.Tests.ps1` ora o aspettare A3 (che li ri-cita).
|
||||||
|
|
||||||
|
## Riferimenti commit
|
||||||
|
|
||||||
|
- `794db1a` — feat(a2): port leaf PS scripts to ci_orchestrator CLI
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# A3 — Closeout
|
||||||
|
|
||||||
|
Fase A3 di `plans/implementation-plan-A-B.md`: pipeline core (clone → build → collect) in Python.
|
||||||
|
Branch: `feature/python-rewrite-phase-a`.
|
||||||
|
|
||||||
|
## Stato attuale
|
||||||
|
|
||||||
|
- [x] `commands/vm.py` esteso con `vm new` (sostituisce `New-BuildVM.ps1`)
|
||||||
|
- [x] `commands/build.py` con `build run` (sostituisce `Invoke-RemoteBuild.ps1`)
|
||||||
|
- [x] `commands/artifacts.py` con `artifacts collect` (sostituisce `Get-BuildArtifacts.ps1`)
|
||||||
|
- [x] Tutti i 3 `.ps1` ridotti a shim verso la CLI Python
|
||||||
|
- [x] Pester `tests/{New-BuildVM,Wait-VMReady,Remove-BuildVM}.Tests.ps1` rimossi (casi negativi coperti dai pytest equivalenti)
|
||||||
|
- [x] 91 pytest, ruff/mypy --strict clean, coverage 78.27%
|
||||||
|
- [x] Hook Fase C: `build run` + `artifacts collect` accettano `VmHandle`/`vmx` opaco; nessuna assunzione path Windows
|
||||||
|
- [ ] Smoke end-to-end manuale: clone WinBuild2025 → build script trivial → collect artifact ZIP
|
||||||
|
- [ ] Smoke end-to-end Linux equivalente
|
||||||
|
|
||||||
|
## Definizione di "fatto" A3
|
||||||
|
|
||||||
|
| Criterio | Stato |
|
||||||
|
| --- | --- |
|
||||||
|
| Pipeline build completa via Python CLI | ✅ |
|
||||||
|
| Shim PS preservano API per caller esistenti | ✅ |
|
||||||
|
| Smoke build PASS contro VM reale Windows e Linux | ⏳ validazione hardware |
|
||||||
|
| Pester `New-BuildVM.Tests.ps1` rimosso e sostituito | ✅ |
|
||||||
|
|
||||||
|
## Cosa resta a carico utente
|
||||||
|
|
||||||
|
1. Smoke manuale end-to-end Windows: `python -m ci_orchestrator vm new ... && build run ... && artifacts collect ...`
|
||||||
|
2. Stesso smoke su Linux template
|
||||||
|
3. Verificare che `Test-NsinnounpBuild.ps1` continui a passare invocando gli shim
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# A4 — Closeout
|
||||||
|
|
||||||
|
Fase A4 di `plans/implementation-plan-A-B.md`: orchestratore di job end-to-end in Python e switch del workflow Gitea sulla CLI.
|
||||||
|
Branch: `feature/python-rewrite-phase-a`.
|
||||||
|
|
||||||
|
## Stato attuale
|
||||||
|
|
||||||
|
- [x] `commands/job.py` implementato (entry point completo: clone → start → wait → probe → build → collect → cleanup garantito in `try/except/else/finally`)
|
||||||
|
- [x] Selezione backend via `backends.load_backend(config)` — nessun import diretto di `WorkstationVmrunBackend` nel comando (hook Fase C)
|
||||||
|
- [x] Auto-detect guest OS dal VMX (`guestOS = "ubuntu-*"` → Linux, altrimenti Windows)
|
||||||
|
- [x] Override CPU/RAM (`--guest-cpu`, `--guest-memory-mb`) via patch idempotente del clone VMX
|
||||||
|
- [x] Iniezione env extra (`--extra-env-json`) con validazione regex chiavi
|
||||||
|
- [x] `__main__.py` registra il sottocomando `job`
|
||||||
|
- [x] `scripts/Invoke-CIJob.ps1` ridotto a shim PS 5.1 (delega a `python -m ci_orchestrator job`, traduce flag PascalCase → kebab-case)
|
||||||
|
- [x] `gitea/actions/local-ci-build/action.yml` invoca direttamente `& $venvPython -m ci_orchestrator job ...` (più nessuno `@params` splatting verso lo shim PS)
|
||||||
|
- [x] `runner/config.yaml` esporta `PYTHONIOENCODING=utf-8` per garantire stdio UTF-8 nei log act_runner → Gitea
|
||||||
|
- [x] `tests/python/test_commands_job.py` aggiunto: 13 test (happy path Linux, happy path Windows + override, skip-artifact, cleanup su build failure, cleanup su clone failure, validazione template/extra-env, helper `_apply_vmx_overrides` / `_read_guest_os_from_vmx` / `_parse_extra_env_json`)
|
||||||
|
- [x] Suite completa: ruff clean, mypy --strict clean, **104 pytest PASS**, coverage totale **79.25%**, coverage `commands/job.py` **83%**
|
||||||
|
- [ ] Workflow `build-ns7zip.yml` matrix (Win + Linux) end-to-end PASS contro VM reali — richiede host con runner registrato
|
||||||
|
- [ ] Workflow `self-test.yml` / `lint.yml` PASS sul nuovo entry point — stessa dipendenza
|
||||||
|
- [ ] Misurazione tempo job entro ±10% del baseline PowerShell (`Measure-CIBenchmark.ps1`)
|
||||||
|
|
||||||
|
## Definizione di "fatto" A4
|
||||||
|
|
||||||
|
| Criterio | Stato |
|
||||||
|
| --------------------------------------------------- | ------------------------------------ |
|
||||||
|
| `action.yml` chiama Python direttamente | ✅ |
|
||||||
|
| Shim `Invoke-CIJob.ps1` preservato per uso manuale | ✅ |
|
||||||
|
| `runner/config.yaml` ha `PYTHONIOENCODING=utf-8` | ✅ |
|
||||||
|
| Coverage `commands/job.py` ≥80% | ✅ 83% |
|
||||||
|
| Cleanup VM garantito su tutti i path di errore | ✅ (test `test_job_cleanup_on_*`) |
|
||||||
|
| Hook Fase C (factory backend, no import workstation diretto) | ✅ |
|
||||||
|
| Workflow `build-ns7zip.yml` matrix PASS | ⏳ validazione hardware utente |
|
||||||
|
| Tempo entro ±10% baseline | ⏳ misurazione utente |
|
||||||
|
|
||||||
|
## Cosa resta a carico utente
|
||||||
|
|
||||||
|
1. Eseguire `build-ns7zip.yml` (matrix Win + Linux) sul runner host registrato e verificare che il job concluda con artifact pubblicato — il flusso deve passare dal nuovo entry point Python.
|
||||||
|
2. Lanciare `self-test.yml` e `lint.yml` per validare che gli altri workflow restino verdi dopo lo switch.
|
||||||
|
3. Eseguire `scripts/Measure-CIBenchmark.ps1` (resta in PS) prima/dopo lo switch e confermare delta tempo entro ±10%.
|
||||||
|
4. Aggiornare `Measure-CIBenchmark.ps1` se la wall-clock del nuovo entry point la rende necessaria; in caso di regressione >10%, aprire issue in `TODO.md` per profiling A5.
|
||||||
|
5. Coverage `commands/job.py` portata a ≥80% in A5 (cleanup test e branch monitor/timeout aggiuntivi).
|
||||||
|
|
||||||
|
## Riferimenti commit
|
||||||
|
|
||||||
|
- `feat(a4): port Invoke-CIJob.ps1 to ci_orchestrator job` — `commands/job.py`, `__main__.py`, shim `scripts/Invoke-CIJob.ps1`, `tests/python/test_commands_job.py`
|
||||||
|
- `ci(a4): switch local-ci-build action.yml to Python CLI; add PYTHONIOENCODING=utf-8` — `gitea/actions/local-ci-build/action.yml`, `runner/config.yaml`
|
||||||
|
- `docs(a4): mark A4 as code-complete; add A4-closeout.md` — `plans/implementation-plan-A-B.md`, `plans/A4-closeout.md`
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
# A5 — Closeout
|
||||||
|
|
||||||
|
Fase A5 di `plans/implementation-plan-A-B.md`: polish, regression test
|
||||||
|
per gli errori frequenti `AGENTS.md` #9-#12, documentazione Python
|
||||||
|
allineata, gate coverage alzato a 80%.
|
||||||
|
Branch: `feature/python-rewrite-phase-a`.
|
||||||
|
|
||||||
|
## Stato attuale
|
||||||
|
|
||||||
|
- [x] `tests/python/test_agents_errors.py` aggiunto: 12 test dedicati che
|
||||||
|
coprono regressioni per `AGENTS.md` "Errori frequenti da evitare"
|
||||||
|
#9, #10, #11, #12.
|
||||||
|
- **#9** (snapshot template con VM accesa): `clone_linked` propaga
|
||||||
|
"should not be powered on" come `BackendOperationFailed` (subclass
|
||||||
|
di `BackendError`) con `operation == "clone"` e `output` che
|
||||||
|
contiene il messaggio originale di `vmrun`.
|
||||||
|
- **#10** (`vmrun getGuestIPAddress` lento): `is_running` usa solo
|
||||||
|
`vmrun list`. Tre asserzioni — happy path, VM non listata, errore
|
||||||
|
di `vmrun list` — verificano che `getGuestIPAddress` non venga MAI
|
||||||
|
chiamato come fallback.
|
||||||
|
- **#11** (machine-id duplicato → DHCP collision): la safety net
|
||||||
|
orchestrator-side è che `vm new` non riusi mai un clone-name
|
||||||
|
(timestamp suffix); il test invoca `vm new` due volte con lo stesso
|
||||||
|
`--job-id` e verifica che `clone_linked` riceva due
|
||||||
|
`name`/`destination` distinti. Il fix template-side
|
||||||
|
(`truncate -s 0 /etc/machine-id`) resta documentato in
|
||||||
|
`template/Prepare-LinuxBuild2404.ps1`.
|
||||||
|
- **#12** (`& nativecmd 2>$null` non sopprime stderr in PS 5.1):
|
||||||
|
`SshTransport` usa `paramiko.AutoAddPolicy` di default con
|
||||||
|
`known_hosts=None`, non chiama mai `subprocess.run`/`Popen`
|
||||||
|
(incluso `ssh-keygen`/`ssh.exe`/`scp.exe`), e wrappa errori
|
||||||
|
paramiko (import mancante, `exec_command` exception, SFTP
|
||||||
|
`put`/`get`) come `TransportConnectError` invece di propagare
|
||||||
|
eccezioni nude.
|
||||||
|
- [x] `AGENTS.md` aggiornato: nuova sezione "Python development" subito
|
||||||
|
dopo "PowerShell 5.1 — Vincoli critici", con tabella parametri,
|
||||||
|
convenzioni (type hints, no `Any`/globali/`print`/`shell=True`),
|
||||||
|
mappatura completa PS → Python sub-command, elenco PS che restano
|
||||||
|
e gate coverage 80%.
|
||||||
|
- [x] `docs/ARCHITECTURE.md` esteso con sezione "Python orchestrator
|
||||||
|
(Phase A)": layout package, `VmBackend` Protocol, factory
|
||||||
|
`load_backend(config)`, ricetta Phase C per backend ESXi
|
||||||
|
(`backends/esxi.py` + selettore `[backend]` in `config.toml`),
|
||||||
|
transport selection, gate CI.
|
||||||
|
- [x] `README.md` ampliato: setup venv produzione + dev, esempi CLI per
|
||||||
|
tutti i 10 sub-comandi (`wait-ready`, `vm new/remove/cleanup`,
|
||||||
|
`build run`, `artifacts collect`, `monitor disk/runner`,
|
||||||
|
`report job`, `job`), comandi validazione locale con gate 80%,
|
||||||
|
link a `test_agents_errors.py` e `pyproject.toml`.
|
||||||
|
- [x] Suite completa: ruff clean, mypy --strict clean, **116 pytest
|
||||||
|
PASS**, coverage totale **80.10%** (gate 80% raggiunto).
|
||||||
|
- [ ] Burn-in 4 job concorrenti × 10 round (versione Python di
|
||||||
|
`Test-CapacityBurnIn`) — **richiede VM reali, lasciato a carico
|
||||||
|
utente**.
|
||||||
|
- [ ] Workflow `build-ns7zip.yml` matrix (Win + Linux) end-to-end
|
||||||
|
PASS — eredita pendenza da A4 (richiede runner registrato).
|
||||||
|
|
||||||
|
## Definizione di "fatto" A5
|
||||||
|
|
||||||
|
| Criterio | Stato |
|
||||||
|
| ----------------------------------------------------- | -------------------------------- |
|
||||||
|
| Errori #9–#12 coperti da test pytest | ✅ |
|
||||||
|
| `AGENTS.md` ha sezione "Python development" | ✅ |
|
||||||
|
| `docs/ARCHITECTURE.md` aggiornato (layout + hook C) | ✅ |
|
||||||
|
| `README.md` aggiornato (setup + esempi CLI) | ✅ |
|
||||||
|
| Coverage globale ≥80% (gate alzato in pyproject) | ✅ 80.10% |
|
||||||
|
| Shim PS non referenziati rimossi | ⏭️ nessuno candidato (vedi nota) |
|
||||||
|
| Burn-in 4×10 PASS | ⏳ utente — richiede VM reali |
|
||||||
|
|
||||||
|
**Nota shim PS**: tutti i 10 shim (`Wait-VMReady.ps1`, `New-BuildVM.ps1`,
|
||||||
|
`Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`,
|
||||||
|
`Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1`, `Watch-DiskSpace.ps1`,
|
||||||
|
`Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1`, `Invoke-CIJob.ps1`)
|
||||||
|
sono ancora referenziati: `Register-CIScheduledTasks.ps1` registra
|
||||||
|
scheduled task che invocano `Watch-DiskSpace.ps1`,
|
||||||
|
`Watch-RunnerHealth.ps1`, `Cleanup-OrphanedBuildVMs.ps1`; `README.md`
|
||||||
|
documenta `Invoke-CIJob.ps1` per uso manuale; `Test-*.ps1` di smoke
|
||||||
|
chiamano gli shim. Nessuno shim è candidato a rimozione in A5 senza
|
||||||
|
prima migrare anche `Register-CIScheduledTasks.ps1` (Fase B5 lo
|
||||||
|
sostituirà con `*.service` + `*.timer` su systemd).
|
||||||
|
|
||||||
|
## Cosa resta a carico utente
|
||||||
|
|
||||||
|
1. Eseguire un burn-in concorrente (4 job × 10 round) usando il workflow
|
||||||
|
`build-ns7zip.yml` matrix o `scripts/Test-CapacityBurnIn.ps1` (che
|
||||||
|
ora delega via shim alla CLI Python `job`) e verificare:
|
||||||
|
- PASS senza VM orfane (`vm cleanup` non rimuove nulla a fine round).
|
||||||
|
- Wall-clock entro ±20% del baseline pre-Python (`Measure-CIBenchmark.ps1`).
|
||||||
|
2. Validare `build-ns7zip.yml` matrix (Win + Linux) end-to-end PASS
|
||||||
|
sul runner reale — pendenza ereditata da A4.
|
||||||
|
3. Validare `self-test.yml` e `lint.yml` post-switch.
|
||||||
|
|
||||||
|
## Riferimenti commit
|
||||||
|
|
||||||
|
- `test(a5): add pytest regression tests for AGENTS.md errors #9-#12` —
|
||||||
|
`tests/python/test_agents_errors.py`
|
||||||
|
- `docs(a5): document Python orchestrator in AGENTS.md, ARCHITECTURE.md, README.md` —
|
||||||
|
`AGENTS.md`, `docs/ARCHITECTURE.md`, `README.md`
|
||||||
|
- `docs(a5): mark A5 as code-complete; add A5-closeout.md` —
|
||||||
|
`plans/implementation-plan-A-B.md`, `plans/A5-closeout.md`
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
# B5 — Closeout (parziale: artefatti repo)
|
||||||
|
|
||||||
|
Fase B5 di `plans/implementation-plan-A-B.md`: conversione scheduled task
|
||||||
|
Windows in coppie `*.service` + `*.timer` systemd su Linux.
|
||||||
|
|
||||||
|
## Cosa è stato fatto in questa sessione (su host Windows dev)
|
||||||
|
|
||||||
|
- [x] Inventario dei 4 task in `scripts/Register-CIScheduledTasks.ps1` (cadenze + comandi)
|
||||||
|
- [x] 5 coppie `*.service` + `*.timer` create in `deploy/systemd/`:
|
||||||
|
- `ci-cleanup-orphans.{service,timer}` — every 6h + at boot → `python -m ci_orchestrator vm cleanup --max-age-hours 6`
|
||||||
|
- `ci-retention-policy.{service,timer}` — daily 03:00 + 30min jitter → `pwsh Invoke-RetentionPolicy.ps1`
|
||||||
|
- `ci-watch-disk-space.{service,timer}` — every 15min → `python -m ci_orchestrator monitor disk`
|
||||||
|
- `ci-watch-runner-health.{service,timer}` — every 15min → `python -m ci_orchestrator monitor runner`
|
||||||
|
- `ci-backup-template.{service,timer}` — weekly Sun 02:00 + 1h jitter → `pwsh Backup-CITemplate.ps1` (NEW, non era in Register-CIScheduledTasks.ps1)
|
||||||
|
- [x] `deploy/systemd/README.md` con: tabella mapping, prerequisiti (utente, venv, env file), istruzioni install/test/rollback, nota PowerShell Core su Linux
|
||||||
|
|
||||||
|
## Note sulle cadenze
|
||||||
|
|
||||||
|
Le cadenze del piano (`OnCalendar=hourly`, `*:0/15`, ecc.) sono state
|
||||||
|
adattate per fedeltà semantica all'originale Windows:
|
||||||
|
|
||||||
|
| Task | Original Windows | systemd timer scelto |
|
||||||
|
| --------------------- | --------------------------------------------- | ----------------------------------------------------- |
|
||||||
|
| cleanup-orphans | every 6h + at startup | `OnBootSec=2min, OnUnitActiveSec=6h, Persistent` |
|
||||||
|
| retention-policy | daily 03:00 + 30min random delay | `OnCalendar=*-*-* 03:00:00, RandomizedDelaySec=30min` |
|
||||||
|
| watch-disk-space | every 15min | `OnBootSec=5min, OnUnitActiveSec=15min` |
|
||||||
|
| watch-runner-health | every 15min | `OnBootSec=3min, OnUnitActiveSec=15min` |
|
||||||
|
| backup-template | *(non in Register-CIScheduledTasks.ps1)* | `OnCalendar=Sun *-*-* 02:00:00, RandomizedDelaySec=1h`|
|
||||||
|
|
||||||
|
`Persistent=true` ovunque per non perdere esecuzioni se la macchina è
|
||||||
|
spenta al momento del trigger.
|
||||||
|
|
||||||
|
## Decisione — script che restano in PowerShell
|
||||||
|
|
||||||
|
`Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` non sono stati
|
||||||
|
portati a Python (resta scelta documentata in `AGENTS.md` "Mappatura").
|
||||||
|
Su Linux quindi le rispettive `.service` invocano `pwsh` (PowerShell
|
||||||
|
Core). Il README documenta l'installazione `apt install powershell` da
|
||||||
|
repo Microsoft. Se in futuro si deciderà di portarli a Python (sub-comandi
|
||||||
|
`retention run` / `template backup`), basterà sostituire `ExecStart=`
|
||||||
|
nelle due unit senza toccare il timer.
|
||||||
|
|
||||||
|
## Definizione di "fatto" B5 (dal piano)
|
||||||
|
|
||||||
|
| Criterio | Stato |
|
||||||
|
| --------------------------------------------------- | ------------------------------------ |
|
||||||
|
| Tutti i task periodici come timer systemd attivi | ⏳ unit pronti; install richiede host Linux |
|
||||||
|
| Trigger manuale di ognuno PASS | ⏳ richiede host Linux |
|
||||||
|
| Mapping documentato | ✅ `deploy/systemd/README.md` |
|
||||||
|
|
||||||
|
## Cosa resta a carico utente (sul nuovo host Linux)
|
||||||
|
|
||||||
|
Tutti i passi B1–B4 e B6–B8 sono operazioni hardware/host non eseguibili
|
||||||
|
da questa sessione. Per chiudere B5 una volta sull'host Linux:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Prerequisito: aver completato B1 (ci-runner user, /opt/ci/venv, /etc/ci/environment)
|
||||||
|
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \
|
||||||
|
ci-watch-runner-health ci-backup-template; do
|
||||||
|
sudo systemctl enable --now "${t}.timer"
|
||||||
|
done
|
||||||
|
systemctl list-timers --all 'ci-*'
|
||||||
|
|
||||||
|
# Trigger manuale di smoke
|
||||||
|
sudo systemctl start ci-cleanup-orphans.service
|
||||||
|
journalctl -u ci-cleanup-orphans.service -n 50 --no-pager
|
||||||
|
```
|
||||||
|
|
||||||
|
## Riferimenti
|
||||||
|
|
||||||
|
- Spec systemd timer: `man systemd.timer`, `man systemd.time`
|
||||||
|
- Original PS task scheduler: `scripts/Register-CIScheduledTasks.ps1`
|
||||||
|
- Mapping completo: `deploy/systemd/README.md`
|
||||||
|
- Piano: `plans/implementation-plan-A-B.md` sezione B5
|
||||||
@@ -0,0 +1,341 @@
|
|||||||
|
# Fase A — Checklist finale per l'utente
|
||||||
|
|
||||||
|
La Fase A è **code-complete e pushata** sul branch `feature/python-rewrite-and-linux-migration`.
|
||||||
|
Restano solo le validazioni che richiedono accesso fisico all'host CI con VMware
|
||||||
|
Workstation, snapshot template attivi e act_runner registrato.
|
||||||
|
|
||||||
|
Questo documento elenca i passi da eseguire in ordine, con checklist e comandi
|
||||||
|
copiabili. Stop a primo errore: ogni passo è un gate per il successivo.
|
||||||
|
|
||||||
|
> **Tempo stimato totale**: 1–2 ore (escluse build lunghe nel passo 5).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 1 — Aggiornare il venv di produzione sull'host CI
|
||||||
|
|
||||||
|
Il codice nuovo va installato nel venv che act_runner usa
|
||||||
|
(`F:\CI\python\venv\`).
|
||||||
|
|
||||||
|
- [x] Aprire PowerShell **come amministratore** sull'host CI.
|
||||||
|
- [x] Posizionarsi nella copia git aggiornata del repo (pull del branch
|
||||||
|
`feature/python-rewrite-and-linux-migration`):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd <path-locale-repo>
|
||||||
|
git fetch origin
|
||||||
|
git checkout feature/python-rewrite-and-linux-migration
|
||||||
|
git pull
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Installare il package nel venv di produzione (**non editable**):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m pip install .
|
||||||
|
```
|
||||||
|
|
||||||
|
> ⚠️ **Non usare `pip install -e .` su questo host.** act_runner gira
|
||||||
|
> come `LocalSystem`. Un install editable scrive un `.pth` verso la
|
||||||
|
> sorgente; se il `cd` era in una dir di lavoro di act_runner
|
||||||
|
> (`F:\CI\RunnerWork\<hash>\hostexecutor`, transitoria) o sul drive
|
||||||
|
> utente `N:` (non risolto da LocalSystem), il runner fallisce con
|
||||||
|
> `No module named ci_orchestrator`. L'install regolare copia il
|
||||||
|
> package in `site-packages` su `F:` (accessibile a LocalSystem).
|
||||||
|
> Conseguenza: ogni modifica al codice richiede di ri-eseguire
|
||||||
|
> `pip install .` — è uno step di deploy. Assicurarsi che il `cd` del
|
||||||
|
> passo precedente punti alla copia git reale del repo, non a una dir
|
||||||
|
> sotto `F:\CI\RunnerWork`.
|
||||||
|
|
||||||
|
- [x] Smoke check rapido che la CLI risponda:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: lista degli 11 sub-comandi (`wait-ready`, `vm`, `build`,
|
||||||
|
`artifacts`, `monitor`, `report`, `job`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 2 — Riavviare act_runner
|
||||||
|
|
||||||
|
L'act_runner gira come servizio SYSTEM e legge `runner/config.yaml`
|
||||||
|
all'avvio. La modifica di A4 (`PYTHONIOENCODING=utf-8`) ha effetto solo
|
||||||
|
dopo restart.
|
||||||
|
|
||||||
|
- [x] Trovare il servizio:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-Service | Where-Object { $_.Name -like '*act*runner*' }
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Restart (sostituisci `<NomeServizio>` con quello trovato):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Restart-Service -Name '<NomeServizio>'
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Verificare che sia partito:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-Service -Name '<NomeServizio>'
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: `Status = Running`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 3 — Smoke `wait-ready` su VM Windows reale
|
||||||
|
|
||||||
|
PoC pendente da A1.
|
||||||
|
|
||||||
|
- [x] Clonare manualmente un template Windows:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||||
|
-T ws clone `
|
||||||
|
'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
||||||
|
'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' `
|
||||||
|
linked -snapshot=BaseClean -cloneName=smoke-win
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Avviare la VM:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||||
|
-T ws start 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' nogui
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Eseguire `wait-ready`:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready `
|
||||||
|
--vmx 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' `
|
||||||
|
--guest-os windows --timeout 180
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: exit code `0` entro 3 minuti.
|
||||||
|
|
||||||
|
- [x] Cleanup VM di smoke:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm remove `
|
||||||
|
--vmx 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' --force
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 4 — Smoke `wait-ready` su VM Linux reale
|
||||||
|
|
||||||
|
Stesso schema, template Linux. PoC pendente da A1.
|
||||||
|
|
||||||
|
- [x] Clonare:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||||
|
-T ws clone `
|
||||||
|
'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
||||||
|
'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' `
|
||||||
|
linked -snapshot=BaseClean-Linux -cloneName=smoke-linux
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Start + wait-ready:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' `
|
||||||
|
-T ws start 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' nogui
|
||||||
|
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready `
|
||||||
|
--vmx 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' `
|
||||||
|
--guest-os linux --timeout 180
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: exit code `0`.
|
||||||
|
|
||||||
|
> **Prerequisito scoperto**: `F:\CI\config.toml` deve esistere con `ssh_key_path`
|
||||||
|
> impostato — altrimenti SSH fallisce con `No authentication methods available`.
|
||||||
|
> Creato durante questa validazione; vedi HOST-SETUP.md step 6.
|
||||||
|
|
||||||
|
- [x] Cleanup:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm remove `
|
||||||
|
--vmx 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' --force
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 5 — Smoke end-to-end pipeline (`job`)
|
||||||
|
|
||||||
|
Valida A3+A4 insieme: clone → wait → build → collect → cleanup.
|
||||||
|
|
||||||
|
- [x] Lanciare un job sul template Windows con uno script trivial:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator job `
|
||||||
|
--template-path 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
|
||||||
|
--snapshot-name BaseClean `
|
||||||
|
--job-id smoke-job-win `
|
||||||
|
--repo-url 'http://10.10.20.11:3100/Simone/your-repo.git' `
|
||||||
|
--branch main `
|
||||||
|
--build-command 'echo hello > artifact.txt' `
|
||||||
|
--guest-artifact-source 'artifact.txt' `
|
||||||
|
--artifact-base-dir 'F:\CI\Artifacts\smoke-job-win'
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: exit code `0`, file `F:\CI\Artifacts\smoke-job-win\artifact.txt` presente,
|
||||||
|
VM smoke-job-win **non** più esistente in `F:\CI\BuildVMs\` (cleanup garantito).
|
||||||
|
|
||||||
|
- [x] Ripetere su Linux:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator job `
|
||||||
|
--template-path 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' `
|
||||||
|
--snapshot-name BaseClean-Linux `
|
||||||
|
--guest-os linux `
|
||||||
|
--job-id smoke-job-linux `
|
||||||
|
--repo-url 'http://10.10.20.11:3100/Simone/your-repo.git' `
|
||||||
|
--branch main `
|
||||||
|
--build-command 'echo hello > artifact.txt' `
|
||||||
|
--guest-artifact-source 'artifact.txt' `
|
||||||
|
--artifact-base-dir 'F:\CI\Artifacts\smoke-job-linux'
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: exit code `0`, `F:\CI\Artifacts\smoke-job-linux\smoke-job-linux\artifact.txt` presente.
|
||||||
|
|
||||||
|
- [x] Verificare assenza VM orfane:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm cleanup --what-if
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: nessun candidato.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 6 — Workflow Gitea end-to-end
|
||||||
|
|
||||||
|
Validazione di A4 lato runner reale.
|
||||||
|
|
||||||
|
- [x] Aprire la UI Gitea, repo `local-ci-cd-system`, branch
|
||||||
|
`feature/python-rewrite-and-linux-migration`.
|
||||||
|
- [x] Triggerare manualmente i 3 workflow (Actions → Run workflow):
|
||||||
|
- [x] `lint.yml` → atteso PASS
|
||||||
|
- [x] `self-test.yml` → PASS — validati **entrambi i transport**
|
||||||
|
(in-guest clone default + host-side clone+zip) su Win e Linux:
|
||||||
|
|
||||||
|
| Job | Transport | Esito |
|
||||||
|
| ---------------------------- | ----------------------- | ----------- |
|
||||||
|
| smoke-windows | in-guest git clone | SUCCESS 26s |
|
||||||
|
| smoke-linux | in-guest git clone | SUCCESS 51s |
|
||||||
|
| smoke-windows-hostclone | host-side clone + zip | SUCCESS 28s |
|
||||||
|
| smoke-linux-hostclone | host-side clone + zip | SUCCESS 46s |
|
||||||
|
|
||||||
|
Note: artifact github-free (no upload-artifact; file raw su
|
||||||
|
`F:\CI\Artifacts\<job>`); `use-git-clone`/`submodules` default ON.
|
||||||
|
- [x] `build-ns7zip.yml` matrix Win+Linux → PASS (target repo
|
||||||
|
`Simone/nsis-plugin-ns7zip`, in-guest clone, `--dist`):
|
||||||
|
|
||||||
|
| Job | Esito |
|
||||||
|
| ------------------ | --------------------------------------- |
|
||||||
|
| build (windows) | SUCCESS — DLL in dist/, artifact host |
|
||||||
|
| build (linux) | SUCCESS — 8 vCPU/8 GB, wall ~121s |
|
||||||
|
| release (tag push) | SUCCESS — release Gitea + 2 asset upload |
|
||||||
|
|
||||||
|
Note: `release` validato con tag throwaway `v0.0.0-phaseA-test`
|
||||||
|
(poi rimosso); usa il token Actions automatico (Default Token
|
||||||
|
Permissions = Permissive) + REST API Gitea, legge gli artifact
|
||||||
|
host-side (no download-artifact). Linux perf: vCPU-starvation a 4
|
||||||
|
vCPU risolta con guest-cpu=8 (≈ Windows VM ~120s); gap col Windows
|
||||||
|
nativo MSVC (~30s) è inerente MinGW, fuori scope CI.
|
||||||
|
|
||||||
|
> Tutti i workflow PASS — nessun fallimento da triagare. (Procedura
|
||||||
|
> storica: in caso di fallimento, scaricare i log del run e aprire una
|
||||||
|
> voce in `TODO.md` con sha + output errore.)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 7 — Burn-in capacità (4 job concorrenti)
|
||||||
|
|
||||||
|
Validazione finale di A5 (definizione di "fatto Fase A").
|
||||||
|
|
||||||
|
- [x] Burn-in via `scripts/Start-BurnInTest.ps1` (wrapper lab che
|
||||||
|
hardcoda repo `burnin-dummy` + `BuildCommand` = `build.ps1`;
|
||||||
|
`Test-CapacityBurnIn.ps1` diretto senza `-BuildCommand` usa il
|
||||||
|
default dotnet → fallisce su repo non-.NET):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\scripts\Start-BurnInTest.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
- [x] Esito (4 job concorrenti × 3 round):
|
||||||
|
|
||||||
|
| Round | Pass | Fail | Elapsed | Status |
|
||||||
|
| ----- | ---- | ---- | ------- | ------ |
|
||||||
|
| 1 | 4 | 0 | 76s | PASS |
|
||||||
|
| 2 | 4 | 0 | 88s | PASS |
|
||||||
|
| 3 | 4 | 0 | 73s | PASS |
|
||||||
|
|
||||||
|
Total 12/12 PASS — **OVERALL: PASS**. Cleanup VM + lock OK ogni
|
||||||
|
round (0 orfani). A5 / Passo 7 validato.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 8 — Benchmark wall-clock
|
||||||
|
|
||||||
|
Confronto Python vs PowerShell. Pendenza A4.
|
||||||
|
|
||||||
|
- [x] Eseguito `.\scripts\Measure-CIBenchmark.ps1` (4 iter):
|
||||||
|
|
||||||
|
| Iter | Clone | Start | IP | WinRM | Destroy | Boot tot |
|
||||||
|
| ---- | ----- | ----- | ----- | ----- | ------- | -------- |
|
||||||
|
| 1 | 0.63 | 1.75 | 66.57 | 0.01 | 4.81 | 68.96 |
|
||||||
|
| 2 | 0.63 | 1.89 | 20.21 | 0.01 | 6.39 | 22.74 |
|
||||||
|
| 3 | 0.62 | 1.72 | 85.07 | 0.01 | 4.50 | 87.42 |
|
||||||
|
| 4 | 0.61 | 1.72 | 60.97 | 0.01 | 4.20 | 63.31 |
|
||||||
|
|
||||||
|
Nessun errore, destroy OK (~4-6s), 0 orfani. Clone/start/WinRM
|
||||||
|
trascurabili; **fase IP (20-85s) = costo dominante e variabile**
|
||||||
|
(detect IP guest via VMware Tools / ci-report-ip) — noto, non
|
||||||
|
bloccante. Prerequisito scoperto: lo shim `Remove-BuildVM.ps1`
|
||||||
|
inoltrava `-ErrorAction` come `--error-action` (destroy falliva ->
|
||||||
|
orfana); risolto per tutti gli shim blind-loop (commit
|
||||||
|
`fix(shims): skip PowerShell common params`).
|
||||||
|
|
||||||
|
- [x] Confronto col baseline pre-Python — **SALTATO** (deciso): nessun
|
||||||
|
baseline PS affidabile registrato; metriche infra appese a
|
||||||
|
`F:\CI\Logs\benchmark.jsonl` per trend futuri. Non bloccante.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 9 — Merge della Fase A in `main`
|
||||||
|
|
||||||
|
Solo dopo che i passi 3–7 sono tutti `[x]` PASS.
|
||||||
|
|
||||||
|
- [ ] Aprire PR su Gitea: `feature/python-rewrite-and-linux-migration` → `main`.
|
||||||
|
- [ ] Verificare diff atteso: ~20 file Python nuovi sotto
|
||||||
|
`src/ci_orchestrator/`, ~10 shim PS in `scripts/`, doc in
|
||||||
|
`plans/A*-closeout.md`, AGENTS.md/ARCHITECTURE.md/README.md
|
||||||
|
aggiornati.
|
||||||
|
- [ ] Self-review della PR: nessun secret, nessun path hardcodato fuori
|
||||||
|
da `config.py`, nessun `.venv/` committato.
|
||||||
|
- [ ] Merge (squash sconsigliato — preserva la storia per-fase).
|
||||||
|
- [ ] Tag release: `git tag v2.0.0-phaseA` (o convenzione preferita).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tracciamento globale
|
||||||
|
|
||||||
|
| Passo | Descrizione | Stato |
|
||||||
|
| ----- | ----------------------------------------- | ----- |
|
||||||
|
| 1 | Aggiorna venv di produzione | [x] |
|
||||||
|
| 2 | Restart act_runner | [x] |
|
||||||
|
| 3 | Smoke `wait-ready` Windows | [x] |
|
||||||
|
| 4 | Smoke `wait-ready` Linux | [x] |
|
||||||
|
| 5 | Smoke `job` end-to-end (Win + Linux) | [x] |
|
||||||
|
| 6 | Workflow Gitea (lint + self-test + build) | [ ] |
|
||||||
|
| 7 | Burn-in 4 job × 10 round | [ ] |
|
||||||
|
| 8 | Benchmark wall-clock | [ ] |
|
||||||
|
| 9 | Merge PR e tag release | [ ] |
|
||||||
|
|
||||||
|
Quando tutti i passi sono `[x]`, la Fase A è **definitivamente chiusa** e
|
||||||
|
si può aprire la Fase B (porting su host Linux).
|
||||||
@@ -0,0 +1,211 @@
|
|||||||
|
# Fase A → Fase B — Bridge checklist per l'utente
|
||||||
|
|
||||||
|
Questa checklist copre i passi **intermedi** tra "Fase A done" e "B1
|
||||||
|
inizia". Sono attività che girano sull'host **Windows attuale** (o sono
|
||||||
|
logistiche fuori dal repo) e non rientrano né in
|
||||||
|
[PhaseA-user-checklist.md](PhaseA-user-checklist.md) (validazione del
|
||||||
|
codice Python) né in [PhaseB-user-checklist.md](PhaseB-user-checklist.md)
|
||||||
|
(setup del nuovo host Linux).
|
||||||
|
|
||||||
|
> **Quando eseguirla**: dopo Passo 9 di Fase A (merge + tag), prima di
|
||||||
|
> Passo 1 di Fase B (setup host Linux Mint).
|
||||||
|
|
||||||
|
> **Tempo stimato**: 1–2 ore (escluso `tar` di backup, che dipende
|
||||||
|
> dalla dimensione di `F:\CI\`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Sezione 1 — Salva il baseline pre-migrazione
|
||||||
|
|
||||||
|
Senza un baseline registrato, il confronto richiesto da B7 ("tempo
|
||||||
|
medio entro ±20% baseline Windows") è impossibile.
|
||||||
|
|
||||||
|
- [ ] Eseguire benchmark sull'host Windows (post Fase A, runner Python attivo):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd <path-locale-repo>
|
||||||
|
.\scripts\Measure-CIBenchmark.ps1 | Tee-Object -FilePath "baseline-windows-$(Get-Date -Format 'yyyyMMdd').txt"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Annotare in `docs/RUNBOOK.md` (sezione nuova "Windows host baseline"):
|
||||||
|
- Data
|
||||||
|
- Tempo medio per job (Windows e Linux template)
|
||||||
|
- Success rate
|
||||||
|
- Versione `act_runner` e `ci_orchestrator` (output di `python -m ci_orchestrator --version` se disponibile, altrimenti SHA del commit)
|
||||||
|
- Hardware: i9-10900X, 64 GB RAM, NVMe SSD (da `AGENTS.md`)
|
||||||
|
|
||||||
|
- [ ] Committare:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
git add docs/RUNBOOK.md
|
||||||
|
git commit -m "docs(runbook): record Windows host pre-migration baseline"
|
||||||
|
git push
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Sezione 2 — Pre-requisiti Windows per Fase B
|
||||||
|
|
||||||
|
Tutti questi vanno fatti **prima** di iniziare B1, perché B2/B3/B6 li
|
||||||
|
presuppongono già pronti.
|
||||||
|
|
||||||
|
### 2.1 Inventario chiavi SSH
|
||||||
|
|
||||||
|
- [ ] Verificare presenza chiavi:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-ChildItem F:\CI\keys\ci_linux*
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: `ci_linux` (privata, perm utente owner) e `ci_linux.pub`. Se
|
||||||
|
mancano, rigenerarle con `ssh-keygen -t ed25519 -f F:\CI\keys\ci_linux -N ""`
|
||||||
|
e re-installare `ci_linux.pub` nel template `LinuxBuild2404` su
|
||||||
|
`~ci_build/.ssh/authorized_keys`.
|
||||||
|
|
||||||
|
### 2.2 Recupero password guest Windows (`BuildVMGuest`)
|
||||||
|
|
||||||
|
La password salvata in Windows Credential Manager **non è
|
||||||
|
esportabile**. Devi conoscerla per re-inserirla con `secret-tool` su
|
||||||
|
Linux (Passo 3 di Fase B).
|
||||||
|
|
||||||
|
- [ ] Verificare di averla nel password manager personale.
|
||||||
|
- [ ] Se non la ricordi: bootare il template `WinBuild2025` (e `WinBuild2022`),
|
||||||
|
resettare la password dell'utente CI (`net user <user> <newpass>`),
|
||||||
|
ricatturare snapshot `BaseClean` da stato powered-off (vedi
|
||||||
|
`AGENTS.md` errore #9), aggiornare il Credential Manager Windows con
|
||||||
|
la nuova password, e annotarla in modo sicuro.
|
||||||
|
|
||||||
|
### 2.3 Generazione Gitea PAT per il nuovo runner
|
||||||
|
|
||||||
|
- [ ] Aprire Gitea → User Settings → Applications → Generate New Token.
|
||||||
|
- [ ] Nome: `ci-runner-linux`. Scope: `read:repository` + `write:package`
|
||||||
|
(o gli stessi del runner Windows attuale).
|
||||||
|
- [ ] Annotare il token in modo sicuro (verrà inserito al Passo 3 di
|
||||||
|
Fase B con `secret-tool store ... target GiteaPAT`).
|
||||||
|
|
||||||
|
### 2.4 OpenSSH Server su Windows (per rsync pull dal Linux)
|
||||||
|
|
||||||
|
Necessario solo se al Passo 2 di Fase B (transfer template) preferisci
|
||||||
|
fare `rsync pull` dal Linux. Alternativa: `rsync push` da Windows con
|
||||||
|
WSL/Cygwin/MSYS2.
|
||||||
|
|
||||||
|
- [ ] Installare OpenSSH Server:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
|
||||||
|
Start-Service sshd
|
||||||
|
Set-Service sshd -StartupType Automatic
|
||||||
|
New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server' -Enabled True `
|
||||||
|
-Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Validare connessione da un altro host: `ssh user@<windows-ip> "whoami"`.
|
||||||
|
- [ ] (Opzionale) Configurare `authorized_keys` per il futuro user
|
||||||
|
`ci-runner` del Linux box.
|
||||||
|
|
||||||
|
### 2.5 Template fully powered-off (gate per B2)
|
||||||
|
|
||||||
|
> AGENTS.md errore #9: snapshot catturato con VM accesa produce
|
||||||
|
> `*.vmem` / `*.vmsn` che rompono il clone su host diverso.
|
||||||
|
|
||||||
|
- [ ] Verificare che nessun template sia running:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' list
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: nessun path sotto `F:\CI\Templates\` nell'output.
|
||||||
|
|
||||||
|
- [ ] Verificare assenza file di runtime:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-ChildItem F:\CI\Templates -Recurse -Include *.vmem,*.vmsn,*.lck
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: nessun risultato. Se ci sono `.vmem`/`.vmsn`, lo snapshot
|
||||||
|
non è clean → ricatturare `BaseClean` / `BaseClean-Linux` da stato
|
||||||
|
fully powered-off.
|
||||||
|
|
||||||
|
### 2.6 Backup precauzionale di `F:\CI\`
|
||||||
|
|
||||||
|
Da fare **prima** del cutover B6, ma anche prima del transfer B2 è una
|
||||||
|
buona pratica.
|
||||||
|
|
||||||
|
- [ ] Eseguire backup completo:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$stamp = Get-Date -Format 'yyyyMMdd'
|
||||||
|
$archive = "E:\Backup\CI-pre-migration-$stamp.tar"
|
||||||
|
& 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\'
|
||||||
|
Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Verifica integrità (lista contenuto, primi 20 record):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
& 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Annotare path archivio + checksum SHA256.
|
||||||
|
|
||||||
|
### 2.7 Annotare cadenze scheduled task (se custom)
|
||||||
|
|
||||||
|
- [ ] Esportare l'inventario corrente:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-ScheduledTask -TaskName 'CI-*' |
|
||||||
|
Select-Object TaskName,
|
||||||
|
@{n='Trigger';e={ $_.Triggers | ForEach-Object { $_.PSObject.Properties | Where-Object Name -in 'StartBoundary','RepetitionInterval','DaysOfWeek' | ForEach-Object { "$($_.Name)=$($_.Value)" } } -join '; ' }},
|
||||||
|
@{n='Action'; e={ "$($_.Actions.Execute) $($_.Actions.Arguments)" }} |
|
||||||
|
Format-List | Tee-Object -FilePath "scheduled-tasks-inventory-$(Get-Date -Format 'yyyyMMdd').txt"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Confrontare con i `.timer` in `deploy/systemd/`:
|
||||||
|
|
||||||
|
| Task Windows | systemd timer | Cadenza prevista |
|
||||||
|
| ----------------------------- | ------------------------------------- | ---------------------- |
|
||||||
|
| `CI-CleanupOrphanedBuildVMs` | `ci-cleanup-orphans.timer` | ogni 6h + boot |
|
||||||
|
| `CI-Invoke-RetentionPolicy` | `ci-retention-policy.timer` | giornaliero 03:00 |
|
||||||
|
| `CI-Watch-DiskSpace` | `ci-watch-disk-space.timer` | ogni 15min |
|
||||||
|
| `CI-Watch-RunnerHealth` | `ci-watch-runner-health.timer` | ogni 15min |
|
||||||
|
| (nuovo) | `ci-backup-template.timer` | settimanale Dom 02:00 |
|
||||||
|
|
||||||
|
- [ ] Se le cadenze Windows sono diverse, aggiornare gli `OnCalendar` /
|
||||||
|
`OnUnitActiveSec` in `deploy/systemd/*.timer` **prima** del Passo
|
||||||
|
5 di Fase B. Committare la modifica.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Sezione 3 — Logistica e finestra di manutenzione
|
||||||
|
|
||||||
|
Questi non sono comandi, ma checkpoint prima di iniziare B1 e B6.
|
||||||
|
|
||||||
|
- [ ] **Hardware Linux disponibile**: il box target è acquisito,
|
||||||
|
racked, raggiungibile in rete dall'host Windows e da Gitea.
|
||||||
|
- [ ] **DNS / IP statico** assegnato al nuovo host Linux (per
|
||||||
|
registrazione runner + raggiungibilità da admin).
|
||||||
|
- [ ] **Finestra di manutenzione per cutover B6 concordata**: 30–60 min
|
||||||
|
di stop Gitea Actions, in un orario a basso traffico.
|
||||||
|
- [ ] **Comunicazione utenti del repo**: avviso che durante la
|
||||||
|
finestra B6 i workflow saranno temporaneamente fermi.
|
||||||
|
- [ ] **Piano di rollback compreso**: rileggere la sezione "Rollback"
|
||||||
|
del Passo 6 di [PhaseB-user-checklist.md](PhaseB-user-checklist.md).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tracciamento globale
|
||||||
|
|
||||||
|
| Sezione | Descrizione | Stato |
|
||||||
|
| ------- | ---------------------------------------------- | ----- |
|
||||||
|
| 1 | Baseline benchmark salvato in `docs/RUNBOOK.md` | [ ] |
|
||||||
|
| 2.1 | Chiavi SSH presenti | [ ] |
|
||||||
|
| 2.2 | Password `BuildVMGuest` recuperata | [ ] |
|
||||||
|
| 2.3 | Gitea PAT generato per `ci-runner-linux` | [ ] |
|
||||||
|
| 2.4 | OpenSSH Server attivo (o alt. rsync) | [ ] |
|
||||||
|
| 2.5 | Template fully powered-off, no `.vmem` | [ ] |
|
||||||
|
| 2.6 | Backup `F:\CI\` con SHA256 | [ ] |
|
||||||
|
| 2.7 | Cadenze scheduled task confrontate con timer | [ ] |
|
||||||
|
| 3 | Hardware + DNS + finestra + comunicazione OK | [ ] |
|
||||||
|
|
||||||
|
Quando tutte le righe sono `[x]`, puoi iniziare il Passo 1 di
|
||||||
|
[PhaseB-user-checklist.md](PhaseB-user-checklist.md).
|
||||||
@@ -0,0 +1,484 @@
|
|||||||
|
# Fase B — Checklist finale per l'utente
|
||||||
|
|
||||||
|
La Fase B è la migrazione del runner CI dall'host Windows attuale a un
|
||||||
|
nuovo host **Linux Mint LTS** con VMware Workstation Pro Linux.
|
||||||
|
|
||||||
|
> **Stato repo**: solo lo step **B5** (unit `systemd` per i task
|
||||||
|
> periodici) è già committato in `deploy/systemd/` sul branch
|
||||||
|
> `feature/python-rewrite-and-linux-migration` (commit `50d37b5`).
|
||||||
|
> Tutti gli altri step (B1–B4, B6–B8) sono **operazioni hardware**
|
||||||
|
> sull'host Linux nuovo e devono essere eseguiti dall'utente.
|
||||||
|
|
||||||
|
> **Pre-requisito**: la Fase A deve essere `done` (vedi
|
||||||
|
> `plans/PhaseA-user-checklist.md` Passo 9, merge + tag).
|
||||||
|
> B1, B2, B3 possono partire in parallelo a A3/A4/A5; B4–B8 sono
|
||||||
|
> sequenziali e iniziano solo dopo "A done".
|
||||||
|
|
||||||
|
> **Tempo stimato totale**: 1–2 giorni di lavoro distribuiti su una
|
||||||
|
> finestra di 1–2 settimane (per consentire il burn-in tra B6 e B8).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 1 — Setup host Linux Mint (B1)
|
||||||
|
|
||||||
|
**Obiettivo**: hardware target con Linux Mint LTS, VMware Workstation
|
||||||
|
Pro Linux, layout storage e venv Python pronti.
|
||||||
|
|
||||||
|
- [ ] Installare Linux Mint LTS sull'hardware target.
|
||||||
|
- [ ] `sudo apt update && sudo apt full-upgrade -y && sudo reboot`
|
||||||
|
- [ ] Scaricare e installare VMware Workstation Pro Linux (bundle
|
||||||
|
`.bundle` da Broadcow):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo bash VMware-Workstation-Full-*.bundle
|
||||||
|
vmrun -T ws --version # deve ritornare versione coerente
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Smoke test Workstation (UI): creare VM trivia, clone, start,
|
||||||
|
stop, delete.
|
||||||
|
- [ ] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato a
|
||||||
|
Windows) editando `/etc/vmware/vmnet8/dhcpd/dhcpd.conf` o via
|
||||||
|
`vmware-netcfg`.
|
||||||
|
- [ ] Creare utente di servizio `ci-runner`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo useradd -r -m -s /bin/bash ci-runner
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Creare layout storage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /var/lib/ci/{build-vms,artifacts,templates,keys}
|
||||||
|
sudo chown -R ci-runner:ci-runner /var/lib/ci
|
||||||
|
sudo chmod 750 /var/lib/ci /var/lib/ci/*
|
||||||
|
sudo setfacl -d -m u:ci-runner:rwX /var/lib/ci/build-vms
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Installare Python 3.11+ e creare venv produzione:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo apt install -y python3.11 python3.11-venv git
|
||||||
|
sudo mkdir -p /opt/ci
|
||||||
|
sudo python3.11 -m venv /opt/ci/venv
|
||||||
|
sudo chown -R ci-runner:ci-runner /opt/ci
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Clonare il repo e installare il package nel venv:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner git clone https://gitea.emulab.it/Simone/local-ci-cd-system.git /opt/ci/local-ci-cd-system
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/pip install -e /opt/ci/local-ci-cd-system
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: lista degli 11 sub-comandi.
|
||||||
|
|
||||||
|
- [ ] Installare PowerShell Core (richiesto da `Invoke-RetentionPolicy.ps1`
|
||||||
|
e `Backup-CITemplate.ps1`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo apt install -y wget apt-transport-https software-properties-common
|
||||||
|
source /etc/os-release
|
||||||
|
wget -q https://packages.microsoft.com/config/ubuntu/${VERSION_ID}/packages-microsoft-prod.deb
|
||||||
|
sudo dpkg -i packages-microsoft-prod.deb
|
||||||
|
sudo apt update && sudo apt install -y powershell
|
||||||
|
pwsh --version
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 2 — Trasferimento template VM (B2)
|
||||||
|
|
||||||
|
**Obiettivo**: copiare i template VMware da `F:\CI\Templates\` su
|
||||||
|
`/var/lib/ci/templates/` mantenendo gli snapshot.
|
||||||
|
|
||||||
|
> ⚠️ AGENTS.md errore #9: i template devono essere **fully
|
||||||
|
> powered-off** prima della copia. Verificare assenza di `*.vmem` /
|
||||||
|
> `*.vmsn` di runtime.
|
||||||
|
|
||||||
|
- [ ] Sull'host Windows:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-ChildItem 'F:\CI\Templates' -Recurse -Include *.vmem,*.vmsn
|
||||||
|
# atteso: nessun risultato
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Sull'host Linux, eseguire rsync via SSH dall'host Windows
|
||||||
|
(richiede OpenSSH Server attivo su Windows o, in alternativa,
|
||||||
|
`scp -r` lanciato da Windows verso Linux):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner rsync -av --progress \
|
||||||
|
user@windows-host:/cygdrive/f/CI/Templates/ \
|
||||||
|
/var/lib/ci/templates/
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Validare integrità snapshot:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
find /var/lib/ci/templates -name '*.vmx' -exec vmrun -T ws listSnapshots {} \;
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: `BaseClean` su `WinBuild2025.vmx` e `WinBuild2022.vmx`,
|
||||||
|
`BaseClean-Linux` su `LinuxBuild2404.vmx`.
|
||||||
|
|
||||||
|
- [ ] Aprire ciascun `.vmx` su Workstation Linux per registrarli;
|
||||||
|
al prompt rispondere "**I copied it**".
|
||||||
|
- [ ] Smoke test pipeline VM:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm new \
|
||||||
|
--template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \
|
||||||
|
--snapshot BaseClean --name smoke-b2-win
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator wait-ready \
|
||||||
|
--vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx \
|
||||||
|
--guest-os windows --timeout 180
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm remove \
|
||||||
|
--vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx --force
|
||||||
|
```
|
||||||
|
|
||||||
|
Ripetere per `LinuxBuild2404` (snapshot `BaseClean-Linux`,
|
||||||
|
`--guest-os linux`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 3 — Trasferimento credenziali e chiavi (B3)
|
||||||
|
|
||||||
|
**Obiettivo**: chiavi SSH e credenziali guest disponibili al user
|
||||||
|
`ci-runner` headless.
|
||||||
|
|
||||||
|
- [ ] Copiare le chiavi SSH guest Linux:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /etc/ci/keys
|
||||||
|
sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux /etc/ci/keys/
|
||||||
|
sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux.pub /etc/ci/keys/
|
||||||
|
sudo chown ci-runner:ci-runner /etc/ci/keys/*
|
||||||
|
sudo chmod 600 /etc/ci/keys/ci_linux
|
||||||
|
sudo chmod 644 /etc/ci/keys/ci_linux.pub
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Re-store credenziale guest Windows (sostituire la password al
|
||||||
|
prompt — **NON** inserirla in chat o issue):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner secret-tool store --label='BuildVMGuest' service ci target BuildVMGuest
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Re-store Gitea PAT:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner secret-tool store --label='GiteaPAT' service ci target GiteaPAT
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Validare lettura interattiva:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -c \
|
||||||
|
"import keyring; print(keyring.get_credential('BuildVMGuest', None))"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **PoC headless** (critico): verificare lettura keyring da
|
||||||
|
contesto systemd (no D-Bus user session):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemd-run --uid=ci-runner --pipe \
|
||||||
|
/opt/ci/venv/bin/python -c \
|
||||||
|
"import keyring; print(keyring.get_credential('BuildVMGuest', None))"
|
||||||
|
```
|
||||||
|
|
||||||
|
Se fallisce → implementare backend keyring file-based con `age` o
|
||||||
|
`sops` come fallback. Documentare la scelta finale in
|
||||||
|
`docs/HOST-SETUP.md` prima di proseguire.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 4 — Setup act_runner come systemd service (B4)
|
||||||
|
|
||||||
|
**Obiettivo**: act_runner Linux registrato verso Gitea, gestito da
|
||||||
|
systemd.
|
||||||
|
|
||||||
|
- [ ] Scaricare il binario act_runner Linux ≥ v1.0.2:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /opt/ci/act_runner
|
||||||
|
sudo wget -O /opt/ci/act_runner/act_runner \
|
||||||
|
https://gitea.com/gitea/act_runner/releases/download/v1.0.2/act_runner-1.0.2-linux-amd64
|
||||||
|
sudo chmod +x /opt/ci/act_runner/act_runner
|
||||||
|
sudo chown -R ci-runner:ci-runner /opt/ci/act_runner
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Generare token registrazione su Gitea (Admin → Runners → Create new runner).
|
||||||
|
- [ ] Registrare il runner (sostituire `<TOKEN>`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner mkdir -p /var/lib/ci/runner
|
||||||
|
cd /var/lib/ci/runner
|
||||||
|
sudo -u ci-runner /opt/ci/act_runner/act_runner register \
|
||||||
|
--no-interactive \
|
||||||
|
--instance https://gitea.emulab.it \
|
||||||
|
--token <TOKEN> \
|
||||||
|
--name ci-linux \
|
||||||
|
--labels windows-build:host,linux-build:host
|
||||||
|
```
|
||||||
|
|
||||||
|
> ⚠️ Lasciare il runner in stato **paused** lato Gitea UI fino al
|
||||||
|
> cutover (Passo 6), per non intercettare job di produzione.
|
||||||
|
|
||||||
|
- [ ] Creare `/etc/systemd/system/act-runner.service`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
[Unit]
|
||||||
|
Description=Gitea Act Runner
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=ci-runner
|
||||||
|
WorkingDirectory=/var/lib/ci/runner
|
||||||
|
Environment="PYTHONIOENCODING=utf-8"
|
||||||
|
Environment="CI_ROOT=/var/lib/ci"
|
||||||
|
Environment="CI_TEMPLATES=/var/lib/ci/templates"
|
||||||
|
Environment="CI_BUILD_VMS=/var/lib/ci/build-vms"
|
||||||
|
Environment="CI_ARTIFACTS=/var/lib/ci/artifacts"
|
||||||
|
Environment="CI_KEYS=/etc/ci/keys"
|
||||||
|
ExecStart=/opt/ci/act_runner/act_runner daemon
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=10
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Creare `/etc/ci/environment` (letto dalle unit dei task periodici
|
||||||
|
di B5; vedi `deploy/systemd/README.md`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo tee /etc/ci/environment <<'EOF'
|
||||||
|
PYTHONIOENCODING=utf-8
|
||||||
|
CI_ROOT=/var/lib/ci
|
||||||
|
CI_TEMPLATES=/var/lib/ci/templates
|
||||||
|
CI_BUILD_VMS=/var/lib/ci/build-vms
|
||||||
|
CI_ARTIFACTS=/var/lib/ci/artifacts
|
||||||
|
CI_KEYS=/etc/ci/keys
|
||||||
|
EOF
|
||||||
|
sudo chmod 644 /etc/ci/environment
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Abilitare e avviare:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now act-runner.service
|
||||||
|
sudo systemctl status act-runner
|
||||||
|
sudo journalctl -u act-runner -f # Ctrl+C dopo conferma OK
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Smoke job: triggerare manualmente `self-test.yml` selezionando
|
||||||
|
il runner Linux (mettere temporaneamente in pausa quello Windows
|
||||||
|
o usare label distinte). Atteso: PASS.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 5 — Installare i timer systemd (B5)
|
||||||
|
|
||||||
|
**Obiettivo**: attivare le coppie `.service`+`.timer` già committate
|
||||||
|
in `deploy/systemd/` per replicare i task periodici di
|
||||||
|
`Register-CIScheduledTasks.ps1`.
|
||||||
|
|
||||||
|
- [ ] Copiare le unit:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /opt/ci/local-ci-cd-system
|
||||||
|
sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Abilitare e avviare tutti i timer:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for t in ci-cleanup-orphans ci-retention-policy \
|
||||||
|
ci-watch-disk-space ci-watch-runner-health \
|
||||||
|
ci-backup-template; do
|
||||||
|
sudo systemctl enable --now "${t}.timer"
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Verificare lo schedule:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
systemctl list-timers --all 'ci-*'
|
||||||
|
```
|
||||||
|
|
||||||
|
Atteso: 5 timer in stato `active`.
|
||||||
|
|
||||||
|
- [ ] Trigger manuale di ciascun service per validare l'esecuzione
|
||||||
|
one-shot:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
for s in ci-cleanup-orphans ci-watch-disk-space ci-watch-runner-health; do
|
||||||
|
sudo systemctl start "${s}.service"
|
||||||
|
sudo systemctl status "${s}.service" --no-pager
|
||||||
|
done
|
||||||
|
```
|
||||||
|
|
||||||
|
> `ci-retention-policy` e `ci-backup-template` chiamano `pwsh`:
|
||||||
|
> eseguirli solo dopo aver verificato che `pwsh --version` funzioni
|
||||||
|
> sotto user `ci-runner`.
|
||||||
|
|
||||||
|
- [ ] Per maggiori dettagli (mapping Windows→Linux, troubleshooting,
|
||||||
|
rollback) vedi `deploy/systemd/README.md`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 6 — Cutover (B6)
|
||||||
|
|
||||||
|
**Obiettivo**: spostare la produzione dall'host Windows a Linux in una
|
||||||
|
finestra di manutenzione concordata.
|
||||||
|
|
||||||
|
> ⚠️ Eseguire solo dopo che i Passi 1–5 sono tutti `[x]` PASS.
|
||||||
|
|
||||||
|
- [ ] Annunciare la finestra di manutenzione (durata stimata: 30–60 min).
|
||||||
|
- [ ] Verificare nessun job in coda lato Gitea (Admin → Actions → Tasks).
|
||||||
|
- [ ] **Stop** act_runner sull'host Windows:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Stop-Service actions-runner
|
||||||
|
Get-Service actions-runner # atteso: Stopped
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Disabilitare** scheduled task Windows:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Get-ScheduledTask -TaskName 'CI-*' | Unregister-ScheduledTask -Confirm:$false
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Rsync incrementale finale degli artifact:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner rsync -av --delete \
|
||||||
|
user@windows-host:/cygdrive/f/CI/Artifacts/ \
|
||||||
|
/var/lib/ci/artifacts/
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Sbloccare il runner Linux su Gitea UI (rimuovere `paused` o
|
||||||
|
ripristinare le label di produzione `windows-build:host,linux-build:host`).
|
||||||
|
- [ ] Verificare runner Linux online su Gitea UI.
|
||||||
|
- [ ] Smoke trigger:
|
||||||
|
- [ ] `self-test.yml` → PASS dal runner Linux
|
||||||
|
- [ ] `build-ns7zip.yml` matrix Win+Linux → PASS dal runner Linux
|
||||||
|
- [ ] Monitorare per ≥30 min:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo journalctl -u act-runner -f --since "30min ago"
|
||||||
|
```
|
||||||
|
|
||||||
|
Nessun errore di severità critica atteso.
|
||||||
|
|
||||||
|
> **Rollback (entro 30 min se PASS critico fallisce)**:
|
||||||
|
> 1. `sudo systemctl stop act-runner` su Linux
|
||||||
|
> 2. `Start-Service actions-runner` su Windows
|
||||||
|
> 3. Re-registrare scheduled task con `scripts\Register-CIScheduledTasks.ps1`
|
||||||
|
> 4. Documentare incidente in `TODO.md` e tornare a B1–B5 per fix
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 7 — Capacity burn-in (B7)
|
||||||
|
|
||||||
|
**Obiettivo**: validare il carico target con tempi paragonabili al
|
||||||
|
baseline Windows (entro ±20%).
|
||||||
|
|
||||||
|
- [ ] Burn-in 4 job concorrenti × 10 round su `WinBuild2025`
|
||||||
|
(lo script `Test-CapacityBurnIn.ps1` ora delega via shim alla CLI
|
||||||
|
Python `job` e gira anche da `pwsh` su Linux):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \
|
||||||
|
-Concurrency 4 -Rounds 10 -Template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Burn-in 4 × 10 su `LinuxBuild2404` (analogo, `--Template`
|
||||||
|
sul VMX Linux).
|
||||||
|
- [ ] Misurare:
|
||||||
|
- [ ] Tempo medio per job, confronto con baseline A5
|
||||||
|
- [ ] Tutti i 80 job (2 × 4 × 10) PASS
|
||||||
|
- [ ] Zero VM orfane in `/var/lib/ci/build-vms/`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Spazio disco `/var/lib/ci/build-vms/` torna al baseline post-cleanup
|
||||||
|
- [ ] Documentare i risultati in `docs/RUNBOOK.md` (sezione "Linux
|
||||||
|
host baseline").
|
||||||
|
- [ ] Se delta > 20% vs baseline Windows: aprire issue in `TODO.md`
|
||||||
|
con dettagli per profiling (probabile candidato: filesystem ext4
|
||||||
|
vs NTFS — valutare XFS/BTRFS).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 8 — Stabilità ≥1 settimana (gate per B8)
|
||||||
|
|
||||||
|
**Obiettivo**: verificare ≥1 settimana di esercizio Linux senza
|
||||||
|
incidenti critici prima di dismettere l'host Windows.
|
||||||
|
|
||||||
|
- [ ] Esercizio normale per ≥7 giorni con runner Linux primario.
|
||||||
|
- [ ] Verifica giornaliera (bastano 2 minuti):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo systemctl --failed # nessun service in failed
|
||||||
|
systemctl list-timers --all 'ci-*' # tutti gli ultimi run OK
|
||||||
|
sudo journalctl -u act-runner --since "24h ago" -p err
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Nessun rollback effettuato durante la settimana.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Passo 9 — Decommissioning host Windows (B8, opzionale)
|
||||||
|
|
||||||
|
> ⚠️ Procedere solo dopo Passo 8 `[x]`. L'host Windows va lasciato
|
||||||
|
> spento ma intatto per ≥1 mese come rollback.
|
||||||
|
|
||||||
|
- [ ] Backup finale di `F:\CI\` su archivio esterno:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$stamp = Get-Date -Format 'yyyyMMdd'
|
||||||
|
$archive = "E:\Backup\CI-$stamp.tar"
|
||||||
|
& 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\'
|
||||||
|
Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256"
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Verifica integrità (estrazione di prova in tmp):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$test = "$env:TEMP\ci-restore-test"
|
||||||
|
& 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Spegnere host Windows (`shutdown /s /t 0`).
|
||||||
|
- [ ] Lasciare hardware integro e spento per ≥1 mese.
|
||||||
|
- [ ] Aggiornare `docs/RUNBOOK.md` con la procedura di riaccensione +
|
||||||
|
ri-registrazione scheduled task come rollback.
|
||||||
|
- [ ] Dopo 1 mese senza rollback: riallocare hardware.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tracciamento globale
|
||||||
|
|
||||||
|
| Passo | Step | Descrizione | Stato |
|
||||||
|
| ----- | ---- | -------------------------------------------------- | ----- |
|
||||||
|
| 1 | B1 | Setup host Linux Mint (OS, Workstation, venv, ACL) | [ ] |
|
||||||
|
| 2 | B2 | Trasferimento template VM + smoke `vm new` | [ ] |
|
||||||
|
| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [ ] |
|
||||||
|
| 4 | B4 | act_runner systemd service registrato | [ ] |
|
||||||
|
| 5 | B5 | Timer systemd installati e attivi | [ ] |
|
||||||
|
| 6 | B6 | Cutover (stop Windows, runner Linux primario) | [ ] |
|
||||||
|
| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux) | [ ] |
|
||||||
|
| 8 | — | ≥1 settimana di stabilità senza rollback | [ ] |
|
||||||
|
| 9 | B8 | Decommissioning host Windows (opzionale) | [ ] |
|
||||||
|
|
||||||
|
Quando tutti i passi sono `[x]`, la **Fase B è chiusa** e si può
|
||||||
|
valutare l'apertura della Fase C (backend ESXi, vedi
|
||||||
|
`plans/idea-3-esxi-support.md`).
|
||||||
@@ -202,7 +202,7 @@ gitea/ # workflow + actions invariati
|
|||||||
- [ ] `pytest` verde con coverage minima 70% su `src/ci_orchestrator/`
|
- [ ] `pytest` verde con coverage minima 70% su `src/ci_orchestrator/`
|
||||||
- [ ] `ruff check` + `mypy --strict` puliti su tutto `src/`
|
- [ ] `ruff check` + `mypy --strict` puliti su tutto `src/`
|
||||||
- [ ] `lint.yml` aggiornato: PSSA per file PS legacy + ruff/mypy per Python
|
- [ ] `lint.yml` aggiornato: PSSA per file PS legacy + ruff/mypy per Python
|
||||||
- [ ] Workflow `build-nsInnoUnp.yml` PASS su matrice Windows + Linux
|
- [ ] Workflow `build-ns7zip.yml` PASS su matrice Windows + Linux
|
||||||
- [ ] Smoke test (`Test-Smoke.ps1` o equivalente Python) PASS
|
- [ ] Smoke test (`Test-Smoke.ps1` o equivalente Python) PASS
|
||||||
- [ ] Capacity burn-in 4 job concorrenti PASS
|
- [ ] Capacity burn-in 4 job concorrenti PASS
|
||||||
- [ ] `README.md`, `AGENTS.md`, `docs/ARCHITECTURE.md`, `docs/HOST-SETUP.md` aggiornati
|
- [ ] `README.md`, `AGENTS.md`, `docs/ARCHITECTURE.md`, `docs/HOST-SETUP.md` aggiornati
|
||||||
|
|||||||
@@ -132,7 +132,7 @@ Finestra di manutenzione concordata:
|
|||||||
3. Verificare nessun job in coda lato Gitea
|
3. Verificare nessun job in coda lato Gitea
|
||||||
4. Avviare act_runner Linux (`systemctl start act-runner`)
|
4. Avviare act_runner Linux (`systemctl start act-runner`)
|
||||||
5. Trigger manuale di un workflow di smoke test
|
5. Trigger manuale di un workflow di smoke test
|
||||||
6. Trigger del workflow matrix `build-nsInnoUnp.yml` (Win + Linux build)
|
6. Trigger del workflow matrix `build-ns7zip.yml` (Win + Linux build)
|
||||||
|
|
||||||
Se PASS → cutover confermato. Se FAIL → rollback (riavvio runner Windows,
|
Se PASS → cutover confermato. Se FAIL → rollback (riavvio runner Windows,
|
||||||
nessun dato è stato distrutto).
|
nessun dato è stato distrutto).
|
||||||
@@ -205,7 +205,7 @@ Costo: codebase divisa in due ambienti, maggiore overhead operativo.
|
|||||||
- [ ] act_runner gira come systemd service `act-runner.service` su Linux Mint
|
- [ ] act_runner gira come systemd service `act-runner.service` su Linux Mint
|
||||||
- [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`)
|
- [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`)
|
||||||
operativi dal nuovo host con snapshot integri
|
operativi dal nuovo host con snapshot integri
|
||||||
- [ ] Workflow `build-nsInnoUnp.yml` (matrix Win+Linux) PASS dal nuovo host
|
- [ ] Workflow `build-ns7zip.yml` (matrix Win+Linux) PASS dal nuovo host
|
||||||
- [ ] Burn-in capacity 4 job concorrenti PASS, tempi entro ±20% del baseline
|
- [ ] Burn-in capacity 4 job concorrenti PASS, tempi entro ±20% del baseline
|
||||||
- [ ] Storage migrato a `/var/lib/ci/`, host Windows in stand-by come rollback
|
- [ ] Storage migrato a `/var/lib/ci/`, host Windows in stand-by come rollback
|
||||||
- [ ] Tutti i timer systemd attivi e schedulati
|
- [ ] Tutti i timer systemd attivi e schedulati
|
||||||
|
|||||||
+103
-103
@@ -13,7 +13,7 @@ Sintesi fasi:
|
|||||||
- **Fase A — Rewrite Python dell'orchestratore (host Windows attuale)**
|
- **Fase A — Rewrite Python dell'orchestratore (host Windows attuale)**
|
||||||
- Output verificabile: `python -m ci_orchestrator job ...` sostituisce
|
- Output verificabile: `python -m ci_orchestrator job ...` sostituisce
|
||||||
`Invoke-CIJob.ps1`; `pytest` ≥70% coverage; workflow
|
`Invoke-CIJob.ps1`; `pytest` ≥70% coverage; workflow
|
||||||
`build-nsInnoUnp.yml` PASS.
|
`build-ns7zip.yml` PASS.
|
||||||
- Stato: da iniziare.
|
- Stato: da iniziare.
|
||||||
- **Fase B — Migrazione host a Linux Mint + Workstation Pro Linux**
|
- **Fase B — Migrazione host a Linux Mint + Workstation Pro Linux**
|
||||||
- Output verificabile: act_runner come `act-runner.service` systemd;
|
- Output verificabile: act_runner come `act-runner.service` systemd;
|
||||||
@@ -43,35 +43,35 @@ A4; cambiano solo path/env vars in Fase B).
|
|||||||
|
|
||||||
## Checklist riassuntiva master
|
## Checklist riassuntiva master
|
||||||
|
|
||||||
- [ ] [A1] Creare `pyproject.toml` + package `src/ci_orchestrator/` + venv `F:\CI\python\venv\`
|
- [x] [A1] Creare `pyproject.toml` + package `src/ci_orchestrator/` + venv `F:\CI\python\venv\`
|
||||||
- [ ] [A1] Implementare `config.py` (env vars + `config.toml`) con default Windows e hook path Linux
|
- [x] [A1] Implementare `config.py` (env vars + `config.toml`) con default Windows e hook path Linux
|
||||||
- [ ] [A1] Implementare `backends/protocol.py` con Protocol `VmBackend` (firma neutra, no `vmrun_*` nei nomi pubblici)
|
- [x] [A1] Implementare `backends/protocol.py` con Protocol `VmBackend` (firma neutra, no `vmrun_*` nei nomi pubblici)
|
||||||
- [ ] [A1] Implementare `backends/workstation.py` (`WorkstationVmrunBackend`) usando `subprocess` + `shutil.which('vmrun')`
|
- [x] [A1] Implementare `backends/workstation.py` (`WorkstationVmrunBackend`) usando `subprocess` + `shutil.which('vmrun')`
|
||||||
- [ ] [A1] Implementare `transport/winrm.py` con wrapper `pypsrp.client.Client`
|
- [x] [A1] Implementare `transport/winrm.py` con wrapper `pypsrp.client.Client`
|
||||||
- [ ] [A1] Implementare `transport/ssh.py` con wrapper `paramiko.SSHClient` / `SFTPClient`
|
- [x] [A1] Implementare `transport/ssh.py` con wrapper `paramiko.SSHClient` / `SFTPClient`
|
||||||
- [ ] [A1] Implementare `credentials.py` con Protocol `CredentialStore` + `KeyringCredentialStore`
|
- [x] [A1] Implementare `credentials.py` con Protocol `CredentialStore` + `KeyringCredentialStore`
|
||||||
- [ ] [A1] PoC `wait-ready` end-to-end via `pypsrp` contro un guest Windows reale
|
- [ ] [A1] PoC `wait-ready` end-to-end via `pypsrp` contro un guest Windows reale
|
||||||
- [ ] [A1] Test pytest unitari per `vmrun.py`, `winrm.py`, `ssh.py`, `credentials.py` con mock
|
- [x] [A1] Test pytest unitari per `vmrun.py`, `winrm.py`, `ssh.py`, `credentials.py` con mock
|
||||||
- [ ] [A1] Aggiungere job pytest a `gitea/workflows/lint.yml`
|
- [x] [A1] Aggiungere job pytest a `gitea/workflows/lint.yml`
|
||||||
- [ ] [A2] Portare `Wait-VMReady.ps1` → `python -m ci_orchestrator wait-ready`
|
- [x] [A2] Portare `Wait-VMReady.ps1` → `python -m ci_orchestrator wait-ready`
|
||||||
- [ ] [A2] Portare `Remove-BuildVM.ps1` → `vm remove`
|
- [x] [A2] Portare `Remove-BuildVM.ps1` → `vm remove`
|
||||||
- [ ] [A2] Portare `Cleanup-OrphanedBuildVMs.ps1` → `vm cleanup`
|
- [x] [A2] Portare `Cleanup-OrphanedBuildVMs.ps1` → `vm cleanup`
|
||||||
- [ ] [A2] Portare `Watch-DiskSpace.ps1` → `monitor disk`
|
- [x] [A2] Portare `Watch-DiskSpace.ps1` → `monitor disk`
|
||||||
- [ ] [A2] Portare `Watch-RunnerHealth.ps1` → `monitor runner`
|
- [x] [A2] Portare `Watch-RunnerHealth.ps1` → `monitor runner`
|
||||||
- [ ] [A2] Portare `Get-CIJobSummary.ps1` → `report job`
|
- [x] [A2] Portare `Get-CIJobSummary.ps1` → `report job`
|
||||||
- [ ] [A2] Sostituire ognuno dei `.ps1` portati con shim a 3 righe verso la CLI Python
|
- [x] [A2] Sostituire ognuno dei `.ps1` portati con shim a 3 righe verso la CLI Python
|
||||||
- [ ] [A3] Portare `New-BuildVM.ps1` → `vm new`
|
- [x] [A3] Portare `New-BuildVM.ps1` → `vm new`
|
||||||
- [ ] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run`
|
- [x] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run`
|
||||||
- [ ] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect`
|
- [x] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect`
|
||||||
- [ ] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest
|
- [x] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest
|
||||||
- [ ] [A4] Portare `Invoke-CIJob.ps1` → `python -m ci_orchestrator job`
|
- [x] [A4] Portare `Invoke-CIJob.ps1` → `python -m ci_orchestrator job`
|
||||||
- [ ] [A4] Aggiornare `gitea/actions/local-ci-build/action.yml` per invocare la CLI Python direttamente
|
- [x] [A4] Aggiornare `gitea/actions/local-ci-build/action.yml` per invocare la CLI Python direttamente
|
||||||
- [ ] [A4] Forzare `PYTHONIOENCODING=utf-8` in `runner/config.yaml`
|
- [x] [A4] Forzare `PYTHONIOENCODING=utf-8` in `runner/config.yaml`
|
||||||
- [ ] [A4] Eseguire workflow `build-nsInnoUnp.yml` (matrix Win+Linux) end-to-end PASS
|
- [ ] [A4] Eseguire workflow `build-ns7zip.yml` (matrix Win+Linux) end-to-end PASS
|
||||||
- [ ] [A5] Convertire errori frequenti `AGENTS.md` #9, #10, #11, #12 in test pytest dedicati
|
- [x] [A5] Convertire errori frequenti `AGENTS.md` #9, #10, #11, #12 in test pytest dedicati
|
||||||
- [ ] [A5] Aggiornare `AGENTS.md` con sezione "Python development" (venv, ruff, mypy, pytest)
|
- [x] [A5] Aggiornare `AGENTS.md` con sezione "Python development" (venv, ruff, mypy, pytest)
|
||||||
- [ ] [A5] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout package
|
- [x] [A5] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout package
|
||||||
- [ ] [A5] Aggiornare `README.md` con setup Python
|
- [x] [A5] Aggiornare `README.md` con setup Python
|
||||||
- [ ] [A5] Eseguire `Test-CapacityBurnIn` (versione Python) 4 job concorrenti PASS
|
- [ ] [A5] Eseguire `Test-CapacityBurnIn` (versione Python) 4 job concorrenti PASS
|
||||||
- [ ] [B1] Installare Linux Mint LTS + aggiornamenti su hardware target
|
- [ ] [B1] Installare Linux Mint LTS + aggiornamenti su hardware target
|
||||||
- [ ] [B1] Installare VMware Workstation Pro Linux (bundle ufficiale Broadcom) e validare `vmrun` su VM di test
|
- [ ] [B1] Installare VMware Workstation Pro Linux (bundle ufficiale Broadcom) e validare `vmrun` su VM di test
|
||||||
@@ -89,11 +89,11 @@ A4; cambiano solo path/env vars in Fase B).
|
|||||||
- [ ] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host`
|
- [ ] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host`
|
||||||
- [ ] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8`
|
- [ ] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8`
|
||||||
- [ ] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f`
|
- [ ] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f`
|
||||||
- [ ] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`)
|
- [x] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`)
|
||||||
- [ ] [B5] Abilitare tutti i timer con `systemctl enable --now` e validare `systemctl list-timers`
|
- [ ] [B5] Abilitare tutti i timer con `systemctl enable --now` e validare `systemctl list-timers`
|
||||||
- [ ] [B6] Stop act_runner sull'host Windows e verifica coda Gitea vuota
|
- [ ] [B6] Stop act_runner sull'host Windows e verifica coda Gitea vuota
|
||||||
- [ ] [B6] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/`
|
- [ ] [B6] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/`
|
||||||
- [ ] [B6] Trigger smoke workflow + `build-nsInnoUnp.yml` matrix da host Linux PASS
|
- [ ] [B6] Trigger smoke workflow + `build-ns7zip.yml` matrix da host Linux PASS
|
||||||
- [ ] [B7] Eseguire burn-in 4 job concorrenti × 10 round su template Windows e Linux con tempi entro ±20% baseline
|
- [ ] [B7] Eseguire burn-in 4 job concorrenti × 10 round su template Windows e Linux con tempi entro ±20% baseline
|
||||||
- [ ] [B8] Backup finale di `F:\CI\` su archivio
|
- [ ] [B8] Backup finale di `F:\CI\` su archivio
|
||||||
- [ ] [B8] Lasciare host Windows spento ma reinstallato per ≥1 mese come rollback
|
- [ ] [B8] Lasciare host Windows spento ma reinstallato per ≥1 mese come rollback
|
||||||
@@ -128,19 +128,19 @@ WinRM/SSH e credential store funzionanti contro l'ambiente reale via PoC
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Creare `pyproject.toml` (build backend `setuptools` o `hatchling`) con dipendenze `pypsrp`, `paramiko`, `keyring`, `click`, `tomli` (se Python <3.11), `rich` opzionale per logging
|
- [x] Creare `pyproject.toml` (build backend `setuptools` o `hatchling`) con dipendenze `pypsrp`, `paramiko`, `keyring`, `click`, `tomli` (se Python <3.11), `rich` opzionale per logging
|
||||||
- [ ] Creare layout `src/ci_orchestrator/` con `__init__.py`, `__main__.py` (entry point `click`)
|
- [x] Creare layout `src/ci_orchestrator/` con `__init__.py`, `__main__.py` (entry point `click`)
|
||||||
- [ ] Creare venv in `F:\CI\python\venv\` e installare il package in editable (`pip install -e .[dev]`)
|
- [x] Creare venv in `F:\CI\python\venv\` e installare il package in editable (`pip install -e .[dev]`)
|
||||||
- [ ] Implementare `config.py`: caricamento env vars (`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`) con merge da `config.toml` opzionale; default OS-aware (Windows → `F:\CI\...`, Linux → `/var/lib/ci/...`)
|
- [x] Implementare `config.py`: caricamento env vars (`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`) con merge da `config.toml` opzionale; default OS-aware (Windows → `F:\CI\...`, Linux → `/var/lib/ci/...`)
|
||||||
- [ ] Implementare `backends/protocol.py` con Protocol `VmBackend` (metodi `clone_linked`, `start`, `stop`, `delete`, `get_ip`, `list_snapshots`) e dataclass `VmHandle` (path/identificatore opaco)
|
- [x] Implementare `backends/protocol.py` con Protocol `VmBackend` (metodi `clone_linked`, `start`, `stop`, `delete`, `get_ip`, `list_snapshots`) e dataclass `VmHandle` (path/identificatore opaco)
|
||||||
- [ ] Implementare `backends/workstation.py`: `WorkstationVmrunBackend` che usa `subprocess.run([vmrun, '-T', 'ws', op, *args], check=False, capture_output=True, text=True, encoding='utf-8')` con check esplicito su `returncode`
|
- [x] Implementare `backends/workstation.py`: `WorkstationVmrunBackend` che usa `subprocess.run([vmrun, '-T', 'ws', op, *args], check=False, capture_output=True, text=True, encoding='utf-8')` con check esplicito su `returncode`
|
||||||
- [ ] Implementare `transport/winrm.py`: wrapper su `pypsrp.client.Client(host, username, password, ssl=True, cert_validation=False)` con metodi `run`, `copy`, `fetch`
|
- [x] Implementare `transport/winrm.py`: wrapper su `pypsrp.client.Client(host, username, password, ssl=True, cert_validation=False)` con metodi `run`, `copy`, `fetch`
|
||||||
- [ ] Implementare `transport/ssh.py`: wrapper su `paramiko.SSHClient` con `set_missing_host_key_policy(AutoAddPolicy)`, `known_hosts` configurabile (default `None` per non interferire con clone ephemeri — vedi `AGENTS.md` errore #12)
|
- [x] Implementare `transport/ssh.py`: wrapper su `paramiko.SSHClient` con `set_missing_host_key_policy(AutoAddPolicy)`, `known_hosts` configurabile (default `None` per non interferire con clone ephemeri — vedi `AGENTS.md` errore #12)
|
||||||
- [ ] Implementare `credentials.py`: Protocol `CredentialStore` + `KeyringCredentialStore` che usa `keyring.get_credential(target, None)` e ritorna oggetto `Credential(username, password)`
|
- [x] Implementare `credentials.py`: Protocol `CredentialStore` + `KeyringCredentialStore` che usa `keyring.get_credential(target, None)` e ritorna oggetto `Credential(username, password)`
|
||||||
- [ ] PoC end-to-end: comando `python -m ci_orchestrator wait-ready --vmx <path> --timeout 120` su un clone reale del template `WinBuild2025` (validare WinRM HTTPS self-signed)
|
- [ ] PoC end-to-end: comando `python -m ci_orchestrator wait-ready --vmx <path> --timeout 120` su un clone reale del template `WinBuild2025` (validare WinRM HTTPS self-signed)
|
||||||
- [ ] Setup `pytest`, `pytest-mock`, `ruff`, `mypy` come dev dependencies; configurare `pyproject.toml` con `[tool.ruff]` e `[tool.mypy]` strict
|
- [x] Setup `pytest`, `pytest-mock`, `ruff`, `mypy` come dev dependencies; configurare `pyproject.toml` con `[tool.ruff]` e `[tool.mypy]` strict
|
||||||
- [ ] Test pytest unitari: `test_vmrun.py` (mock subprocess, casi `vmrun list` con e senza VMX target — copre errore #10), `test_winrm.py` (mock `pypsrp`), `test_ssh.py` (mock `paramiko`, copre errore #12), `test_credentials.py` (mock `keyring`)
|
- [x] Test pytest unitari: `test_vmrun.py` (mock subprocess, casi `vmrun list` con e senza VMX target — copre errore #10), `test_winrm.py` (mock `pypsrp`), `test_ssh.py` (mock `paramiko`, copre errore #12), `test_credentials.py` (mock `keyring`)
|
||||||
- [ ] Aggiungere job `python` a `gitea/workflows/lint.yml`: setup venv → `ruff check src/ tests/` → `mypy --strict src/` → `pytest --cov=ci_orchestrator --cov-fail-under=70`
|
- [x] Aggiungere job `python` a `gitea/workflows/lint.yml`: setup venv → `ruff check src/ tests/` → `mypy --strict src/` → `pytest --cov=ci_orchestrator --cov-fail-under=70`
|
||||||
|
|
||||||
**Hook futuri Fase C**: il Protocol `VmBackend` deve usare nomi neutri
|
**Hook futuri Fase C**: il Protocol `VmBackend` deve usare nomi neutri
|
||||||
(`clone_linked`, non `vmrun_clone`), accettare `template`/`snapshot`/`name`
|
(`clone_linked`, non `vmrun_clone`), accettare `template`/`snapshot`/`name`
|
||||||
@@ -166,10 +166,10 @@ modificato in A1.
|
|||||||
**Definizione di fatto step A1**:
|
**Definizione di fatto step A1**:
|
||||||
|
|
||||||
- [ ] PoC `wait-ready` PASS contro VM reale
|
- [ ] PoC `wait-ready` PASS contro VM reale
|
||||||
- [ ] Coverage pytest ≥70% sui moduli core
|
- [x] Coverage pytest ≥70% sui moduli core
|
||||||
- [ ] `lint.yml` aggiornato e verde
|
- [x] `lint.yml` aggiornato e verde
|
||||||
- [ ] Protocol `VmBackend` reviewato e congelato
|
- [x] Protocol `VmBackend` reviewato e congelato
|
||||||
- [ ] Documentato setup venv in `README.md` (sezione minima)
|
- [x] Documentato setup venv in `README.md` (sezione minima)
|
||||||
|
|
||||||
### A2 — Script "foglia" (no state condiviso)
|
### A2 — Script "foglia" (no state condiviso)
|
||||||
|
|
||||||
@@ -182,15 +182,15 @@ stato con l'orchestratore, sostituendoli con shim minimi.
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Implementare `commands/wait.py` con sottocomando `wait-ready` (parametri `--vmx`, `--timeout`, `--guest-os`)
|
- [x] Implementare `commands/wait.py` con sottocomando `wait-ready` (parametri `--vmx`, `--timeout`, `--guest-os`)
|
||||||
- [ ] Implementare `commands/vm.py` con sottocomando `vm remove` (parametri `--vmx`, `--force`)
|
- [x] Implementare `commands/vm.py` con sottocomando `vm remove` (parametri `--vmx`, `--force`)
|
||||||
- [ ] Implementare `vm cleanup` (scan `CI_BUILD_VMS`, riconoscimento clone orfani per pattern naming + età)
|
- [x] Implementare `vm cleanup` (scan `CI_BUILD_VMS`, riconoscimento clone orfani per pattern naming + età)
|
||||||
- [ ] Implementare `commands/monitor.py` con `monitor disk` (soglie configurabili, output JSON o human)
|
- [x] Implementare `commands/monitor.py` con `monitor disk` (soglie configurabili, output JSON o human)
|
||||||
- [ ] Implementare `monitor runner` (controllo processo act_runner attivo + ultimo heartbeat)
|
- [x] Implementare `monitor runner` (controllo processo act_runner attivo + ultimo heartbeat)
|
||||||
- [ ] Implementare `commands/report.py` con `report job` (read-only su artifact dir + log job)
|
- [x] Implementare `commands/report.py` con `report job` (read-only su artifact dir + log job)
|
||||||
- [ ] Sostituire `scripts/Wait-VMReady.ps1` con shim PS 5.1 di 3 righe: `& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready @args; exit $LASTEXITCODE`
|
- [x] Sostituire `scripts/Wait-VMReady.ps1` con shim PS 5.1 di 3 righe: `& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready @args; exit $LASTEXITCODE`
|
||||||
- [ ] Idem per `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`, `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1`
|
- [x] Idem per `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`, `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1`
|
||||||
- [ ] Aggiungere test pytest per ogni comando (mock backend + mock filesystem via `tmp_path`)
|
- [x] Aggiungere test pytest per ogni comando (mock backend + mock filesystem via `tmp_path`)
|
||||||
- [ ] Convertire `tests/Wait-VMReady.Tests.ps1` in `tests/test_commands_wait.py` (preservando i casi negativi)
|
- [ ] Convertire `tests/Wait-VMReady.Tests.ps1` in `tests/test_commands_wait.py` (preservando i casi negativi)
|
||||||
- [ ] Convertire `tests/Remove-BuildVM.Tests.ps1` in `tests/test_commands_vm_remove.py`
|
- [ ] Convertire `tests/Remove-BuildVM.Tests.ps1` in `tests/test_commands_vm_remove.py`
|
||||||
- [ ] Verificare che gli scheduled task esistenti (`Register-CIScheduledTasks.ps1`) continuino a funzionare invocando gli shim PS
|
- [ ] Verificare che gli scheduled task esistenti (`Register-CIScheduledTasks.ps1`) continuino a funzionare invocando gli shim PS
|
||||||
@@ -229,15 +229,15 @@ Python preservando il comportamento dei `.ps1` esistenti.
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`)
|
- [x] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`)
|
||||||
- [ ] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip`
|
- [x] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip`
|
||||||
- [ ] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux
|
- [x] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux
|
||||||
- [ ] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`)
|
- [x] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`)
|
||||||
- [ ] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`)
|
- [x] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`)
|
||||||
- [ ] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim
|
- [x] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim
|
||||||
- [ ] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py`
|
- [x] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py`
|
||||||
- [ ] Aggiungere test pytest per `build run` (mock transport + capture stdout)
|
- [x] Aggiungere test pytest per `build run` (mock transport + capture stdout)
|
||||||
- [ ] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path)
|
- [x] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path)
|
||||||
- [ ] Validare end-to-end: clone WinBuild2025 → build script PowerShell trivial → collect artifact ZIP
|
- [ ] Validare end-to-end: clone WinBuild2025 → build script PowerShell trivial → collect artifact ZIP
|
||||||
|
|
||||||
**Hook futuri Fase C**: `build run` deve ricevere un `VmHandle` opaco,
|
**Hook futuri Fase C**: `build run` deve ricevere un `VmHandle` opaco,
|
||||||
@@ -257,10 +257,10 @@ Python; nessuna modifica ai workflow YAML in A3 (solo shim).
|
|||||||
|
|
||||||
**Definizione di fatto step A3**:
|
**Definizione di fatto step A3**:
|
||||||
|
|
||||||
- [ ] Pipeline build completa accessibile via Python CLI
|
- [x] Pipeline build completa accessibile via Python CLI
|
||||||
- [ ] Shim PS preservano l'API per i caller esistenti
|
- [x] Shim PS preservano l'API per i caller esistenti
|
||||||
- [ ] Smoke build PASS contro VM reale Windows e Linux
|
- [ ] Smoke build PASS contro VM reale Windows e Linux
|
||||||
- [ ] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito
|
- [x] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito
|
||||||
|
|
||||||
### A4 — Orchestratore + switch workflow
|
### A4 — Orchestratore + switch workflow
|
||||||
|
|
||||||
@@ -272,13 +272,13 @@ Gitea per chiamare la CLI Python direttamente.
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Implementare `commands/job.py` con sottocomando `job` (entry point completo: parsing parametri job, clone, wait, build, collect, cleanup)
|
- [x] Implementare `commands/job.py` con sottocomando `job` (entry point completo: parsing parametri job, clone, wait, build, collect, cleanup)
|
||||||
- [ ] Gestione errori e cleanup garantito (try/finally con `vm remove` anche su failure)
|
- [x] Gestione errori e cleanup garantito (try/finally con `vm remove` anche su failure)
|
||||||
- [ ] Aggiornare `gitea/actions/local-ci-build/action.yml`: cambiare `shell: powershell` → `shell: cmd` o invocazione diretta `python -m ci_orchestrator job ...`
|
- [x] Aggiornare `gitea/actions/local-ci-build/action.yml`: cambiare `shell: powershell` → `shell: cmd` o invocazione diretta `python -m ci_orchestrator job ...`
|
||||||
- [ ] Aggiungere `PYTHONIOENCODING=utf-8` in `runner/config.yaml` env
|
- [x] Aggiungere `PYTHONIOENCODING=utf-8` in `runner/config.yaml` env
|
||||||
- [ ] Lasciare `Invoke-CIJob.ps1` come shim per scheduled task / chiamate manuali esterne (rimosso in A5)
|
- [x] Lasciare `Invoke-CIJob.ps1` come shim per scheduled task / chiamate manuali esterne (rimosso in A5)
|
||||||
- [ ] Test pytest end-to-end con mock backend + transport: `test_commands_job.py`
|
- [x] Test pytest end-to-end con mock backend + transport: `test_commands_job.py`
|
||||||
- [ ] Eseguire workflow `build-nsInnoUnp.yml` matrix (Win + Linux) PASS
|
- [ ] Eseguire workflow `build-ns7zip.yml` matrix (Win + Linux) PASS
|
||||||
- [ ] Eseguire workflow `self-test.yml` PASS
|
- [ ] Eseguire workflow `self-test.yml` PASS
|
||||||
- [ ] Eseguire workflow `lint.yml` PASS
|
- [ ] Eseguire workflow `lint.yml` PASS
|
||||||
|
|
||||||
@@ -290,7 +290,7 @@ factory `backends.load_backend(config)`.
|
|||||||
**Test / validazione**:
|
**Test / validazione**:
|
||||||
|
|
||||||
- `pytest tests/test_commands_job.py` PASS con coverage ≥80% sul comando `job`
|
- `pytest tests/test_commands_job.py` PASS con coverage ≥80% sul comando `job`
|
||||||
- Workflow `build-nsInnoUnp.yml` end-to-end PASS dal nuovo entry point
|
- Workflow `build-ns7zip.yml` end-to-end PASS dal nuovo entry point
|
||||||
- Output catturato da act_runner leggibile (no encoding broken, no ANSI rotti)
|
- Output catturato da act_runner leggibile (no encoding broken, no ANSI rotti)
|
||||||
- Tempi job entro ±10% rispetto al baseline PS (misurati con `Measure-CIBenchmark` ancora in PS, lecito)
|
- Tempi job entro ±10% rispetto al baseline PS (misurati con `Measure-CIBenchmark` ancora in PS, lecito)
|
||||||
|
|
||||||
@@ -299,10 +299,10 @@ factory `backends.load_backend(config)`.
|
|||||||
|
|
||||||
**Definizione di fatto step A4**:
|
**Definizione di fatto step A4**:
|
||||||
|
|
||||||
- [ ] `action.yml` chiama Python direttamente
|
- [x] `action.yml` chiama Python direttamente
|
||||||
- [ ] Workflow matrix Win+Linux PASS
|
- [ ] Workflow matrix Win+Linux PASS
|
||||||
- [ ] `runner/config.yaml` ha `PYTHONIOENCODING=utf-8`
|
- [x] `runner/config.yaml` ha `PYTHONIOENCODING=utf-8`
|
||||||
- [ ] Coverage `job.py` ≥80%
|
- [x] Coverage `job.py` ≥80%
|
||||||
- [ ] Misurazione tempo entro ±10% baseline
|
- [ ] Misurazione tempo entro ±10% baseline
|
||||||
|
|
||||||
### A5 — Test, documentazione, cleanup
|
### A5 — Test, documentazione, cleanup
|
||||||
@@ -314,15 +314,15 @@ aggiornare documentazione, eseguire burn-in finale.
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Implementare test pytest dedicati per ognuno degli errori frequenti `AGENTS.md` #9 (snapshot powered-off), #10 (`vmrun list` vs `getGuestIPAddress`), #11 (machine-id reset), #12 (stderr handling — N/A per Python ma documentare)
|
- [x] Implementare test pytest dedicati per ognuno degli errori frequenti `AGENTS.md` #9 (snapshot powered-off), #10 (`vmrun list` vs `getGuestIPAddress`), #11 (machine-id reset → unique clone-name guard), #12 (stderr handling → AutoAddPolicy + no shell-out)
|
||||||
- [ ] Rimuovere shim PS che non hanno call site esterni (verificare con `grep_search` che nessun task / doc li referenzia)
|
- [x] Verificare con `grep_search` che gli shim PS abbiano ancora call site esterni (`Register-CIScheduledTasks.ps1`, doc, `Test-*.ps1`) → nessuno candidato a rimozione in A5; rimozione differita a B5 (systemd timers)
|
||||||
- [ ] Lasciare `Install-CIToolchain-*.ps1`, `Prepare-*.ps1`, `Deploy-*.ps1` in PowerShell (girano dentro guest o per ricostruzione template — vedi §4 di [idea-1](idea-1-python-rewrite.md))
|
- [x] Lasciare `Install-CIToolchain-*.ps1`, `Prepare-*.ps1`, `Deploy-*.ps1` in PowerShell (girano dentro guest o per ricostruzione template — vedi §4 di [idea-1](idea-1-python-rewrite.md))
|
||||||
- [ ] Aggiornare `AGENTS.md`: aggiungere sezione "Python development" (venv path, ruff, mypy strict, pytest, encoding utf-8)
|
- [x] Aggiornare `AGENTS.md`: aggiungere sezione "Python development" (venv path, ruff, mypy strict, pytest, encoding utf-8, mappatura PS → Python)
|
||||||
- [ ] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout `src/ci_orchestrator/`
|
- [x] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout `src/ci_orchestrator/` + Phase C extension point
|
||||||
- [ ] Aggiornare `README.md` con setup Python e quick-start CLI
|
- [x] Aggiornare `README.md` con setup Python e quick-start CLI per tutti i 10 sub-comandi
|
||||||
- [ ] Portare `Test-CapacityBurnIn.ps1` in `commands/burnin.py` (o lasciare in PS se ancora utile come driver esterno)
|
- [ ] Portare `Test-CapacityBurnIn.ps1` in `commands/burnin.py` (o lasciare in PS se ancora utile come driver esterno) — non implementato in A5; lo shim attuale già delega via `job` Python
|
||||||
- [ ] Eseguire burn-in 4 job concorrenti × 10 round PASS sull'host Windows
|
- [ ] Eseguire burn-in 4 job concorrenti × 10 round PASS sull'host Windows — richiede VM reali, a carico utente
|
||||||
- [ ] Aggiornare `docs/RUNBOOK.md` con sezione "Operare da Python CLI"
|
- [ ] Aggiornare `docs/RUNBOOK.md` con sezione "Operare da Python CLI" — deferito a chiusura Fase B (RUNBOOK riscritto end-to-end in B-finale)
|
||||||
|
|
||||||
**Hook futuri Fase C**: documentare in `docs/ARCHITECTURE.md` il
|
**Hook futuri Fase C**: documentare in `docs/ARCHITECTURE.md` il
|
||||||
contratto `VmBackend` come "extension point" e il selettore `[backend]`
|
contratto `VmBackend` come "extension point" e il selettore `[backend]`
|
||||||
@@ -341,11 +341,11 @@ qualche caller esterno non ancora migrato emerge.
|
|||||||
|
|
||||||
**Definizione di fatto step A5**:
|
**Definizione di fatto step A5**:
|
||||||
|
|
||||||
- [ ] Errori #9–#12 coperti da test pytest
|
- [x] Errori #9–#12 coperti da test pytest (`tests/python/test_agents_errors.py`, 12 test)
|
||||||
- [ ] Documentazione aggiornata
|
- [x] Documentazione aggiornata (`AGENTS.md`, `docs/ARCHITECTURE.md`, `README.md`)
|
||||||
- [ ] Shim non referenziati rimossi
|
- [x] Shim non referenziati rimossi → nessuno candidato (tutti referenziati da `Register-CIScheduledTasks.ps1`/doc/`Test-*.ps1`); rimozione differita a B5 con sostituzione systemd
|
||||||
- [ ] Burn-in PASS
|
- [ ] Burn-in PASS — pendenza utente (richiede VM reali, vedi `plans/A5-closeout.md`)
|
||||||
- [ ] **Fase A done**: tutti i criteri di "fatto" §7 soddisfatti per la parte A
|
- [ ] **Fase A done**: code-complete; pending solo validazioni hardware (burn-in + workflow matrix end-to-end)
|
||||||
|
|
||||||
## 2. Fase B — Migrazione host Linux Mint
|
## 2. Fase B — Migrazione host Linux Mint
|
||||||
|
|
||||||
@@ -541,15 +541,15 @@ per l'inventario dei task.
|
|||||||
|
|
||||||
**Attività**:
|
**Attività**:
|
||||||
|
|
||||||
- [ ] Inventariare i task in `Register-CIScheduledTasks.ps1` (identificare cadenza e comando per ognuno)
|
- [x] Inventariare i task in `Register-CIScheduledTasks.ps1` (identificare cadenza e comando per ognuno)
|
||||||
- [ ] Per `cleanup-orphaned-vms`: creare `ci-cleanup-orphaned-vms.service` (oneshot, `ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup`) + `.timer` (`OnCalendar=hourly`)
|
- [x] Per `cleanup-orphaned-vms`: creare `ci-cleanup-orphaned-vms.service` (oneshot, `ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup`) + `.timer` (`OnCalendar=hourly`)
|
||||||
- [ ] Per `retention-policy`: creare `ci-retention-policy.service` + `.timer` (`OnCalendar=daily`)
|
- [x] Per `retention-policy`: creare `ci-retention-policy.service` + `.timer` (`OnCalendar=daily`)
|
||||||
- [ ] Per `watch-disk-space`: `ci-watch-disk-space.service` + `.timer` (`OnCalendar=*:0/15`)
|
- [x] Per `watch-disk-space`: `ci-watch-disk-space.service` + `.timer` (`OnCalendar=*:0/15`)
|
||||||
- [ ] Per `watch-runner-health`: `ci-watch-runner-health.service` + `.timer` (`OnCalendar=*:0/5`)
|
- [x] Per `watch-runner-health`: `ci-watch-runner-health.service` + `.timer` (`OnCalendar=*:0/5`)
|
||||||
- [ ] Per `backup-template`: `ci-backup-template.service` + `.timer` (`OnCalendar=weekly`)
|
- [x] Per `backup-template`: `ci-backup-template.service` + `.timer` (`OnCalendar=weekly`)
|
||||||
- [ ] `systemctl daemon-reload` e `systemctl enable --now <ognuno>.timer`
|
- [ ] `systemctl daemon-reload` e `systemctl enable --now <ognuno>.timer`
|
||||||
- [ ] Validare `systemctl list-timers` mostra tutti i timer schedulati
|
- [ ] Validare `systemctl list-timers` mostra tutti i timer schedulati
|
||||||
- [ ] Documentare il mapping in `docs/HOST-SETUP.md`
|
- [x] Documentare il mapping in `docs/HOST-SETUP.md`
|
||||||
|
|
||||||
**Hook futuri Fase C**: `vm cleanup` con backend ESXi userà la stessa
|
**Hook futuri Fase C**: `vm cleanup` con backend ESXi userà la stessa
|
||||||
unit — il selettore backend è in `config.toml`, non nel comando.
|
unit — il selettore backend è in `config.toml`, non nel comando.
|
||||||
@@ -589,7 +589,7 @@ una finestra di manutenzione concordata.
|
|||||||
- [ ] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/` (preserva permessi)
|
- [ ] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/` (preserva permessi)
|
||||||
- [ ] Verificare che il runner Linux sia "online" e non "paused" su Gitea
|
- [ ] Verificare che il runner Linux sia "online" e non "paused" su Gitea
|
||||||
- [ ] Trigger manuale di workflow smoke (`self-test.yml`) PASS
|
- [ ] Trigger manuale di workflow smoke (`self-test.yml`) PASS
|
||||||
- [ ] Trigger manuale `build-nsInnoUnp.yml` matrix Win + Linux PASS
|
- [ ] Trigger manuale `build-ns7zip.yml` matrix Win + Linux PASS
|
||||||
- [ ] Monitorare `journalctl -u act-runner -f` per ≥30 min sotto carico reale
|
- [ ] Monitorare `journalctl -u act-runner -f` per ≥30 min sotto carico reale
|
||||||
|
|
||||||
**Hook futuri Fase C**: nessuno specifico in B6. Cutover non tocca
|
**Hook futuri Fase C**: nessuno specifico in B6. Cutover non tocca
|
||||||
@@ -603,7 +603,7 @@ backend.
|
|||||||
**Test / validazione**:
|
**Test / validazione**:
|
||||||
|
|
||||||
- `self-test.yml` PASS dal runner Linux
|
- `self-test.yml` PASS dal runner Linux
|
||||||
- `build-nsInnoUnp.yml` matrix Win+Linux PASS dal runner Linux
|
- `build-ns7zip.yml` matrix Win+Linux PASS dal runner Linux
|
||||||
- Nessun errore in `journalctl -u act-runner --since "1h ago"` di severità critica
|
- Nessun errore in `journalctl -u act-runner --since "1h ago"` di severità critica
|
||||||
- Artifact correttamente in `/var/lib/ci/artifacts/`
|
- Artifact correttamente in `/var/lib/ci/artifacts/`
|
||||||
|
|
||||||
@@ -987,7 +987,7 @@ Ogni voce: **Rischio** — Fase / Severità / Mitigazione / Owner action item.
|
|||||||
- [ ] `lint.yml` dual-stack (PSSA per PS legacy + ruff/mypy/pytest per Python) PASS
|
- [ ] `lint.yml` dual-stack (PSSA per PS legacy + ruff/mypy/pytest per Python) PASS
|
||||||
- [ ] act_runner gira come `act-runner.service` systemd su Linux Mint
|
- [ ] act_runner gira come `act-runner.service` systemd su Linux Mint
|
||||||
- [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`) operativi dal nuovo host con snapshot integri
|
- [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`) operativi dal nuovo host con snapshot integri
|
||||||
- [ ] Workflow `build-nsInnoUnp.yml` matrix Win+Linux PASS dal nuovo host
|
- [ ] Workflow `build-ns7zip.yml` matrix Win+Linux PASS dal nuovo host
|
||||||
- [ ] Burn-in capacity 4 job concorrenti × 10 round PASS, tempi entro ±20% baseline Windows
|
- [ ] Burn-in capacity 4 job concorrenti × 10 round PASS, tempi entro ±20% baseline Windows
|
||||||
- [ ] Storage migrato a `/var/lib/ci/`, host Windows spento come rollback
|
- [ ] Storage migrato a `/var/lib/ci/`, host Windows spento come rollback
|
||||||
- [ ] Tutti i timer systemd attivi e schedulati equivalenti ai Task Scheduler Windows
|
- [ ] Tutti i timer systemd attivi e schedulati equivalenti ai Task Scheduler Windows
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["hatchling>=1.18"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "ci-orchestrator"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Local CI/CD orchestrator (Phase A: Python rewrite of PowerShell scripts)"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.11"
|
||||||
|
license = { text = "Proprietary" }
|
||||||
|
authors = [{ name = "Local CI/CD System" }]
|
||||||
|
dependencies = [
|
||||||
|
"click>=8.1",
|
||||||
|
"pypsrp>=0.8",
|
||||||
|
"paramiko>=3.4",
|
||||||
|
"keyring>=24.0",
|
||||||
|
"tomli>=2.0; python_version < '3.11'",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = [
|
||||||
|
"pytest>=8.0",
|
||||||
|
"pytest-cov>=5.0",
|
||||||
|
"pytest-mock>=3.12",
|
||||||
|
"ruff>=0.5",
|
||||||
|
"mypy>=1.10",
|
||||||
|
"types-paramiko",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
ci-orchestrator = "ci_orchestrator.__main__:cli"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/ci_orchestrator"]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 100
|
||||||
|
target-version = "py311"
|
||||||
|
src = ["src", "tests"]
|
||||||
|
|
||||||
|
[tool.ruff.lint]
|
||||||
|
select = [
|
||||||
|
"E", # pycodestyle errors
|
||||||
|
"W", # pycodestyle warnings
|
||||||
|
"F", # pyflakes
|
||||||
|
"I", # isort
|
||||||
|
"B", # flake8-bugbear
|
||||||
|
"UP", # pyupgrade
|
||||||
|
"SIM", # flake8-simplify
|
||||||
|
"RUF", # ruff-specific
|
||||||
|
]
|
||||||
|
ignore = [
|
||||||
|
"E501", # line length handled by formatter
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"tests/**" = ["B011", "SIM117"]
|
||||||
|
|
||||||
|
[tool.mypy]
|
||||||
|
python_version = "3.11"
|
||||||
|
strict = true
|
||||||
|
files = ["src"]
|
||||||
|
warn_unused_ignores = true
|
||||||
|
warn_redundant_casts = true
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
module = ["pypsrp.*", "keyring.*"]
|
||||||
|
ignore_missing_imports = true
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
minversion = "8.0"
|
||||||
|
addopts = "-q --strict-markers"
|
||||||
|
testpaths = ["tests"]
|
||||||
|
pythonpath = ["src"]
|
||||||
|
markers = [
|
||||||
|
"integration: requires real VM/network (skipped in unit-test runs)",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.coverage.run]
|
||||||
|
source = ["ci_orchestrator"]
|
||||||
|
branch = true
|
||||||
|
|
||||||
|
[tool.coverage.report]
|
||||||
|
exclude_lines = [
|
||||||
|
"pragma: no cover",
|
||||||
|
"if TYPE_CHECKING:",
|
||||||
|
"raise NotImplementedError",
|
||||||
|
]
|
||||||
@@ -35,6 +35,14 @@ runner:
|
|||||||
GITEA_CI_SSH_KEY_PATH: "F:\\CI\\keys\\ci_linux"
|
GITEA_CI_SSH_KEY_PATH: "F:\\CI\\keys\\ci_linux"
|
||||||
# Root directory of the local-ci-system repository on this host (scripts\ subdir)
|
# Root directory of the local-ci-system repository on this host (scripts\ subdir)
|
||||||
GITEA_CI_SCRIPT_ROOT: "N:\\Code\\Workspace\\Local-CI-CD-System\\scripts"
|
GITEA_CI_SCRIPT_ROOT: "N:\\Code\\Workspace\\Local-CI-CD-System\\scripts"
|
||||||
|
# Force UTF-8 stdio for the Python orchestrator (Phase A4) so non-ASCII
|
||||||
|
# characters in build logs survive the act_runner -> Gitea round-trip.
|
||||||
|
PYTHONIOENCODING: "utf-8"
|
||||||
|
# Absolute path to the system Python interpreter used to bootstrap the
|
||||||
|
# production venv (F:\CI\python\venv). act_runner runs as SYSTEM with a
|
||||||
|
# sanitized PATH where neither `python` nor `py.exe` resolve, so workflows
|
||||||
|
# rely on this single source of truth. Update only when upgrading Python.
|
||||||
|
CI_PYTHON_LAUNCHER: "C:\\Program Files\\Python314\\python.exe"
|
||||||
|
|
||||||
# Optional: load additional env vars from a file (one KEY=VALUE per line)
|
# Optional: load additional env vars from a file (one KEY=VALUE per line)
|
||||||
# env_file: "F:\\CI\\runner.env"
|
# env_file: "F:\\CI\\runner.env"
|
||||||
|
|||||||
@@ -1,118 +1,37 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss).
|
|
||||||
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
.DESCRIPTION
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours.
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
For each stale directory, stops the VM (hard), deletes via vmrun deleteVM,
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
then removes the directory.
|
$commonWithValue = @{
|
||||||
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
Run as a daily scheduled task or on host startup to prevent disk accumulation.
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours.
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
|
pipelinevariable = $true
|
||||||
.PARAMETER CloneBaseDir
|
}
|
||||||
Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
|
|
||||||
.PARAMETER MaxAgeHours
|
$pyArgs = @()
|
||||||
Age threshold in hours. Directories with LastWriteTime older than this are
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
treated as orphaned. Must exceed the longest expected build duration.
|
$a = $args[$i]
|
||||||
Default: 4 (runner.timeout is 2h — 4h gives double margin)
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
|
$name = $a.Substring(1)
|
||||||
.PARAMETER VmrunPath
|
$lname = $name.ToLower()
|
||||||
Path to vmrun.exe.
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
.PARAMETER WhatIf
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
List orphaned VMs without destroying them.
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
|
}
|
||||||
.EXAMPLE
|
else {
|
||||||
# Dry run
|
$pyArgs += $a
|
||||||
.\Cleanup-OrphanedBuildVMs.ps1 -WhatIf
|
}
|
||||||
|
}
|
||||||
# Live cleanup (run elevated)
|
|
||||||
.\Cleanup-OrphanedBuildVMs.ps1
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
#>
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
[CmdletBinding(SupportsShouldProcess)]
|
& $venvPython -m ci_orchestrator vm cleanup @pyArgs
|
||||||
param(
|
exit $LASTEXITCODE
|
||||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
|
||||||
|
|
||||||
[ValidateRange(0, 168)]
|
|
||||||
[int] $MaxAgeHours = 4,
|
|
||||||
|
|
||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors
|
|
||||||
|
|
||||||
if (-not (Test-Path $CloneBaseDir)) {
|
|
||||||
Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do."
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
||||||
Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath"
|
|
||||||
Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only."
|
|
||||||
}
|
|
||||||
|
|
||||||
$cutoff = (Get-Date).AddHours(-$MaxAgeHours)
|
|
||||||
$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
|
||||||
Where-Object { $_.LastWriteTime -lt $cutoff })
|
|
||||||
|
|
||||||
if (-not $orphans) {
|
|
||||||
Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)."
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h."
|
|
||||||
|
|
||||||
foreach ($dir in $orphans) {
|
|
||||||
$vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue |
|
|
||||||
Select-Object -First 1
|
|
||||||
|
|
||||||
if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) {
|
|
||||||
Write-Host "[Cleanup] Processing: $($dir.FullName)"
|
|
||||||
|
|
||||||
if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) {
|
|
||||||
# Hard stop — don't wait for graceful shutdown on an orphan
|
|
||||||
& $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null
|
|
||||||
Start-Sleep -Seconds 2
|
|
||||||
$deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut"
|
|
||||||
Write-Warning " Falling back to directory removal."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
elseif (-not $vmx) {
|
|
||||||
Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only."
|
|
||||||
}
|
|
||||||
|
|
||||||
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
if (Test-Path $dir.FullName) {
|
|
||||||
Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed."
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "[Cleanup] Removed: $($dir.FullName)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[Cleanup] Done."
|
|
||||||
|
|
||||||
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
|
||||||
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
|
||||||
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
|
||||||
# than 30 minutes.
|
|
||||||
$lockFile = 'F:\CI\State\vm-start.lock'
|
|
||||||
$lockCutoff = (Get-Date).AddMinutes(-30)
|
|
||||||
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
|
||||||
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
|
||||||
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
|
||||||
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+30
-195
@@ -1,45 +1,16 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Copies build artifacts from an ephemeral VM to the host artifact directory.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
# Shim: delegates to the Python ci_orchestrator CLI (`artifacts collect`).
|
||||||
Opens a WinRM session to the build VM and uses Copy-Item -FromSession
|
# Original PS implementation moved to git history; see plans/A3-closeout.md.
|
||||||
to transfer the artifact archive and optional log files.
|
#
|
||||||
Validates that at least one file was successfully transferred.
|
# The bound `-Credential` is intentionally discarded — Python uses keyring
|
||||||
Throws on failure so the caller's try/finally can clean up the VM.
|
# via `--credential-target` (defaulting to `BuildVMGuest`).
|
||||||
|
|
||||||
.PARAMETER IPAddress
|
|
||||||
IP of the build VM.
|
|
||||||
|
|
||||||
.PARAMETER Credential
|
|
||||||
PSCredential for the guest build user.
|
|
||||||
|
|
||||||
.PARAMETER GuestArtifactPath
|
|
||||||
Full path to the artifact file or directory inside the VM.
|
|
||||||
Example: C:\CI\output\artifacts.zip
|
|
||||||
|
|
||||||
.PARAMETER HostArtifactDir
|
|
||||||
Directory on the host where artifacts will be stored.
|
|
||||||
The directory is created if it does not exist.
|
|
||||||
Example: F:\CI\Artifacts\run-42
|
|
||||||
|
|
||||||
.PARAMETER IncludeLogs
|
|
||||||
If set, also copies *.log files from the guest work directory
|
|
||||||
(C:\CI\build\*.log) alongside the artifact archive.
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
$cred = Get-Credential
|
|
||||||
.\Get-BuildArtifacts.ps1 `
|
|
||||||
-IPAddress "192.168.11.101" `
|
|
||||||
-Credential $cred `
|
|
||||||
-GuestArtifactPath "C:\CI\output\artifacts.zip" `
|
|
||||||
-HostArtifactDir "F:\CI\Artifacts\run-42"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)]
|
[Parameter(Mandatory)]
|
||||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
||||||
[string] $IPAddress,
|
[string] $IPAddress,
|
||||||
|
|
||||||
[Parameter()]
|
[Parameter()]
|
||||||
@@ -53,175 +24,39 @@ param(
|
|||||||
|
|
||||||
[switch] $IncludeLogs,
|
[switch] $IncludeLogs,
|
||||||
|
|
||||||
# Guest OS type — Linux uses SCP instead of WinRM.
|
|
||||||
[ValidateSet('Windows', 'Linux')]
|
[ValidateSet('Windows', 'Linux')]
|
||||||
[string] $GuestOS = 'Windows',
|
[string] $GuestOS = 'Windows',
|
||||||
|
|
||||||
# SSH private key path (used when GuestOS=Linux).
|
|
||||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||||
|
|
||||||
# SSH username (used when GuestOS=Linux).
|
|
||||||
[string] $SshUser = 'ci_build',
|
[string] $SshUser = 'ci_build',
|
||||||
|
|
||||||
# Persistent known_hosts file for SSH calls (CI jobs).
|
|
||||||
# Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL).
|
|
||||||
[string] $SshKnownHostsFile = '',
|
[string] $SshKnownHostsFile = '',
|
||||||
|
|
||||||
# Optional job ID and commit SHA written into manifest.json.
|
[string] $JobId = '',
|
||||||
# When either is non-empty a manifest.json is written to $HostArtifactDir
|
[string] $Commit = '',
|
||||||
# listing all collected files with name, size, and SHA256.
|
|
||||||
[string] $JobId = '',
|
[string] $CredentialTarget = ''
|
||||||
[string] $Commit = ''
|
|
||||||
)
|
)
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
$pyArgs = @(
|
||||||
$ErrorActionPreference = 'Stop'
|
'--ip-address', $IPAddress,
|
||||||
|
'--guest-os', $GuestOS.ToLower(),
|
||||||
|
'--guest-artifact-path', $GuestArtifactPath,
|
||||||
|
'--host-artifact-dir', $HostArtifactDir,
|
||||||
|
'--ssh-user', $SshUser,
|
||||||
|
'--ssh-key-path', $SshKeyPath,
|
||||||
|
'--job-id', $JobId,
|
||||||
|
'--commit', $Commit
|
||||||
|
)
|
||||||
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) }
|
||||||
|
if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) }
|
||||||
|
if ($IncludeLogs.IsPresent) { $pyArgs += '--include-logs' }
|
||||||
|
|
||||||
# ── Helper: write artifact manifest ─────────────────────────────────────────
|
if ($Credential) {
|
||||||
function Write-ArtifactManifest {
|
Write-Host '[Get-BuildArtifacts] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).'
|
||||||
param([string] $Dir, [string] $JobId, [string] $Commit)
|
|
||||||
$files = @(Get-ChildItem -Path $Dir -Recurse -File -ErrorAction SilentlyContinue |
|
|
||||||
Where-Object { $_.Name -ne 'manifest.json' })
|
|
||||||
$fileEntries = foreach ($f in $files) {
|
|
||||||
[pscustomobject]@{
|
|
||||||
name = $f.FullName.Substring($Dir.TrimEnd('\', '/').Length).TrimStart('\', '/')
|
|
||||||
size = $f.Length
|
|
||||||
sha256 = (Get-FileHash $f.FullName -Algorithm SHA256).Hash.ToLower()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$manifest = [ordered]@{
|
|
||||||
ts = (Get-Date -Format 'o')
|
|
||||||
jobId = $JobId
|
|
||||||
commit = $Commit
|
|
||||||
files = @($fileEntries)
|
|
||||||
}
|
|
||||||
$manifestPath = Join-Path $Dir 'manifest.json'
|
|
||||||
try {
|
|
||||||
$manifest | ConvertTo-Json -Depth 4 |
|
|
||||||
Set-Content -Path $manifestPath -Encoding UTF8 -Force -ErrorAction Stop
|
|
||||||
Write-Host "[Get-BuildArtifacts] Manifest written: $manifestPath ($($files.Count) file(s))"
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[Get-BuildArtifacts] Could not write manifest.json: $_"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($GuestOS -eq 'Windows' -and $null -eq $Credential) {
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
throw "Credential is required when GuestOS is Windows."
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
}
|
& $venvPython -m ci_orchestrator artifacts collect @pyArgs
|
||||||
|
exit $LASTEXITCODE
|
||||||
# ── Linux branch: use scp to transfer artifacts ─────────────────────────────────────────────
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force
|
|
||||||
|
|
||||||
if (-not (Test-Path $HostArtifactDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null
|
|
||||||
Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir"
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[Get-BuildArtifacts] Transferring artifacts via SCP from $GuestArtifactPath ..."
|
|
||||||
|
|
||||||
Copy-SshItem `
|
|
||||||
-Source "$GuestArtifactPath/." `
|
|
||||||
-Destination $HostArtifactDir `
|
|
||||||
-IP $IPAddress `
|
|
||||||
-User $SshUser `
|
|
||||||
-KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Direction 'FromGuest' `
|
|
||||||
-Recurse
|
|
||||||
|
|
||||||
$transferred = @(Get-ChildItem -Path $HostArtifactDir -Recurse -File -ErrorAction SilentlyContinue)
|
|
||||||
if ($transferred.Count -eq 0) {
|
|
||||||
throw "[Get-BuildArtifacts] No files found in $HostArtifactDir after SCP transfer."
|
|
||||||
}
|
|
||||||
Write-Host "[Get-BuildArtifacts] $($transferred.Count) file(s) collected from Linux guest."
|
|
||||||
if ($JobId -ne '' -or $Commit -ne '') {
|
|
||||||
Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
# Ensure destination exists on host
|
|
||||||
if (-not (Test-Path $HostArtifactDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null
|
|
||||||
Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir"
|
|
||||||
}
|
|
||||||
|
|
||||||
$sessionOptions = New-CISessionOption
|
|
||||||
|
|
||||||
Write-Host "[Get-BuildArtifacts] Connecting to VM at $IPAddress..."
|
|
||||||
$session = $null
|
|
||||||
$attempts = 3
|
|
||||||
$delay = 10 # seconds between attempts
|
|
||||||
for ($i = 1; $i -le $attempts; $i++) {
|
|
||||||
try {
|
|
||||||
$session = New-PSSession `
|
|
||||||
-ComputerName $IPAddress `
|
|
||||||
-Credential $Credential `
|
|
||||||
-SessionOption $sessionOptions `
|
|
||||||
-UseSSL `
|
|
||||||
-Port 5986 `
|
|
||||||
-Authentication Basic `
|
|
||||||
-ErrorAction Stop
|
|
||||||
break
|
|
||||||
} catch {
|
|
||||||
if ($i -eq $attempts) { throw }
|
|
||||||
Write-Warning "[Get-BuildArtifacts] WinRM connect attempt $i/$attempts failed — retrying in ${delay}s... ($_)"
|
|
||||||
Start-Sleep -Seconds $delay
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── Verify artifact exists in guest ──────────────────────────────────
|
|
||||||
$guestExists = Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($path)
|
|
||||||
Test-Path $path
|
|
||||||
} -ArgumentList $GuestArtifactPath
|
|
||||||
|
|
||||||
if (-not $guestExists) {
|
|
||||||
throw "Artifact not found in guest at: $GuestArtifactPath"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Copy main artifact ────────────────────────────────────────────────
|
|
||||||
$artifactFileName = Split-Path $GuestArtifactPath -Leaf
|
|
||||||
$hostDestPath = Join-Path $HostArtifactDir $artifactFileName
|
|
||||||
|
|
||||||
Write-Host "[Get-BuildArtifacts] Copying $artifactFileName from guest..."
|
|
||||||
Copy-Item -Path $GuestArtifactPath -Destination $hostDestPath -FromSession $session -Force
|
|
||||||
|
|
||||||
# ── Copy logs if requested ────────────────────────────────────────────
|
|
||||||
if ($IncludeLogs) {
|
|
||||||
$logFiles = Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
Get-ChildItem -Path 'C:\CI\build' -Filter '*.log' -Recurse -ErrorAction SilentlyContinue |
|
|
||||||
Select-Object -ExpandProperty FullName
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach ($log in $logFiles) {
|
|
||||||
$logDest = Join-Path $HostArtifactDir (Split-Path $log -Leaf)
|
|
||||||
Write-Host "[Get-BuildArtifacts] Copying log: $(Split-Path $log -Leaf)"
|
|
||||||
Copy-Item -Path $log -Destination $logDest -FromSession $session -Force
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Validate at least the main artifact landed ────────────────────────
|
|
||||||
if (-not (Test-Path $hostDestPath)) {
|
|
||||||
throw "Artifact was not found at expected host path after copy: $hostDestPath"
|
|
||||||
}
|
|
||||||
|
|
||||||
$fileItem = Get-Item $hostDestPath
|
|
||||||
if ($fileItem.Length -eq 0) {
|
|
||||||
throw "Artifact file exists but is empty: $hostDestPath"
|
|
||||||
}
|
|
||||||
$sizeKB = [math]::Round($fileItem.Length / 1KB, 1)
|
|
||||||
|
|
||||||
Write-Host "[Get-BuildArtifacts] Artifact collected: $hostDestPath ($sizeKB KB)"
|
|
||||||
if ($JobId -ne '' -or $Commit -ne '') {
|
|
||||||
Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit
|
|
||||||
}
|
|
||||||
return $hostDestPath
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|||||||
+215
-181
@@ -1,181 +1,215 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Displays a summary table of CI job results parsed from JSONL log files.
|
|
||||||
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
.DESCRIPTION
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
prints a compact table with job ID, status, total elapsed time, and the
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
last error message (if any).
|
$commonWithValue = @{
|
||||||
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
Each row corresponds to one completed (or failed) CI job.
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
Use -JobId to inspect a single job in detail.
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
|
pipelinevariable = $true
|
||||||
.PARAMETER LogDir
|
}
|
||||||
Base directory containing per-job log subdirectories.
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
Default: F:\CI\Logs
|
|
||||||
|
$pyArgs = @()
|
||||||
.PARAMETER Last
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
Show only the N most recent jobs (sorted by job-start timestamp).
|
$a = $args[$i]
|
||||||
Default: 20
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
|
$name = $a.Substring(1)
|
||||||
.PARAMETER JobId
|
$lname = $name.ToLower()
|
||||||
When specified, shows a detailed phase-by-phase breakdown for that job.
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
.PARAMETER Failed
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
When specified, filters to failed jobs only.
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
.EXAMPLE
|
}
|
||||||
# Show last 20 jobs
|
else {
|
||||||
.\Get-CIJobSummary.ps1
|
$pyArgs += $a
|
||||||
|
}
|
||||||
.EXAMPLE
|
}
|
||||||
# Show last 5 failed jobs
|
|
||||||
.\Get-CIJobSummary.ps1 -Last 5 -Failed
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
.EXAMPLE
|
& $venvPython -m ci_orchestrator report job @pyArgs
|
||||||
# Detailed breakdown for a specific job
|
exit $LASTEXITCODE
|
||||||
.\Get-CIJobSummary.ps1 -JobId 'run-12345'
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
.DESCRIPTION
|
||||||
param(
|
Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and
|
||||||
[string] $LogDir = 'F:\CI\Logs',
|
prints a compact table with job ID, status, total elapsed time, and the
|
||||||
|
last error message (if any).
|
||||||
[ValidateRange(1, 1000)]
|
|
||||||
[int] $Last = 20,
|
Each row corresponds to one completed (or failed) CI job.
|
||||||
|
Use -JobId to inspect a single job in detail.
|
||||||
[string] $JobId = '',
|
|
||||||
|
.PARAMETER LogDir
|
||||||
[switch] $Failed
|
Base directory containing per-job log subdirectories.
|
||||||
)
|
Default: F:\CI\Logs
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
.PARAMETER Last
|
||||||
$ErrorActionPreference = 'Stop'
|
Show only the N most recent jobs (sorted by job-start timestamp).
|
||||||
|
Default: 20
|
||||||
# ── Helpers ────────────────────────────────────────────────────────────────────
|
|
||||||
function Format-SecToHMS {
|
.PARAMETER JobId
|
||||||
param([int] $Seconds)
|
When specified, shows a detailed phase-by-phase breakdown for that job.
|
||||||
if ($Seconds -lt 0) { return '?' }
|
|
||||||
$h = [math]::Floor($Seconds / 3600)
|
.PARAMETER Failed
|
||||||
$m = [math]::Floor(($Seconds % 3600) / 60)
|
When specified, filters to failed jobs only.
|
||||||
$s = $Seconds % 60
|
|
||||||
if ($h -gt 0) { return '{0}h{1:00}m{2:00}s' -f $h, $m, $s }
|
.EXAMPLE
|
||||||
return '{0}m{1:00}s' -f $m, $s
|
# Show last 20 jobs
|
||||||
}
|
.\Get-CIJobSummary.ps1
|
||||||
|
|
||||||
function Read-JobEvents {
|
.EXAMPLE
|
||||||
param([string] $JsonlPath)
|
# Show last 5 failed jobs
|
||||||
$events = [System.Collections.Generic.List[PSCustomObject]]::new()
|
.\Get-CIJobSummary.ps1 -Last 5 -Failed
|
||||||
foreach ($line in (Get-Content $JsonlPath -Encoding UTF8 -ErrorAction SilentlyContinue)) {
|
|
||||||
$line = $line.Trim()
|
.EXAMPLE
|
||||||
if (-not $line) { continue }
|
# Detailed breakdown for a specific job
|
||||||
try {
|
.\Get-CIJobSummary.ps1 -JobId 'run-12345'
|
||||||
$events.Add(($line | ConvertFrom-Json))
|
#>
|
||||||
} catch {
|
[CmdletBinding()]
|
||||||
Write-Debug "[Get-CIJobSummary] Skipping malformed JSONL line in $JsonlPath"
|
param(
|
||||||
}
|
[string] $LogDir = 'F:\CI\Logs',
|
||||||
}
|
|
||||||
return $events
|
[ValidateRange(1, 1000)]
|
||||||
}
|
[int] $Last = 20,
|
||||||
|
|
||||||
# ── Validate log dir ────────────────────────────────────────────────────────
|
[string] $JobId = '',
|
||||||
if (-not (Test-Path $LogDir -PathType Container)) {
|
|
||||||
Write-Warning "Log directory not found: $LogDir"
|
[switch] $Failed
|
||||||
exit 1
|
)
|
||||||
}
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
# ── Single-job detail mode ──────────────────────────────────────────────────
|
$ErrorActionPreference = 'Stop'
|
||||||
if ($JobId -ne '') {
|
|
||||||
$jsonlPath = Join-Path $LogDir (Join-Path $JobId 'invoke-ci.jsonl')
|
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||||||
if (-not (Test-Path $jsonlPath)) {
|
function Format-SecToHMS {
|
||||||
Write-Warning "No JSONL log found for job '$JobId' at: $jsonlPath"
|
param([int] $Seconds)
|
||||||
exit 1
|
if ($Seconds -lt 0) { return '?' }
|
||||||
}
|
$h = [math]::Floor($Seconds / 3600)
|
||||||
$events = Read-JobEvents -JsonlPath $jsonlPath
|
$m = [math]::Floor(($Seconds % 3600) / 60)
|
||||||
if ($events.Count -eq 0) {
|
$s = $Seconds % 60
|
||||||
Write-Warning "JSONL file is empty or unreadable: $jsonlPath"
|
if ($h -gt 0) { return '{0}h{1:00}m{2:00}s' -f $h, $m, $s }
|
||||||
exit 1
|
return '{0}m{1:00}s' -f $m, $s
|
||||||
}
|
}
|
||||||
Write-Host "`nJob: $JobId"
|
|
||||||
Write-Host ('=' * 60)
|
function Read-JobEvents {
|
||||||
Write-Host ('{0,-35} {1,-10} {2}' -f 'Phase', 'Status', 'Timestamp')
|
param([string] $JsonlPath)
|
||||||
Write-Host ('{0,-35} {1,-10} {2}' -f ('-'*35), ('-'*10), ('-'*24))
|
$events = [System.Collections.Generic.List[PSCustomObject]]::new()
|
||||||
foreach ($ev in $events) {
|
foreach ($line in (Get-Content $JsonlPath -Encoding UTF8 -ErrorAction SilentlyContinue)) {
|
||||||
$ts = if ($ev.ts) { $ev.ts } else { '' }
|
$line = $line.Trim()
|
||||||
Write-Host ('{0,-35} {1,-10} {2}' -f $ev.phase, $ev.status, $ts)
|
if (-not $line) { continue }
|
||||||
}
|
try {
|
||||||
# Print error if present
|
$events.Add(($line | ConvertFrom-Json))
|
||||||
$failEvent = $events | Where-Object { $_.status -eq 'failure' } | Select-Object -Last 1
|
} catch {
|
||||||
if ($failEvent -and $failEvent.data -and $failEvent.data.error) {
|
Write-Debug "[Get-CIJobSummary] Skipping malformed JSONL line in $JsonlPath"
|
||||||
Write-Host "`nError: $($failEvent.data.error)"
|
}
|
||||||
}
|
}
|
||||||
exit 0
|
return $events
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Scan all job JSONL files ─────────────────────────────────────────────────
|
# ── Validate log dir ────────────────────────────────────────────────────────
|
||||||
$jsonlFiles = Get-ChildItem -Path $LogDir -Recurse -Filter 'invoke-ci.jsonl' -ErrorAction SilentlyContinue |
|
if (-not (Test-Path $LogDir -PathType Container)) {
|
||||||
Sort-Object { $_.LastWriteTime } -Descending
|
Write-Warning "Log directory not found: $LogDir"
|
||||||
|
exit 1
|
||||||
if ($jsonlFiles.Count -eq 0) {
|
}
|
||||||
Write-Host "No invoke-ci.jsonl files found under $LogDir"
|
|
||||||
exit 0
|
# ── Single-job detail mode ──────────────────────────────────────────────────
|
||||||
}
|
if ($JobId -ne '') {
|
||||||
|
$jsonlPath = Join-Path $LogDir (Join-Path $JobId 'invoke-ci.jsonl')
|
||||||
$rows = [System.Collections.Generic.List[PSCustomObject]]::new()
|
if (-not (Test-Path $jsonlPath)) {
|
||||||
|
Write-Warning "No JSONL log found for job '$JobId' at: $jsonlPath"
|
||||||
foreach ($file in $jsonlFiles) {
|
exit 1
|
||||||
$events = Read-JobEvents -JsonlPath $file.FullName
|
}
|
||||||
if ($events.Count -eq 0) { continue }
|
$events = Read-JobEvents -JsonlPath $jsonlPath
|
||||||
|
if ($events.Count -eq 0) {
|
||||||
$jobEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'start' } | Select-Object -First 1
|
Write-Warning "JSONL file is empty or unreadable: $jsonlPath"
|
||||||
$successEvent= $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'success' } | Select-Object -First 1
|
exit 1
|
||||||
$failEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'failure' } | Select-Object -First 1
|
}
|
||||||
|
Write-Host "`nJob: $JobId"
|
||||||
$jId = if ($events[0].jobId) { $events[0].jobId } else { (Split-Path (Split-Path $file.FullName -Parent) -Leaf) }
|
Write-Host ('=' * 60)
|
||||||
$started = if ($jobEvent -and $jobEvent.ts) { $jobEvent.ts } else { '' }
|
Write-Host ('{0,-35} {1,-10} {2}' -f 'Phase', 'Status', 'Timestamp')
|
||||||
|
Write-Host ('{0,-35} {1,-10} {2}' -f ('-'*35), ('-'*10), ('-'*24))
|
||||||
if ($successEvent) {
|
foreach ($ev in $events) {
|
||||||
$status = 'success'
|
$ts = if ($ev.ts) { $ev.ts } else { '' }
|
||||||
$elSec = if ($successEvent.data -and $null -ne $successEvent.data.elapsedSec) { [int]$successEvent.data.elapsedSec } else { -1 }
|
Write-Host ('{0,-35} {1,-10} {2}' -f $ev.phase, $ev.status, $ts)
|
||||||
$errMsg = ''
|
}
|
||||||
} elseif ($failEvent) {
|
# Print error if present
|
||||||
$status = 'FAILED'
|
$failEvent = $events | Where-Object { $_.status -eq 'failure' } | Select-Object -Last 1
|
||||||
$elSec = if ($failEvent.data -and $null -ne $failEvent.data.elapsedSec) { [int]$failEvent.data.elapsedSec } else { -1 }
|
if ($failEvent -and $failEvent.data -and $failEvent.data.error) {
|
||||||
$errMsg = if ($failEvent.data -and $failEvent.data.error) { "$($failEvent.data.error)" } else { '' }
|
Write-Host "`nError: $($failEvent.data.error)"
|
||||||
# Truncate error for table display
|
}
|
||||||
if ($errMsg.Length -gt 60) { $errMsg = $errMsg.Substring(0, 57) + '...' }
|
exit 0
|
||||||
} else {
|
}
|
||||||
$status = 'in-progress'
|
|
||||||
$elSec = -1
|
# ── Scan all job JSONL files ─────────────────────────────────────────────────
|
||||||
$errMsg = ''
|
$jsonlFiles = Get-ChildItem -Path $LogDir -Recurse -Filter 'invoke-ci.jsonl' -ErrorAction SilentlyContinue |
|
||||||
}
|
Sort-Object { $_.LastWriteTime } -Descending
|
||||||
|
|
||||||
$rows.Add([PSCustomObject]@{
|
if ($jsonlFiles.Count -eq 0) {
|
||||||
JobId = $jId
|
Write-Host "No invoke-ci.jsonl files found under $LogDir"
|
||||||
Status = $status
|
exit 0
|
||||||
Elapsed = Format-SecToHMS -Seconds $elSec
|
}
|
||||||
Started = $started
|
|
||||||
Error = $errMsg
|
$rows = [System.Collections.Generic.List[PSCustomObject]]::new()
|
||||||
})
|
|
||||||
}
|
foreach ($file in $jsonlFiles) {
|
||||||
|
$events = Read-JobEvents -JsonlPath $file.FullName
|
||||||
# ── Apply filters + limit ────────────────────────────────────────────────────
|
if ($events.Count -eq 0) { continue }
|
||||||
if ($Failed) {
|
|
||||||
$rows = @($rows | Where-Object { $_.Status -eq 'FAILED' })
|
$jobEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'start' } | Select-Object -First 1
|
||||||
}
|
$successEvent= $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'success' } | Select-Object -First 1
|
||||||
|
$failEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'failure' } | Select-Object -First 1
|
||||||
$rows = @($rows | Select-Object -First $Last)
|
|
||||||
|
$jId = if ($events[0].jobId) { $events[0].jobId } else { (Split-Path (Split-Path $file.FullName -Parent) -Leaf) }
|
||||||
if ($rows.Count -eq 0) {
|
$started = if ($jobEvent -and $jobEvent.ts) { $jobEvent.ts } else { '' }
|
||||||
Write-Host "No matching jobs found."
|
|
||||||
exit 0
|
if ($successEvent) {
|
||||||
}
|
$status = 'success'
|
||||||
|
$elSec = if ($successEvent.data -and $null -ne $successEvent.data.elapsedSec) { [int]$successEvent.data.elapsedSec } else { -1 }
|
||||||
# ── Print table ──────────────────────────────────────────────────────────────
|
$errMsg = ''
|
||||||
Write-Host "`nCI Job Summary (last $($rows.Count) jobs):"
|
} elseif ($failEvent) {
|
||||||
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f 'JobId', 'Status', 'Elapsed', 'Started', 'Error')
|
$status = 'FAILED'
|
||||||
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f ('-'*40), ('-'*12), ('-'*10), ('-'*26), ('-'*30))
|
$elSec = if ($failEvent.data -and $null -ne $failEvent.data.elapsedSec) { [int]$failEvent.data.elapsedSec } else { -1 }
|
||||||
foreach ($r in $rows) {
|
$errMsg = if ($failEvent.data -and $failEvent.data.error) { "$($failEvent.data.error)" } else { '' }
|
||||||
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f $r.JobId, $r.Status, $r.Elapsed, $r.Started, $r.Error)
|
# Truncate error for table display
|
||||||
}
|
if ($errMsg.Length -gt 60) { $errMsg = $errMsg.Substring(0, 57) + '...' }
|
||||||
Write-Host ''
|
} else {
|
||||||
|
$status = 'in-progress'
|
||||||
|
$elSec = -1
|
||||||
|
$errMsg = ''
|
||||||
|
}
|
||||||
|
|
||||||
|
$rows.Add([PSCustomObject]@{
|
||||||
|
JobId = $jId
|
||||||
|
Status = $status
|
||||||
|
Elapsed = Format-SecToHMS -Seconds $elSec
|
||||||
|
Started = $started
|
||||||
|
Error = $errMsg
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Apply filters + limit ────────────────────────────────────────────────────
|
||||||
|
if ($Failed) {
|
||||||
|
$rows = @($rows | Where-Object { $_.Status -eq 'FAILED' })
|
||||||
|
}
|
||||||
|
|
||||||
|
$rows = @($rows | Select-Object -First $Last)
|
||||||
|
|
||||||
|
if ($rows.Count -eq 0) {
|
||||||
|
Write-Host "No matching jobs found."
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Print table ──────────────────────────────────────────────────────────────
|
||||||
|
Write-Host "`nCI Job Summary (last $($rows.Count) jobs):"
|
||||||
|
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f 'JobId', 'Status', 'Elapsed', 'Started', 'Error')
|
||||||
|
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f ('-'*40), ('-'*12), ('-'*10), ('-'*26), ('-'*30))
|
||||||
|
foreach ($r in $rows) {
|
||||||
|
Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f $r.JobId, $r.Status, $r.Elapsed, $r.Started, $r.Error)
|
||||||
|
}
|
||||||
|
Write-Host ''
|
||||||
|
|||||||
+37
-691
@@ -1,691 +1,37 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Main CI orchestrator: creates an ephemeral VM, runs a build, collects
|
|
||||||
artifacts, and destroys the VM — in a guaranteed try/finally pattern.
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
|
# Original PS implementation moved to git history; see plans/A4-closeout.md.
|
||||||
.DESCRIPTION
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
This script is called by act_runner as the primary build step.
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
It coordinates all sub-scripts:
|
$commonWithValue = @{
|
||||||
1. New-BuildVM.ps1 — linked clone from template
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
2. vmrun start — start the clone
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
3. Wait-VMReady.ps1 — poll until WinRM is ready
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
4. Invoke-RemoteBuild.ps1 — execute build inside VM
|
pipelinevariable = $true
|
||||||
5. Get-BuildArtifacts.ps1 — copy artifacts to host
|
}
|
||||||
6. Remove-BuildVM.ps1 — destroy VM (always runs via finally)
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
|
|
||||||
Exit codes:
|
$pyArgs = @()
|
||||||
0 = success
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
1 = build or orchestration failure (VM destroyed, artifacts may be partial)
|
$a = $args[$i]
|
||||||
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
.PARAMETER JobId
|
$name = $a.Substring(1)
|
||||||
Unique job identifier (use ${{ github.run_id }} from Gitea Actions).
|
$lname = $name.ToLower()
|
||||||
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
.PARAMETER RepoUrl
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
Git URL of the repository to build (reachable from the build VM).
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
.PARAMETER Branch
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
Branch to build.
|
}
|
||||||
|
else {
|
||||||
.PARAMETER Commit
|
$pyArgs += $a
|
||||||
Optional specific commit SHA to check out.
|
}
|
||||||
|
}
|
||||||
.PARAMETER Configuration
|
|
||||||
Build configuration. Default: Release
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
.PARAMETER TemplatePath
|
& $venvPython -m ci_orchestrator job @pyArgs
|
||||||
Full path to the template VM .vmx file.
|
exit $LASTEXITCODE
|
||||||
Can also be set via env var GITEA_CI_TEMPLATE_PATH.
|
|
||||||
|
|
||||||
.PARAMETER SnapshotName
|
|
||||||
Template snapshot to clone from. Default: BaseClean
|
|
||||||
|
|
||||||
.PARAMETER CloneBaseDir
|
|
||||||
Base directory for clone VMs.
|
|
||||||
Can also be set via env var GITEA_CI_CLONE_BASE_DIR.
|
|
||||||
Default: F:\CI\BuildVMs
|
|
||||||
|
|
||||||
.PARAMETER ArtifactBaseDir
|
|
||||||
Host directory where artifacts are stored per job.
|
|
||||||
Default: F:\CI\Artifacts
|
|
||||||
|
|
||||||
.PARAMETER VMIPAddress
|
|
||||||
IP address to assign/expect for the build VM.
|
|
||||||
Must be unique per concurrent build.
|
|
||||||
Can be passed explicitly or derived from a registry/file-based IP allocator.
|
|
||||||
|
|
||||||
.PARAMETER GuestCredentialTarget
|
|
||||||
Target name in Windows Credential Manager for guest credentials.
|
|
||||||
Default: BuildVMGuest
|
|
||||||
|
|
||||||
.PARAMETER VmrunPath
|
|
||||||
Path to vmrun.exe.
|
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
# Called from Gitea Actions workflow:
|
|
||||||
.\scripts\Invoke-CIJob.ps1 `
|
|
||||||
-JobId "${{ github.run_id }}" `
|
|
||||||
-RepoUrl "http://gitea.local/org/myrepo.git" `
|
|
||||||
-Branch "${{ github.ref_name }}" `
|
|
||||||
-Commit "${{ github.sha }}" `
|
|
||||||
-VMIPAddress "192.168.11.101"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
||||||
[string] $JobId,
|
|
||||||
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $RepoUrl,
|
|
||||||
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $Branch,
|
|
||||||
|
|
||||||
[string] $Commit = '',
|
|
||||||
|
|
||||||
[string] $Configuration = 'Release',
|
|
||||||
|
|
||||||
# Pass -Submodules to clone with --recurse-submodules
|
|
||||||
[switch] $Submodules,
|
|
||||||
|
|
||||||
# Custom build command to run inside the VM (empty = dotnet build)
|
|
||||||
[string] $BuildCommand = '',
|
|
||||||
|
|
||||||
# Artifact source pattern relative to GuestWorkDir (used with BuildCommand)
|
|
||||||
[string] $GuestArtifactSource = 'dist',
|
|
||||||
|
|
||||||
# Clone repo IN the VM instead of on host (requires Git in template §6.6)
|
|
||||||
# Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone.
|
|
||||||
# Prerequisite: Git for Windows installed in template.
|
|
||||||
[switch] $UseGitClone,
|
|
||||||
|
|
||||||
# Use VMware shared folders for NuGet/pip cache (§SharedCache).
|
|
||||||
# Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
|
|
||||||
[switch] $UseSharedCache,
|
|
||||||
|
|
||||||
# Skip artifact packaging (Invoke-RemoteBuild) and collection (Phase 6).
|
|
||||||
# Use for jobs that intentionally produce no output (lint, test-only, etc.).
|
|
||||||
# When absent, the build must produce GuestArtifactSource or the job fails.
|
|
||||||
[switch] $SkipArtifact,
|
|
||||||
|
|
||||||
# Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
|
|
||||||
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
|
||||||
# Ignored when -UseGitClone is not set.
|
|
||||||
[string] $GiteaCredentialTarget = 'GiteaPAT',
|
|
||||||
|
|
||||||
[string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,
|
|
||||||
|
|
||||||
[string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),
|
|
||||||
|
|
||||||
[string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),
|
|
||||||
|
|
||||||
[string] $ArtifactBaseDir = 'F:\CI\Artifacts',
|
|
||||||
|
|
||||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
||||||
[string] $VMIPAddress = '', # if empty, auto-detected via vmrun getGuestIPAddress
|
|
||||||
|
|
||||||
[string] $GuestCredentialTarget = 'BuildVMGuest',
|
|
||||||
|
|
||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
|
||||||
|
|
||||||
# Directory for job execution logs. Each job gets its own sub-directory:
|
|
||||||
# $LogDir\$JobId\invoke-ci.log
|
|
||||||
[string] $LogDir = 'F:\CI\Logs',
|
|
||||||
|
|
||||||
# Days to retain job logs. Directories older than this are purged at job end.
|
|
||||||
# Set to 0 to disable retention/purge.
|
|
||||||
[int] $LogRetentionDays = 30,
|
|
||||||
|
|
||||||
# Guest OS type — determines transport (WinRM for Windows, SSH for Linux).
|
|
||||||
# When 'Auto', detected from the VMX guestOS field after clone.
|
|
||||||
[ValidateSet('Windows', 'Linux', 'Auto')]
|
|
||||||
[string] $GuestOS = 'Auto',
|
|
||||||
|
|
||||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
|
||||||
[string] $SshUser = 'ci_build',
|
|
||||||
|
|
||||||
# Persistent known_hosts file used for CI-job SSH calls.
|
|
||||||
# Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL).
|
|
||||||
# Ephemeral CI VMs on a local NAT network do not need strict host-key verification.
|
|
||||||
# Pass a file path only if you need accept-new mode for a specific deployment.
|
|
||||||
[string] $SshKnownHostsFile = '',
|
|
||||||
|
|
||||||
# Extra environment variables injected into the guest VM before the build command (§6.5).
|
|
||||||
# Keys must be valid environment variable names. Values must be plain strings.
|
|
||||||
# Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
|
|
||||||
[hashtable] $ExtraGuestEnv = @{},
|
|
||||||
|
|
||||||
# Optional webhook URL (Discord/Gitea). When set, a background job fires a
|
|
||||||
# [WARNING] once the job has been running for 90 minutes.
|
|
||||||
[string] $WebhookUrl = '',
|
|
||||||
|
|
||||||
# Optional per-job VMX CPU and RAM override (default 0 = use template value).
|
|
||||||
# Applied to the linked clone after New-BuildVM.ps1, before vmrun start.
|
|
||||||
# The template VMX is never modified.
|
|
||||||
# Example: -GuestCPU 4 -GuestMemoryMB 8192
|
|
||||||
[ValidateRange(0, 32)]
|
|
||||||
[int] $GuestCPU = 0,
|
|
||||||
|
|
||||||
[ValidateRange(0, 131072)]
|
|
||||||
[int] $GuestMemoryMB = 0
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
|
|
||||||
# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
|
|
||||||
$scriptDir = $PSScriptRoot
|
|
||||||
|
|
||||||
# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
|
|
||||||
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
|
|
||||||
|
|
||||||
# ── Validate required config ──────────────────────────────────────────────────
|
|
||||||
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
|
|
||||||
Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
|
|
||||||
Write-Error "Template VMX not found: $TemplatePath"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
# Sanity check: warn if template is outside the expected CI tree
|
|
||||||
if ($TemplatePath -notlike 'F:\CI\Templates\*') {
|
|
||||||
Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional."
|
|
||||||
}
|
|
||||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
||||||
Write-Error "vmrun.exe not found: $VmrunPath"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
|
|
||||||
foreach ($key in $ExtraGuestEnv.Keys) {
|
|
||||||
if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
|
|
||||||
Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Load guest credentials from Windows Credential Manager ───────────────────
|
|
||||||
# Requires the CredentialManager module: Install-Module CredentialManager
|
|
||||||
# In a lab setup you can fall back to prompting, but never hardcode credentials.
|
|
||||||
$credential = $null
|
|
||||||
try {
|
|
||||||
$credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop
|
|
||||||
Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)."
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)."
|
|
||||||
Write-Warning " Using interactive prompt as fallback. Store credentials with:"
|
|
||||||
Write-Warning " cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password"
|
|
||||||
$credential = Get-Credential -Message "Enter guest VM credentials for job $JobId"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Setup paths ───────────────────────────────────────────────────────────────
|
|
||||||
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
|
|
||||||
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
|
|
||||||
$cloneVmxPath = $null
|
|
||||||
$leaseFile = $null # §2.1 IP lease file path — released in finally
|
|
||||||
$jsonLog = $null # §4.1 JSONL structured log path
|
|
||||||
$startTime = Get-Date
|
|
||||||
|
|
||||||
# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
|
|
||||||
$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new()
|
|
||||||
$phaseStart = $startTime
|
|
||||||
|
|
||||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
|
||||||
$transcriptStarted = $false
|
|
||||||
try {
|
|
||||||
$jobLogDir = Join-Path $LogDir $JobId
|
|
||||||
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
|
|
||||||
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
|
|
||||||
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
|
|
||||||
# -Append allows starting a new transcript even when VS Code (or another host)
|
|
||||||
# has already opened a transcript on this PowerShell session.
|
|
||||||
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
|
|
||||||
$transcriptStarted = $true
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── §4.1 Structured event emitter ────────────────────────────────────────────
|
|
||||||
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
|
|
||||||
# Silent no-op if $jsonLog is null (transcript setup failed).
|
|
||||||
function Write-JobEvent {
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)] [string] $Phase,
|
|
||||||
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
|
|
||||||
[hashtable] $Data = @{}
|
|
||||||
)
|
|
||||||
if (-not $script:jsonLog) { return }
|
|
||||||
$logEntry = [ordered]@{
|
|
||||||
ts = (Get-Date -Format 'o')
|
|
||||||
jobId = $script:JobId
|
|
||||||
phase = $Phase
|
|
||||||
status = $Status
|
|
||||||
data = $Data
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$logEntry | ConvertTo-Json -Compress -Depth 5 |
|
|
||||||
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
|
|
||||||
} catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" } # never let logging fail the build
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] Job : $JobId"
|
|
||||||
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
|
|
||||||
Write-Host "[Invoke-CIJob] Branch : $Branch"
|
|
||||||
if ($Commit) { Write-Host "[Invoke-CIJob] Commit : $Commit" }
|
|
||||||
Write-Host "[Invoke-CIJob] Build VM IP : $VMIPAddress"
|
|
||||||
Write-Host "[Invoke-CIJob] Template : $TemplatePath"
|
|
||||||
Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
|
|
||||||
Write-Host "[Invoke-CIJob] Started : $startTime"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
|
|
||||||
repo = $RepoUrl
|
|
||||||
branch = $Branch
|
|
||||||
commit = $Commit
|
|
||||||
template = $TemplatePath
|
|
||||||
snapshot = $SnapshotName
|
|
||||||
}
|
|
||||||
|
|
||||||
$exitCode = 0
|
|
||||||
$durationWarnJob = $null
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── 90-minute duration warning (fires once via background job) ───────────────
|
|
||||||
if ($WebhookUrl -ne '') {
|
|
||||||
$warnMsg = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck."
|
|
||||||
$warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress)
|
|
||||||
$durationWarnJob = Start-Job -ScriptBlock {
|
|
||||||
param($url, $body)
|
|
||||||
Start-Sleep -Seconds 5400 # 90 minutes
|
|
||||||
try {
|
|
||||||
Invoke-RestMethod -Uri $url -Method Post -Body $body `
|
|
||||||
-ContentType 'application/json' -TimeoutSec 10
|
|
||||||
} catch { }
|
|
||||||
} -ArgumentList $WebhookUrl, $warnBody
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 1: Clone repo (HOST or GUEST) ───────────────────────────────
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# §3.3: Clone in guest VM instead of host. Skip host clone phase.
|
|
||||||
# PAT will be injected at runtime (Phase 5) from Credential Manager.
|
|
||||||
Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..."
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' }
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# Default: Clone on host, zip, transfer to guest via WinRM.
|
|
||||||
Write-Host "`n[Phase 1/6] Cloning repository on host..."
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
|
|
||||||
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
|
|
||||||
$gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch)
|
|
||||||
if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') }
|
|
||||||
$gitArgs += @($RepoUrl, $hostCloneDir)
|
|
||||||
Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')"
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
# Stream git stdout/stderr live (otherwise heavy clones with submodules
|
|
||||||
# look frozen for a long time). Tee for diagnostic in case of failure.
|
|
||||||
$gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
||||||
}
|
|
||||||
if ($Commit -ne '') {
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
# Commit not in shallow history (e.g. manual run on older SHA).
|
|
||||||
# Fetch full history and retry.
|
|
||||||
Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..."
|
|
||||||
& git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null
|
|
||||||
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
}
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
|
||||||
}
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
|
||||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
|
||||||
# and produce two VMs with the same DHCP lease. A file-based mutex ensures
|
|
||||||
# only one job is in the clone→start→IP phase at a time. After the IP is
|
|
||||||
# acquired and written to a lease file the lock is released, so build phases
|
|
||||||
# (the slow part) run fully in parallel. Lease is released in finally.
|
|
||||||
$stateDir = 'F:\CI\State'
|
|
||||||
$leasesDir = Join-Path $stateDir 'ip-leases'
|
|
||||||
$lockPath = Join-Path $stateDir 'vm-start.lock'
|
|
||||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
|
||||||
|
|
||||||
# ── Pre-clone disk space gate ─────────────────────────────────────────
|
|
||||||
$cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
|
|
||||||
$cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
|
|
||||||
if ($null -ne $cloneVol) {
|
|
||||||
$freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
|
|
||||||
if ($freeGB -lt 20) {
|
|
||||||
throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
|
||||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
|
||||||
$lockStream = $null
|
|
||||||
$lockDeadline = (Get-Date).AddMinutes(10)
|
|
||||||
while ($true) {
|
|
||||||
try {
|
|
||||||
$lockStream = [System.IO.File]::Open(
|
|
||||||
$lockPath,
|
|
||||||
[System.IO.FileMode]::OpenOrCreate,
|
|
||||||
[System.IO.FileAccess]::ReadWrite,
|
|
||||||
[System.IO.FileShare]::None)
|
|
||||||
break
|
|
||||||
} catch [System.IO.IOException] {
|
|
||||||
if ((Get-Date) -ge $lockDeadline) {
|
|
||||||
throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock acquired."
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── Phase 2: Clone VM ─────────────────────────────────────────────
|
|
||||||
Write-Host "[Phase 2/6] Cloning VM..."
|
|
||||||
$cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
|
|
||||||
-TemplatePath $TemplatePath `
|
|
||||||
-SnapshotName $SnapshotName `
|
|
||||||
-CloneBaseDir $CloneBaseDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-VmrunPath $VmrunPath
|
|
||||||
|
|
||||||
# ── Optional VMX CPU/RAM override (post-clone, pre-start) ─────────────
|
|
||||||
if ($GuestCPU -gt 0 -or $GuestMemoryMB -gt 0) {
|
|
||||||
Write-Host "[Invoke-CIJob] Applying VMX override: CPU=$GuestCPU, RAM=${GuestMemoryMB}MB"
|
|
||||||
$vmxLines = Get-Content -LiteralPath $cloneVmxPath
|
|
||||||
if ($GuestCPU -gt 0) {
|
|
||||||
if ($vmxLines -imatch '^numvcpus\s*=') {
|
|
||||||
$vmxLines = $vmxLines -replace '(?i)^numvcpus\s*=.*', "numvcpus = `"$GuestCPU`""
|
|
||||||
} else {
|
|
||||||
$vmxLines = $vmxLines + "numvcpus = `"$GuestCPU`""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($GuestMemoryMB -gt 0) {
|
|
||||||
if ($vmxLines -imatch '^memsize\s*=') {
|
|
||||||
$vmxLines = $vmxLines -replace '(?i)^memsize\s*=.*', "memsize = `"$GuestMemoryMB`""
|
|
||||||
} else {
|
|
||||||
$vmxLines = $vmxLines + "memsize = `"$GuestMemoryMB`""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Set-Content -LiteralPath $cloneVmxPath -Value $vmxLines -Encoding UTF8
|
|
||||||
Write-Host "[Invoke-CIJob] VMX updated: $cloneVmxPath"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 3: Start VM ─────────────────────────────────────────────
|
|
||||||
Write-Host "`n[Phase 3/6] Starting VM..."
|
|
||||||
$startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
||||||
-Operation start `
|
|
||||||
-Arguments @($cloneVmxPath, 'nogui') `
|
|
||||||
-TimeoutSeconds 180
|
|
||||||
if ($startResult.TimedOut) {
|
|
||||||
throw "vmrun start timed out after 180s for: $cloneVmxPath"
|
|
||||||
}
|
|
||||||
if ($startResult.ExitCode -ne 0) {
|
|
||||||
throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
|
|
||||||
|
|
||||||
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
|
|
||||||
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
||||||
$detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
|
|
||||||
-VmxPath $cloneVmxPath -TimeoutSeconds 120
|
|
||||||
if ($detectedIP -eq '') {
|
|
||||||
throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
|
|
||||||
}
|
|
||||||
$VMIPAddress = $detectedIP
|
|
||||||
Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Lease IP ──────────────────────────────────────────────────────
|
|
||||||
# If two VMs somehow got the same IP (DHCP edge case), fail fast here
|
|
||||||
# rather than letting both jobs write to the same WinRM target.
|
|
||||||
$leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
|
|
||||||
if (Test-Path $leaseFile) {
|
|
||||||
$holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
|
|
||||||
throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
|
|
||||||
}
|
|
||||||
[System.IO.File]::WriteAllText($leaseFile, $JobId)
|
|
||||||
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
|
|
||||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
|
|
||||||
|
|
||||||
} finally {
|
|
||||||
# Release lock — next waiting job can now enter clone→start→IP phase
|
|
||||||
if ($lockStream) {
|
|
||||||
$lockStream.Close()
|
|
||||||
$lockStream.Dispose()
|
|
||||||
$lockStream = $null
|
|
||||||
}
|
|
||||||
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock released."
|
|
||||||
}
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
# ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
|
|
||||||
if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
|
|
||||||
$vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
|
|
||||||
if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') {
|
|
||||||
$detectedOS = $Matches[1]
|
|
||||||
$GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' }
|
|
||||||
Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)"
|
|
||||||
} else {
|
|
||||||
$GuestOS = 'Windows'
|
|
||||||
Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# ── Phase 4: Wait for readiness ───────────────────────────────────────
|
|
||||||
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
|
|
||||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
|
|
||||||
$waitParams = @{
|
|
||||||
VMPath = $cloneVmxPath
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
VmrunPath = $VmrunPath
|
|
||||||
}
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
$waitParams['Transport'] = 'SSH'
|
|
||||||
$waitParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$waitParams['SshUser'] = $SshUser
|
|
||||||
}
|
|
||||||
& "$scriptDir\Wait-VMReady.ps1" @waitParams
|
|
||||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
|
||||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
|
||||||
Write-JobEvent -Phase 'phase5.build' -Status 'start'
|
|
||||||
$remoteBuildParams = @{
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
Credential = $credential
|
|
||||||
Configuration = $Configuration
|
|
||||||
}
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
$remoteBuildParams['GuestOS'] = 'Linux'
|
|
||||||
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$remoteBuildParams['SshUser'] = $SshUser
|
|
||||||
$remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
|
|
||||||
$remoteBuildParams.Remove('Credential')
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
|
|
||||||
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
||||||
$remoteBuildParams['CloneBranch'] = $Branch
|
|
||||||
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
||||||
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
||||||
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
||||||
} else {
|
|
||||||
# Default: host-cloned source shipped to guest via tar+scp
|
|
||||||
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir
|
|
||||||
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
||||||
$remoteBuildParams['CloneBranch'] = $Branch
|
|
||||||
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
||||||
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
||||||
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# Default: Use pre-cloned source from host
|
|
||||||
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
|
||||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
|
||||||
if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv }
|
|
||||||
if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true }
|
|
||||||
if ($SkipArtifact) { $remoteBuildParams['SkipArtifact'] = $true }
|
|
||||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
|
||||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
|
||||||
if ($SkipArtifact) {
|
|
||||||
Write-Host "`n[Phase 6/6] Artifact collection skipped (-SkipArtifact)."
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'skipped'
|
|
||||||
} else {
|
|
||||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
||||||
-IPAddress $VMIPAddress `
|
|
||||||
-GuestOS 'Linux' `
|
|
||||||
-SshKeyPath $SshKeyPath `
|
|
||||||
-SshUser $SshUser `
|
|
||||||
-SshKnownHostsFile $SshKnownHostsFile `
|
|
||||||
-GuestArtifactPath '/opt/ci/output' `
|
|
||||||
-HostArtifactDir $jobArtifactDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-Commit $Commit `
|
|
||||||
-IncludeLogs
|
|
||||||
} else {
|
|
||||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
||||||
-IPAddress $VMIPAddress `
|
|
||||||
-Credential $credential `
|
|
||||||
-GuestArtifactPath 'C:\CI\output\artifacts.zip' `
|
|
||||||
-HostArtifactDir $jobArtifactDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-Commit $Commit `
|
|
||||||
-IncludeLogs
|
|
||||||
}
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
|
||||||
}
|
|
||||||
$elapsed = (Get-Date) - $startTime
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
Write-Host "`n============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] Phase durations:"
|
|
||||||
foreach ($pt in $phaseTimings) {
|
|
||||||
Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
|
|
||||||
}
|
|
||||||
Write-Host (" {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds)
|
|
||||||
Write-Host "============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
|
||||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
$exitCode = 1
|
|
||||||
$elapsed = (Get-Date) - $startTime
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
|
|
||||||
elapsedSec = [int]$elapsed.TotalSeconds
|
|
||||||
error = "$_"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Best-effort guest diagnostics (skip if VM was never started) ─────────
|
|
||||||
# Only collect when a VM IP is known (Phase 3b completed). Never throws.
|
|
||||||
if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
||||||
try {
|
|
||||||
$diagDir = Join-Path $jobArtifactDir 'diagnostics'
|
|
||||||
$diagParams = @{
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
|
|
||||||
DestinationDir = $diagDir
|
|
||||||
}
|
|
||||||
if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
|
|
||||||
$diagParams['Credential'] = $credential
|
|
||||||
}
|
|
||||||
if ($diagParams.GuestOS -eq 'Linux') {
|
|
||||||
$diagParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$diagParams['SshUser'] = $SshUser
|
|
||||||
}
|
|
||||||
Get-GuestDiagnostics @diagParams
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "`n============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
|
||||||
Write-Host "Error: $_"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
# ── Release IP lease (§2.1) ───────────────────────────────────────────
|
|
||||||
if ($leaseFile -and (Test-Path $leaseFile)) {
|
|
||||||
Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Cleanup] IP lease released: $VMIPAddress"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Always destroy the VM ─────────────────────────────────────────────
|
|
||||||
if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
|
|
||||||
Write-Host "`n[Cleanup] Destroying ephemeral VM..."
|
|
||||||
& "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)."
|
|
||||||
}
|
|
||||||
# ── Clean up host-side git clone (unless in-VM clone was used) ────────
|
|
||||||
# When -UseGitClone, $hostCloneDir is never created
|
|
||||||
if (-not $UseGitClone -and (Test-Path $hostCloneDir)) {
|
|
||||||
Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..."
|
|
||||||
Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Cancel 90-minute warning job (build finished before it fired) ──────────
|
|
||||||
if ($null -ne $durationWarnJob) {
|
|
||||||
Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue
|
|
||||||
$durationWarnJob = $null
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Stop transcript ───────────────────────────────────────────────────
|
|
||||||
if ($transcriptStarted) {
|
|
||||||
try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" }
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Log retention: purge directories older than $LogRetentionDays days ─
|
|
||||||
if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) {
|
|
||||||
$cutoff = (Get-Date).AddDays(-$LogRetentionDays)
|
|
||||||
Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue |
|
|
||||||
Where-Object { $_.LastWriteTime -lt $cutoff } |
|
|
||||||
ForEach-Object {
|
|
||||||
Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Cleanup] Purged old log dir: $($_.FullName)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
exit $exitCode
|
|
||||||
|
|||||||
+48
-500
@@ -1,543 +1,91 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Executes a build inside a guest VM (via WinRM). Supports two clone modes:
|
|
||||||
(1) Host-side clone: copy pre-cloned source via WinRM zip, or
|
|
||||||
(2) Guest-side clone: clone repo directly in VM via git (§3.3, requires Git)
|
|
||||||
|
|
||||||
.DESCRIPTION
|
# Shim: delegates to the Python ci_orchestrator CLI (`build run`).
|
||||||
Called by Invoke-CIJob.ps1 to orchestrate the build inside the VM.
|
# Original PS implementation moved to git history; see plans/A3-closeout.md.
|
||||||
|
#
|
||||||
|
# Typed parameters (`[PSCredential]`, `[hashtable]`, `[switch]`) cannot cross
|
||||||
|
# a process boundary as-is; this shim translates them. Credentials are pulled
|
||||||
|
# inside Python via keyring (`--credential-target`), so the bound `-Credential`
|
||||||
|
# is intentionally discarded — callers should set `-CredentialTarget` instead
|
||||||
|
# (or rely on the default `BuildVMGuest` resolved by the Python CLI).
|
||||||
|
|
||||||
**Mode 1: Host-side (default)** — Caller provides pre-cloned source:
|
|
||||||
1. Compresses the source tree on host, copies archive into VM via WinRM
|
|
||||||
2. Runs dotnet restore + dotnet build inside VM
|
|
||||||
3. Packages output as C:\CI\output\artifacts.zip
|
|
||||||
|
|
||||||
**Mode 2: Guest-side (§3.3, -UseGitClone in Invoke-CIJob)** — Repo cloned in VM:
|
|
||||||
1. Clone repo directly into VM via git (eliminates host-zip-transfer overhead)
|
|
||||||
2. Runs dotnet restore + dotnet build inside VM
|
|
||||||
3. Packages output same as Mode 1
|
|
||||||
|
|
||||||
Supply either -HostSourceDir (Mode 1) or -CloneUrl (Mode 2), not both.
|
|
||||||
|
|
||||||
In Mode 2, PAT for private repos is read from Credential Manager inside
|
|
||||||
this script's WinRM session (§1.5: no argv, no log, cleanup at end).
|
|
||||||
|
|
||||||
.PARAMETER IPAddress
|
|
||||||
IP of the build VM (e.g. 192.168.11.101).
|
|
||||||
|
|
||||||
.PARAMETER Credential
|
|
||||||
PSCredential for the build user inside the VM.
|
|
||||||
Retrieve from Windows Credential Manager in production:
|
|
||||||
$cred = Get-StoredCredential -Target 'BuildVMGuest'
|
|
||||||
|
|
||||||
.PARAMETER HostSourceDir
|
|
||||||
[Mode 1] Full path on HOST to already-cloned repository (host-side clone mode).
|
|
||||||
Mutually exclusive with -CloneUrl. When omitted, -CloneUrl must be provided.
|
|
||||||
Example: C:\Users\user\AppData\Local\Temp\ci-src-run-42
|
|
||||||
|
|
||||||
.PARAMETER CloneUrl
|
|
||||||
[Mode 2, §3.3] Git URL to clone directly in VM (guest-side clone mode).
|
|
||||||
Mutually exclusive with -HostSourceDir. When provided, repo cloned via git inside VM.
|
|
||||||
PAT (if needed) read from Credential Manager at runtime.
|
|
||||||
Example: https://gitea.emulab.it/Simone/myrepo.git
|
|
||||||
|
|
||||||
.PARAMETER CloneBranch
|
|
||||||
[Mode 2] Branch to clone. Ignored in Mode 1. Default: main
|
|
||||||
|
|
||||||
.PARAMETER CloneCommit
|
|
||||||
[Mode 2] Specific commit SHA to checkout after clone. Ignored in Mode 1.
|
|
||||||
|
|
||||||
.PARAMETER CloneSubmodules
|
|
||||||
[Mode 2] Pass --recurse-submodules to git clone. Ignored in Mode 1.
|
|
||||||
|
|
||||||
.PARAMETER Configuration
|
|
||||||
MSBuild/dotnet build configuration. Default: Release
|
|
||||||
|
|
||||||
.PARAMETER GuestWorkDir
|
|
||||||
Working directory inside the VM where source will be placed.
|
|
||||||
Default: C:\CI\build
|
|
||||||
|
|
||||||
.PARAMETER GuestArtifactZip
|
|
||||||
Full path inside the VM where the artifact archive will be written.
|
|
||||||
Default: C:\CI\output\artifacts.zip
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
$cred = Get-Credential
|
|
||||||
.\Invoke-RemoteBuild.ps1 `
|
|
||||||
-IPAddress "192.168.11.101" `
|
|
||||||
-Credential $cred `
|
|
||||||
-HostSourceDir "C:\Temp\ci-src-run-42"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)]
|
[Parameter(Mandatory)]
|
||||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
||||||
[string] $IPAddress,
|
[string] $IPAddress,
|
||||||
|
|
||||||
[Parameter()]
|
[Parameter()]
|
||||||
[System.Management.Automation.PSCredential] $Credential,
|
[System.Management.Automation.PSCredential] $Credential,
|
||||||
|
|
||||||
# [Mode 1] Host-side clone — pre-cloned source directory on HOST
|
|
||||||
[ValidateScript({
|
|
||||||
if ($_ -and -not (Test-Path $_ -PathType Container)) {
|
|
||||||
throw "HostSourceDir path does not exist: $_"
|
|
||||||
}
|
|
||||||
return $true
|
|
||||||
})]
|
|
||||||
[string] $HostSourceDir = '',
|
[string] $HostSourceDir = '',
|
||||||
|
|
||||||
# [Mode 2, §3.3] Guest-side clone — Git URL for in-VM clone
|
|
||||||
[string] $CloneUrl = '',
|
[string] $CloneUrl = '',
|
||||||
|
|
||||||
# [Mode 2] Branch to clone (default: main). Ignored if HostSourceDir provided.
|
|
||||||
[string] $CloneBranch = 'main',
|
[string] $CloneBranch = 'main',
|
||||||
|
|
||||||
# [Mode 2] Specific commit SHA to checkout. Ignored if HostSourceDir provided.
|
|
||||||
[string] $CloneCommit = '',
|
[string] $CloneCommit = '',
|
||||||
|
|
||||||
# [Mode 2] Clone with --recurse-submodules. Ignored if HostSourceDir provided.
|
|
||||||
[switch] $CloneSubmodules,
|
[switch] $CloneSubmodules,
|
||||||
|
|
||||||
# [Mode 2] Credential Manager target for Gitea PAT (private repos).
|
|
||||||
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
|
||||||
# Leave empty for public repos (no auth needed).
|
|
||||||
[string] $GiteaCredentialTarget = 'GiteaPAT',
|
[string] $GiteaCredentialTarget = 'GiteaPAT',
|
||||||
|
|
||||||
[string] $Configuration = 'Release',
|
[string] $Configuration = 'Release',
|
||||||
|
|
||||||
# Custom build command to run inside the VM (working dir = GuestWorkDir).
|
|
||||||
# When empty, defaults to: dotnet restore + dotnet build --configuration $Configuration
|
|
||||||
# Example: 'python build_plugin.py --final'
|
|
||||||
[string] $BuildCommand = '',
|
[string] $BuildCommand = '',
|
||||||
|
|
||||||
# Glob pattern (relative to GuestWorkDir) of files to include in the artifact zip.
|
|
||||||
# Used only when BuildCommand is set. Default: collect everything under 'dist'.
|
|
||||||
# Example: 'dist\**' or 'Contrib\nsInnoUnp\Build\**'
|
|
||||||
[string] $GuestArtifactSource = 'dist',
|
[string] $GuestArtifactSource = 'dist',
|
||||||
|
|
||||||
[string] $GuestWorkDir = 'C:\CI\build',
|
[string] $GuestWorkDir = 'C:\CI\build',
|
||||||
|
|
||||||
[string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip',
|
[string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip',
|
||||||
|
|
||||||
# Redirect dotnet NuGet package store and pip download cache to VMware shared
|
|
||||||
# folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip).
|
|
||||||
# Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1
|
|
||||||
# and VMware Tools HGFS driver present in the guest.
|
|
||||||
[switch] $UseSharedCache,
|
[switch] $UseSharedCache,
|
||||||
|
|
||||||
# Skip artifact packaging entirely. Use for jobs that produce no output.
|
|
||||||
# When absent, missing GuestArtifactSource after a successful build is an error.
|
|
||||||
[switch] $SkipArtifact,
|
[switch] $SkipArtifact,
|
||||||
|
|
||||||
# Guest OS type — Linux uses SSH instead of WinRM.
|
|
||||||
[ValidateSet('Windows', 'Linux')]
|
[ValidateSet('Windows', 'Linux')]
|
||||||
[string] $GuestOS = 'Windows',
|
[string] $GuestOS = 'Windows',
|
||||||
|
|
||||||
# SSH private key path (used when GuestOS=Linux).
|
|
||||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||||
|
|
||||||
# SSH username (used when GuestOS=Linux).
|
|
||||||
[string] $SshUser = 'ci_build',
|
[string] $SshUser = 'ci_build',
|
||||||
|
|
||||||
# Persistent known_hosts file for SSH calls (CI jobs).
|
|
||||||
# Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL).
|
|
||||||
[string] $SshKnownHostsFile = '',
|
[string] $SshKnownHostsFile = '',
|
||||||
|
|
||||||
# Work directory inside the Linux guest.
|
|
||||||
[string] $GuestLinuxWorkDir = '/opt/ci/build',
|
[string] $GuestLinuxWorkDir = '/opt/ci/build',
|
||||||
|
|
||||||
# Output directory inside the Linux guest.
|
|
||||||
[string] $GuestLinuxOutputDir = '/opt/ci/output',
|
[string] $GuestLinuxOutputDir = '/opt/ci/output',
|
||||||
|
|
||||||
# Extra environment variables injected into the guest before the build command (§6.5).
|
[hashtable] $ExtraGuestEnv = @{},
|
||||||
# Keys must be valid env var names. Values are plain strings (never logged).
|
|
||||||
[hashtable] $ExtraGuestEnv = @{}
|
# Optional override for the Credential Manager target (overrides any
|
||||||
|
# fallback resolved by Python). Discards $Credential.
|
||||||
|
[string] $CredentialTarget = ''
|
||||||
)
|
)
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
$pyArgs = @(
|
||||||
$ErrorActionPreference = 'Stop'
|
'--ip-address', $IPAddress,
|
||||||
|
'--guest-os', $GuestOS.ToLower(),
|
||||||
|
'--ssh-user', $SshUser,
|
||||||
|
'--ssh-key-path', $SshKeyPath,
|
||||||
|
'--clone-branch', $CloneBranch,
|
||||||
|
'--gitea-credential-target', $GiteaCredentialTarget,
|
||||||
|
'--configuration', $Configuration,
|
||||||
|
'--guest-work-dir', $GuestWorkDir,
|
||||||
|
'--guest-artifact-zip', $GuestArtifactZip,
|
||||||
|
'--guest-linux-work-dir', $GuestLinuxWorkDir,
|
||||||
|
'--guest-linux-output-dir', $GuestLinuxOutputDir,
|
||||||
|
'--guest-artifact-source', $GuestArtifactSource
|
||||||
|
)
|
||||||
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
if ($HostSourceDir) { $pyArgs += @('--host-source-dir', $HostSourceDir) }
|
||||||
|
if ($CloneUrl) { $pyArgs += @('--clone-url', $CloneUrl) }
|
||||||
|
if ($CloneCommit) { $pyArgs += @('--clone-commit', $CloneCommit) }
|
||||||
|
if ($BuildCommand) { $pyArgs += @('--build-command', $BuildCommand) }
|
||||||
|
if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) }
|
||||||
|
if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) }
|
||||||
|
|
||||||
# ── Validate mutually exclusive modes ─────────────────────────────────────
|
if ($CloneSubmodules.IsPresent) { $pyArgs += '--clone-submodules' }
|
||||||
$hostCloneMode = -not [string]::IsNullOrWhiteSpace($HostSourceDir)
|
if ($UseSharedCache.IsPresent) { $pyArgs += '--use-shared-cache' }
|
||||||
$guestCloneMode = -not [string]::IsNullOrWhiteSpace($CloneUrl)
|
if ($SkipArtifact.IsPresent) { $pyArgs += '--skip-artifact' }
|
||||||
if ($hostCloneMode -and $guestCloneMode) {
|
|
||||||
throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone), not both."
|
|
||||||
}
|
|
||||||
if (-not $hostCloneMode -and -not $guestCloneMode) {
|
|
||||||
throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone)."
|
|
||||||
}
|
|
||||||
if ($guestCloneMode -and -not $CloneBranch) {
|
|
||||||
throw "CloneBranch is required when using -CloneUrl mode."
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($GuestOS -eq 'Windows' -and $null -eq $Credential) {
|
if ($ExtraGuestEnv -and $ExtraGuestEnv.Count -gt 0) {
|
||||||
throw "Credential is required when GuestOS is Windows."
|
foreach ($key in $ExtraGuestEnv.Keys) {
|
||||||
}
|
$pyArgs += @('--extra-env', "$key=$($ExtraGuestEnv[$key])")
|
||||||
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force
|
|
||||||
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Linux build mode — SSH transport"
|
|
||||||
|
|
||||||
# Step 1: Clean work dir (parent permissions handled by template setup)
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Command "rm -rf '$GuestLinuxWorkDir' && mkdir -p '$GuestLinuxWorkDir'"
|
|
||||||
|
|
||||||
if ($hostCloneMode) {
|
|
||||||
# ── Mode 1 (Linux): tar host source -> scp -> untar in guest ──
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Packaging host source ($HostSourceDir) ..."
|
|
||||||
$tarName = "ci-src-$([Guid]::NewGuid().ToString('N').Substring(0,8)).tar.gz"
|
|
||||||
$hostTar = Join-Path $env:TEMP $tarName
|
|
||||||
$guestTar = "/tmp/$tarName"
|
|
||||||
if (Test-Path $hostTar) { Remove-Item $hostTar -Force }
|
|
||||||
# bsdtar on Windows 10/11 supports -C and gz natively.
|
|
||||||
& tar -czf $hostTar -C $HostSourceDir .
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "[Invoke-RemoteBuild] tar failed packaging $HostSourceDir (exit $LASTEXITCODE)"
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Copying source archive to guest:$guestTar ..."
|
|
||||||
Copy-SshItem -Source $hostTar -Destination $guestTar `
|
|
||||||
-IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile -Direction ToGuest
|
|
||||||
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Extracting source in guest ($GuestLinuxWorkDir) ..."
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Command "tar -xzf '$guestTar' -C '$GuestLinuxWorkDir' && rm -f '$guestTar'"
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
if (Test-Path $hostTar) { Remove-Item $hostTar -Force -ErrorAction SilentlyContinue }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# ── Mode 2 (Linux): in-guest git clone ──
|
|
||||||
$cloneCmd = "git clone --depth 1 --branch '$CloneBranch'"
|
|
||||||
if ($CloneSubmodules) { $cloneCmd += ' --recurse-submodules' }
|
|
||||||
|
|
||||||
$cloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" # non-authed URL for logging
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Cloning $CloneUrl into guest $GuestLinuxWorkDir ..."
|
|
||||||
|
|
||||||
# PAT injection via git http.extraHeader — PAT never appears in argv or /proc/environ
|
|
||||||
$authHeader = ''
|
|
||||||
if ($GiteaCredentialTarget -ne '') {
|
|
||||||
try {
|
|
||||||
Import-Module CredentialManager -ErrorAction Stop
|
|
||||||
$credObj = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
|
|
||||||
if ($credObj) {
|
|
||||||
$b64 = [Convert]::ToBase64String(
|
|
||||||
[System.Text.Encoding]::UTF8.GetBytes(
|
|
||||||
"$($credObj.UserName):$($credObj.GetNetworkCredential().Password)"))
|
|
||||||
$authHeader = "Authorization: Basic $b64"
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[Invoke-RemoteBuild] Could not load PAT from Credential Manager ($GiteaCredentialTarget) — cloning without auth (public repo?)."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($authHeader -ne '') {
|
|
||||||
$authCloneCmd = "git -c credential.helper= -c 'http.extraHeader=$authHeader' clone --depth 1 --branch '$CloneBranch'"
|
|
||||||
if ($CloneSubmodules) { $authCloneCmd += ' --recurse-submodules' }
|
|
||||||
$authCloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'"
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile -Command $authCloneCmd
|
|
||||||
} else {
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile -Command $cloneCmd
|
|
||||||
}
|
|
||||||
|
|
||||||
# Checkout specific commit if requested
|
|
||||||
if ($CloneCommit -ne '') {
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Command "git -C '$GuestLinuxWorkDir' checkout '$CloneCommit'"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Step 3: Run build command (always cd into workdir first)
|
|
||||||
# Prepend extra env exports (§6.5) — scoped to this SSH command only
|
|
||||||
$envPrefix = ''
|
|
||||||
foreach ($envKey in $ExtraGuestEnv.Keys) {
|
|
||||||
$envVal = $ExtraGuestEnv[$envKey] -replace "'", "'\''" # escape ' for POSIX single-quoted string
|
|
||||||
$envPrefix += "export $envKey='$envVal'; "
|
|
||||||
}
|
|
||||||
if ($BuildCommand -ne '') {
|
|
||||||
$buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && $BuildCommand"
|
|
||||||
} else {
|
|
||||||
$buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && make"
|
|
||||||
}
|
|
||||||
$displayCmd = if ($BuildCommand -ne '') { $BuildCommand } else { 'make' }
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Running build (env vars redacted): cd '$GuestLinuxWorkDir' && $displayCmd"
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile -Command $buildCmd
|
|
||||||
|
|
||||||
# Step 4: Stage artifact tree under $GuestLinuxOutputDir for SCP collection
|
|
||||||
$outDir = $GuestLinuxOutputDir
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Command "rm -rf '$outDir' && mkdir -p '$outDir'"
|
|
||||||
|
|
||||||
$artSrc = if ($GuestArtifactSource -ne '') { $GuestArtifactSource } else { 'dist' }
|
|
||||||
Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath `
|
|
||||||
-KnownHostsFile $SshKnownHostsFile `
|
|
||||||
-Command "if [ -d '$GuestLinuxWorkDir/$artSrc' ]; then cp -r '$GuestLinuxWorkDir/$artSrc/.' '$outDir/'; elif [ -e '$GuestLinuxWorkDir/$artSrc' ]; then cp '$GuestLinuxWorkDir/$artSrc' '$outDir/'; else echo '[Invoke-RemoteBuild] WARNING: artifact source not found: $artSrc' >&2; fi"
|
|
||||||
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Linux build complete."
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
function Compress-BuildArtifact {
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $Path,
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $DestinationPath
|
|
||||||
)
|
|
||||||
$7zip = 'C:\Program Files\7-Zip\7z.exe'
|
|
||||||
if (Test-Path $7zip) {
|
|
||||||
Write-Host "[Compress-BuildArtifact] Using 7-Zip (multi-threaded)..."
|
|
||||||
& $7zip a -mmt=on -mx1 $DestinationPath $Path 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ }
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "7-Zip compression failed (exit $LASTEXITCODE)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "[Compress-BuildArtifact] 7-Zip not found, using Compress-Archive (fallback)..."
|
|
||||||
Compress-Archive -Path $Path -DestinationPath $DestinationPath -CompressionLevel Fastest -Force
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$sessionOptions = New-CISessionOption
|
if ($Credential) {
|
||||||
|
Write-Host '[Invoke-RemoteBuild] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).'
|
||||||
Write-Host "[Invoke-RemoteBuild] Connecting to VM at $IPAddress..."
|
|
||||||
|
|
||||||
$session = New-PSSession `
|
|
||||||
-ComputerName $IPAddress `
|
|
||||||
-Credential $Credential `
|
|
||||||
-SessionOption $sessionOptions `
|
|
||||||
-UseSSL `
|
|
||||||
-Port 5986 `
|
|
||||||
-Authentication Basic `
|
|
||||||
-ErrorAction Stop
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── Ensure working directories exist on guest ────────────────────────
|
|
||||||
Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($workDir, $outputDir)
|
|
||||||
# Ensure artifact output parent exists
|
|
||||||
$outParent = Split-Path $outputDir -Parent
|
|
||||||
if (-not (Test-Path $outParent)) {
|
|
||||||
New-Item -ItemType Directory -Path $outParent -Force | Out-Null
|
|
||||||
}
|
|
||||||
# Clean and recreate build dir (ephemeral VM, but guard against partial prior run)
|
|
||||||
if (Test-Path $workDir) { Remove-Item $workDir -Recurse -Force }
|
|
||||||
New-Item -ItemType Directory -Path $workDir -Force | Out-Null
|
|
||||||
} -ArgumentList $GuestWorkDir, $GuestArtifactZip
|
|
||||||
|
|
||||||
# ── Source preparation: Host clone (compress+transfer) or Guest clone (git) ──
|
|
||||||
if ($hostCloneMode) {
|
|
||||||
# Mode 1: Copy pre-cloned source from host to guest via WinRM zip
|
|
||||||
# Single-file transfer over WinRM is far faster than recursive Copy-Item.
|
|
||||||
$hostZip = [System.IO.Path]::GetTempFileName() -replace '\.tmp$', '.zip'
|
|
||||||
try {
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Compressing source on host..."
|
|
||||||
Compress-BuildArtifact -Path "$HostSourceDir\*" -DestinationPath $hostZip
|
|
||||||
$zipSize = [math]::Round((Get-Item $hostZip).Length / 1MB, 1)
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Transferring archive to guest ($zipSize MB)..."
|
|
||||||
$guestZip = 'C:\CI\src-transfer.zip'
|
|
||||||
Copy-Item -Path $hostZip -Destination $guestZip -ToSession $session -Force
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Expanding archive in guest..."
|
|
||||||
Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($zip, $dest)
|
|
||||||
Expand-Archive -Path $zip -DestinationPath $dest -Force
|
|
||||||
Remove-Item $zip -Force
|
|
||||||
} -ArgumentList $guestZip, $GuestWorkDir
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Source transfer complete."
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
Remove-Item $hostZip -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# Mode 2 (§3.3): Clone repo directly in guest via git
|
|
||||||
# PAT (if needed) read from Credential Manager inside session, never logged
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Cloning repository in guest (§3.3)..."
|
|
||||||
|
|
||||||
# Read PAT from host Credential Manager (§1.5: never in argv or logs)
|
|
||||||
$gitPatUser = $null
|
|
||||||
$gitPatPass = $null
|
|
||||||
if (-not [string]::IsNullOrWhiteSpace($GiteaCredentialTarget)) {
|
|
||||||
try {
|
|
||||||
$stored = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
|
|
||||||
$gitPatUser = $stored.UserName
|
|
||||||
$gitPatPass = $stored.GetNetworkCredential().Password
|
|
||||||
Write-Host "[Invoke-RemoteBuild] PAT loaded from Credential Manager ($GiteaCredentialTarget)."
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[Invoke-RemoteBuild] Could not read PAT from Credential Manager '$GiteaCredentialTarget' — proceeding without auth (public repo only)."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Clone in guest via WinRM
|
|
||||||
Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($cloneUrl, $branch, $commit, $workDir, $patUser, $patPass, $submodules)
|
|
||||||
|
|
||||||
Set-Location (Split-Path $workDir -Parent)
|
|
||||||
$gitArgs = @('clone', '--depth', '1', '--branch', $branch)
|
|
||||||
if ($submodules) { $gitArgs += '--recurse-submodules' }
|
|
||||||
$gitArgs += @($cloneUrl, (Split-Path $workDir -Leaf))
|
|
||||||
|
|
||||||
# Disable interactive credential prompts — WinRM has no TTY
|
|
||||||
$env:GIT_TERMINAL_PROMPT = '0'
|
|
||||||
|
|
||||||
# Build git config args:
|
|
||||||
# - credential.helper= (empty) clears GCM and any system credential helper
|
|
||||||
# - http.extraHeader injects Basic auth without credential helpers, temp files,
|
|
||||||
# or modifying the clone URL (§1.5: token never in URL, argv display, or logs)
|
|
||||||
$gitConfigArgs = @('-c', 'credential.helper=')
|
|
||||||
if ($patUser -and $patPass) {
|
|
||||||
$b64 = [Convert]::ToBase64String(
|
|
||||||
[Text.Encoding]::UTF8.GetBytes("${patUser}:${patPass}"))
|
|
||||||
$gitConfigArgs += @('-c', "http.extraHeader=Authorization: Basic $b64")
|
|
||||||
}
|
|
||||||
|
|
||||||
$gitArgs = $gitConfigArgs + $gitArgs
|
|
||||||
|
|
||||||
Write-Host "[Guest] Cloning $cloneUrl (branch: $branch)..."
|
|
||||||
& git @gitArgs 2>&1 | ForEach-Object { Write-Host "[Guest Clone] $_" }
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "git clone failed in guest (exit $LASTEXITCODE)"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Checkout specific commit if provided
|
|
||||||
if ($commit) {
|
|
||||||
Write-Host "[Guest] Checking out commit: $commit"
|
|
||||||
Set-Location $workDir
|
|
||||||
& git checkout $commit 2>&1 | ForEach-Object { Write-Host "[Guest Checkout] $_" }
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
throw "git checkout $commit failed in guest (exit $LASTEXITCODE)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} -ArgumentList $CloneUrl, $CloneBranch, $CloneCommit, $GuestWorkDir, $gitPatUser, $gitPatPass, $CloneSubmodules.IsPresent
|
|
||||||
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Guest clone complete."
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Build ──────────────────────────────────────────────────────────
|
|
||||||
if ($BuildCommand -ne '') {
|
|
||||||
# ── Custom build command (e.g. python, msbuild, cmake…) ──────────
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Running build command: $BuildCommand"
|
|
||||||
$buildExit = Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($workDir, $cmd, $artifactZip, $artifactSrc, $useCache, $extraEnv, $skipArt)
|
|
||||||
# Unbuffered Python output — lines appear in real-time instead of being held in buffer
|
|
||||||
$env:PYTHONUNBUFFERED = '1'
|
|
||||||
if ($useCache) {
|
|
||||||
$env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache'
|
|
||||||
}
|
|
||||||
# Inject extra env vars (§6.5 secret injection)
|
|
||||||
foreach ($pair in $extraEnv.GetEnumerator()) {
|
|
||||||
[System.Environment]::SetEnvironmentVariable($pair.Key, $pair.Value, 'Process')
|
|
||||||
}
|
|
||||||
Set-Location $workDir
|
|
||||||
# Stream each output line via Write-Host — WinRM forwards Write-Host in real-time
|
|
||||||
& cmd /c "$cmd 2>&1" | ForEach-Object { Write-Host $_ }
|
|
||||||
$exit = $LASTEXITCODE
|
|
||||||
|
|
||||||
if ($exit -eq 0 -and -not $skipArt) {
|
|
||||||
$archiveDir = Split-Path $artifactZip -Parent
|
|
||||||
if (-not (Test-Path $archiveDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null
|
|
||||||
}
|
|
||||||
$srcPath = Join-Path $workDir $artifactSrc
|
|
||||||
if (-not (Test-Path $srcPath)) {
|
|
||||||
throw "Artifact source directory not found: $srcPath. Build succeeded (exit 0) but produced no output. Check GuestArtifactSource or set skip-artifact: 'true' in the workflow."
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$7zip = 'C:\Program Files\7-Zip\7z.exe'
|
|
||||||
if (Test-Path $7zip) {
|
|
||||||
Write-Host "[Build] Compressing artifacts with 7-Zip..."
|
|
||||||
& $7zip a -mmt=on -mx1 $artifactZip $srcPath 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ }
|
|
||||||
if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" }
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Compress-Archive -Path $srcPath -DestinationPath $artifactZip -Force
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $exit
|
|
||||||
} -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource, $UseSharedCache.IsPresent, $ExtraGuestEnv, $SkipArtifact.IsPresent
|
|
||||||
|
|
||||||
if ($buildExit -ne 0) {
|
|
||||||
throw "Build command failed (exit $buildExit)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# ── Default: dotnet restore + dotnet build ────────────────────────
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Running dotnet restore..."
|
|
||||||
$restoreExit = Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($workDir, $useCache)
|
|
||||||
if ($useCache) {
|
|
||||||
$env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache'
|
|
||||||
}
|
|
||||||
Set-Location $workDir
|
|
||||||
dotnet restore 2>&1 | ForEach-Object { Write-Host $_ }
|
|
||||||
return $LASTEXITCODE
|
|
||||||
} -ArgumentList $GuestWorkDir, $UseSharedCache.IsPresent
|
|
||||||
|
|
||||||
if ($restoreExit -ne 0) {
|
|
||||||
throw "dotnet restore failed (exit $restoreExit)"
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Running dotnet build (configuration: $Configuration)..."
|
|
||||||
$buildExit = Invoke-Command -Session $session -ScriptBlock {
|
|
||||||
param($workDir, $config, $artifactZip, $useCache)
|
|
||||||
if ($useCache) {
|
|
||||||
$env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache'
|
|
||||||
}
|
|
||||||
Set-Location $workDir
|
|
||||||
$outputDir = Join-Path $workDir 'bin\CIOutput'
|
|
||||||
dotnet build --configuration $config --output $outputDir 2>&1 | ForEach-Object { Write-Host $_ }
|
|
||||||
$exit = $LASTEXITCODE
|
|
||||||
|
|
||||||
if ($exit -eq 0) {
|
|
||||||
$archiveDir = Split-Path $artifactZip -Parent
|
|
||||||
if (-not (Test-Path $archiveDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null
|
|
||||||
}
|
|
||||||
if (-not (Test-Path $outputDir)) {
|
|
||||||
Write-Host "[Build] Output dir '$outputDir' not found — skipping packaging."
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$7zip = 'C:\Program Files\7-Zip\7z.exe'
|
|
||||||
if (Test-Path $7zip) {
|
|
||||||
Write-Host "[Build] Compressing output with 7-Zip..."
|
|
||||||
& $7zip a -mmt=on -mx1 $artifactZip "$outputDir\*" 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ }
|
|
||||||
if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" }
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Compress-Archive -Path "$outputDir\*" -DestinationPath $artifactZip -Force
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return $exit
|
|
||||||
} -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip, $UseSharedCache.IsPresent
|
|
||||||
|
|
||||||
if ($buildExit -ne 0) {
|
|
||||||
throw "dotnet build failed (exit $buildExit)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($SkipArtifact) {
|
|
||||||
Write-Host '[Invoke-RemoteBuild] Build succeeded. Artifact packaging skipped (-SkipArtifact).'
|
|
||||||
} else {
|
|
||||||
Write-Host "[Invoke-RemoteBuild] Build succeeded. Artifact at $GuestArtifactZip"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
|
& $venvPython -m ci_orchestrator build run @pyArgs
|
||||||
|
exit $LASTEXITCODE
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ for ($i = 1; $i -le $Iterations; $i++) {
|
|||||||
|
|
||||||
Write-Host "[bench] Cloning..."
|
Write-Host "[bench] Cloning..."
|
||||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||||
$r = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
|
$null = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
|
||||||
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) `
|
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) `
|
||||||
-ThrowOnError
|
-ThrowOnError
|
||||||
$t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
$t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2)
|
||||||
|
|||||||
+15
-82
@@ -1,43 +1,10 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Creates a linked clone of the template VM for an ephemeral CI build.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
# Shim: delegates to the Python ci_orchestrator CLI (`vm new`).
|
||||||
Uses vmrun.exe to create a linked clone from a frozen template snapshot.
|
# Original PS implementation moved to git history; see plans/A3-closeout.md.
|
||||||
Returns the full path to the new clone's .vmx file on success.
|
|
||||||
The template VM and its snapshot are never modified.
|
|
||||||
|
|
||||||
.PARAMETER TemplatePath
|
|
||||||
Full path to the template VM's .vmx file.
|
|
||||||
Example: F:\CI\Templates\WinBuild\WinBuild.vmx
|
|
||||||
|
|
||||||
.PARAMETER SnapshotName
|
|
||||||
Name of the snapshot on the template VM to clone from.
|
|
||||||
Default: BaseClean
|
|
||||||
|
|
||||||
.PARAMETER CloneBaseDir
|
|
||||||
Directory under which the new clone folder will be created.
|
|
||||||
The clone folder is named after the job ID and timestamp.
|
|
||||||
Example: F:\CI\BuildVMs
|
|
||||||
|
|
||||||
.PARAMETER JobId
|
|
||||||
Unique identifier for the CI job (e.g. run ID from Gitea Actions).
|
|
||||||
Used to name the clone: Clone_{JobId}_{yyyyMMdd_HHmmss}
|
|
||||||
|
|
||||||
.PARAMETER VmrunPath
|
|
||||||
Full path to vmrun.exe.
|
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
||||||
|
|
||||||
.OUTPUTS
|
|
||||||
[string] Full path to the new clone's .vmx file.
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
$vmxPath = .\New-BuildVM.ps1 `
|
|
||||||
-TemplatePath "F:\CI\Templates\WinBuild\WinBuild.vmx" `
|
|
||||||
-CloneBaseDir "F:\CI\BuildVMs" `
|
|
||||||
-JobId "run-42"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)]
|
[Parameter(Mandatory)]
|
||||||
@@ -55,49 +22,15 @@ param(
|
|||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||||
)
|
)
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
$pyArgs = @(
|
||||||
$ErrorActionPreference = 'Stop'
|
'--template-path', $TemplatePath,
|
||||||
|
'--snapshot-name', $SnapshotName,
|
||||||
|
'--clone-base-dir', $CloneBaseDir,
|
||||||
|
'--job-id', $JobId,
|
||||||
|
'--vmrun-path', $VmrunPath
|
||||||
|
)
|
||||||
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null
|
& $venvPython -m ci_orchestrator vm new @pyArgs
|
||||||
|
exit $LASTEXITCODE
|
||||||
# --- Build clone path ---
|
|
||||||
$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss'
|
|
||||||
$cloneName = "Clone_${JobId}_${timestamp}"
|
|
||||||
$cloneDir = Join-Path $CloneBaseDir $cloneName
|
|
||||||
$cloneVmx = Join-Path $cloneDir "$cloneName.vmx"
|
|
||||||
|
|
||||||
# Ensure clone base dir exists
|
|
||||||
if (-not (Test-Path $CloneBaseDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[New-BuildVM] Creating linked clone..."
|
|
||||||
Write-Host " Template : $TemplatePath"
|
|
||||||
Write-Host " Snapshot : $SnapshotName"
|
|
||||||
Write-Host " Clone VMX : $cloneVmx"
|
|
||||||
|
|
||||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
|
||||||
|
|
||||||
$cloneResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
|
|
||||||
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName)
|
|
||||||
|
|
||||||
$sw.Stop()
|
|
||||||
|
|
||||||
if ($cloneResult.ExitCode -ne 0) {
|
|
||||||
# Clean up partial clone directory if it was created
|
|
||||||
if (Test-Path $cloneDir) {
|
|
||||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
throw "vmrun clone failed (exit $($cloneResult.ExitCode)).`nOutput: $($cloneResult.Output)"
|
|
||||||
}
|
|
||||||
|
|
||||||
if (-not (Test-Path $cloneVmx -PathType Leaf)) {
|
|
||||||
throw "vmrun reported success but clone VMX not found at: $cloneVmx"
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[New-BuildVM] Clone created in $($sw.Elapsed.TotalSeconds.ToString('F1'))s : $cloneVmx"
|
|
||||||
|
|
||||||
# Return the VMX path as output
|
|
||||||
return $cloneVmx
|
|
||||||
|
|||||||
+32
-122
@@ -1,130 +1,40 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Stops and permanently deletes an ephemeral build VM.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
Performs a graceful shutdown (soft stop) with a timeout fallback to
|
|
||||||
a hard stop, then uses vmrun deleteVM to remove all VM files.
|
|
||||||
Finally, removes the clone directory from disk.
|
|
||||||
|
|
||||||
This script is designed to be called from a try/finally block so
|
|
||||||
it always runs, even on build failure.
|
|
||||||
|
|
||||||
.PARAMETER VMPath
|
|
||||||
Full path to the clone VM's .vmx file.
|
|
||||||
|
|
||||||
.PARAMETER VmrunPath
|
|
||||||
Full path to vmrun.exe.
|
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
||||||
|
|
||||||
.PARAMETER GracefulTimeoutSeconds
|
|
||||||
Seconds to wait for graceful shutdown before forcing power-off.
|
|
||||||
Default: 15
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx"
|
|
||||||
#>
|
|
||||||
[CmdletBinding(SupportsShouldProcess)]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $VMPath,
|
|
||||||
|
|
||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
|
||||||
|
|
||||||
[ValidateRange(1, 60)]
|
|
||||||
[int] $GracefulTimeoutSeconds = 15
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
Set-StrictMode -Version Latest
|
||||||
$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
|
#
|
||||||
|
# Callers (e.g. Measure-CIBenchmark.ps1) may pass PowerShell common
|
||||||
|
# parameters like -ErrorAction SilentlyContinue. Those must NOT be
|
||||||
|
# forwarded as --error-action etc. (the Python CLI rejects them).
|
||||||
|
# Value-taking common params consume the following token too.
|
||||||
|
$commonWithValue = @{
|
||||||
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
|
pipelinevariable = $true
|
||||||
|
}
|
||||||
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
|
|
||||||
$cloneDir = Split-Path $VMPath -Parent
|
$pyArgs = @()
|
||||||
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
|
$a = $args[$i]
|
||||||
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
# ── Step 1: Check if VM exists at all ────────────────────────────────────────
|
$name = $a.Substring(1)
|
||||||
if (-not (Test-Path $VMPath -PathType Leaf)) {
|
$lname = $name.ToLower()
|
||||||
Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath"
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
# Still attempt to remove the directory in case of partial clone
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) {
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
}
|
}
|
||||||
return
|
else {
|
||||||
}
|
$pyArgs += $a
|
||||||
|
|
||||||
if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
|
|
||||||
|
|
||||||
# ── Step 2: Graceful stop ─────────────────────────────────────────────────────
|
|
||||||
Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..."
|
|
||||||
$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
||||||
-Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds
|
|
||||||
if ($stopSoft.TimedOut) {
|
|
||||||
Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s."
|
|
||||||
} elseif ($stopSoft.ExitCode -eq 0) {
|
|
||||||
Write-Host "[Remove-BuildVM] VM stopped gracefully."
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 3: Force stop if still running ──────────────────────────────────────
|
|
||||||
if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) {
|
|
||||||
Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..."
|
|
||||||
$stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
||||||
-Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60
|
|
||||||
if ($stopHard.TimedOut) {
|
|
||||||
Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s."
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
# after stop. Waiting for the entry to disappear ensures deleteVM won't race the
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
# process exit.
|
& $venvPython -m ci_orchestrator vm remove @pyArgs
|
||||||
$vmxNorm = $VMPath.Trim().ToLower()
|
exit $LASTEXITCODE
|
||||||
$releaseDeadline = (Get-Date).AddSeconds(20)
|
|
||||||
while ((Get-Date) -lt $releaseDeadline) {
|
|
||||||
$listedVMs = & $VmrunPath -T ws list 2>&1 |
|
|
||||||
Where-Object { $_ -notmatch '^Total running' } |
|
|
||||||
ForEach-Object { "$_".Trim().ToLower() }
|
|
||||||
if ($listedVMs -notcontains $vmxNorm) { break }
|
|
||||||
Start-Sleep -Seconds 2
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 4: Delete VM via vmrun — retry up to 3× with backoff ────────────────
|
|
||||||
Write-Host "[Remove-BuildVM] Deleting VM..."
|
|
||||||
$deleteOutput = ''
|
|
||||||
$deleteExit = -1
|
|
||||||
foreach ($delay in @(0, 3, 6)) {
|
|
||||||
if ($delay -gt 0) {
|
|
||||||
Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..."
|
|
||||||
Start-Sleep -Seconds $delay
|
|
||||||
}
|
|
||||||
$delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
||||||
-Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90
|
|
||||||
$deleteOutput = $delResult.Output
|
|
||||||
$deleteExit = $delResult.ExitCode
|
|
||||||
if ($delResult.TimedOut) {
|
|
||||||
Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s."
|
|
||||||
$deleteExit = -1
|
|
||||||
}
|
|
||||||
if ($deleteExit -eq 0) { break }
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($deleteExit -ne 0) {
|
|
||||||
Write-Warning "[Remove-BuildVM] vmrun deleteVM failed after retries (exit $deleteExit): $deleteOutput"
|
|
||||||
Write-Warning "[Remove-BuildVM] Will attempt manual directory removal."
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 5: Remove clone directory (catches any leftover files) ───────────────
|
|
||||||
if (Test-Path $cloneDir) {
|
|
||||||
Write-Host "[Remove-BuildVM] Removing clone directory: $cloneDir"
|
|
||||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
if (Test-Path $cloneDir) {
|
|
||||||
Write-Warning "[Remove-BuildVM] Clone directory could not be fully removed: $cloneDir"
|
|
||||||
Write-Warning " Manual cleanup required."
|
|
||||||
} else {
|
|
||||||
Write-Host "[Remove-BuildVM] Clone directory removed."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "[Remove-BuildVM] VM destruction complete."
|
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
#Requires -Version 5.1
|
||||||
|
#Requires -RunAsAdministrator
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Stores a CI guest-VM credential into the LocalSystem keyring vault.
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
act_runner runs as the LocalSystem service account. Windows Credential
|
||||||
|
Manager / keyring vaults are per-user, so a credential added with
|
||||||
|
`cmdkey` or the Credential Manager UI under an interactive user is
|
||||||
|
invisible to the runner. The orchestrator then fails with:
|
||||||
|
|
||||||
|
Credential '<target>' not found in keyring.
|
||||||
|
|
||||||
|
This script writes the credential into the SYSTEM vault using the
|
||||||
|
production venv's `keyring` itself (the exact backend
|
||||||
|
`ci_orchestrator` reads with `keyring.get_credential(target, None)`),
|
||||||
|
via a transient SYSTEM scheduled task. Using keyring for both write
|
||||||
|
and read avoids cmdkey/keyring format and persistence mismatches.
|
||||||
|
|
||||||
|
Known-issue references:
|
||||||
|
plans/A1-closeout.md (KeyError: 'BuildVMGuest')
|
||||||
|
plans/implementation-plan-A-B.md (keyring under SYSTEM)
|
||||||
|
|
||||||
|
The plaintext password unavoidably transits a temporary .py file.
|
||||||
|
The file is created with an Administrators/SYSTEM-only ACL, overwritten
|
||||||
|
with random bytes, then deleted. The password is never placed in a
|
||||||
|
scheduled-task argument or the command line.
|
||||||
|
|
||||||
|
.PARAMETER Target
|
||||||
|
Credential target name. Must match the runner env
|
||||||
|
GITEA_CI_GUEST_CRED_TARGET. Default: BuildVMGuest
|
||||||
|
|
||||||
|
.PARAMETER UserName
|
||||||
|
Guest login username in the form the transport expects (e.g.
|
||||||
|
'WINBUILD\ci_build', '.\ci_build', or 'ci_build').
|
||||||
|
|
||||||
|
.PARAMETER Password
|
||||||
|
Guest password as a SecureString. If omitted, prompted securely.
|
||||||
|
|
||||||
|
.PARAMETER VenvPython
|
||||||
|
Path to the production venv interpreter act_runner uses. Default:
|
||||||
|
CI_VENV_PYTHON env var, else F:\CI\python\venv\Scripts\python.exe
|
||||||
|
|
||||||
|
.PARAMETER SkipVerify
|
||||||
|
Skip the post-write read-back verification (also run as SYSTEM).
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\Set-CIGuestCredential.ps1 -UserName 'WINBUILD\ci_build'
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
# Non-interactive (e.g. from another provisioning script)
|
||||||
|
$pw = Read-Host 'pw' -AsSecureString
|
||||||
|
.\Set-CIGuestCredential.ps1 -Target BuildVMGuest -UserName '.\ci_build' -Password $pw
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\Set-CIGuestCredential.ps1 -UserName '.\ci_build' -WhatIf
|
||||||
|
#>
|
||||||
|
[CmdletBinding(SupportsShouldProcess)]
|
||||||
|
param(
|
||||||
|
[ValidateNotNullOrEmpty()]
|
||||||
|
[string] $Target = 'BuildVMGuest',
|
||||||
|
|
||||||
|
[Parameter(Mandatory)]
|
||||||
|
[ValidateNotNullOrEmpty()]
|
||||||
|
[string] $UserName,
|
||||||
|
|
||||||
|
[System.Security.SecureString] $Password,
|
||||||
|
|
||||||
|
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
|
||||||
|
else { 'F:\CI\python\venv\Scripts\python.exe' }),
|
||||||
|
|
||||||
|
[switch] $SkipVerify
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $VenvPython)) {
|
||||||
|
throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON."
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $Password) {
|
||||||
|
$Password = Read-Host "Password for '$UserName'" -AsSecureString
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Run a script block as SYSTEM via a transient scheduled task ----------
|
||||||
|
function Invoke-AsSystem {
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory)] [string] $TaskName,
|
||||||
|
[Parameter(Mandatory)] [string] $Execute,
|
||||||
|
[Parameter(Mandatory)] [string] $Argument,
|
||||||
|
[int] $TimeoutSec = 30
|
||||||
|
)
|
||||||
|
$action = New-ScheduledTaskAction -Execute $Execute -Argument $Argument
|
||||||
|
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
|
||||||
|
-LogonType ServiceAccount -RunLevel Highest
|
||||||
|
Register-ScheduledTask -TaskName $TaskName -Action $action `
|
||||||
|
-Principal $principal -Force | Out-Null
|
||||||
|
try {
|
||||||
|
Start-ScheduledTask -TaskName $TaskName
|
||||||
|
$deadline = (Get-Date).AddSeconds($TimeoutSec)
|
||||||
|
do {
|
||||||
|
Start-Sleep -Milliseconds 500
|
||||||
|
$info = Get-ScheduledTaskInfo -TaskName $TaskName
|
||||||
|
$state = (Get-ScheduledTask -TaskName $TaskName).State
|
||||||
|
} while ($state -eq 'Running' -and (Get-Date) -lt $deadline)
|
||||||
|
return $info.LastTaskResult
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false `
|
||||||
|
-ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Harden + shred helper for the transient secret file ------------------
|
||||||
|
function Remove-Securely {
|
||||||
|
# Best-effort cleanup helper; ShouldProcess prompting is undesirable
|
||||||
|
# for an internal shred-and-delete of our own temp file.
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute(
|
||||||
|
'PSUseShouldProcessForStateChangingFunctions', '')]
|
||||||
|
[CmdletBinding()]
|
||||||
|
param([Parameter(Mandatory)] [string] $Path)
|
||||||
|
if (-not (Test-Path -LiteralPath $Path)) { return }
|
||||||
|
try {
|
||||||
|
$len = (Get-Item -LiteralPath $Path).Length
|
||||||
|
if ($len -gt 0) {
|
||||||
|
$rnd = New-Object byte[] $len
|
||||||
|
[Security.Cryptography.RandomNumberGenerator]::Create().GetBytes($rnd)
|
||||||
|
[IO.File]::WriteAllBytes($Path, $rnd)
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
Write-Verbose "secure overwrite failed (continuing to delete): $_"
|
||||||
|
}
|
||||||
|
Remove-Item -LiteralPath $Path -Force -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
|
||||||
|
$tmpDir = $env:TEMP
|
||||||
|
$setPy = Join-Path $tmpDir ("setcred_{0}.py" -f ([guid]::NewGuid().ToString('N')))
|
||||||
|
$verPy = Join-Path $tmpDir ("vercred_{0}.py" -f ([guid]::NewGuid().ToString('N')))
|
||||||
|
$verOut = Join-Path $tmpDir ("vercred_{0}.txt" -f ([guid]::NewGuid().ToString('N')))
|
||||||
|
|
||||||
|
if (-not $PSCmdlet.ShouldProcess("$Target ($UserName)",
|
||||||
|
"store credential in SYSTEM keyring vault")) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
# Decrypt SecureString only at the moment of writing the temp script.
|
||||||
|
$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($Password)
|
||||||
|
$plain = [Runtime.InteropServices.Marshal]::PtrToStringAuto($bstr)
|
||||||
|
[Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr)
|
||||||
|
|
||||||
|
# Python literals: pass via triple-quoted raw-ish strings. Reject quotes
|
||||||
|
# that would break out of the literal rather than risk silent corruption.
|
||||||
|
if ($UserName -match "'''" -or $plain -match "'''") {
|
||||||
|
throw "UserName/Password contains a triple-quote sequence; unsupported."
|
||||||
|
}
|
||||||
|
$setBody = @"
|
||||||
|
import keyring
|
||||||
|
keyring.set_password(r'''$Target''', r'''$UserName''', r'''$plain''')
|
||||||
|
print('OK')
|
||||||
|
"@
|
||||||
|
Set-Content -LiteralPath $setPy -Value $setBody -Encoding UTF8
|
||||||
|
Remove-Variable plain -ErrorAction SilentlyContinue
|
||||||
|
|
||||||
|
# Lock the secret file down to SYSTEM + Administrators before it runs.
|
||||||
|
$acl = New-Object Security.AccessControl.FileSecurity
|
||||||
|
$acl.SetAccessRuleProtection($true, $false)
|
||||||
|
foreach ($id in 'SYSTEM', 'Administrators') {
|
||||||
|
$acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule(
|
||||||
|
$id, 'FullControl', 'Allow')))
|
||||||
|
}
|
||||||
|
Set-Acl -LiteralPath $setPy -AclObject $acl
|
||||||
|
|
||||||
|
$rc = Invoke-AsSystem -TaskName 'CI-SetGuestCred' `
|
||||||
|
-Execute $VenvPython -Argument "`"$setPy`""
|
||||||
|
if ($rc -ne 0) {
|
||||||
|
throw "keyring.set_password task exited with code $rc."
|
||||||
|
}
|
||||||
|
Write-Host "Stored '$Target' (user '$UserName') in the SYSTEM vault." `
|
||||||
|
-ForegroundColor Green
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Remove-Securely -Path $setPy
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($SkipVerify) { return }
|
||||||
|
|
||||||
|
try {
|
||||||
|
$verBody = @"
|
||||||
|
import keyring
|
||||||
|
c = keyring.get_credential(r'''$Target''', None)
|
||||||
|
open(r'''$verOut''', 'w', encoding='utf-8').write(
|
||||||
|
'NONE' if c is None else 'OK:' + c.username)
|
||||||
|
"@
|
||||||
|
Set-Content -LiteralPath $verPy -Value $verBody -Encoding UTF8
|
||||||
|
Invoke-AsSystem -TaskName 'CI-VerifyGuestCred' `
|
||||||
|
-Execute $VenvPython -Argument "`"$verPy`"" | Out-Null
|
||||||
|
|
||||||
|
$result = if (Test-Path -LiteralPath $verOut) {
|
||||||
|
(Get-Content -LiteralPath $verOut -Raw).Trim()
|
||||||
|
} else { 'NONE' }
|
||||||
|
|
||||||
|
if ($result -like 'OK:*') {
|
||||||
|
Write-Host "Verified: SYSTEM keyring returns user '$($result.Substring(3))'." `
|
||||||
|
-ForegroundColor Green
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
throw "Verification failed: SYSTEM keyring.get_credential('$Target', None) " +
|
||||||
|
"returned None. Credential did not persist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Remove-Item -LiteralPath $verPy -Force -ErrorAction SilentlyContinue
|
||||||
|
Remove-Item -LiteralPath $verOut -Force -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
#Requires -Version 5.1
|
||||||
|
#Requires -RunAsAdministrator
|
||||||
|
<#
|
||||||
|
.SYNOPSIS
|
||||||
|
Diagnoses WinRM reachability to a CI build VM from the runner's context.
|
||||||
|
|
||||||
|
.DESCRIPTION
|
||||||
|
The orchestrator's readiness probe (WinRmTransport.is_ready) swallows
|
||||||
|
connection and auth errors and returns False, so a failing guest just
|
||||||
|
hangs "waiting for windows transport readiness" until timeout with no
|
||||||
|
cause in the log.
|
||||||
|
|
||||||
|
This script reproduces the exact probe the runner performs, but
|
||||||
|
surfaces the real error:
|
||||||
|
|
||||||
|
1. TCP test to the WinRM HTTPS port (network / listener / firewall).
|
||||||
|
2. A real pypsrp `execute_ps('$true')` against the guest, run as
|
||||||
|
SYSTEM (the act_runner account) so it reads the same SYSTEM
|
||||||
|
keyring vault credential the job uses. The full Python traceback
|
||||||
|
is captured and classified.
|
||||||
|
|
||||||
|
Classifications:
|
||||||
|
CONNECT — port closed / refused / unreachable (guest WinRM
|
||||||
|
listener or firewall; check template WinRM 5986 setup).
|
||||||
|
SSL — TLS handshake/cert problem.
|
||||||
|
AUTH — credential rejected (wrong password, or username form;
|
||||||
|
try 'HOSTNAME\ci_build' instead of bare 'ci_build').
|
||||||
|
KEYRING — credential missing from the SYSTEM vault (run
|
||||||
|
Set-CIGuestCredential.ps1).
|
||||||
|
OK — WinRM works; the hang is elsewhere.
|
||||||
|
|
||||||
|
.PARAMETER IpAddress
|
||||||
|
Guest IP shown in the job log ("[job] guest IP: ...").
|
||||||
|
|
||||||
|
.PARAMETER Target
|
||||||
|
Credential target in the SYSTEM keyring. Default: BuildVMGuest
|
||||||
|
|
||||||
|
.PARAMETER Port
|
||||||
|
WinRM HTTPS port. Default: 5986
|
||||||
|
|
||||||
|
.PARAMETER VenvPython
|
||||||
|
Production venv interpreter. Default: CI_VENV_PYTHON env, else
|
||||||
|
F:\CI\python\venv\Scripts\python.exe
|
||||||
|
|
||||||
|
.PARAMETER TimeoutSec
|
||||||
|
Per-probe connection timeout (also the task wait budget). Default: 30
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237
|
||||||
|
|
||||||
|
.EXAMPLE
|
||||||
|
.\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 -Target BuildVMGuest -Port 5986
|
||||||
|
#>
|
||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory)]
|
||||||
|
[ValidateNotNullOrEmpty()]
|
||||||
|
[string] $IpAddress,
|
||||||
|
|
||||||
|
[ValidateNotNullOrEmpty()]
|
||||||
|
[string] $Target = 'BuildVMGuest',
|
||||||
|
|
||||||
|
[ValidateRange(1, 65535)]
|
||||||
|
[int] $Port = 5986,
|
||||||
|
|
||||||
|
# Optional username override (password still read from the SYSTEM
|
||||||
|
# keyring). Use to test forms like 'HOSTNAME\ci_build' without
|
||||||
|
# re-storing the credential. Empty = use the keyring username.
|
||||||
|
[string] $UserName = '',
|
||||||
|
|
||||||
|
# Auth packages to try, in order. Local/workgroup accounts usually
|
||||||
|
# need 'ntlm' (Negotiate->Kerberos is meaningless without a domain).
|
||||||
|
[ValidateNotNullOrEmpty()]
|
||||||
|
[string[]] $AuthOrder = @('ntlm', 'negotiate'),
|
||||||
|
|
||||||
|
[string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON }
|
||||||
|
else { 'F:\CI\python\venv\Scripts\python.exe' }),
|
||||||
|
|
||||||
|
[ValidateRange(5, 300)]
|
||||||
|
[int] $TimeoutSec = 30
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $VenvPython)) {
|
||||||
|
throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON."
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── 1. TCP reachability ───────────────────────────────────────────────────
|
||||||
|
Write-Host "[1/2] TCP $IpAddress`:$Port ..." -ForegroundColor Cyan
|
||||||
|
$tcp = Test-NetConnection -ComputerName $IpAddress -Port $Port `
|
||||||
|
-WarningAction SilentlyContinue
|
||||||
|
if (-not $tcp.TcpTestSucceeded) {
|
||||||
|
Write-Host " RESULT: CONNECT — port $Port closed/unreachable." `
|
||||||
|
-ForegroundColor Red
|
||||||
|
Write-Host " WinRM HTTPS listener or guest firewall. Check the" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " Windows template WinRM 5986 setup (WINDOWS-TEMPLATE-SETUP.md)." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
return
|
||||||
|
}
|
||||||
|
Write-Host " port open." -ForegroundColor Green
|
||||||
|
|
||||||
|
# ── 2a. Discover guest hostname from the WinRM TLS certificate CN ──────────
|
||||||
|
# WinRM's self-signed cert CN is normally the guest computer name. Used to
|
||||||
|
# build the COMPUTERNAME\user form NTLM needs for a local/workgroup account.
|
||||||
|
$guestCn = ''
|
||||||
|
try {
|
||||||
|
$tcpc = New-Object Net.Sockets.TcpClient
|
||||||
|
$tcpc.Connect($IpAddress, $Port)
|
||||||
|
$ssl = New-Object Net.Security.SslStream(
|
||||||
|
$tcpc.GetStream(), $false,
|
||||||
|
([Net.Security.RemoteCertificateValidationCallback] { $true }))
|
||||||
|
$ssl.AuthenticateAsClient($IpAddress)
|
||||||
|
$cn = $ssl.RemoteCertificate.Subject # e.g. "CN=WINBUILD"
|
||||||
|
if ($cn -match 'CN=([^,]+)') { $guestCn = $Matches[1].Trim() }
|
||||||
|
$ssl.Dispose(); $tcpc.Close()
|
||||||
|
} catch {
|
||||||
|
Write-Verbose "cert CN probe failed (hostname auto-detect skipped): $_"
|
||||||
|
}
|
||||||
|
if ($guestCn) {
|
||||||
|
Write-Host " guest cert CN (hostname): $guestCn" -ForegroundColor Cyan
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── 2b. Build the username candidate matrix ───────────────────────────────
|
||||||
|
$authList = ($AuthOrder | ForEach-Object { "'" + $_ + "'" }) -join ', '
|
||||||
|
Write-Host "[2/2] pypsrp execute_ps as SYSTEM — auth order: $($AuthOrder -join ', ')" `
|
||||||
|
-ForegroundColor Cyan
|
||||||
|
if ($UserName) {
|
||||||
|
Write-Host " username override: $UserName" -ForegroundColor Cyan
|
||||||
|
}
|
||||||
|
|
||||||
|
$probePy = Join-Path $env:TEMP ("winrmprobe_{0}.py" -f ([guid]::NewGuid().ToString('N')))
|
||||||
|
$probeOut = Join-Path $env:TEMP ("winrmprobe_{0}.txt" -f ([guid]::NewGuid().ToString('N')))
|
||||||
|
|
||||||
|
$body = @"
|
||||||
|
import keyring, traceback
|
||||||
|
out = r'''$probeOut'''
|
||||||
|
auths = [$authList]
|
||||||
|
override = r'''$UserName'''
|
||||||
|
cn = r'''$guestCn'''
|
||||||
|
c = keyring.get_credential(r'''$Target''', None)
|
||||||
|
if c is None:
|
||||||
|
open(out, 'w', encoding='utf-8').write('KEYRING')
|
||||||
|
else:
|
||||||
|
base = override if override else c.username
|
||||||
|
leaf = base.split('\\')[-1] # strip any DOMAIN\ prefix
|
||||||
|
users = [base]
|
||||||
|
if cn:
|
||||||
|
users.append(cn + '\\' + leaf) # COMPUTERNAME\user
|
||||||
|
users.append('.\\' + leaf) # .\user
|
||||||
|
if cn:
|
||||||
|
users.append(leaf + '@' + cn) # user@COMPUTERNAME
|
||||||
|
# de-dup, keep order
|
||||||
|
seen, cand = set(), []
|
||||||
|
for u in users:
|
||||||
|
if u not in seen:
|
||||||
|
seen.add(u); cand.append(u)
|
||||||
|
from pypsrp.client import Client
|
||||||
|
lines, success = [], None
|
||||||
|
lines.append('cred user=%r pwlen=%d' % (c.username, len(c.password or '')))
|
||||||
|
for u in cand:
|
||||||
|
for a in auths:
|
||||||
|
try:
|
||||||
|
cl = Client(r'''$IpAddress''', username=u, password=c.password,
|
||||||
|
port=$Port, ssl=True, cert_validation=False,
|
||||||
|
auth=a, connection_timeout=$TimeoutSec)
|
||||||
|
so, se, rc = cl.execute_ps('`$true')
|
||||||
|
lines.append('user=%r auth=%s OK rc=%s' % (u, a, rc))
|
||||||
|
success = (u, a)
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
msg = (str(e).splitlines() or [''])[0]
|
||||||
|
lines.append('user=%r auth=%s FAIL %s: %s'
|
||||||
|
% (u, a, type(e).__name__, msg))
|
||||||
|
if success:
|
||||||
|
break
|
||||||
|
if success:
|
||||||
|
hdr = 'OK user=%r auth=%s' % success
|
||||||
|
tb = ''
|
||||||
|
else:
|
||||||
|
hdr = 'EXC (all combos failed)'
|
||||||
|
tb = '\n' + traceback.format_exc()
|
||||||
|
open(out, 'w', encoding='utf-8').write(
|
||||||
|
hdr + '\n' + '\n'.join(lines) + tb)
|
||||||
|
"@
|
||||||
|
Set-Content -LiteralPath $probePy -Value $body -Encoding UTF8
|
||||||
|
|
||||||
|
try {
|
||||||
|
$action = New-ScheduledTaskAction -Execute $VenvPython -Argument "`"$probePy`""
|
||||||
|
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' `
|
||||||
|
-LogonType ServiceAccount -RunLevel Highest
|
||||||
|
Register-ScheduledTask -TaskName 'CI-WinRmProbe' -Action $action `
|
||||||
|
-Principal $principal -Force | Out-Null
|
||||||
|
try {
|
||||||
|
Start-ScheduledTask -TaskName 'CI-WinRmProbe'
|
||||||
|
$deadline = (Get-Date).AddSeconds($TimeoutSec + 10)
|
||||||
|
do {
|
||||||
|
Start-Sleep -Milliseconds 500
|
||||||
|
$state = (Get-ScheduledTask -TaskName 'CI-WinRmProbe').State
|
||||||
|
} while ($state -eq 'Running' -and (Get-Date) -lt $deadline)
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Unregister-ScheduledTask -TaskName 'CI-WinRmProbe' -Confirm:$false `
|
||||||
|
-ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
|
|
||||||
|
$raw = if (Test-Path -LiteralPath $probeOut) {
|
||||||
|
(Get-Content -LiteralPath $probeOut -Raw)
|
||||||
|
} else { '' }
|
||||||
|
|
||||||
|
if (-not $raw) {
|
||||||
|
Write-Host " RESULT: probe produced no output (task may have failed)." `
|
||||||
|
-ForegroundColor Red
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($raw -eq 'KEYRING') {
|
||||||
|
Write-Host " RESULT: KEYRING — '$Target' not in the SYSTEM vault." `
|
||||||
|
-ForegroundColor Red
|
||||||
|
Write-Host " Run: .\Set-CIGuestCredential.ps1 -UserName '<guest user>'" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ($raw -like 'OK *') {
|
||||||
|
$okUser = if ($raw -match "OK user='([^']+)'") { $Matches[1] } else { '?' }
|
||||||
|
$okAuth = if ($raw -match 'auth=(\w+)') { $Matches[1] } else { '?' }
|
||||||
|
Write-Host " RESULT: OK — WinRM works." -ForegroundColor Green
|
||||||
|
Write-Host " Winning combo: user='$okUser' auth='$okAuth'" `
|
||||||
|
-ForegroundColor Green
|
||||||
|
Write-Host $raw
|
||||||
|
Write-Host ""
|
||||||
|
$okLeaf = $okUser.Split('\')[-1].Split('@')[0]
|
||||||
|
$needsQualified = ($okUser -ne $okLeaf)
|
||||||
|
if ($needsQualified) {
|
||||||
|
Write-Host " -> FIX (credential): the stored bare username is rejected;" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " re-store in the working form:" -ForegroundColor Yellow
|
||||||
|
Write-Host " .\scripts\Set-CIGuestCredential.ps1 -UserName '$okUser'" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
if ($okAuth -ne 'negotiate') {
|
||||||
|
Write-Host " -> FIX (code): WinRmTransport (src/ci_orchestrator/transport/winrm.py)" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " builds pypsrp Client() without auth= (default 'negotiate')." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " Pass auth='$okAuth' for local-account guests." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
if (-not $needsQualified -and $okAuth -eq 'negotiate') {
|
||||||
|
Write-Host " Transport auth/connect is fine; the hang is elsewhere." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
# EXC ... : classify the traceback.
|
||||||
|
Write-Host " RESULT: probe raised an exception:" -ForegroundColor Red
|
||||||
|
Write-Host $raw
|
||||||
|
$allFailed = $raw -like 'EXC (all combos failed)*'
|
||||||
|
$cls = switch -Regex ($raw) {
|
||||||
|
'CertificateError|SSLError|SSL:|certificate' { 'SSL'; break }
|
||||||
|
'2146893043|UNKNOWN_CREDENTIALS|401|403|AuthenticationError|the user name or password|AccessDenied|access is denied' { 'AUTH'; break }
|
||||||
|
'ConnectionError|timed out|TimeoutError|refused|No route|Failed to establish|getaddrinfo|Max retries' { 'CONNECT'; break }
|
||||||
|
default { 'UNKNOWN' }
|
||||||
|
}
|
||||||
|
Write-Host ""
|
||||||
|
switch ($cls) {
|
||||||
|
'AUTH' {
|
||||||
|
if ($allFailed) {
|
||||||
|
Write-Host " -> AUTH: EVERY user-form x auth combo was rejected with" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " SEC_E_UNKNOWN_CREDENTIALS. Username form is NOT the cause." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " Most likely: the password stored in the SYSTEM keyring is" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " wrong / does not match the guest 'ci_build' account." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " 1. Confirm the real ci_build password in the template." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " 2. Re-run: .\scripts\Set-CIGuestCredential.ps1 -UserName 'ci_build'" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " If the password is definitely correct, suspect the" `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " pyspnego SSPI provider under the SYSTEM account." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
Write-Host " -> AUTH: credential rejected for the tested form(s)." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
Write-Host " Try a hostname-qualified form via Set-CIGuestCredential.ps1." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
'CONNECT' {
|
||||||
|
Write-Host " -> CONNECT: WinRM service/listener not answering on the guest." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
'SSL' {
|
||||||
|
Write-Host " -> SSL: TLS/cert problem (probe uses cert_validation=False; investigate listener cert)." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
default {
|
||||||
|
Write-Host " -> UNKNOWN: inspect the traceback above." `
|
||||||
|
-ForegroundColor Yellow
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
Remove-Item -LiteralPath $probePy -Force -ErrorAction SilentlyContinue
|
||||||
|
Remove-Item -LiteralPath $probeOut -Force -ErrorAction SilentlyContinue
|
||||||
|
}
|
||||||
@@ -89,7 +89,7 @@ function Format-Elapsed {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function Run-Test {
|
function Invoke-Test {
|
||||||
param(
|
param(
|
||||||
[string] $JobId,
|
[string] $JobId,
|
||||||
[string] $Mode,
|
[string] $Mode,
|
||||||
@@ -164,11 +164,11 @@ $baselineResult = $null
|
|||||||
$guestCloneResult = $null
|
$guestCloneResult = $null
|
||||||
|
|
||||||
if (-not $SkipBaseline) {
|
if (-not $SkipBaseline) {
|
||||||
$baselineResult = Run-Test -JobId $BaselineJobId -Mode "BASELINE (host-side zip)" -CommonParams $commonParams
|
$baselineResult = Invoke-Test -JobId $BaselineJobId -Mode "BASELINE (host-side zip)" -CommonParams $commonParams
|
||||||
}
|
}
|
||||||
|
|
||||||
if (-not $SkipGuestClone) {
|
if (-not $SkipGuestClone) {
|
||||||
$guestCloneResult = Run-Test -JobId $GuestCloneJobId -Mode "GUEST CLONE (-UseGitClone)" -UseGitClone -CommonParams $commonParams
|
$guestCloneResult = Invoke-Test -JobId $GuestCloneJobId -Mode "GUEST CLONE (-UseGitClone)" -UseGitClone -CommonParams $commonParams
|
||||||
}
|
}
|
||||||
|
|
||||||
# ────────────────────────────────────────────────────────────────────────────
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
+37
-250
@@ -1,250 +1,37 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Waits until a build VM is fully ready to accept WinRM connections.
|
|
||||||
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
.DESCRIPTION
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
Performs a three-phase readiness check in a polling loop:
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
1. vmrun getState returns "running"
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH)
|
$commonWithValue = @{
|
||||||
3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux)
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
Throws if the VM does not become ready within the timeout period.
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
|
pipelinevariable = $true
|
||||||
Transport modes:
|
}
|
||||||
WinRM (default) — existing Windows VM path: ping + WinRM port 5986.
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
SSH — Linux VM path: port 22 check + ssh echo test.
|
|
||||||
|
$pyArgs = @()
|
||||||
.PARAMETER VMPath
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
Full path to the clone VM's .vmx file.
|
$a = $args[$i]
|
||||||
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
.PARAMETER IPAddress
|
$name = $a.Substring(1)
|
||||||
IP address of the VM on the build network (e.g. 192.168.11.101).
|
$lname = $name.ToLower()
|
||||||
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
.PARAMETER TimeoutSeconds
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
Maximum seconds to wait for the VM to become ready.
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
Default: 300 (5 minutes)
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
.PARAMETER PollIntervalSeconds
|
}
|
||||||
Seconds between readiness checks.
|
else {
|
||||||
Default: 5
|
$pyArgs += $a
|
||||||
|
}
|
||||||
.PARAMETER VmrunPath
|
}
|
||||||
Full path to vmrun.exe.
|
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
.PARAMETER Transport
|
& $venvPython -m ci_orchestrator wait-ready @pyArgs
|
||||||
Transport protocol for readiness checks.
|
exit $LASTEXITCODE
|
||||||
WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs).
|
|
||||||
SSH: checks TCP 22 + ssh echo (Linux VMs).
|
|
||||||
|
|
||||||
.PARAMETER Credential
|
|
||||||
Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs
|
|
||||||
a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming
|
|
||||||
that WinRM authentication (not just port) is working. Useful to detect the
|
|
||||||
window where the WinRM listener is up but the LocalSystem session is not yet
|
|
||||||
accepting logins. When omitted the TCP check alone is used (existing behaviour).
|
|
||||||
|
|
||||||
.PARAMETER SshKeyPath
|
|
||||||
Path to the SSH private key for key-based auth when Transport=SSH.
|
|
||||||
Default: F:\CI\keys\ci_linux
|
|
||||||
|
|
||||||
.PARAMETER SshUser
|
|
||||||
SSH username when Transport=SSH.
|
|
||||||
Default: ci_build
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
.\Wait-VMReady.ps1 `
|
|
||||||
-VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" `
|
|
||||||
-IPAddress "192.168.11.101"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $VMPath,
|
|
||||||
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
||||||
[string] $IPAddress,
|
|
||||||
|
|
||||||
[ValidateRange(3, 600)]
|
|
||||||
[int] $TimeoutSeconds = 300,
|
|
||||||
|
|
||||||
[ValidateRange(1, 30)]
|
|
||||||
[int] $PollIntervalSeconds = 5,
|
|
||||||
|
|
||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
|
||||||
|
|
||||||
# Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP
|
|
||||||
# but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM)
|
|
||||||
# are still checked.
|
|
||||||
[switch] $SkipPing,
|
|
||||||
|
|
||||||
# Transport protocol for phase 2/3 checks.
|
|
||||||
# WinRM (default) = Windows VM; SSH = Linux VM.
|
|
||||||
[ValidateSet('WinRM', 'SSH')]
|
|
||||||
[string] $Transport = 'WinRM',
|
|
||||||
|
|
||||||
# SSH private key path (used when Transport=SSH).
|
|
||||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
|
||||||
|
|
||||||
# SSH username (used when Transport=SSH).
|
|
||||||
[string] $SshUser = 'ci_build',
|
|
||||||
|
|
||||||
# Optional credential for WinRM authentication probe (Transport=WinRM).
|
|
||||||
# When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds.
|
|
||||||
[PSCredential] $Credential = $null
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
|
||||||
|
|
||||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
||||||
throw "vmrun.exe not found at: $VmrunPath"
|
|
||||||
}
|
|
||||||
|
|
||||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
|
||||||
$attempt = 0
|
|
||||||
$phase = 'vmrun-state' # tracks current phase for informative output
|
|
||||||
$lastPhase = ''
|
|
||||||
|
|
||||||
Write-Host "[Wait-VMReady] Waiting for VM ready (timeout: ${TimeoutSeconds}s, IP: $IPAddress)..."
|
|
||||||
|
|
||||||
while ((Get-Date) -lt $deadline) {
|
|
||||||
$attempt++
|
|
||||||
|
|
||||||
# ── Phase 1: VM must be running ─────────────────────────────────────
|
|
||||||
# vmrun has no getState. We previously used `getGuestIPAddress`, but that
|
|
||||||
# requires VMware Tools to be up and responding inside the guest. In
|
|
||||||
# headless boots Tools can take 30-60s to come online, during which the
|
|
||||||
# call exits non-zero even though the VM is fully powered on. That caused
|
|
||||||
# Phase 1 to appear "stuck" until a GUI was opened manually.
|
|
||||||
# `vmrun list` reports actual power state without needing Tools.
|
|
||||||
$vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue)
|
|
||||||
if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath }
|
|
||||||
$listOut = & $VmrunPath list 2>&1
|
|
||||||
$isRunning = $false
|
|
||||||
foreach ($line in $listOut) {
|
|
||||||
if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
||||||
$isRunning = $true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (-not $isRunning) {
|
|
||||||
if ($phase -ne 'vmrun-state') {
|
|
||||||
$phase = 'vmrun-state'
|
|
||||||
}
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 1/3: waiting for VM to enter running state..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 2: Network check ──────────────────────────────────────────
|
|
||||||
if ($Transport -eq 'SSH') {
|
|
||||||
$port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 `
|
|
||||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
|
||||||
-ErrorAction SilentlyContinue
|
|
||||||
if (-not $port22Open) {
|
|
||||||
$phase = 'ssh-port'
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
elseif (-not $SkipPing) {
|
|
||||||
$pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue
|
|
||||||
|
|
||||||
if (-not $pingOk) {
|
|
||||||
$phase = 'ping'
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for network (ping $IPAddress)..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 3: Transport-specific login check ────────────────────────
|
|
||||||
if ($Transport -eq 'SSH') {
|
|
||||||
$sshArgs = @(
|
|
||||||
'-i', $SshKeyPath,
|
|
||||||
'-o', 'StrictHostKeyChecking=no',
|
|
||||||
'-o', 'UserKnownHostsFile=NUL',
|
|
||||||
'-o', 'ConnectTimeout=5',
|
|
||||||
'-o', 'BatchMode=yes',
|
|
||||||
"$SshUser@$IPAddress",
|
|
||||||
'echo ready'
|
|
||||||
)
|
|
||||||
# ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts")
|
|
||||||
# to stderr. Under $ErrorActionPreference='Stop' (set at script top),
|
|
||||||
# those stderr lines captured via 2>&1 are promoted to terminating errors
|
|
||||||
# and abort the script. Wrap the call to demote stderr to non-terminating.
|
|
||||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
|
||||||
$sshOut = & ssh @sshArgs 2>&1
|
|
||||||
$sshExit = $LASTEXITCODE
|
|
||||||
$ErrorActionPreference = $savedEap
|
|
||||||
if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') {
|
|
||||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
|
||||||
return
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$phase = 'ssh-echo'
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# ── WinRM port 5986 accepting TCP connections ─────────────────────
|
|
||||||
# Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for
|
|
||||||
# self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready.
|
|
||||||
$winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 `
|
|
||||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
|
||||||
-ErrorAction SilentlyContinue
|
|
||||||
if ($winrmUp) {
|
|
||||||
# Optional auth probe — ensures WinRM login is functional, not just TCP open
|
|
||||||
if ($null -ne $Credential) {
|
|
||||||
try {
|
|
||||||
$so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
|
||||||
$null = Invoke-Command -ComputerName $IPAddress -Credential $Credential `
|
|
||||||
-Authentication Basic -UseSSL -Port 5986 -SessionOption $so `
|
|
||||||
-ScriptBlock { 'ready' } -ErrorAction Stop
|
|
||||||
Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress."
|
|
||||||
} catch {
|
|
||||||
$phase = 'winrm-auth'
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
|
||||||
return
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
$phase = 'winrm'
|
|
||||||
if ($lastPhase -ne $phase) {
|
|
||||||
Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..."
|
|
||||||
$lastPhase = $phase
|
|
||||||
}
|
|
||||||
Start-Sleep -Seconds $PollIntervalSeconds
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw "[Wait-VMReady] Timed out after ${TimeoutSeconds}s waiting for VM at $IPAddress to become ready. Last phase: $phase"
|
|
||||||
|
|||||||
+37
-96
@@ -1,96 +1,37 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook).
|
|
||||||
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
.DESCRIPTION
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
Intended to run as a scheduled task every 15 minutes (registered by
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB,
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
writes a Warning entry to the Windows Application Event Log under source
|
$commonWithValue = @{
|
||||||
'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
marks the task as failed (visible in Task Scheduler history).
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
If WebhookUrl is provided, also POSTs a JSON payload to that URL
|
pipelinevariable = $true
|
||||||
(compatible with Discord and Gitea webhooks — content field only).
|
}
|
||||||
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
A full drive silently causes vmrun clone to fail with cryptic errors
|
|
||||||
(not enough disk space for linked clone delta files). Alert early.
|
$pyArgs = @()
|
||||||
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
.PARAMETER MinFreeGB
|
$a = $args[$i]
|
||||||
Alert threshold in GB. Default: 50
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
|
$name = $a.Substring(1)
|
||||||
.PARAMETER DriveLetter
|
$lname = $name.ToLower()
|
||||||
Single letter (no colon) of the drive to monitor. Default: F
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
.PARAMETER WebhookUrl
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
.EXAMPLE
|
}
|
||||||
# Manual check
|
else {
|
||||||
.\Watch-DiskSpace.ps1 -MinFreeGB 50
|
$pyArgs += $a
|
||||||
|
}
|
||||||
# With Discord webhook
|
}
|
||||||
.\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
|
||||||
#>
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
[CmdletBinding()]
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
param(
|
& $venvPython -m ci_orchestrator monitor disk @pyArgs
|
||||||
[ValidateRange(1, 2000)]
|
exit $LASTEXITCODE
|
||||||
[int] $MinFreeGB = 50,
|
|
||||||
|
|
||||||
[ValidatePattern('^[A-Za-z]$')]
|
|
||||||
[string] $DriveLetter = 'F',
|
|
||||||
|
|
||||||
[string] $WebhookUrl = ''
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
|
|
||||||
$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue
|
|
||||||
if (-not $drive) {
|
|
||||||
Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found."
|
|
||||||
exit 2
|
|
||||||
}
|
|
||||||
|
|
||||||
$freeGB = [math]::Round($drive.Free / 1GB, 1)
|
|
||||||
$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1)
|
|
||||||
$pctFree = [math]::Round($freeGB / $totalGB * 100, 0)
|
|
||||||
|
|
||||||
if ($freeGB -ge $MinFreeGB) {
|
|
||||||
Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)"
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Alert ─────────────────────────────────────────────────────────────────────
|
|
||||||
$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " +
|
|
||||||
"the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " +
|
|
||||||
"Run Invoke-RetentionPolicy.ps1 or free disk space manually."
|
|
||||||
|
|
||||||
Write-Warning "[DiskSpace] $msg"
|
|
||||||
|
|
||||||
# Windows Event Log
|
|
||||||
$logSource = 'CI-DiskSpaceAlert'
|
|
||||||
try {
|
|
||||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
|
||||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
|
||||||
}
|
|
||||||
Write-EventLog -LogName Application -Source $logSource `
|
|
||||||
-EventId 1001 -EntryType Warning -Message $msg
|
|
||||||
Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)."
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[DiskSpace] Could not write Event Log: $_"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Optional webhook (Discord / Gitea)
|
|
||||||
if ($WebhookUrl) {
|
|
||||||
$body = @{ content = "[WARNING] CI Disk Alert -- $msg" } | ConvertTo-Json -Compress
|
|
||||||
try {
|
|
||||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
|
||||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
|
||||||
Write-Host "[DiskSpace] Webhook notified: $WebhookUrl"
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[DiskSpace] Webhook POST failed: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer)
|
|
||||||
|
|||||||
+37
-214
@@ -1,214 +1,37 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
Set-StrictMode -Version Latest
|
||||||
.SYNOPSIS
|
$ErrorActionPreference = 'Stop'
|
||||||
Monitors the act_runner service and auto-restarts it if stopped.
|
|
||||||
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
.DESCRIPTION
|
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||||
Intended to run as a scheduled task every 15 minutes (registered by
|
# PowerShell common params (-ErrorAction etc.) must not be forwarded
|
||||||
Register-CIScheduledTasks.ps1 as CI-RunnerHealth).
|
# to the Python CLI. Value-taking ones also consume the next token.
|
||||||
|
$commonWithValue = @{
|
||||||
Behaviour:
|
erroraction = $true; warningaction = $true; informationaction = $true
|
||||||
- If act_runner is Running: exits 0, no action.
|
progressaction = $true; errorvariable = $true; warningvariable = $true
|
||||||
- If not Running: writes a Warning to the Windows Application Event Log
|
informationvariable = $true; outvariable = $true; outbuffer = $true
|
||||||
(source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
|
pipelinevariable = $true
|
||||||
- Restart attempts are rate-limited to MaxRestarts per hour using a JSON
|
}
|
||||||
state file in StateDir. Once the limit is hit, further runs write an
|
$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true }
|
||||||
alert (EventId 1004) and exit 1 without restarting — Task Scheduler
|
|
||||||
history shows the failure so the operator is alerted.
|
$pyArgs = @()
|
||||||
- On successful restart: writes EventId 1003 (Information).
|
for ($i = 0; $i -lt $args.Count; $i++) {
|
||||||
- Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
|
$a = $args[$i]
|
||||||
Gitea webhooks on every restart or limit-exceeded event.
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
|
$name = $a.Substring(1)
|
||||||
.PARAMETER MaxRestarts
|
$lname = $name.ToLower()
|
||||||
Maximum number of auto-restart attempts allowed per rolling 60-minute
|
if ($commonSwitch.ContainsKey($lname)) { continue }
|
||||||
window before giving up and requiring manual intervention. Default: 3
|
if ($commonWithValue.ContainsKey($lname)) { $i++; continue }
|
||||||
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
.PARAMETER ServiceName
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
Windows service name to monitor. Default: act_runner
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
|
}
|
||||||
.PARAMETER StateDir
|
else {
|
||||||
Directory used for the cooldown state file (runner-restart-log.json).
|
$pyArgs += $a
|
||||||
Default: F:\CI\State
|
}
|
||||||
|
}
|
||||||
.PARAMETER WebhookUrl
|
|
||||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
.EXAMPLE
|
& $venvPython -m ci_orchestrator monitor runner @pyArgs
|
||||||
# Manual health check
|
exit $LASTEXITCODE
|
||||||
.\Watch-RunnerHealth.ps1
|
|
||||||
|
|
||||||
# With Discord webhook
|
|
||||||
.\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[ValidateRange(1, 10)]
|
|
||||||
[int] $MaxRestarts = 3,
|
|
||||||
|
|
||||||
[string] $ServiceName = 'act_runner',
|
|
||||||
|
|
||||||
[string] $StateDir = 'F:\CI\State',
|
|
||||||
|
|
||||||
[string] $WebhookUrl = '',
|
|
||||||
|
|
||||||
# Optional Gitea API runner-online verification.
|
|
||||||
# When $GiteaUrl is non-empty and the act_runner service is Running, this script
|
|
||||||
# queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online.
|
|
||||||
# Requires an admin Personal Access Token stored in Windows Credential Manager
|
|
||||||
# under $GiteaCredentialTarget (same module: CredentialManager).
|
|
||||||
# Silently skipped if GiteaUrl is empty or the credential is unavailable.
|
|
||||||
[string] $GiteaUrl = '',
|
|
||||||
|
|
||||||
[string] $GiteaCredentialTarget = ''
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
|
|
||||||
$logSource = 'CI-RunnerHealth'
|
|
||||||
|
|
||||||
# ── Helper: write to Event Log ────────────────────────────────────────────────
|
|
||||||
function Write-CIEvent {
|
|
||||||
param([int] $EventId, [string] $EntryType, [string] $Message)
|
|
||||||
try {
|
|
||||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
|
||||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
|
||||||
}
|
|
||||||
Write-EventLog -LogName Application -Source $logSource `
|
|
||||||
-EventId $EventId -EntryType $EntryType -Message $Message
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[RunnerHealth] Could not write Event Log: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Helper: POST webhook ──────────────────────────────────────────────────────
|
|
||||||
function Send-Webhook {
|
|
||||||
param([string] $Content)
|
|
||||||
if (-not $WebhookUrl) { return }
|
|
||||||
$body = @{ content = $Content } | ConvertTo-Json -Compress
|
|
||||||
try {
|
|
||||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
|
||||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
|
||||||
Write-Host "[RunnerHealth] Webhook notified."
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[RunnerHealth] Webhook POST failed: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Maintenance flag check ────────────────────────────────────────────────────
|
|
||||||
# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic.
|
|
||||||
# Create the flag before planned maintenance; delete it when done.
|
|
||||||
$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag'
|
|
||||||
if (Test-Path $maintenanceFlag) {
|
|
||||||
Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic."
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 1: Check service ─────────────────────────────────────────────────────
|
|
||||||
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
|
||||||
if (-not $svc) {
|
|
||||||
Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
|
|
||||||
exit 2
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($svc.Status -eq 'Running') {
|
|
||||||
Write-Host "[RunnerHealth] $ServiceName is Running — OK"
|
|
||||||
|
|
||||||
# ── Optional Gitea runner-online API check ────────────────────────────────
|
|
||||||
if ($GiteaUrl -ne '') {
|
|
||||||
$pat = $null
|
|
||||||
if ($GiteaCredentialTarget -ne '') {
|
|
||||||
try {
|
|
||||||
$cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
|
|
||||||
$pat = $cred.GetNetworkCredential().Password
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($pat) {
|
|
||||||
try {
|
|
||||||
$apiUrl = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners"
|
|
||||||
$runners = Invoke-RestMethod -Uri $apiUrl `
|
|
||||||
-Headers @{ Authorization = "token $pat" } `
|
|
||||||
-Method Get -TimeoutSec 10 -ErrorAction Stop
|
|
||||||
$online = @($runners | Where-Object { $_.online -eq $true })
|
|
||||||
if ($online.Count -gt 0) {
|
|
||||||
Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK"
|
|
||||||
} else {
|
|
||||||
$msg = "Gitea API ($apiUrl) reports 0 online runners. " +
|
|
||||||
"$ServiceName service is Running — possible registration issue."
|
|
||||||
Write-Warning "[RunnerHealth] $msg"
|
|
||||||
Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg
|
|
||||||
Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg"
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[RunnerHealth] Gitea API check failed: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 2: Service not running — load cooldown state ─────────────────────────
|
|
||||||
if (-not (Test-Path $StateDir)) {
|
|
||||||
New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
|
|
||||||
}
|
|
||||||
|
|
||||||
$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
|
|
||||||
$restartLog = @()
|
|
||||||
|
|
||||||
if (Test-Path $cooldownFile) {
|
|
||||||
try {
|
|
||||||
$raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
|
|
||||||
if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
|
|
||||||
} catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" }
|
|
||||||
}
|
|
||||||
|
|
||||||
# Prune timestamps older than 1 hour
|
|
||||||
$cutoff = (Get-Date).AddHours(-1)
|
|
||||||
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
|
|
||||||
$recentCount = $restartLog.Count
|
|
||||||
|
|
||||||
$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
|
|
||||||
Write-Warning "[RunnerHealth] $stateMsg"
|
|
||||||
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg
|
|
||||||
|
|
||||||
# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
|
|
||||||
if ($recentCount -ge $MaxRestarts) {
|
|
||||||
$limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
|
|
||||||
"Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
|
|
||||||
Write-Warning "[RunnerHealth] $limitMsg"
|
|
||||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
|
|
||||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Step 4: Attempt restart ───────────────────────────────────────────────────
|
|
||||||
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
|
|
||||||
try {
|
|
||||||
Restart-Service -Name $ServiceName -Force -ErrorAction Stop
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
$newStatus = (Get-Service -Name $ServiceName).Status
|
|
||||||
$restartMsg = "$ServiceName restarted — new status: $newStatus."
|
|
||||||
Write-Host "[RunnerHealth] $restartMsg"
|
|
||||||
|
|
||||||
# Persist this restart in the cooldown log
|
|
||||||
$restartLog += (Get-Date -Format 'o')
|
|
||||||
$restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue
|
|
||||||
|
|
||||||
$evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
|
|
||||||
Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg
|
|
||||||
|
|
||||||
$prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' }
|
|
||||||
Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg"
|
|
||||||
|
|
||||||
exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
$errMsg = "$ServiceName restart failed: $_"
|
|
||||||
Write-Warning "[RunnerHealth] $errMsg"
|
|
||||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
|
|
||||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
"""Local CI/CD orchestrator package.
|
||||||
|
|
||||||
|
Phase A: Python rewrite of the PowerShell-based orchestrator.
|
||||||
|
See plans/implementation-plan-A-B.md for the full scope.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__version__ = "0.1.0"
|
||||||
|
|
||||||
|
__all__ = ["__version__"]
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""CLI entry point.
|
||||||
|
|
||||||
|
Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor``
|
||||||
|
(disk/runner) and ``report`` (job). Phase A3 adds ``vm new``, ``build``,
|
||||||
|
``artifacts``. Subsequent phases add ``job``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time as time
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator import __version__
|
||||||
|
from ci_orchestrator.commands.artifacts import artifacts
|
||||||
|
from ci_orchestrator.commands.build import build
|
||||||
|
from ci_orchestrator.commands.job import job
|
||||||
|
from ci_orchestrator.commands.monitor import monitor
|
||||||
|
from ci_orchestrator.commands.report import report
|
||||||
|
from ci_orchestrator.commands.vm import vm
|
||||||
|
|
||||||
|
# Re-export so legacy A1 tests that patched ``cli_module.<name>`` keep working.
|
||||||
|
from ci_orchestrator.commands.wait import ( # noqa: F401 (intentional re-exports)
|
||||||
|
KeyringCredentialStore,
|
||||||
|
SshTransport,
|
||||||
|
WinRmTransport,
|
||||||
|
load_backend,
|
||||||
|
wait_ready,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@click.group()
|
||||||
|
@click.version_option(__version__, prog_name="ci-orchestrator")
|
||||||
|
def cli() -> None:
|
||||||
|
"""Local CI/CD orchestrator."""
|
||||||
|
|
||||||
|
|
||||||
|
cli.add_command(wait_ready)
|
||||||
|
cli.add_command(vm)
|
||||||
|
cli.add_command(build)
|
||||||
|
cli.add_command(artifacts)
|
||||||
|
cli.add_command(monitor)
|
||||||
|
cli.add_command(report)
|
||||||
|
cli.add_command(job)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
cli()
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""VM backend implementations.
|
||||||
|
|
||||||
|
The :class:`~ci_orchestrator.backends.protocol.VmBackend` Protocol defines
|
||||||
|
the neutral interface every backend must satisfy. Phase A ships only the
|
||||||
|
:class:`~ci_orchestrator.backends.workstation.WorkstationVmrunBackend`;
|
||||||
|
Phase C will add an ESXi backend behind the same interface.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend, VmHandle, VmState
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
__all__ = ["VmBackend", "VmHandle", "VmState", "WorkstationVmrunBackend", "load_backend"]
|
||||||
|
|
||||||
|
|
||||||
|
def load_backend(config: object) -> VmBackend:
|
||||||
|
"""Factory: pick a backend implementation from ``config.backend.type``.
|
||||||
|
|
||||||
|
Phase C hook: when the ESXi backend lands, this function will dispatch
|
||||||
|
on ``config.backend.type``. For now only ``workstation`` is supported.
|
||||||
|
"""
|
||||||
|
backend_type = getattr(getattr(config, "backend", None), "type", "workstation")
|
||||||
|
if backend_type != "workstation":
|
||||||
|
raise NotImplementedError(
|
||||||
|
f"Backend type '{backend_type}' is not implemented. "
|
||||||
|
"Only 'workstation' is available in Phase A."
|
||||||
|
)
|
||||||
|
vmrun_path = getattr(config, "vmrun_path", None)
|
||||||
|
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""Backend error hierarchy."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
class BackendError(RuntimeError):
|
||||||
|
"""Base error for VM backend failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class BackendNotAvailable(BackendError):
|
||||||
|
"""Backend tooling (e.g. ``vmrun``) is missing or unusable."""
|
||||||
|
|
||||||
|
|
||||||
|
class BackendOperationFailed(BackendError):
|
||||||
|
"""A backend operation (clone, start, ...) returned a non-zero status."""
|
||||||
|
|
||||||
|
def __init__(self, operation: str, returncode: int, output: str) -> None:
|
||||||
|
super().__init__(
|
||||||
|
f"Backend operation '{operation}' failed "
|
||||||
|
f"(exit={returncode}): {output.strip() or '<no output>'}"
|
||||||
|
)
|
||||||
|
self.operation = operation
|
||||||
|
self.returncode = returncode
|
||||||
|
self.output = output
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Neutral VM backend Protocol.
|
||||||
|
|
||||||
|
Phase C hook: keep names hypervisor-agnostic. ``clone_linked``, ``start``,
|
||||||
|
``stop``, ``delete``, ``get_ip``, ``list_snapshots`` map cleanly onto both
|
||||||
|
``vmrun`` and pyVmomi (vSphere). Identifiers are opaque strings (a VMX path
|
||||||
|
on Workstation, a managed object reference on ESXi).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import StrEnum
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class VmState(StrEnum):
|
||||||
|
"""Coarse VM power state, normalised across backends."""
|
||||||
|
|
||||||
|
POWERED_OFF = "powered_off"
|
||||||
|
POWERED_ON = "powered_on"
|
||||||
|
SUSPENDED = "suspended"
|
||||||
|
UNKNOWN = "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class VmHandle:
|
||||||
|
"""Opaque reference to a VM managed by a backend.
|
||||||
|
|
||||||
|
``identifier`` is backend-specific (a VMX path for Workstation, a
|
||||||
|
managed object reference for vSphere). Callers MUST treat it as opaque.
|
||||||
|
"""
|
||||||
|
|
||||||
|
identifier: str
|
||||||
|
name: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class VmBackend(Protocol):
|
||||||
|
"""Hypervisor-agnostic VM operations.
|
||||||
|
|
||||||
|
All methods raise :class:`~ci_orchestrator.backends.errors.BackendError`
|
||||||
|
on failure. Implementations are free to add backend-specific kwargs but
|
||||||
|
MUST honour the documented signature for the listed parameters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def clone_linked(
|
||||||
|
self,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
"""Create a linked clone of ``template`` at snapshot ``snapshot``."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def start(self, handle: VmHandle, headless: bool = True) -> None:
|
||||||
|
"""Power on the VM."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
"""Power off the VM. ``hard=True`` requests a forced power-off."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
"""Remove the VM from inventory and delete its files."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
"""Return the guest's primary IP, or ``None`` if not yet known."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def list_snapshots(self, handle: VmHandle) -> list[str]:
|
||||||
|
"""Return snapshot names defined on ``handle``."""
|
||||||
|
...
|
||||||
|
|
||||||
|
def is_running(self, handle: VmHandle) -> bool:
|
||||||
|
"""Return ``True`` if the VM appears in the running-VM list.
|
||||||
|
|
||||||
|
Implementations MUST NOT rely on guest tools (e.g. ``vmrun
|
||||||
|
getGuestIPAddress``) for this check — see ``AGENTS.md`` error #10.
|
||||||
|
"""
|
||||||
|
...
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
"""VMware Workstation backend (``vmrun -T ws`` wrapper).
|
||||||
|
|
||||||
|
Mirrors the behaviour of ``scripts/_Common.psm1`` :func:`Invoke-Vmrun` plus
|
||||||
|
the higher-level helpers used by ``New-BuildVM.ps1`` and
|
||||||
|
``Wait-VMReady.ps1``. Only ``subprocess`` is used — no PowerShell.
|
||||||
|
|
||||||
|
Cross-platform: ``vmrun`` lives at different paths on Windows vs Linux;
|
||||||
|
when ``vmrun_path`` is not provided, :func:`shutil.which` is consulted and
|
||||||
|
finally a Windows default is tried.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ci_orchestrator.backends.errors import BackendNotAvailable, BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
_DEFAULT_WIN_VMRUN = r"C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe"
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_vmrun(explicit: str | None) -> str:
|
||||||
|
"""Pick a usable ``vmrun`` binary path."""
|
||||||
|
if explicit:
|
||||||
|
if not Path(explicit).is_file():
|
||||||
|
raise BackendNotAvailable(f"vmrun not found at: {explicit}")
|
||||||
|
return explicit
|
||||||
|
found = shutil.which("vmrun")
|
||||||
|
if found:
|
||||||
|
return found
|
||||||
|
if Path(_DEFAULT_WIN_VMRUN).is_file():
|
||||||
|
return _DEFAULT_WIN_VMRUN
|
||||||
|
raise BackendNotAvailable(
|
||||||
|
"vmrun is not available. Install VMware Workstation Pro or set vmrun_path."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class WorkstationVmrunBackend:
|
||||||
|
"""``vmrun -T ws`` backend for VMware Workstation Pro."""
|
||||||
|
|
||||||
|
def __init__(self, vmrun_path: str | None = None) -> None:
|
||||||
|
self._vmrun_path = _resolve_vmrun(vmrun_path)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ utils
|
||||||
|
@property
|
||||||
|
def vmrun_path(self) -> str:
|
||||||
|
return self._vmrun_path
|
||||||
|
|
||||||
|
def _run(
|
||||||
|
self,
|
||||||
|
operation: str,
|
||||||
|
*args: str,
|
||||||
|
check: bool = True,
|
||||||
|
timeout: float | None = None,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
"""Run ``vmrun -T ws <operation> <args...>`` and capture output."""
|
||||||
|
cmd = [self._vmrun_path, "-T", "ws", operation, *args]
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
encoding="utf-8",
|
||||||
|
errors="replace",
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
if check and result.returncode != 0:
|
||||||
|
raise BackendOperationFailed(
|
||||||
|
operation=operation,
|
||||||
|
returncode=result.returncode,
|
||||||
|
output=(result.stdout or "") + (result.stderr or ""),
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
# ------------------------------------------------------------- VmBackend
|
||||||
|
def clone_linked(
|
||||||
|
self,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
if destination is None:
|
||||||
|
destination = str(Path(template).parent.parent / name / f"{name}.vmx")
|
||||||
|
# vmrun clone <srcVMX> <destVMX> linked -snapshot=<name> -cloneName=<name>
|
||||||
|
self._run(
|
||||||
|
"clone",
|
||||||
|
template,
|
||||||
|
destination,
|
||||||
|
"linked",
|
||||||
|
f"-snapshot={snapshot}",
|
||||||
|
f"-cloneName={name}",
|
||||||
|
)
|
||||||
|
return VmHandle(identifier=destination, name=name)
|
||||||
|
|
||||||
|
def start(self, handle: VmHandle, headless: bool = True) -> None:
|
||||||
|
mode = "nogui" if headless else "gui"
|
||||||
|
self._run("start", handle.identifier, mode)
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
mode = "hard" if hard else "soft"
|
||||||
|
# stop may legitimately fail if the VM is already off; treat that as ok
|
||||||
|
result = self._run("stop", handle.identifier, mode, check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
stderr = (result.stderr or "").lower()
|
||||||
|
if "not powered on" in stderr or "is not running" in stderr:
|
||||||
|
return
|
||||||
|
raise BackendOperationFailed(
|
||||||
|
"stop",
|
||||||
|
result.returncode,
|
||||||
|
(result.stdout or "") + (result.stderr or ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
self._run("deleteVM", handle.identifier)
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
# ``getGuestIPAddress`` requires VMware Tools; tolerate failure.
|
||||||
|
args: list[str] = [handle.identifier]
|
||||||
|
if timeout > 0:
|
||||||
|
args.extend(["-wait"])
|
||||||
|
try:
|
||||||
|
result = self._run("getGuestIPAddress", *args, check=False, timeout=timeout or None)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return None
|
||||||
|
if result.returncode != 0:
|
||||||
|
return None
|
||||||
|
ip = (result.stdout or "").strip()
|
||||||
|
if not ip or ip.lower().startswith("error:"):
|
||||||
|
return None
|
||||||
|
return ip
|
||||||
|
|
||||||
|
def list_snapshots(self, handle: VmHandle) -> list[str]:
|
||||||
|
result = self._run("listSnapshots", handle.identifier)
|
||||||
|
lines = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()]
|
||||||
|
# First line is "Total snapshots: N"
|
||||||
|
if lines and lines[0].lower().startswith("total snapshots"):
|
||||||
|
lines = lines[1:]
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def is_running(self, handle: VmHandle) -> bool:
|
||||||
|
"""``vmrun list`` lookup — does NOT call ``getGuestIPAddress``.
|
||||||
|
|
||||||
|
See ``AGENTS.md`` error #10: ``getGuestIPAddress`` blocks for 30-60s
|
||||||
|
without VMware Tools. ``vmrun list`` returns immediately.
|
||||||
|
"""
|
||||||
|
result = self._run("list", check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return False
|
||||||
|
target = Path(handle.identifier).resolve()
|
||||||
|
for line in (result.stdout or "").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.lower().startswith("total running"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if Path(line).resolve() == target:
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return False
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""Sub-command modules for the ``ci_orchestrator`` CLI.
|
||||||
|
|
||||||
|
Each module exposes one or more :class:`click.Command` instances that the
|
||||||
|
top-level :mod:`ci_orchestrator.__main__` registers on the root group.
|
||||||
|
Phase A2 ports the leaf PowerShell scripts (Wait-VMReady, Remove-BuildVM,
|
||||||
|
Cleanup-OrphanedBuildVMs, Watch-DiskSpace, Watch-RunnerHealth,
|
||||||
|
Get-CIJobSummary).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
"""``artifacts`` sub-commands.
|
||||||
|
|
||||||
|
Phase A3 ports the artifact-collection script:
|
||||||
|
|
||||||
|
* ``artifacts collect`` — replaces ``scripts/Get-BuildArtifacts.ps1``
|
||||||
|
|
||||||
|
Uses the abstract :meth:`~ci_orchestrator.transport.winrm.WinRmTransport.fetch`
|
||||||
|
and the SSH transport (via a tar+fetch helper) so no platform-specific
|
||||||
|
``Copy-Item`` / ``scp.exe`` invocations leak into command code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.config import load_config
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- group
|
||||||
|
|
||||||
|
|
||||||
|
@click.group("artifacts")
|
||||||
|
def artifacts() -> None:
|
||||||
|
"""Artifact transfer commands."""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _sh_quote(value: str) -> str:
|
||||||
|
return shlex.quote(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _ps_quote(value: str) -> str:
|
||||||
|
return "'" + value.replace("'", "''") + "'"
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256_hex(path: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with path.open("rb") as fh:
|
||||||
|
for chunk in iter(lambda: fh.read(64 * 1024), b""):
|
||||||
|
h.update(chunk)
|
||||||
|
return h.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_manifest(host_dir: Path, job_id: str, commit: str) -> int:
|
||||||
|
"""Write ``manifest.json`` summarising every collected file. Returns count."""
|
||||||
|
files = [
|
||||||
|
p
|
||||||
|
for p in host_dir.rglob("*")
|
||||||
|
if p.is_file() and p.name != "manifest.json"
|
||||||
|
]
|
||||||
|
entries = []
|
||||||
|
for f in files:
|
||||||
|
rel = f.relative_to(host_dir).as_posix()
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"name": rel,
|
||||||
|
"size": f.stat().st_size,
|
||||||
|
"sha256": _sha256_hex(f),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
payload = {
|
||||||
|
"ts": datetime.now(tz=UTC).isoformat(),
|
||||||
|
"jobId": job_id,
|
||||||
|
"commit": commit,
|
||||||
|
"files": entries,
|
||||||
|
}
|
||||||
|
out = host_dir / "manifest.json"
|
||||||
|
out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
||||||
|
return len(entries)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- Linux flow
|
||||||
|
|
||||||
|
|
||||||
|
def _linux_collect(
|
||||||
|
*,
|
||||||
|
ip_address: str,
|
||||||
|
ssh_user: str,
|
||||||
|
key_path: str | None,
|
||||||
|
known_hosts: str | None,
|
||||||
|
guest_path: str,
|
||||||
|
host_dir: Path,
|
||||||
|
) -> None:
|
||||||
|
"""Tar the guest directory, fetch the single archive, extract locally."""
|
||||||
|
click.echo(f"[artifacts collect] tarring guest:{guest_path}")
|
||||||
|
archive_remote = f"/tmp/ci-artifacts-{ip_address.replace('.', '-')}.tar.gz"
|
||||||
|
with SshTransport(
|
||||||
|
ip_address,
|
||||||
|
username=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=known_hosts,
|
||||||
|
) as t:
|
||||||
|
# Build a tarball whose contents are *inside* guest_path. Strip the
|
||||||
|
# parent component so files land directly under host_dir.
|
||||||
|
t.run(
|
||||||
|
f"tar -czf {_sh_quote(archive_remote)} "
|
||||||
|
f"-C {_sh_quote(guest_path)} ."
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
local_tar = Path(td) / "artifacts.tar.gz"
|
||||||
|
t.fetch(archive_remote, str(local_tar))
|
||||||
|
with tarfile.open(local_tar, "r:gz") as tf:
|
||||||
|
# ``data`` filter (Python 3.12+) protects against path
|
||||||
|
# traversal in malicious tarballs.
|
||||||
|
tf.extractall(host_dir, filter="data")
|
||||||
|
finally:
|
||||||
|
t.run(f"rm -f {_sh_quote(archive_remote)}", check=False)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- Windows flow
|
||||||
|
|
||||||
|
|
||||||
|
def _windows_collect(
|
||||||
|
*,
|
||||||
|
ip_address: str,
|
||||||
|
credential_target: str,
|
||||||
|
guest_path: str,
|
||||||
|
host_dir: Path,
|
||||||
|
include_logs: bool,
|
||||||
|
) -> None:
|
||||||
|
cred = KeyringCredentialStore().get(credential_target)
|
||||||
|
remote_zip = "C:\\CI\\ci-artifacts-transfer.zip"
|
||||||
|
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||||
|
# guest_path is the collect directory (raw build outputs).
|
||||||
|
# Verify it exists and is non-empty.
|
||||||
|
probe = t.run(
|
||||||
|
f"if ((Test-Path {_ps_quote(guest_path)}) -and "
|
||||||
|
f"(Get-ChildItem -Force -- {_ps_quote(guest_path)})) "
|
||||||
|
f"{{ 'OK' }} else {{ 'MISSING' }}",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if "MISSING" in probe.stdout:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"artifact not found or empty in guest: {guest_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Zip the directory contents as a single transport file, fetch,
|
||||||
|
# extract into host_dir. Mirrors the Linux tar flow so both OSes
|
||||||
|
# deposit raw files; the final single archive is produced by
|
||||||
|
# actions/upload-artifact (no zip-in-zip).
|
||||||
|
t.run(
|
||||||
|
f"if (Test-Path {_ps_quote(remote_zip)}) "
|
||||||
|
f"{{ Remove-Item {_ps_quote(remote_zip)} -Force }}; "
|
||||||
|
f"Compress-Archive -Path (Join-Path {_ps_quote(guest_path)} '*') "
|
||||||
|
f"-DestinationPath {_ps_quote(remote_zip)} -Force"
|
||||||
|
)
|
||||||
|
click.echo(f"[artifacts collect] fetching artifacts from {guest_path}")
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
local_zip = Path(td) / "artifacts.zip"
|
||||||
|
t.fetch(remote_zip, str(local_zip))
|
||||||
|
with zipfile.ZipFile(local_zip) as zf:
|
||||||
|
zf.extractall(host_dir)
|
||||||
|
finally:
|
||||||
|
t.run(
|
||||||
|
f"Remove-Item {_ps_quote(remote_zip)} -Force "
|
||||||
|
f"-ErrorAction SilentlyContinue",
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
if include_logs:
|
||||||
|
# List log files in the build dir and fetch each one.
|
||||||
|
list_ps = (
|
||||||
|
"Get-ChildItem -Path 'C:\\CI\\build' -Filter '*.log' -Recurse "
|
||||||
|
"-ErrorAction SilentlyContinue | "
|
||||||
|
"ForEach-Object { $_.FullName }"
|
||||||
|
)
|
||||||
|
listing = t.run(list_ps, check=False)
|
||||||
|
for line in (listing.stdout or "").splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
dest = host_dir / Path(line).name
|
||||||
|
click.echo(f"[artifacts collect] fetching log: {Path(line).name}")
|
||||||
|
try:
|
||||||
|
t.fetch(line, str(dest))
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[artifacts collect] log fetch failed ({line}): {exc}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not any(host_dir.iterdir()):
|
||||||
|
raise click.ClickException(
|
||||||
|
f"no artifacts after collect from guest: {guest_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- CLI
|
||||||
|
|
||||||
|
|
||||||
|
@artifacts.command("collect")
|
||||||
|
@click.option("--ip-address", "ip_address", required=True)
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||||
|
default="windows",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"guest_artifact_path",
|
||||||
|
required=True,
|
||||||
|
help="Guest collect directory (Windows and Linux).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--host-artifact-dir",
|
||||||
|
"host_artifact_dir",
|
||||||
|
required=True,
|
||||||
|
help="Local directory to deposit artifacts (created if missing).",
|
||||||
|
)
|
||||||
|
@click.option("--include-logs", "include_logs", is_flag=True, default=False)
|
||||||
|
@click.option("--credential-target", "credential_target", default=None)
|
||||||
|
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||||
|
@click.option("--ssh-key-path", "ssh_key_path", default=None)
|
||||||
|
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
|
||||||
|
@click.option("--job-id", "job_id", default="")
|
||||||
|
@click.option("--commit", "commit", default="")
|
||||||
|
def artifacts_collect(
|
||||||
|
ip_address: str,
|
||||||
|
guest_os: str,
|
||||||
|
guest_artifact_path: str,
|
||||||
|
host_artifact_dir: str,
|
||||||
|
include_logs: bool,
|
||||||
|
credential_target: str | None,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_known_hosts_file: str | None,
|
||||||
|
job_id: str,
|
||||||
|
commit: str,
|
||||||
|
) -> None:
|
||||||
|
"""Copy build artifacts from a guest VM to a local directory.
|
||||||
|
|
||||||
|
Mirrors ``scripts/Get-BuildArtifacts.ps1``. Writes ``manifest.json``
|
||||||
|
when ``--job-id`` or ``--commit`` is provided.
|
||||||
|
"""
|
||||||
|
guest_os = guest_os.lower()
|
||||||
|
host_dir = Path(host_artifact_dir)
|
||||||
|
host_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
config = load_config()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if guest_os == "linux":
|
||||||
|
key_path: str | None = ssh_key_path
|
||||||
|
if key_path is None and config.ssh_key_path is not None:
|
||||||
|
key_path = str(config.ssh_key_path)
|
||||||
|
_linux_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
guest_path=guest_artifact_path,
|
||||||
|
host_dir=host_dir,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
target = credential_target or config.guest_cred_target
|
||||||
|
_windows_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
guest_path=guest_artifact_path,
|
||||||
|
host_dir=host_dir,
|
||||||
|
include_logs=include_logs,
|
||||||
|
)
|
||||||
|
except (TransportError, TransportCommandError) as exc:
|
||||||
|
raise click.ClickException(str(exc)) from exc
|
||||||
|
|
||||||
|
transferred = [p for p in host_dir.rglob("*") if p.is_file()]
|
||||||
|
if not transferred:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"no files were transferred into {host_dir}"
|
||||||
|
)
|
||||||
|
click.echo(f"[artifacts collect] {len(transferred)} file(s) collected.")
|
||||||
|
|
||||||
|
if job_id or commit:
|
||||||
|
count = _write_manifest(host_dir, job_id, commit)
|
||||||
|
click.echo(f"[artifacts collect] manifest.json written ({count} file(s)).")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["artifacts", "artifacts_collect"]
|
||||||
@@ -0,0 +1,595 @@
|
|||||||
|
"""``build`` sub-commands.
|
||||||
|
|
||||||
|
Phase A3 ports the remote build pipeline:
|
||||||
|
|
||||||
|
* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1``
|
||||||
|
|
||||||
|
The original PS script supports two source-provisioning modes (host-side
|
||||||
|
clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom
|
||||||
|
build command and optional artifact packaging. This Python port preserves
|
||||||
|
the mechanics while keeping the guest workdir and artifact paths fully
|
||||||
|
parameterised — see Phase C hooks below.
|
||||||
|
|
||||||
|
Phase C hook: the guest workdir / artifact paths are passed as opaque
|
||||||
|
strings; this module makes no assumption about Windows vs POSIX path
|
||||||
|
syntax beyond a sensible default per ``--guest-os``. ``build run`` does
|
||||||
|
not import ``WorkstationVmrunBackend`` directly; it relies on the
|
||||||
|
transport layer to talk to whatever guest the orchestrator points it at.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shlex
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import zipfile
|
||||||
|
from collections.abc import Iterable
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.config import load_config
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- group
|
||||||
|
|
||||||
|
|
||||||
|
@click.group("build")
|
||||||
|
def build() -> None:
|
||||||
|
"""Remote build commands."""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_extra_env(items: Iterable[str]) -> dict[str, str]:
|
||||||
|
"""Parse ``KEY=VALUE`` pairs into a dict.
|
||||||
|
|
||||||
|
Empty entries are ignored. A pair without ``=`` raises ``BadParameter``.
|
||||||
|
"""
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for raw in items:
|
||||||
|
if not raw:
|
||||||
|
continue
|
||||||
|
if "=" not in raw:
|
||||||
|
raise click.BadParameter(
|
||||||
|
f"--extra-env value must be KEY=VALUE, got: {raw!r}"
|
||||||
|
)
|
||||||
|
key, value = raw.split("=", 1)
|
||||||
|
if not key:
|
||||||
|
raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}")
|
||||||
|
out[key] = value
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_dir(source_dir: Path, archive_path: Path) -> None:
|
||||||
|
"""Create a deterministic ZIP archive containing the contents of ``source_dir``."""
|
||||||
|
with zipfile.ZipFile(
|
||||||
|
archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1
|
||||||
|
) as zf:
|
||||||
|
for item in source_dir.rglob("*"):
|
||||||
|
if item.is_file():
|
||||||
|
zf.write(item, arcname=item.relative_to(source_dir))
|
||||||
|
|
||||||
|
|
||||||
|
def _tar_dir(source_dir: Path, archive_path: Path) -> None:
|
||||||
|
"""Create a gzip tar archive containing the contents of ``source_dir``."""
|
||||||
|
with tarfile.open(archive_path, "w:gz") as tf:
|
||||||
|
for item in source_dir.iterdir():
|
||||||
|
tf.add(item, arcname=item.name)
|
||||||
|
|
||||||
|
|
||||||
|
def _ps_quote(value: str) -> str:
|
||||||
|
"""Quote a value for embedding in a PowerShell single-quoted string."""
|
||||||
|
return "'" + value.replace("'", "''") + "'"
|
||||||
|
|
||||||
|
|
||||||
|
def _sh_quote(value: str) -> str:
|
||||||
|
"""POSIX shell single-quote escaping."""
|
||||||
|
return shlex.quote(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _report_build_output(stdout: str, stderr: str) -> str:
|
||||||
|
"""Emit guest build stdout/stderr to the CI log, flushed.
|
||||||
|
|
||||||
|
Returns the last few lines for the failure message so the cause is
|
||||||
|
visible even if stream buffering hides the body on a non-zero exit.
|
||||||
|
"""
|
||||||
|
click.echo("[build run] ----- build output -----")
|
||||||
|
if stdout:
|
||||||
|
sys.stdout.write(stdout if stdout.endswith("\n") else stdout + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
if stderr:
|
||||||
|
sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
click.echo("[build run] ----- end build output -----")
|
||||||
|
return _tail_lines(stdout, stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _tail_lines(stdout: str, stderr: str) -> str:
|
||||||
|
"""Last few output lines for a failure message (no printing)."""
|
||||||
|
combined = (stderr or "").strip() or (stdout or "").strip()
|
||||||
|
lines = combined.splitlines()
|
||||||
|
return " | ".join(lines[-5:]) if lines else "(no build output)"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- Linux flow
|
||||||
|
|
||||||
|
|
||||||
|
def _linux_build(
|
||||||
|
*,
|
||||||
|
ip_address: str,
|
||||||
|
ssh_user: str,
|
||||||
|
key_path: str | None,
|
||||||
|
known_hosts: str | None,
|
||||||
|
workdir: str,
|
||||||
|
output_dir: str,
|
||||||
|
host_source_dir: str | None,
|
||||||
|
clone_url: str | None,
|
||||||
|
clone_branch: str,
|
||||||
|
clone_commit: str,
|
||||||
|
clone_submodules: bool,
|
||||||
|
build_command: str,
|
||||||
|
artifact_source: str,
|
||||||
|
extra_env: dict[str, str],
|
||||||
|
skip_artifact: bool,
|
||||||
|
) -> None:
|
||||||
|
click.echo("[build run] Linux build mode (SSH transport)")
|
||||||
|
|
||||||
|
with SshTransport(
|
||||||
|
ip_address,
|
||||||
|
username=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=known_hosts,
|
||||||
|
) as t:
|
||||||
|
t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
|
||||||
|
|
||||||
|
if host_source_dir:
|
||||||
|
click.echo(f"[build run] packaging host source: {host_source_dir}")
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
tar_path = Path(td) / "ci-src.tar.gz"
|
||||||
|
_tar_dir(Path(host_source_dir), tar_path)
|
||||||
|
guest_tar = f"/tmp/{tar_path.name}"
|
||||||
|
t.copy(str(tar_path), guest_tar)
|
||||||
|
click.echo(f"[build run] extracting source in guest: {workdir}")
|
||||||
|
t.run(
|
||||||
|
f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} "
|
||||||
|
f"&& rm -f {_sh_quote(guest_tar)}"
|
||||||
|
)
|
||||||
|
elif clone_url:
|
||||||
|
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
|
||||||
|
cmd = (
|
||||||
|
f"GIT_TERMINAL_PROMPT=0 "
|
||||||
|
f"git clone --depth 1 --branch {_sh_quote(clone_branch)} "
|
||||||
|
f"{'--recurse-submodules ' if clone_submodules else ''}"
|
||||||
|
f"{_sh_quote(clone_url)} {_sh_quote(workdir)}"
|
||||||
|
)
|
||||||
|
t.run(cmd)
|
||||||
|
if clone_commit:
|
||||||
|
# The shallow --depth 1 --branch clone only has the branch
|
||||||
|
# tip. If the branch advanced between workflow trigger and
|
||||||
|
# runner pickup, the pinned commit is absent ("reference is
|
||||||
|
# not a tree"). Fetch exactly that commit, then check out.
|
||||||
|
t.run(
|
||||||
|
f"git -C {_sh_quote(workdir)} fetch --depth 1 origin "
|
||||||
|
f"{_sh_quote(clone_commit)} && "
|
||||||
|
f"git -C {_sh_quote(workdir)} checkout "
|
||||||
|
f"{_sh_quote(clone_commit)}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise click.ClickException(
|
||||||
|
"Linux build requires either --host-source-dir or --clone-url."
|
||||||
|
)
|
||||||
|
|
||||||
|
env_prefix = "".join(
|
||||||
|
f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
|
||||||
|
)
|
||||||
|
cmd = build_command or "make"
|
||||||
|
# PYTHONUNBUFFERED so the guest python flushes promptly and the
|
||||||
|
# streamed output is timely rather than emitted in one block.
|
||||||
|
full_cmd = (
|
||||||
|
f"{env_prefix}export PYTHONUNBUFFERED=1; "
|
||||||
|
f"cd {_sh_quote(workdir)} && {cmd}"
|
||||||
|
)
|
||||||
|
click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}")
|
||||||
|
click.echo("[build run] ----- build output (live) -----")
|
||||||
|
result = t.run_streaming(full_cmd, check=False)
|
||||||
|
click.echo("[build run] ----- end build output -----")
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise click.ClickException(
|
||||||
|
"build command failed "
|
||||||
|
f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if skip_artifact:
|
||||||
|
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Resolve the artifact source: absolute path as-is, else relative
|
||||||
|
# to the build workdir. (artifact_source may be absolute, e.g.
|
||||||
|
# /opt/ci/output; the old `workdir + '/' + src` produced
|
||||||
|
# /opt/ci/build//opt/ci/output and silently found nothing.)
|
||||||
|
src = artifact_source or "dist"
|
||||||
|
srcpath = src.rstrip("/") if src.startswith("/") else workdir.rstrip("/") + "/" + src
|
||||||
|
out = output_dir.rstrip("/")
|
||||||
|
|
||||||
|
if srcpath == out or srcpath.startswith(out + "/"):
|
||||||
|
# The build wrote directly into the collect dir; do NOT wipe it
|
||||||
|
# (that deleted the build output) and nothing to copy. Fail
|
||||||
|
# loudly if it is missing or empty rather than collecting an
|
||||||
|
# empty artifact.
|
||||||
|
click.echo(
|
||||||
|
f"[build run] artifact source is the output dir ({out}); collecting in place"
|
||||||
|
)
|
||||||
|
t.run(
|
||||||
|
f"if [ ! -d {_sh_quote(out)} ] || "
|
||||||
|
f'[ -z "$(ls -A {_sh_quote(out)} 2>/dev/null)" ]; then '
|
||||||
|
f"echo '[build run] artifact source not found or empty: {out}' >&2; "
|
||||||
|
f"exit 1; fi"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
t.run(f"rm -rf {_sh_quote(out)} && mkdir -p {_sh_quote(out)}")
|
||||||
|
cp_cmd = (
|
||||||
|
f"if [ -d {_sh_quote(srcpath)} ]; then "
|
||||||
|
f"cp -r {_sh_quote(srcpath + '/.')} {_sh_quote(out + '/')}; "
|
||||||
|
f"elif [ -e {_sh_quote(srcpath)} ]; then "
|
||||||
|
f"cp {_sh_quote(srcpath)} {_sh_quote(out + '/')}; "
|
||||||
|
f"else echo '[build run] artifact source not found: {srcpath}' >&2; "
|
||||||
|
f"exit 1; fi"
|
||||||
|
)
|
||||||
|
t.run(cp_cmd)
|
||||||
|
click.echo(f"[build run] artifact staged at {out}")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- Windows flow
|
||||||
|
|
||||||
|
|
||||||
|
def _windows_build(
|
||||||
|
*,
|
||||||
|
ip_address: str,
|
||||||
|
credential_target: str,
|
||||||
|
workdir: str,
|
||||||
|
output_dir: str,
|
||||||
|
host_source_dir: str | None,
|
||||||
|
clone_url: str | None,
|
||||||
|
clone_branch: str,
|
||||||
|
clone_commit: str,
|
||||||
|
clone_submodules: bool,
|
||||||
|
build_command: str,
|
||||||
|
artifact_source: str,
|
||||||
|
configuration: str,
|
||||||
|
extra_env: dict[str, str],
|
||||||
|
skip_artifact: bool,
|
||||||
|
use_shared_cache: bool,
|
||||||
|
) -> None:
|
||||||
|
click.echo("[build run] Windows build mode (WinRM transport)")
|
||||||
|
cred = KeyringCredentialStore().get(credential_target)
|
||||||
|
|
||||||
|
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||||
|
# Prepare workdir + artifact output directory.
|
||||||
|
prep_ps = (
|
||||||
|
f"$work = {_ps_quote(workdir)}; "
|
||||||
|
f"$out = {_ps_quote(output_dir)}; "
|
||||||
|
"if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; "
|
||||||
|
"if (Test-Path $work) { Remove-Item $work -Recurse -Force }; "
|
||||||
|
"New-Item -ItemType Directory -Path $work -Force | Out-Null"
|
||||||
|
)
|
||||||
|
t.run(prep_ps)
|
||||||
|
|
||||||
|
if host_source_dir:
|
||||||
|
click.echo(f"[build run] zipping host source: {host_source_dir}")
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
zip_path = Path(td) / "ci-src.zip"
|
||||||
|
_zip_dir(Path(host_source_dir), zip_path)
|
||||||
|
size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1)
|
||||||
|
click.echo(f"[build run] uploading source archive ({size_mb} MB)")
|
||||||
|
guest_zip = "C:\\CI\\src-transfer.zip"
|
||||||
|
t.copy(str(zip_path), guest_zip)
|
||||||
|
t.run(
|
||||||
|
f"Expand-Archive -Path {_ps_quote(guest_zip)} "
|
||||||
|
f"-DestinationPath {_ps_quote(workdir)} -Force; "
|
||||||
|
f"Remove-Item {_ps_quote(guest_zip)} -Force"
|
||||||
|
)
|
||||||
|
elif clone_url:
|
||||||
|
click.echo(f"[build run] cloning {clone_url} into guest:{workdir}")
|
||||||
|
sub_flag = " --recurse-submodules" if clone_submodules else ""
|
||||||
|
t.run(
|
||||||
|
f"$env:GIT_TERMINAL_PROMPT = '0'; "
|
||||||
|
f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); "
|
||||||
|
f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} "
|
||||||
|
f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)"
|
||||||
|
)
|
||||||
|
if clone_commit:
|
||||||
|
# Shallow clone has only the branch tip; fetch the pinned
|
||||||
|
# commit explicitly so checkout works even if the branch
|
||||||
|
# advanced between trigger and runner pickup.
|
||||||
|
t.run(
|
||||||
|
f"git -C {_ps_quote(workdir)} fetch --depth 1 origin "
|
||||||
|
f"{_ps_quote(clone_commit)}; "
|
||||||
|
f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise click.ClickException(
|
||||||
|
"Windows build requires either --host-source-dir or --clone-url."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Inject env vars + run the build.
|
||||||
|
env_setup = "; ".join(
|
||||||
|
f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items()
|
||||||
|
)
|
||||||
|
if env_setup:
|
||||||
|
env_setup += "; "
|
||||||
|
if use_shared_cache:
|
||||||
|
env_setup += (
|
||||||
|
"$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; "
|
||||||
|
"$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; "
|
||||||
|
)
|
||||||
|
if build_command:
|
||||||
|
run_cmd = build_command
|
||||||
|
click.echo(f"[build run] custom build command: {run_cmd}")
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[build run] default build: dotnet restore + build (configuration={configuration})"
|
||||||
|
)
|
||||||
|
run_cmd = (
|
||||||
|
"dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; "
|
||||||
|
f"dotnet build --configuration {_ps_quote(configuration)} "
|
||||||
|
"--output (Join-Path (Get-Location) 'bin\\CIOutput')"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Run the build command as PowerShell on the Windows guest. It is
|
||||||
|
# authored in PS by the workflow (New-Item, Set-Content, ...) and
|
||||||
|
# may also invoke native exes (python, dotnet). Wrapping it in
|
||||||
|
# `cmd /c` broke PS-native commands ("'New-Item' is not
|
||||||
|
# recognized"). LASTEXITCODE is seeded to 0 so a PS-only command
|
||||||
|
# (which never sets it) exits cleanly; native tools still set it.
|
||||||
|
full_ps = (
|
||||||
|
f"{env_setup}Set-Location {_ps_quote(workdir)}; "
|
||||||
|
f"$ErrorActionPreference='Continue'; "
|
||||||
|
f"$global:LASTEXITCODE = 0; "
|
||||||
|
f"{run_cmd}; "
|
||||||
|
"if ($null -eq $LASTEXITCODE) { exit 0 } else { exit $LASTEXITCODE }"
|
||||||
|
)
|
||||||
|
result = t.run(full_ps, check=False)
|
||||||
|
tail = _report_build_output(result.stdout, result.stderr)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"build command failed (exit {result.returncode}): {tail}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if skip_artifact:
|
||||||
|
click.echo("[build run] artifact packaging skipped (--skip-artifact).")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Stage the artifact source into the collect dir as raw files
|
||||||
|
# (no intermediate zip — `actions/upload-artifact` produces the
|
||||||
|
# single final archive; mirrors the Linux flow, avoids zip-in-zip).
|
||||||
|
# artifact_source may be absolute (e.g. C:\CI\output) or relative
|
||||||
|
# to the build workdir; PowerShell Join-Path does not treat a
|
||||||
|
# rooted segment as absolute, so resolve explicitly. When the
|
||||||
|
# source IS the collect dir (build wrote there directly), collect
|
||||||
|
# in place — do NOT wipe it (that deleted the build output).
|
||||||
|
src_rel = (artifact_source or "dist") if build_command else "bin\\CIOutput"
|
||||||
|
|
||||||
|
pkg_ps = (
|
||||||
|
f"$work = {_ps_quote(workdir)}; "
|
||||||
|
f"$rel = {_ps_quote(src_rel)}; "
|
||||||
|
"$src = if ([System.IO.Path]::IsPathRooted($rel)) "
|
||||||
|
"{ $rel } else { Join-Path $work $rel }; "
|
||||||
|
f"$out = {_ps_quote(output_dir)}; "
|
||||||
|
"$sf = [System.IO.Path]::GetFullPath($src).TrimEnd('\\'); "
|
||||||
|
"$of = [System.IO.Path]::GetFullPath($out).TrimEnd('\\'); "
|
||||||
|
"$inPlace = ($sf -ieq $of) -or "
|
||||||
|
"$sf.StartsWith($of + '\\', "
|
||||||
|
"[System.StringComparison]::OrdinalIgnoreCase); "
|
||||||
|
"if ($inPlace) { "
|
||||||
|
" if (-not (Test-Path $out) -or "
|
||||||
|
" -not (Get-ChildItem -Force -- $out)) { "
|
||||||
|
" throw \"[build run] artifact source not found or empty: $out\" } "
|
||||||
|
"} else { "
|
||||||
|
" if (-not (Test-Path $src)) { "
|
||||||
|
" throw \"[build run] artifact source not found: $src\" }; "
|
||||||
|
" if (Test-Path $out) { Remove-Item $out -Recurse -Force }; "
|
||||||
|
" New-Item -ItemType Directory -Path $out -Force | Out-Null; "
|
||||||
|
" if (Test-Path $src -PathType Leaf) { "
|
||||||
|
" Copy-Item $src -Destination $out -Force "
|
||||||
|
" } else { "
|
||||||
|
" Copy-Item (Join-Path $src '*') -Destination $out -Recurse -Force } "
|
||||||
|
"}"
|
||||||
|
)
|
||||||
|
t.run(pkg_ps)
|
||||||
|
click.echo(f"[build run] artifact staged at {output_dir}")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- CLI
|
||||||
|
|
||||||
|
|
||||||
|
@build.command("run")
|
||||||
|
@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.")
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||||
|
default="windows",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--credential-target",
|
||||||
|
"credential_target",
|
||||||
|
default=None,
|
||||||
|
help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.",
|
||||||
|
)
|
||||||
|
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||||
|
@click.option(
|
||||||
|
"--ssh-key-path",
|
||||||
|
"ssh_key_path",
|
||||||
|
default=None,
|
||||||
|
help="SSH private key path (Linux). Defaults to config.ssh_key_path.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--ssh-known-hosts-file",
|
||||||
|
"ssh_known_hosts_file",
|
||||||
|
default=None,
|
||||||
|
help="Persistent known_hosts file (Linux). Default: permissive (None).",
|
||||||
|
)
|
||||||
|
@click.option("--host-source-dir", "host_source_dir", default=None)
|
||||||
|
@click.option("--clone-url", "clone_url", default=None)
|
||||||
|
@click.option("--clone-branch", "clone_branch", default="main", show_default=True)
|
||||||
|
@click.option("--clone-commit", "clone_commit", default="")
|
||||||
|
@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False)
|
||||||
|
@click.option(
|
||||||
|
"--gitea-credential-target",
|
||||||
|
"gitea_credential_target",
|
||||||
|
default="GiteaPAT",
|
||||||
|
show_default=True,
|
||||||
|
help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.",
|
||||||
|
)
|
||||||
|
@click.option("--build-command", "build_command", default="")
|
||||||
|
@click.option(
|
||||||
|
"--guest-work-dir",
|
||||||
|
"guest_work_dir",
|
||||||
|
default=None,
|
||||||
|
help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-output-dir",
|
||||||
|
"guest_output_dir",
|
||||||
|
default=None,
|
||||||
|
help="Collect dir inside the Windows guest (default: C:\\CI\\output). "
|
||||||
|
"Raw build outputs are staged here; upload-artifact zips them.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-linux-work-dir",
|
||||||
|
"guest_linux_work_dir",
|
||||||
|
default=None,
|
||||||
|
help="Alias for --guest-work-dir, kept for shim compatibility (Linux).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-linux-output-dir",
|
||||||
|
"guest_linux_output_dir",
|
||||||
|
default="/opt/ci/output",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
|
||||||
|
@click.option("--configuration", default="Release", show_default=True)
|
||||||
|
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||||
|
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||||
|
@click.option(
|
||||||
|
"--extra-env",
|
||||||
|
"extra_env",
|
||||||
|
multiple=True,
|
||||||
|
help="Repeatable KEY=VALUE pairs injected into the guest environment.",
|
||||||
|
)
|
||||||
|
def build_run(
|
||||||
|
ip_address: str,
|
||||||
|
guest_os: str,
|
||||||
|
credential_target: str | None,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_known_hosts_file: str | None,
|
||||||
|
host_source_dir: str | None,
|
||||||
|
clone_url: str | None,
|
||||||
|
clone_branch: str,
|
||||||
|
clone_commit: str,
|
||||||
|
clone_submodules: bool,
|
||||||
|
gitea_credential_target: str,
|
||||||
|
build_command: str,
|
||||||
|
guest_work_dir: str | None,
|
||||||
|
guest_output_dir: str | None,
|
||||||
|
guest_linux_work_dir: str | None,
|
||||||
|
guest_linux_output_dir: str,
|
||||||
|
guest_artifact_source: str,
|
||||||
|
configuration: str,
|
||||||
|
skip_artifact: bool,
|
||||||
|
use_shared_cache: bool,
|
||||||
|
extra_env: tuple[str, ...],
|
||||||
|
) -> None:
|
||||||
|
"""Run a build inside a guest VM, optionally packaging the result.
|
||||||
|
|
||||||
|
Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via
|
||||||
|
one of:
|
||||||
|
|
||||||
|
* ``--host-source-dir`` — local directory zipped/tarred and uploaded;
|
||||||
|
* ``--clone-url`` — ``git clone`` invoked inside the guest.
|
||||||
|
|
||||||
|
Phase C hook: ``--guest-work-dir`` and ``--guest-output-dir`` are
|
||||||
|
parameterised; defaults are applied here, never inside the transport
|
||||||
|
layer.
|
||||||
|
"""
|
||||||
|
del gitea_credential_target # not yet honoured by Python port
|
||||||
|
guest_os = guest_os.lower()
|
||||||
|
|
||||||
|
if host_source_dir and clone_url:
|
||||||
|
raise click.BadParameter(
|
||||||
|
"Provide either --host-source-dir or --clone-url, not both."
|
||||||
|
)
|
||||||
|
if not host_source_dir and not clone_url:
|
||||||
|
raise click.BadParameter(
|
||||||
|
"Provide either --host-source-dir or --clone-url."
|
||||||
|
)
|
||||||
|
if host_source_dir and not Path(host_source_dir).is_dir():
|
||||||
|
raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}")
|
||||||
|
|
||||||
|
extra = _parse_extra_env(extra_env)
|
||||||
|
config = load_config()
|
||||||
|
|
||||||
|
if guest_os == "linux":
|
||||||
|
workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build"
|
||||||
|
key_path: str | None = ssh_key_path
|
||||||
|
if key_path is None and config.ssh_key_path is not None:
|
||||||
|
key_path = str(config.ssh_key_path)
|
||||||
|
try:
|
||||||
|
_linux_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
workdir=workdir,
|
||||||
|
output_dir=guest_linux_output_dir,
|
||||||
|
host_source_dir=host_source_dir,
|
||||||
|
clone_url=clone_url,
|
||||||
|
clone_branch=clone_branch,
|
||||||
|
clone_commit=clone_commit,
|
||||||
|
clone_submodules=clone_submodules,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
extra_env=extra,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
)
|
||||||
|
except (TransportError, TransportCommandError) as exc:
|
||||||
|
raise click.ClickException(str(exc)) from exc
|
||||||
|
else:
|
||||||
|
workdir = guest_work_dir or "C:\\CI\\build"
|
||||||
|
output_dir = guest_output_dir or "C:\\CI\\output"
|
||||||
|
target = credential_target or config.guest_cred_target
|
||||||
|
try:
|
||||||
|
_windows_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
workdir=workdir,
|
||||||
|
output_dir=output_dir,
|
||||||
|
host_source_dir=host_source_dir,
|
||||||
|
clone_url=clone_url,
|
||||||
|
clone_branch=clone_branch,
|
||||||
|
clone_commit=clone_commit,
|
||||||
|
clone_submodules=clone_submodules,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
configuration=configuration,
|
||||||
|
extra_env=extra,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
use_shared_cache=use_shared_cache,
|
||||||
|
)
|
||||||
|
except (TransportError, TransportCommandError) as exc:
|
||||||
|
raise click.ClickException(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["build", "build_run"]
|
||||||
@@ -0,0 +1,638 @@
|
|||||||
|
"""``job`` sub-command — full CI orchestration.
|
||||||
|
|
||||||
|
Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a
|
||||||
|
template snapshot, wait for the guest to become reachable, run the build
|
||||||
|
inside the guest, optionally collect artifacts, and **always** destroy the
|
||||||
|
clone (even on failure or ``KeyboardInterrupt``).
|
||||||
|
|
||||||
|
Composition over re-spawn: this module imports the in-process helpers
|
||||||
|
from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather
|
||||||
|
than re-invoking ``python -m ci_orchestrator`` for each phase.
|
||||||
|
|
||||||
|
Phase C hook: the backend is selected via :func:`load_backend` so that an
|
||||||
|
ESXi backend can be plugged in by changing ``[backend].type`` in
|
||||||
|
``config.toml``. ``WorkstationVmrunBackend`` is never imported directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.backends import load_backend
|
||||||
|
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.commands.artifacts import (
|
||||||
|
_linux_collect,
|
||||||
|
_windows_collect,
|
||||||
|
_write_manifest,
|
||||||
|
)
|
||||||
|
from ci_orchestrator.commands.build import _linux_build, _windows_build
|
||||||
|
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
|
||||||
|
from ci_orchestrator.config import Config, load_config
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
from ci_orchestrator.transport.errors import TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
|
|
||||||
|
|
||||||
|
_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||||
|
_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse")
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> float:
|
||||||
|
return time.monotonic()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_guest_os_from_vmx(vmx_path: Path) -> str:
|
||||||
|
"""Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line."""
|
||||||
|
try:
|
||||||
|
text = vmx_path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
except OSError:
|
||||||
|
return "windows"
|
||||||
|
m = re.search(r'guestOS\s*=\s*"([^"]+)"', text)
|
||||||
|
if not m:
|
||||||
|
return "windows"
|
||||||
|
detected = m.group(1).lower()
|
||||||
|
if any(h in detected for h in _GUEST_OS_LINUX_HINTS):
|
||||||
|
return "linux"
|
||||||
|
return "windows"
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None:
|
||||||
|
"""Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0."""
|
||||||
|
if cpu <= 0 and memory_mb <= 0:
|
||||||
|
return
|
||||||
|
lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||||
|
out: list[str] = []
|
||||||
|
seen_cpu = False
|
||||||
|
seen_mem = False
|
||||||
|
for line in lines:
|
||||||
|
low = line.strip().lower()
|
||||||
|
if cpu > 0 and low.startswith("numvcpus"):
|
||||||
|
out.append(f'numvcpus = "{cpu}"')
|
||||||
|
seen_cpu = True
|
||||||
|
elif memory_mb > 0 and low.startswith("memsize"):
|
||||||
|
out.append(f'memsize = "{memory_mb}"')
|
||||||
|
seen_mem = True
|
||||||
|
else:
|
||||||
|
out.append(line)
|
||||||
|
if cpu > 0 and not seen_cpu:
|
||||||
|
out.append(f'numvcpus = "{cpu}"')
|
||||||
|
if memory_mb > 0 and not seen_mem:
|
||||||
|
out.append(f'memsize = "{memory_mb}"')
|
||||||
|
vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_extra_env_json(raw: str) -> dict[str, str]:
|
||||||
|
"""Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict."""
|
||||||
|
raw = (raw or "").strip()
|
||||||
|
if not raw or raw == "{}":
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
loaded = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc
|
||||||
|
if not isinstance(loaded, dict):
|
||||||
|
raise click.BadParameter("--extra-env-json must be a JSON object.")
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for key, value in loaded.items():
|
||||||
|
if not isinstance(key, str) or not _VALID_ENV_KEY.match(key):
|
||||||
|
raise click.BadParameter(
|
||||||
|
f"--extra-env-json key {key!r} is not a valid env var name."
|
||||||
|
)
|
||||||
|
out[key] = "" if value is None else str(value)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_running(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> bool:
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
if backend.is_running(handle):
|
||||||
|
return True
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[job] backend probe error: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_ip(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> str | None:
|
||||||
|
while (now := _now()) < deadline:
|
||||||
|
call_timeout = min(poll, deadline - now)
|
||||||
|
try:
|
||||||
|
ip = backend.get_ip(handle, timeout=call_timeout)
|
||||||
|
except BackendError:
|
||||||
|
ip = None
|
||||||
|
if ip:
|
||||||
|
return ip
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_transport(
|
||||||
|
*,
|
||||||
|
guest_os: str,
|
||||||
|
ip_address: str,
|
||||||
|
deadline: float,
|
||||||
|
poll: float,
|
||||||
|
credential_target: str,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_known_hosts: str | None,
|
||||||
|
) -> bool:
|
||||||
|
"""Loop until the guest answers WinRM (Windows) or SSH (Linux)."""
|
||||||
|
if guest_os == "windows":
|
||||||
|
cred = KeyringCredentialStore().get(credential_target)
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
return True
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[job] WinRM probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
with SshTransport(
|
||||||
|
ip_address,
|
||||||
|
username=ssh_user,
|
||||||
|
key_path=ssh_key_path,
|
||||||
|
known_hosts=ssh_known_hosts,
|
||||||
|
) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
return True
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[job] SSH probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _make_clone_name(job_id: str) -> str:
|
||||||
|
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||||
|
# uuid4 suffix avoids collisions when several clones share a JobId.
|
||||||
|
suffix = uuid.uuid4().hex[:6]
|
||||||
|
return f"Clone_{job_id}_{timestamp}_{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def _redact_url_creds(text: str) -> str:
|
||||||
|
"""Mask ``scheme://user:pass@host`` credentials in arbitrary text."""
|
||||||
|
return re.sub(r"(\w+://)[^/\s:@]+:[^/\s@]+@", r"\1***@", text)
|
||||||
|
|
||||||
|
|
||||||
|
def _host_clone(
|
||||||
|
repo_url: str, branch: str, commit: str, submodules: bool
|
||||||
|
) -> tuple[Path, Path]:
|
||||||
|
"""Clone the repo on the host into a temp dir (host-side transport).
|
||||||
|
|
||||||
|
Returns ``(tmp_root, src_dir)``. ``src_dir`` is passed as
|
||||||
|
``host_source_dir`` to the build (zipped and transferred to the
|
||||||
|
guest). The caller must remove ``tmp_root`` when done. git stderr is
|
||||||
|
surfaced with embedded URL credentials redacted.
|
||||||
|
"""
|
||||||
|
tmp_root = Path(tempfile.mkdtemp(prefix="ci-host-clone-"))
|
||||||
|
src_dir = tmp_root / "src"
|
||||||
|
clone = ["git", "clone", "--depth", "1", "--branch", branch]
|
||||||
|
if submodules:
|
||||||
|
clone.append("--recurse-submodules")
|
||||||
|
clone += [repo_url, str(src_dir)]
|
||||||
|
try:
|
||||||
|
subprocess.run(clone, check=True, capture_output=True, text=True)
|
||||||
|
if commit:
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", str(src_dir), "fetch", "--depth", "1",
|
||||||
|
"origin", commit],
|
||||||
|
check=True, capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-C", str(src_dir), "checkout", commit],
|
||||||
|
check=True, capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
shutil.rmtree(tmp_root, ignore_errors=True)
|
||||||
|
detail = _redact_url_creds((exc.stderr or "").strip())
|
||||||
|
tail = " | ".join(detail.splitlines()[-3:]) if detail else "(no stderr)"
|
||||||
|
raise click.ClickException(
|
||||||
|
f"host-side git clone failed (exit {exc.returncode}): {tail}"
|
||||||
|
) from None
|
||||||
|
return tmp_root, src_dir
|
||||||
|
|
||||||
|
|
||||||
|
def _destroy_clone(
|
||||||
|
backend: VmBackend | None,
|
||||||
|
handle: VmHandle | None,
|
||||||
|
clone_dir: Path | None,
|
||||||
|
) -> None:
|
||||||
|
"""Final cleanup. Tolerates every error — must never raise."""
|
||||||
|
if backend is not None and handle is not None:
|
||||||
|
try:
|
||||||
|
_stop_with_fallback(backend, handle)
|
||||||
|
except Exception as exc: # pragma: no cover - belt and braces
|
||||||
|
click.echo(f"[job] stop ignored: {exc}", err=True)
|
||||||
|
try:
|
||||||
|
backend.delete(handle)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
except Exception as exc: # pragma: no cover
|
||||||
|
click.echo(f"[job] delete ignored: {exc}", err=True)
|
||||||
|
if clone_dir is not None and clone_dir.exists():
|
||||||
|
if _try_remove_dir(clone_dir):
|
||||||
|
click.echo(f"[job] clone directory removed: {clone_dir}")
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[job] could not fully remove: {clone_dir}", err=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────── command
|
||||||
|
|
||||||
|
|
||||||
|
@click.command("job")
|
||||||
|
@click.option("--job-id", "job_id", required=True)
|
||||||
|
@click.option("--repo-url", "repo_url", required=True)
|
||||||
|
@click.option("--branch", required=True)
|
||||||
|
@click.option("--commit", default="")
|
||||||
|
@click.option("--configuration", default="Release", show_default=True)
|
||||||
|
@click.option("--build-command", "build_command", default="")
|
||||||
|
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
|
||||||
|
@click.option(
|
||||||
|
"--submodules/--no-submodules",
|
||||||
|
"submodules",
|
||||||
|
default=True,
|
||||||
|
show_default=True,
|
||||||
|
help="Recurse submodules when cloning (default: on).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--use-git-clone/--host-clone",
|
||||||
|
"use_git_clone",
|
||||||
|
default=True,
|
||||||
|
show_default=True,
|
||||||
|
help="In-guest git clone (default) vs host-side clone + zip transfer.",
|
||||||
|
)
|
||||||
|
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||||
|
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||||
|
@click.option(
|
||||||
|
"--gitea-credential-target",
|
||||||
|
"gitea_credential_target",
|
||||||
|
default="GiteaPAT",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--template-path",
|
||||||
|
"template_path",
|
||||||
|
default=None,
|
||||||
|
help="Template VMX path (or backend-specific identifier).",
|
||||||
|
)
|
||||||
|
@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True)
|
||||||
|
@click.option("--clone-base-dir", "clone_base_dir", default=None)
|
||||||
|
@click.option("--artifact-base-dir", "artifact_base_dir", default=None)
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
type=click.Choice(["windows", "linux", "auto"], case_sensitive=False),
|
||||||
|
default="auto",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-credential-target",
|
||||||
|
"guest_credential_target",
|
||||||
|
default=None,
|
||||||
|
help="Override config.guest_cred_target (Windows).",
|
||||||
|
)
|
||||||
|
@click.option("--ssh-key-path", "ssh_key_path", default=None)
|
||||||
|
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||||
|
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
|
||||||
|
@click.option(
|
||||||
|
"--extra-env-json",
|
||||||
|
"extra_env_json",
|
||||||
|
default="",
|
||||||
|
help="JSON object of env vars to inject into the guest before the build.",
|
||||||
|
)
|
||||||
|
@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0)
|
||||||
|
@click.option(
|
||||||
|
"--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--ready-timeout",
|
||||||
|
"ready_timeout",
|
||||||
|
type=click.FloatRange(10.0, 3600.0),
|
||||||
|
default=600.0,
|
||||||
|
show_default=True,
|
||||||
|
help="Total seconds to wait for VM running + IP + transport ready.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--poll-interval",
|
||||||
|
"poll_interval",
|
||||||
|
type=click.FloatRange(1.0, 60.0),
|
||||||
|
default=5.0,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--vmrun-path", "vmrun_path", default=None)
|
||||||
|
def job(
|
||||||
|
job_id: str,
|
||||||
|
repo_url: str,
|
||||||
|
branch: str,
|
||||||
|
commit: str,
|
||||||
|
configuration: str,
|
||||||
|
build_command: str,
|
||||||
|
guest_artifact_source: str,
|
||||||
|
submodules: bool,
|
||||||
|
use_git_clone: bool,
|
||||||
|
use_shared_cache: bool,
|
||||||
|
skip_artifact: bool,
|
||||||
|
gitea_credential_target: str,
|
||||||
|
template_path: str | None,
|
||||||
|
snapshot_name: str,
|
||||||
|
clone_base_dir: str | None,
|
||||||
|
artifact_base_dir: str | None,
|
||||||
|
guest_os: str,
|
||||||
|
guest_credential_target: str | None,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_known_hosts_file: str | None,
|
||||||
|
extra_env_json: str,
|
||||||
|
guest_cpu: int,
|
||||||
|
guest_memory_mb: int,
|
||||||
|
ready_timeout: float,
|
||||||
|
poll_interval: float,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
) -> None:
|
||||||
|
"""Run the full CI pipeline against an ephemeral build VM.
|
||||||
|
|
||||||
|
Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is
|
||||||
|
guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``.
|
||||||
|
"""
|
||||||
|
# Transport: in-guest git clone (default) or host-side clone + zip
|
||||||
|
# transfer (``--host-clone``). Both honour ``--submodules``.
|
||||||
|
|
||||||
|
# Inject Gitea credentials into HTTP(S) clone URL so git doesn't prompt.
|
||||||
|
authed_repo_url = repo_url
|
||||||
|
if repo_url and repo_url.startswith(("http://", "https://")):
|
||||||
|
from urllib.parse import urlparse, urlunparse
|
||||||
|
|
||||||
|
try:
|
||||||
|
gitea_cred = KeyringCredentialStore().get(gitea_credential_target)
|
||||||
|
parsed = urlparse(repo_url)
|
||||||
|
host = parsed.hostname or ""
|
||||||
|
port_part = f":{parsed.port}" if parsed.port else ""
|
||||||
|
netloc = f"{gitea_cred.username}:{gitea_cred.password}@{host}{port_part}"
|
||||||
|
authed_repo_url = urlunparse(parsed._replace(netloc=netloc))
|
||||||
|
except Exception:
|
||||||
|
pass # no credential stored — try unauthenticated (public repo)
|
||||||
|
|
||||||
|
config: Config = load_config()
|
||||||
|
if not template_path:
|
||||||
|
raise click.ClickException(
|
||||||
|
"--template-path is required."
|
||||||
|
)
|
||||||
|
template_p = Path(template_path)
|
||||||
|
if not template_p.is_file():
|
||||||
|
raise click.ClickException(f"Template VMX not found: {template_path}")
|
||||||
|
|
||||||
|
base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||||
|
base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts
|
||||||
|
base_clone_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
job_artifact_dir = base_artifact_dir / job_id
|
||||||
|
job_artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
extra_env = _parse_extra_env_json(extra_env_json)
|
||||||
|
|
||||||
|
# Build backend from config; allow CLI override for vmrun path.
|
||||||
|
if vmrun_path is not None:
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
backend = load_backend(config)
|
||||||
|
except BackendNotAvailable as exc:
|
||||||
|
raise click.ClickException(f"backend unavailable: {exc}") from exc
|
||||||
|
|
||||||
|
clone_name = _make_clone_name(job_id)
|
||||||
|
clone_dir = base_clone_dir / clone_name
|
||||||
|
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||||
|
|
||||||
|
handle: VmHandle | None = None
|
||||||
|
started = datetime.now(tz=UTC)
|
||||||
|
phase_timings: list[tuple[str, float]] = []
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
click.echo("=" * 60)
|
||||||
|
click.echo(f"[job] Job : {job_id}")
|
||||||
|
click.echo(f"[job] Repository : {repo_url}")
|
||||||
|
click.echo(f"[job] Branch : {branch}")
|
||||||
|
if commit:
|
||||||
|
click.echo(f"[job] Commit : {commit}")
|
||||||
|
click.echo(f"[job] Template : {template_path}")
|
||||||
|
click.echo(f"[job] Snapshot : {snapshot_name}")
|
||||||
|
click.echo(f"[job] Artifacts : {job_artifact_dir}")
|
||||||
|
click.echo(f"[job] Started : {started.isoformat()}")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# ── Phase 1/5: clone VM ─────────────────────────────────────────
|
||||||
|
click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}")
|
||||||
|
try:
|
||||||
|
handle = backend.clone_linked(
|
||||||
|
template=str(template_p),
|
||||||
|
snapshot=snapshot_name,
|
||||||
|
name=clone_name,
|
||||||
|
destination=str(clone_vmx),
|
||||||
|
)
|
||||||
|
except BackendError as exc:
|
||||||
|
raise click.ClickException(f"clone failed: {exc}") from exc
|
||||||
|
if not clone_vmx.is_file():
|
||||||
|
raise click.ClickException(
|
||||||
|
f"backend reported success but clone VMX is missing: {clone_vmx}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if guest_cpu > 0 or guest_memory_mb > 0:
|
||||||
|
click.echo(
|
||||||
|
f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB"
|
||||||
|
)
|
||||||
|
_apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)
|
||||||
|
|
||||||
|
phase_timings.append(("clone", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 2/5: start VM ─────────────────────────────────────────
|
||||||
|
click.echo("\n[job] Phase 2/5 — starting VM (headless)")
|
||||||
|
try:
|
||||||
|
backend.start(handle, headless=True)
|
||||||
|
except BackendError as exc:
|
||||||
|
raise click.ClickException(f"vmrun start failed: {exc}") from exc
|
||||||
|
|
||||||
|
# Resolve guest OS now that the clone exists.
|
||||||
|
if guest_os.lower() == "auto":
|
||||||
|
guest_os = _read_guest_os_from_vmx(clone_vmx)
|
||||||
|
click.echo(f"[job] auto-detected guest OS: {guest_os}")
|
||||||
|
else:
|
||||||
|
guest_os = guest_os.lower()
|
||||||
|
|
||||||
|
deadline = _now() + ready_timeout
|
||||||
|
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||||
|
raise click.ClickException("timeout waiting for VM to be running.")
|
||||||
|
ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||||
|
if not ip_address:
|
||||||
|
raise click.ClickException("timeout waiting for guest IP.")
|
||||||
|
click.echo(f"[job] guest IP: {ip_address}")
|
||||||
|
phase_timings.append(("start", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 3/5: wait for transport readiness ────────────────────
|
||||||
|
click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness")
|
||||||
|
target = guest_credential_target or config.guest_cred_target
|
||||||
|
key_path: str | None = ssh_key_path
|
||||||
|
if key_path is None and config.ssh_key_path is not None:
|
||||||
|
key_path = str(config.ssh_key_path)
|
||||||
|
|
||||||
|
if not _probe_transport(
|
||||||
|
guest_os=guest_os,
|
||||||
|
ip_address=ip_address,
|
||||||
|
deadline=deadline,
|
||||||
|
poll=poll_interval,
|
||||||
|
credential_target=target,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
ssh_key_path=key_path,
|
||||||
|
ssh_known_hosts=ssh_known_hosts_file,
|
||||||
|
):
|
||||||
|
raise click.ClickException(
|
||||||
|
"timeout waiting for guest transport to be ready."
|
||||||
|
)
|
||||||
|
phase_timings.append(("wait-ready", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 4/5: build inside guest ──────────────────────────────
|
||||||
|
click.echo("\n[job] Phase 4/5 — invoking remote build")
|
||||||
|
host_clone_root: Path | None = None
|
||||||
|
if use_git_clone:
|
||||||
|
build_clone_url: str | None = authed_repo_url
|
||||||
|
build_host_src: str | None = None
|
||||||
|
click.echo("[job] transport: in-guest git clone")
|
||||||
|
else:
|
||||||
|
click.echo("[job] transport: host-side clone + zip transfer")
|
||||||
|
host_clone_root, src_dir = _host_clone(
|
||||||
|
authed_repo_url, branch, commit, submodules
|
||||||
|
)
|
||||||
|
build_clone_url = None
|
||||||
|
build_host_src = str(src_dir)
|
||||||
|
try:
|
||||||
|
if guest_os == "linux":
|
||||||
|
_linux_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
workdir="/opt/ci/build",
|
||||||
|
output_dir="/opt/ci/output",
|
||||||
|
host_source_dir=build_host_src,
|
||||||
|
clone_url=build_clone_url,
|
||||||
|
clone_branch=branch,
|
||||||
|
clone_commit=commit,
|
||||||
|
clone_submodules=submodules,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
extra_env=extra_env,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_windows_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
workdir="C:\\CI\\build",
|
||||||
|
output_dir="C:\\CI\\output",
|
||||||
|
host_source_dir=build_host_src,
|
||||||
|
clone_url=build_clone_url,
|
||||||
|
clone_branch=branch,
|
||||||
|
clone_commit=commit,
|
||||||
|
clone_submodules=submodules,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
configuration=configuration,
|
||||||
|
extra_env=extra_env,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
use_shared_cache=use_shared_cache,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
if host_clone_root is not None:
|
||||||
|
shutil.rmtree(host_clone_root, ignore_errors=True)
|
||||||
|
phase_timings.append(("build", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 5/5: collect artifacts ───────────────────────────────
|
||||||
|
if skip_artifact:
|
||||||
|
click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)")
|
||||||
|
else:
|
||||||
|
click.echo("\n[job] Phase 5/5 — collecting artifacts")
|
||||||
|
if guest_os == "linux":
|
||||||
|
_linux_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
guest_path="/opt/ci/output",
|
||||||
|
host_dir=job_artifact_dir,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_windows_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
guest_path="C:\\CI\\output",
|
||||||
|
host_dir=job_artifact_dir,
|
||||||
|
include_logs=True,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
count = _write_manifest(job_artifact_dir, job_id, commit)
|
||||||
|
click.echo(f"[job] manifest.json written ({count} file(s))")
|
||||||
|
except OSError as exc:
|
||||||
|
click.echo(f"[job] manifest write failed: {exc}", err=True)
|
||||||
|
phase_timings.append(("artifacts", _now() - phase_start))
|
||||||
|
|
||||||
|
elapsed = (datetime.now(tz=UTC) - started).total_seconds()
|
||||||
|
click.echo("\n" + "=" * 60)
|
||||||
|
click.echo("[job] phase durations:")
|
||||||
|
for name, secs in phase_timings:
|
||||||
|
click.echo(f" {name:<14} {int(secs):>5}s")
|
||||||
|
click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s")
|
||||||
|
click.echo(f"[job] Artifacts: {job_artifact_dir}")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
click.echo("\n[job] interrupted by user; cleaning up...", err=True)
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
sys.exit(130)
|
||||||
|
except click.ClickException:
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
click.echo(f"\n[job] FAILURE: {exc}", err=True)
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
raise click.ClickException(str(exc)) from exc
|
||||||
|
else:
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["job"]
|
||||||
@@ -0,0 +1,459 @@
|
|||||||
|
"""``monitor`` sub-commands.
|
||||||
|
|
||||||
|
Phase A2 ports the maintenance/monitoring leaf scripts:
|
||||||
|
|
||||||
|
* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1``
|
||||||
|
* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1``
|
||||||
|
|
||||||
|
Both commands are designed to run from a scheduler (Windows Task Scheduler
|
||||||
|
in Phase A, systemd timers in Phase B) and emit warnings via the platform
|
||||||
|
event log when available, plus optional Discord/Gitea webhooks via stdlib
|
||||||
|
``urllib`` (no extra dependency).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool:
|
||||||
|
"""POST a Discord/Gitea-compatible JSON payload. Returns True on success."""
|
||||||
|
if not url:
|
||||||
|
return False
|
||||||
|
payload = json.dumps({"content": content}).encode("utf-8")
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=payload,
|
||||||
|
method="POST",
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
return bool(200 <= resp.status < 300)
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||||
|
logger.warning("Webhook POST failed: %s", exc)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _write_windows_event(
|
||||||
|
source: str, event_id: int, entry_type: str, message: str
|
||||||
|
) -> bool:
|
||||||
|
"""Write a Windows Application Event Log entry via ``eventcreate.exe``.
|
||||||
|
|
||||||
|
Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts.
|
||||||
|
Returns True on success.
|
||||||
|
"""
|
||||||
|
if os.name != "nt":
|
||||||
|
return False
|
||||||
|
eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe")
|
||||||
|
if eventcreate is None:
|
||||||
|
return False
|
||||||
|
type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"}
|
||||||
|
cmd = [
|
||||||
|
eventcreate,
|
||||||
|
"/T", type_map.get(entry_type, "WARNING"),
|
||||||
|
"/ID", str(event_id),
|
||||||
|
"/L", "APPLICATION",
|
||||||
|
"/SO", source,
|
||||||
|
"/D", message,
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd, capture_output=True, text=True, check=False, timeout=10.0
|
||||||
|
)
|
||||||
|
except (subprocess.TimeoutExpired, OSError) as exc:
|
||||||
|
logger.warning("eventcreate failed: %s", exc)
|
||||||
|
return False
|
||||||
|
if result.returncode != 0:
|
||||||
|
logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr)
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────── group
|
||||||
|
|
||||||
|
|
||||||
|
@click.group("monitor")
|
||||||
|
def monitor() -> None:
|
||||||
|
"""Host monitoring commands (disk, runner)."""
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────────────── disk
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_drive_path(drive: str) -> str:
|
||||||
|
"""Map a single-letter Windows drive (``F``) to a usable path.
|
||||||
|
|
||||||
|
On non-Windows the ``drive`` value is treated as a path itself.
|
||||||
|
"""
|
||||||
|
if os.name == "nt" and len(drive) == 1 and drive.isalpha():
|
||||||
|
return f"{drive.upper()}:\\"
|
||||||
|
return drive
|
||||||
|
|
||||||
|
|
||||||
|
@monitor.command("disk")
|
||||||
|
@click.option(
|
||||||
|
"--min-free-gb",
|
||||||
|
"min_free_gb",
|
||||||
|
type=click.IntRange(1, 2000),
|
||||||
|
default=50,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--drive-letter",
|
||||||
|
"drive_letter",
|
||||||
|
default="F" if os.name == "nt" else "/",
|
||||||
|
show_default=True,
|
||||||
|
help="Drive letter (Windows) or mount path (Linux).",
|
||||||
|
)
|
||||||
|
@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.")
|
||||||
|
@click.option(
|
||||||
|
"--json",
|
||||||
|
"as_json",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Emit a single-line JSON object instead of human-readable output.",
|
||||||
|
)
|
||||||
|
def monitor_disk(
|
||||||
|
min_free_gb: int,
|
||||||
|
drive_letter: str,
|
||||||
|
webhook_url: str,
|
||||||
|
as_json: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Check free space on a CI host drive and alert if below threshold.
|
||||||
|
|
||||||
|
Exit codes mirror the original PowerShell script:
|
||||||
|
|
||||||
|
* 0 = drive OK
|
||||||
|
* 1 = below threshold (alert raised)
|
||||||
|
* 2 = drive not found
|
||||||
|
"""
|
||||||
|
target = _resolve_drive_path(drive_letter)
|
||||||
|
try:
|
||||||
|
usage = shutil.disk_usage(target)
|
||||||
|
except (FileNotFoundError, OSError) as exc:
|
||||||
|
click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
gb = 1024.0 ** 3
|
||||||
|
free_gb = round(usage.free / gb, 1)
|
||||||
|
total_gb = round(usage.total / gb, 1)
|
||||||
|
pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"drive": drive_letter,
|
||||||
|
"free_gb": free_gb,
|
||||||
|
"total_gb": total_gb,
|
||||||
|
"percent_free": pct_free,
|
||||||
|
"threshold_gb": min_free_gb,
|
||||||
|
"status": "ok" if free_gb >= min_free_gb else "alert",
|
||||||
|
}
|
||||||
|
|
||||||
|
if free_gb >= min_free_gb:
|
||||||
|
if as_json:
|
||||||
|
click.echo(json.dumps(payload))
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB "
|
||||||
|
f"({pct_free}%) - OK (threshold: {min_free_gb} GB)"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
msg = (
|
||||||
|
f"CI host drive {drive_letter}: free space {free_gb} GB "
|
||||||
|
f"({pct_free}%) is below the {min_free_gb} GB threshold. "
|
||||||
|
"vmrun linked clones may fail silently."
|
||||||
|
)
|
||||||
|
if as_json:
|
||||||
|
payload["message"] = msg
|
||||||
|
click.echo(json.dumps(payload))
|
||||||
|
else:
|
||||||
|
click.echo(f"[disk] WARNING: {msg}", err=True)
|
||||||
|
|
||||||
|
_write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg)
|
||||||
|
if webhook_url:
|
||||||
|
_post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────────────────── runner
|
||||||
|
|
||||||
|
|
||||||
|
def _service_status(service_name: str) -> str | None:
|
||||||
|
"""Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed."""
|
||||||
|
if os.name == "nt":
|
||||||
|
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
[sc, "query", service_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=10.0,
|
||||||
|
)
|
||||||
|
except (subprocess.TimeoutExpired, OSError):
|
||||||
|
return None
|
||||||
|
if r.returncode != 0:
|
||||||
|
return None
|
||||||
|
out = r.stdout.upper()
|
||||||
|
if "RUNNING" in out:
|
||||||
|
return "Running"
|
||||||
|
if "STOPPED" in out:
|
||||||
|
return "Stopped"
|
||||||
|
return "Unknown"
|
||||||
|
# Linux / systemd
|
||||||
|
systemctl = shutil.which("systemctl")
|
||||||
|
if systemctl is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
[systemctl, "is-active", service_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=10.0,
|
||||||
|
)
|
||||||
|
except (subprocess.TimeoutExpired, OSError):
|
||||||
|
return None
|
||||||
|
state = r.stdout.strip().lower()
|
||||||
|
if state == "active":
|
||||||
|
return "Running"
|
||||||
|
if state in {"inactive", "failed", "deactivating"}:
|
||||||
|
return "Stopped"
|
||||||
|
return None # unknown / not installed
|
||||||
|
|
||||||
|
|
||||||
|
def _restart_service(service_name: str) -> bool:
|
||||||
|
"""Attempt to (re)start ``service_name``. Returns True if now running."""
|
||||||
|
if os.name == "nt":
|
||||||
|
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||||
|
# `sc start` returns 1056 if already running; ignore.
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
[sc, "start", service_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=30.0,
|
||||||
|
)
|
||||||
|
except (subprocess.TimeoutExpired, OSError):
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
systemctl = shutil.which("systemctl")
|
||||||
|
if systemctl is None:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
[systemctl, "restart", service_name],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
timeout=30.0,
|
||||||
|
)
|
||||||
|
except (subprocess.TimeoutExpired, OSError):
|
||||||
|
return False
|
||||||
|
time.sleep(5.0)
|
||||||
|
return _service_status(service_name) == "Running"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_restart_log(path: Path) -> list[str]:
|
||||||
|
if not path.is_file():
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
raw = path.read_text(encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
if not raw.strip():
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
return []
|
||||||
|
if isinstance(data, list):
|
||||||
|
return [str(x) for x in data]
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _save_restart_log(path: Path, entries: list[str]) -> None:
|
||||||
|
try:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(entries), encoding="utf-8")
|
||||||
|
except OSError as exc:
|
||||||
|
logger.warning("Could not persist restart log: %s", exc)
|
||||||
|
|
||||||
|
|
||||||
|
def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None:
|
||||||
|
"""Return number of online runners reported by Gitea, or ``None`` on failure."""
|
||||||
|
api = api_url.rstrip("/") + "/api/v1/admin/runners"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
api, headers={"Authorization": f"token {token}"}
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
|
data = json.loads(resp.read().decode("utf-8"))
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||||
|
logger.warning("Gitea API call failed: %s", exc)
|
||||||
|
return None
|
||||||
|
if isinstance(data, list):
|
||||||
|
return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@monitor.command("runner")
|
||||||
|
@click.option(
|
||||||
|
"--service-name",
|
||||||
|
"service_name",
|
||||||
|
default="act_runner",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--max-restarts",
|
||||||
|
type=click.IntRange(1, 10),
|
||||||
|
default=3,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--state-dir",
|
||||||
|
"state_dir",
|
||||||
|
default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--webhook-url", "webhook_url", default="")
|
||||||
|
@click.option("--gitea-url", "gitea_url", default="")
|
||||||
|
@click.option(
|
||||||
|
"--gitea-credential-target",
|
||||||
|
"gitea_credential_target",
|
||||||
|
default="",
|
||||||
|
help="Credential target name for the Gitea admin PAT.",
|
||||||
|
)
|
||||||
|
def monitor_runner(
|
||||||
|
service_name: str,
|
||||||
|
max_restarts: int,
|
||||||
|
state_dir: str,
|
||||||
|
webhook_url: str,
|
||||||
|
gitea_url: str,
|
||||||
|
gitea_credential_target: str,
|
||||||
|
) -> None:
|
||||||
|
"""Monitor act_runner; auto-restart up to ``--max-restarts`` per hour.
|
||||||
|
|
||||||
|
Exit codes:
|
||||||
|
0 = healthy (or restart succeeded)
|
||||||
|
1 = service down + restart failed/limit reached
|
||||||
|
2 = service not installed
|
||||||
|
"""
|
||||||
|
state_dir_path = Path(state_dir)
|
||||||
|
maintenance_flag = state_dir_path / "runner-maintenance.flag"
|
||||||
|
if maintenance_flag.is_file():
|
||||||
|
click.echo(
|
||||||
|
f"[runner] maintenance mode active ({maintenance_flag}) - skipping."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
status = _service_status(service_name)
|
||||||
|
if status is None:
|
||||||
|
click.echo(
|
||||||
|
f"[runner] service {service_name!r} not found - is act_runner installed?",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
if status == "Running":
|
||||||
|
click.echo(f"[runner] {service_name} is Running - OK")
|
||||||
|
if gitea_url and gitea_credential_target:
|
||||||
|
try:
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
|
||||||
|
cred = KeyringCredentialStore().get(gitea_credential_target)
|
||||||
|
online = _gitea_runners_online(gitea_url, cred.password)
|
||||||
|
except KeyError as exc:
|
||||||
|
click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True)
|
||||||
|
online = None
|
||||||
|
if online is None:
|
||||||
|
click.echo("[runner] Gitea API check skipped/failed.")
|
||||||
|
elif online == 0:
|
||||||
|
msg = (
|
||||||
|
f"Gitea API ({gitea_url}) reports 0 online runners. "
|
||||||
|
f"{service_name} is Running - possible registration issue."
|
||||||
|
)
|
||||||
|
click.echo(f"[runner] WARNING: {msg}", err=True)
|
||||||
|
_write_windows_event("CI-RunnerHealth", 1005, "Warning", msg)
|
||||||
|
if webhook_url:
|
||||||
|
_post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}")
|
||||||
|
else:
|
||||||
|
click.echo(f"[runner] Gitea API: {online} runner(s) online - OK")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Not running → check rate limit and try to restart.
|
||||||
|
cooldown_file = state_dir_path / "runner-restart-log.json"
|
||||||
|
log = _load_restart_log(cooldown_file)
|
||||||
|
cutoff = datetime.now(tz=UTC) - timedelta(hours=1)
|
||||||
|
pruned: list[str] = []
|
||||||
|
for entry in log:
|
||||||
|
try:
|
||||||
|
ts = datetime.fromisoformat(entry)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if ts.tzinfo is None:
|
||||||
|
ts = ts.replace(tzinfo=UTC)
|
||||||
|
if ts > cutoff:
|
||||||
|
pruned.append(entry)
|
||||||
|
log = pruned
|
||||||
|
|
||||||
|
state_msg = (
|
||||||
|
f"{service_name} service is '{status}'. "
|
||||||
|
f"Auto-restarts in last hour: {len(log)} / {max_restarts}."
|
||||||
|
)
|
||||||
|
click.echo(f"[runner] WARNING: {state_msg}", err=True)
|
||||||
|
_write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg)
|
||||||
|
|
||||||
|
if len(log) >= max_restarts:
|
||||||
|
limit_msg = (
|
||||||
|
f"{service_name} auto-restart limit ({max_restarts}/h) reached - "
|
||||||
|
"manual intervention required."
|
||||||
|
)
|
||||||
|
click.echo(f"[runner] {limit_msg}", err=True)
|
||||||
|
_write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg)
|
||||||
|
if webhook_url:
|
||||||
|
_post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
click.echo(
|
||||||
|
f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..."
|
||||||
|
)
|
||||||
|
ok = _restart_service(service_name)
|
||||||
|
log.append(datetime.now(tz=UTC).isoformat())
|
||||||
|
_save_restart_log(cooldown_file, log)
|
||||||
|
new_status = _service_status(service_name) or "Unknown"
|
||||||
|
restart_msg = f"{service_name} restarted - new status: {new_status}."
|
||||||
|
entry_type = "Information" if ok else "Warning"
|
||||||
|
click.echo(f"[runner] {restart_msg}")
|
||||||
|
_write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg)
|
||||||
|
prefix = "[WARNING]" if ok else "[ALERT]"
|
||||||
|
if webhook_url:
|
||||||
|
_post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}")
|
||||||
|
sys.exit(0 if ok else 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _ignored(_name: str, _value: Any) -> None:
|
||||||
|
"""Marker so type checkers see we deliberately drop a value."""
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["monitor"]
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
"""``report`` sub-commands.
|
||||||
|
|
||||||
|
Phase A2 ports ``scripts/Get-CIJobSummary.ps1`` to ``report job``. Reads
|
||||||
|
JSONL log files written by the (still-PowerShell) ``Invoke-CIJob.ps1``
|
||||||
|
under ``F:\\CI\\Logs\\<jobId>\\invoke-ci.jsonl`` and prints either a
|
||||||
|
summary table of recent jobs or a per-phase breakdown for one job.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _Event:
|
||||||
|
phase: str
|
||||||
|
status: str
|
||||||
|
ts: str
|
||||||
|
elapsed_sec: int | None
|
||||||
|
error: str | None
|
||||||
|
raw: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
|
def _read_events(jsonl_path: Path) -> list[_Event]:
|
||||||
|
out: list[_Event] = []
|
||||||
|
try:
|
||||||
|
text = jsonl_path.read_text(encoding="utf-8")
|
||||||
|
except OSError:
|
||||||
|
return out
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
continue
|
||||||
|
data = obj.get("data") if isinstance(obj.get("data"), dict) else {}
|
||||||
|
elapsed: int | None = None
|
||||||
|
if isinstance(data, dict):
|
||||||
|
raw_elapsed = data.get("elapsedSec")
|
||||||
|
if isinstance(raw_elapsed, int | float):
|
||||||
|
elapsed = int(raw_elapsed)
|
||||||
|
err: str | None = None
|
||||||
|
if isinstance(data, dict) and isinstance(data.get("error"), str):
|
||||||
|
err = str(data["error"])
|
||||||
|
out.append(
|
||||||
|
_Event(
|
||||||
|
phase=str(obj.get("phase", "")),
|
||||||
|
status=str(obj.get("status", "")),
|
||||||
|
ts=str(obj.get("ts", "")),
|
||||||
|
elapsed_sec=elapsed,
|
||||||
|
error=err,
|
||||||
|
raw=obj,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _format_hms(sec: int) -> str:
|
||||||
|
if sec < 0:
|
||||||
|
return "?"
|
||||||
|
h, rem = divmod(sec, 3600)
|
||||||
|
m, s = divmod(rem, 60)
|
||||||
|
if h > 0:
|
||||||
|
return f"{h}h{m:02d}m{s:02d}s"
|
||||||
|
return f"{m}m{s:02d}s"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _Row:
|
||||||
|
job_id: str
|
||||||
|
status: str
|
||||||
|
elapsed: str
|
||||||
|
started: str
|
||||||
|
error: str
|
||||||
|
|
||||||
|
|
||||||
|
def _summarise(events: list[_Event], default_id: str) -> _Row:
|
||||||
|
job_start = next(
|
||||||
|
(e for e in events if e.phase == "job" and e.status == "start"), None
|
||||||
|
)
|
||||||
|
job_success = next(
|
||||||
|
(e for e in events if e.phase == "job" and e.status == "success"), None
|
||||||
|
)
|
||||||
|
job_failure = next(
|
||||||
|
(e for e in events if e.phase == "job" and e.status == "failure"), None
|
||||||
|
)
|
||||||
|
raw_id = events[0].raw.get("jobId") if events else None
|
||||||
|
job_id = str(raw_id) if isinstance(raw_id, str) and raw_id else default_id
|
||||||
|
started = job_start.ts if job_start else ""
|
||||||
|
if job_success:
|
||||||
|
return _Row(
|
||||||
|
job_id=job_id,
|
||||||
|
status="success",
|
||||||
|
elapsed=_format_hms(job_success.elapsed_sec or -1),
|
||||||
|
started=started,
|
||||||
|
error="",
|
||||||
|
)
|
||||||
|
if job_failure:
|
||||||
|
err = job_failure.error or ""
|
||||||
|
if len(err) > 60:
|
||||||
|
err = err[:57] + "..."
|
||||||
|
return _Row(
|
||||||
|
job_id=job_id,
|
||||||
|
status="FAILED",
|
||||||
|
elapsed=_format_hms(job_failure.elapsed_sec or -1),
|
||||||
|
started=started,
|
||||||
|
error=err,
|
||||||
|
)
|
||||||
|
return _Row(
|
||||||
|
job_id=job_id,
|
||||||
|
status="in-progress",
|
||||||
|
elapsed="?",
|
||||||
|
started=started,
|
||||||
|
error="",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@click.group("report")
|
||||||
|
def report() -> None:
|
||||||
|
"""Reporting commands (job summaries)."""
|
||||||
|
|
||||||
|
|
||||||
|
@report.command("job")
|
||||||
|
@click.option(
|
||||||
|
"--log-dir",
|
||||||
|
"log_dir",
|
||||||
|
default=r"F:\CI\Logs",
|
||||||
|
show_default=True,
|
||||||
|
help="Base directory containing per-job log subdirectories.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--last",
|
||||||
|
type=click.IntRange(1, 1000),
|
||||||
|
default=20,
|
||||||
|
show_default=True,
|
||||||
|
help="Show only the N most recent jobs.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--job-id",
|
||||||
|
"job_id",
|
||||||
|
default="",
|
||||||
|
help="Show a phase-by-phase breakdown for a single job.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--failed",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Filter to failed jobs only.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--json",
|
||||||
|
"as_json",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Emit JSON instead of a human-readable table.",
|
||||||
|
)
|
||||||
|
def report_job(
|
||||||
|
log_dir: str,
|
||||||
|
last: int,
|
||||||
|
job_id: str,
|
||||||
|
failed: bool,
|
||||||
|
as_json: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Display CI job summaries parsed from invoke-ci.jsonl files."""
|
||||||
|
base = Path(log_dir)
|
||||||
|
if not base.is_dir():
|
||||||
|
click.echo(f"Log directory not found: {log_dir}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if job_id:
|
||||||
|
jsonl = base / job_id / "invoke-ci.jsonl"
|
||||||
|
if not jsonl.is_file():
|
||||||
|
click.echo(f"No JSONL log found for job '{job_id}' at: {jsonl}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
|
events = _read_events(jsonl)
|
||||||
|
if not events:
|
||||||
|
click.echo(f"JSONL file is empty or unreadable: {jsonl}", err=True)
|
||||||
|
sys.exit(1)
|
||||||
|
if as_json:
|
||||||
|
click.echo(json.dumps([e.raw for e in events], default=str))
|
||||||
|
return
|
||||||
|
click.echo(f"\nJob: {job_id}")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
click.echo(f"{'Phase':<35} {'Status':<10} {'Timestamp'}")
|
||||||
|
click.echo(f"{'-' * 35:<35} {'-' * 10:<10} {'-' * 24}")
|
||||||
|
for ev in events:
|
||||||
|
click.echo(f"{ev.phase:<35} {ev.status:<10} {ev.ts}")
|
||||||
|
fail = next(
|
||||||
|
(e for e in reversed(events) if e.status == "failure"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if fail and fail.error:
|
||||||
|
click.echo(f"\nError: {fail.error}")
|
||||||
|
return
|
||||||
|
|
||||||
|
files = sorted(
|
||||||
|
base.rglob("invoke-ci.jsonl"),
|
||||||
|
key=lambda p: p.stat().st_mtime,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
if not files:
|
||||||
|
click.echo(f"No invoke-ci.jsonl files found under {log_dir}")
|
||||||
|
return
|
||||||
|
|
||||||
|
rows: list[_Row] = []
|
||||||
|
for f in files:
|
||||||
|
events = _read_events(f)
|
||||||
|
if not events:
|
||||||
|
continue
|
||||||
|
default_id = f.parent.name
|
||||||
|
rows.append(_summarise(events, default_id))
|
||||||
|
|
||||||
|
if failed:
|
||||||
|
rows = [r for r in rows if r.status == "FAILED"]
|
||||||
|
rows = rows[:last]
|
||||||
|
if not rows:
|
||||||
|
click.echo("No matching jobs found.")
|
||||||
|
return
|
||||||
|
|
||||||
|
if as_json:
|
||||||
|
click.echo(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"jobId": r.job_id,
|
||||||
|
"status": r.status,
|
||||||
|
"elapsed": r.elapsed,
|
||||||
|
"started": r.started,
|
||||||
|
"error": r.error,
|
||||||
|
}
|
||||||
|
for r in rows
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
click.echo(f"\nCI Job Summary (last {len(rows)} jobs):")
|
||||||
|
header = f"{'JobId':<40} {'Status':<12} {'Elapsed':<10} {'Started':<26} Error"
|
||||||
|
click.echo(header)
|
||||||
|
click.echo(
|
||||||
|
f"{'-' * 40:<40} {'-' * 12:<12} {'-' * 10:<10} {'-' * 26:<26} {'-' * 30}"
|
||||||
|
)
|
||||||
|
for r in rows:
|
||||||
|
click.echo(
|
||||||
|
f"{r.job_id:<40} {r.status:<12} {r.elapsed:<10} {r.started:<26} {r.error}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["report"]
|
||||||
@@ -0,0 +1,469 @@
|
|||||||
|
"""``vm`` sub-commands.
|
||||||
|
|
||||||
|
Phase A2 ports the leaf VM management scripts:
|
||||||
|
|
||||||
|
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
||||||
|
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
||||||
|
|
||||||
|
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
||||||
|
dependency so an ESXi backend can later supply a folder-scoped VM list
|
||||||
|
instead of scanning the local filesystem.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.backends import load_backend
|
||||||
|
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.config import load_config
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
||||||
|
"""Build a backend honouring an optional vmrun override."""
|
||||||
|
config = load_config()
|
||||||
|
if vmrun_path is not None:
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||||
|
return load_backend(config)
|
||||||
|
|
||||||
|
|
||||||
|
def _try_remove_dir(path: Path) -> bool:
|
||||||
|
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
||||||
|
if not path.exists():
|
||||||
|
return True
|
||||||
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
|
return not path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
||||||
|
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
||||||
|
try:
|
||||||
|
backend.stop(handle, hard=False)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
||||||
|
try:
|
||||||
|
if backend.is_running(handle):
|
||||||
|
click.echo("[vm remove] still running, forcing hard stop...")
|
||||||
|
backend.stop(handle, hard=True)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────── group
|
||||||
|
|
||||||
|
|
||||||
|
@click.group("vm")
|
||||||
|
def vm() -> None:
|
||||||
|
"""VM lifecycle commands."""
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────── remove
|
||||||
|
|
||||||
|
|
||||||
|
@vm.command("remove")
|
||||||
|
@click.option(
|
||||||
|
"--vmx",
|
||||||
|
"--vm-path",
|
||||||
|
"vmx",
|
||||||
|
required=True,
|
||||||
|
help="Path to the clone VMX file to destroy.",
|
||||||
|
)
|
||||||
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||||
|
@click.option(
|
||||||
|
"--graceful-timeout",
|
||||||
|
"--graceful-timeout-seconds",
|
||||||
|
"graceful_timeout",
|
||||||
|
type=click.IntRange(1, 60),
|
||||||
|
default=15,
|
||||||
|
show_default=True,
|
||||||
|
help="Seconds to wait for soft stop before forcing.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--force",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Skip soft stop and go straight to hard stop.",
|
||||||
|
)
|
||||||
|
def vm_remove(
|
||||||
|
vmx: str,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
graceful_timeout: int,
|
||||||
|
force: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Stop and permanently delete an ephemeral build VM."""
|
||||||
|
vmx_path = Path(vmx)
|
||||||
|
clone_dir = vmx_path.parent
|
||||||
|
click.echo(f"[vm remove] destroying VM: {vmx}")
|
||||||
|
|
||||||
|
if not vmx_path.is_file():
|
||||||
|
click.echo(
|
||||||
|
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
# Still try to clean up a partial clone directory.
|
||||||
|
if clone_dir.exists() and clone_dir != vmx_path:
|
||||||
|
_try_remove_dir(clone_dir)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
backend = _make_backend(vmrun_path)
|
||||||
|
except BackendNotAvailable as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
_try_remove_dir(clone_dir)
|
||||||
|
return
|
||||||
|
|
||||||
|
handle = VmHandle(identifier=vmx)
|
||||||
|
|
||||||
|
if force:
|
||||||
|
try:
|
||||||
|
backend.stop(handle, hard=True)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||||
|
else:
|
||||||
|
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
||||||
|
_stop_with_fallback(backend, handle)
|
||||||
|
|
||||||
|
# Brief settle: VMware may keep file locks for a moment after stop.
|
||||||
|
deadline = time.monotonic() + 20.0
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
if not backend.is_running(handle):
|
||||||
|
break
|
||||||
|
except BackendError:
|
||||||
|
break
|
||||||
|
time.sleep(2.0)
|
||||||
|
|
||||||
|
click.echo("[vm remove] deleting VM...")
|
||||||
|
deleted = False
|
||||||
|
last_err: str = ""
|
||||||
|
for delay in (0, 3, 6):
|
||||||
|
if delay > 0:
|
||||||
|
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
||||||
|
time.sleep(delay)
|
||||||
|
try:
|
||||||
|
backend.delete(handle)
|
||||||
|
deleted = True
|
||||||
|
break
|
||||||
|
except BackendError as exc:
|
||||||
|
last_err = str(exc)
|
||||||
|
|
||||||
|
if not deleted:
|
||||||
|
click.echo(
|
||||||
|
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
||||||
|
)
|
||||||
|
click.echo("[vm remove] falling back to manual directory removal.")
|
||||||
|
|
||||||
|
if clone_dir.exists():
|
||||||
|
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
||||||
|
if _try_remove_dir(clone_dir):
|
||||||
|
click.echo("[vm remove] clone directory removed.")
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
click.echo("[vm remove] VM destruction complete.")
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────────────────── cleanup
|
||||||
|
|
||||||
|
|
||||||
|
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||||
|
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||||
|
if not clone_base.is_dir():
|
||||||
|
return []
|
||||||
|
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||||
|
out: list[Path] = []
|
||||||
|
for entry in clone_base.iterdir():
|
||||||
|
if not entry.is_dir():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if mtime < cutoff:
|
||||||
|
out.append(entry)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||||
|
for entry in clone_dir.iterdir():
|
||||||
|
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||||
|
return entry
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup_orphans(
|
||||||
|
clone_base: Path,
|
||||||
|
max_age_hours: int,
|
||||||
|
backend: VmBackend | None,
|
||||||
|
*,
|
||||||
|
dry_run: bool = False,
|
||||||
|
lock_file: Path | None = None,
|
||||||
|
lock_max_age_minutes: int = 30,
|
||||||
|
) -> int:
|
||||||
|
"""Remove orphaned clone directories. Returns count removed.
|
||||||
|
|
||||||
|
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
||||||
|
backend is unavailable, directory removal is still attempted.
|
||||||
|
"""
|
||||||
|
orphans = _list_orphans(clone_base, max_age_hours)
|
||||||
|
if not orphans:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||||
|
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||||
|
)
|
||||||
|
|
||||||
|
removed = 0
|
||||||
|
now = datetime.now(tz=UTC)
|
||||||
|
for entry in orphans:
|
||||||
|
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
||||||
|
if dry_run:
|
||||||
|
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
||||||
|
continue
|
||||||
|
click.echo(f"[vm cleanup] processing: {entry}")
|
||||||
|
vmx = _find_vmx(entry)
|
||||||
|
if vmx is not None and backend is not None:
|
||||||
|
handle = VmHandle(identifier=str(vmx))
|
||||||
|
try:
|
||||||
|
backend.stop(handle, hard=True)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
||||||
|
time.sleep(2.0)
|
||||||
|
try:
|
||||||
|
backend.delete(handle)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
elif vmx is None:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if _try_remove_dir(entry):
|
||||||
|
click.echo(f"[vm cleanup] removed: {entry}")
|
||||||
|
removed += 1
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||||
|
if lock_file is not None and lock_file.is_file():
|
||||||
|
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||||
|
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||||
|
if mtime < cutoff and not dry_run:
|
||||||
|
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||||
|
try:
|
||||||
|
lock_file.unlink()
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
||||||
|
|
||||||
|
click.echo("[vm cleanup] done.")
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
|
@vm.command("cleanup")
|
||||||
|
@click.option(
|
||||||
|
"--clone-base-dir",
|
||||||
|
"--clone-base",
|
||||||
|
"clone_base_dir",
|
||||||
|
default=None,
|
||||||
|
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--max-age-hours",
|
||||||
|
type=click.IntRange(0, 168),
|
||||||
|
default=4,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||||
|
@click.option(
|
||||||
|
"--lock-file",
|
||||||
|
default=None,
|
||||||
|
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--what-if",
|
||||||
|
"--whatif",
|
||||||
|
"what_if",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="List orphans without destroying them.",
|
||||||
|
)
|
||||||
|
def vm_cleanup(
|
||||||
|
clone_base_dir: str | None,
|
||||||
|
max_age_hours: int,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
lock_file: str | None,
|
||||||
|
what_if: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
||||||
|
config = load_config()
|
||||||
|
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||||
|
if not base_path.exists():
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
backend: VmBackend | None = _make_backend(vmrun_path)
|
||||||
|
except BackendNotAvailable as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
backend = None
|
||||||
|
|
||||||
|
cleanup_orphans(
|
||||||
|
clone_base=base_path,
|
||||||
|
max_age_hours=max_age_hours,
|
||||||
|
backend=backend,
|
||||||
|
dry_run=what_if,
|
||||||
|
lock_file=Path(lock_file) if lock_file else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────────────── new
|
||||||
|
|
||||||
|
|
||||||
|
@vm.command("new")
|
||||||
|
@click.option(
|
||||||
|
"--template",
|
||||||
|
"--template-path",
|
||||||
|
"template",
|
||||||
|
required=True,
|
||||||
|
help="Identifier of the template VM (Workstation: VMX path; ESXi: managed-object ref).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--snapshot",
|
||||||
|
"--snapshot-name",
|
||||||
|
"snapshot",
|
||||||
|
default="BaseClean",
|
||||||
|
show_default=True,
|
||||||
|
help="Snapshot name on the template to clone from.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--clone-base-dir",
|
||||||
|
"clone_base_dir",
|
||||||
|
required=True,
|
||||||
|
help="Directory under which the new clone folder will be created.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--job-id",
|
||||||
|
"job_id",
|
||||||
|
required=True,
|
||||||
|
help="Unique job identifier (used to name the clone folder).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--vmrun-path",
|
||||||
|
"vmrun_path",
|
||||||
|
default=None,
|
||||||
|
help="Override vmrun binary path.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
type=click.Choice(["windows", "linux"], case_sensitive=False),
|
||||||
|
default="windows",
|
||||||
|
show_default=True,
|
||||||
|
help="Informational; reserved for backend hints in Phase C.",
|
||||||
|
)
|
||||||
|
def vm_new(
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
clone_base_dir: str,
|
||||||
|
job_id: str,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
guest_os: str,
|
||||||
|
) -> None:
|
||||||
|
"""Create a linked clone of the template VM for an ephemeral CI build.
|
||||||
|
|
||||||
|
Mirrors ``scripts/New-BuildVM.ps1``. Prints the clone identifier
|
||||||
|
(Workstation: VMX path) on stdout on success.
|
||||||
|
|
||||||
|
Phase C hook: ``--template`` is treated as an opaque string; the backend
|
||||||
|
is responsible for interpretation. Workstation builds the destination
|
||||||
|
path under ``--clone-base-dir`` from ``--job-id`` + timestamp.
|
||||||
|
"""
|
||||||
|
del guest_os # currently informational only
|
||||||
|
# Verify the template identifier looks plausible: for Workstation it is
|
||||||
|
# a VMX path. For other backends in Phase C this validation will move
|
||||||
|
# into the backend itself.
|
||||||
|
template_path = Path(template)
|
||||||
|
if not template_path.is_file():
|
||||||
|
raise click.ClickException(f"Template VMX not found: {template}")
|
||||||
|
|
||||||
|
base = Path(clone_base_dir)
|
||||||
|
base.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||||
|
clone_name = f"Clone_{job_id}_{timestamp}"
|
||||||
|
clone_dir = base / clone_name
|
||||||
|
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||||
|
|
||||||
|
click.echo("[vm new] creating linked clone...")
|
||||||
|
click.echo(f" template : {template}")
|
||||||
|
click.echo(f" snapshot : {snapshot}")
|
||||||
|
click.echo(f" clone vmx: {clone_vmx}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
backend = _make_backend(vmrun_path)
|
||||||
|
except BackendNotAvailable as exc:
|
||||||
|
raise click.ClickException(f"vmrun unavailable: {exc}") from exc
|
||||||
|
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
handle = backend.clone_linked(
|
||||||
|
template=str(template_path),
|
||||||
|
snapshot=snapshot,
|
||||||
|
name=clone_name,
|
||||||
|
destination=str(clone_vmx),
|
||||||
|
)
|
||||||
|
except BackendError as exc:
|
||||||
|
# Clean up partial clone directory if vmrun left one behind.
|
||||||
|
if clone_dir.exists():
|
||||||
|
_try_remove_dir(clone_dir)
|
||||||
|
raise click.ClickException(f"clone failed: {exc}") from exc
|
||||||
|
|
||||||
|
if not clone_vmx.is_file():
|
||||||
|
raise click.ClickException(
|
||||||
|
f"backend reported success but clone VMX is missing: {clone_vmx}"
|
||||||
|
)
|
||||||
|
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
click.echo(f"[vm new] clone created in {elapsed:.1f}s")
|
||||||
|
# Final stdout line: the identifier itself, so PS callers can capture it.
|
||||||
|
click.echo(handle.identifier)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["cleanup_orphans", "vm"]
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
"""``wait-ready`` sub-command.
|
||||||
|
|
||||||
|
Replaces ``scripts/Wait-VMReady.ps1``. Polls a guest VM until WinRM
|
||||||
|
(Windows) or SSH (Linux) is responsive. Designed to be invoked from the
|
||||||
|
PowerShell shim ``scripts/Wait-VMReady.ps1`` which translates PS-style
|
||||||
|
PascalCase parameters (``-VMPath``, ``-IPAddress``, ``-Transport``,
|
||||||
|
``-SshKeyPath``, ``-SshUser``) into click kebab-case options.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.backends import load_backend
|
||||||
|
from ci_orchestrator.backends.errors import BackendError
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.config import load_config
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
from ci_orchestrator.transport.errors import TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
|
|
||||||
|
|
||||||
|
# Exit codes (kept in sync with the PowerShell original where possible):
|
||||||
|
# 0 ready, 2 timeout-running, 3 timeout-ip, 4 timeout-transport
|
||||||
|
_EXIT_TIMEOUT_RUNNING = 2
|
||||||
|
_EXIT_TIMEOUT_IP = 3
|
||||||
|
_EXIT_TIMEOUT_TRANSPORT = 4
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_guest_os(value: str) -> str:
|
||||||
|
"""Map Transport/GuestOS aliases to ``windows``/``linux``."""
|
||||||
|
v = value.strip().lower()
|
||||||
|
if v in {"winrm", "windows", "win"}:
|
||||||
|
return "windows"
|
||||||
|
if v in {"ssh", "linux", "lnx"}:
|
||||||
|
return "linux"
|
||||||
|
raise click.BadParameter(f"Unknown guest-os/transport value: {value!r}")
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_running(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> bool:
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
if backend.is_running(handle):
|
||||||
|
return True
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[wait-ready] backend probe error: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_ip(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> str | None:
|
||||||
|
while (now := time.monotonic()) < deadline:
|
||||||
|
call_timeout = min(poll, deadline - now)
|
||||||
|
try:
|
||||||
|
ip = backend.get_ip(handle, timeout=call_timeout)
|
||||||
|
except BackendError:
|
||||||
|
ip = None
|
||||||
|
if ip:
|
||||||
|
return ip
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@click.command("wait-ready")
|
||||||
|
@click.option(
|
||||||
|
"--vmx",
|
||||||
|
"--vm-path",
|
||||||
|
"vmx",
|
||||||
|
required=True,
|
||||||
|
help="Path to the guest VMX file.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--ip-address",
|
||||||
|
"ip_address",
|
||||||
|
default=None,
|
||||||
|
help="Guest IP address. If omitted the backend is polled for it.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
"--transport",
|
||||||
|
"guest_os",
|
||||||
|
default="windows",
|
||||||
|
show_default=True,
|
||||||
|
callback=lambda _ctx, _p, v: _normalise_guest_os(v),
|
||||||
|
help="windows/WinRM or linux/SSH (case-insensitive).",
|
||||||
|
)
|
||||||
|
@click.option("--timeout", "--timeout-seconds", "timeout", type=float, default=300.0, show_default=True)
|
||||||
|
@click.option(
|
||||||
|
"--poll-interval",
|
||||||
|
"--poll-interval-seconds",
|
||||||
|
"poll_interval",
|
||||||
|
type=float,
|
||||||
|
default=5.0,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||||
|
@click.option(
|
||||||
|
"--credential-target",
|
||||||
|
default=None,
|
||||||
|
help="Override credential target name (defaults to config.guest_cred_target).",
|
||||||
|
)
|
||||||
|
@click.option("--ssh-user", default="ci_build", show_default=True)
|
||||||
|
@click.option(
|
||||||
|
"--ssh-key-path",
|
||||||
|
default=None,
|
||||||
|
help="SSH private key (defaults to config.ssh_key_path).",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--skip-ping",
|
||||||
|
is_flag=True,
|
||||||
|
default=False,
|
||||||
|
help="Accepted for PS compatibility; ignored (Python uses transport probe directly).",
|
||||||
|
)
|
||||||
|
def wait_ready(
|
||||||
|
vmx: str,
|
||||||
|
ip_address: str | None,
|
||||||
|
guest_os: str,
|
||||||
|
timeout: float,
|
||||||
|
poll_interval: float,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
credential_target: str | None,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
skip_ping: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Poll a guest until WinRM (Windows) or SSH (Linux) is responsive."""
|
||||||
|
del skip_ping # accepted for PS compatibility, no-op
|
||||||
|
config = load_config()
|
||||||
|
if vmrun_path is not None:
|
||||||
|
# Re-load backend with override; cheap, no real connections.
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||||
|
else:
|
||||||
|
backend = load_backend(config)
|
||||||
|
handle = VmHandle(identifier=vmx)
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
|
||||||
|
click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}")
|
||||||
|
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||||
|
click.echo("[wait-ready] timeout waiting for VM to be running.", err=True)
|
||||||
|
sys.exit(_EXIT_TIMEOUT_RUNNING)
|
||||||
|
|
||||||
|
if ip_address is None:
|
||||||
|
click.echo("[wait-ready] waiting for guest IP (VMware Tools must be running)...")
|
||||||
|
ip = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||||
|
if not ip:
|
||||||
|
click.echo("[wait-ready] timeout waiting for guest IP.", err=True)
|
||||||
|
sys.exit(_EXIT_TIMEOUT_IP)
|
||||||
|
click.echo(f"[wait-ready] guest IP: {ip}")
|
||||||
|
else:
|
||||||
|
ip = ip_address
|
||||||
|
click.echo(f"[wait-ready] using provided IP: {ip}")
|
||||||
|
|
||||||
|
if guest_os == "windows":
|
||||||
|
store = KeyringCredentialStore()
|
||||||
|
target = credential_target or config.guest_cred_target
|
||||||
|
cred = store.get(target)
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with WinRmTransport(ip, cred.username, cred.password) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
click.echo("[wait-ready] WinRM ready.")
|
||||||
|
return
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll_interval)
|
||||||
|
else:
|
||||||
|
key_path: str | None = ssh_key_path
|
||||||
|
if key_path is None and config.ssh_key_path is not None:
|
||||||
|
key_path = str(config.ssh_key_path)
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
with SshTransport(ip, username=ssh_user, key_path=key_path) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
click.echo("[wait-ready] SSH ready.")
|
||||||
|
return
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll_interval)
|
||||||
|
|
||||||
|
click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True)
|
||||||
|
sys.exit(_EXIT_TIMEOUT_TRANSPORT)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"KeyringCredentialStore",
|
||||||
|
"SshTransport",
|
||||||
|
"WinRmTransport",
|
||||||
|
"load_backend",
|
||||||
|
"wait_ready",
|
||||||
|
]
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
"""Configuration loading for the CI orchestrator.
|
||||||
|
|
||||||
|
Resolution order (later wins):
|
||||||
|
1. OS-aware defaults (Windows: F:\\CI\\..., Linux: /var/lib/ci/...).
|
||||||
|
2. Optional ``config.toml`` file (path via ``CI_CONFIG`` env var or
|
||||||
|
``$CI_ROOT/config.toml`` if present).
|
||||||
|
3. Environment variables (``CI_ROOT``, ``CI_TEMPLATES``, ``CI_BUILD_VMS``,
|
||||||
|
``CI_ARTIFACTS``, ``CI_KEYS``).
|
||||||
|
|
||||||
|
Phase C hook: a ``[backend]`` section in ``config.toml`` selects the VM
|
||||||
|
backend implementation; the field is read but currently only the
|
||||||
|
``workstation`` value is supported.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import tomllib
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
_ENV_KEYS = ("CI_ROOT", "CI_TEMPLATES", "CI_BUILD_VMS", "CI_ARTIFACTS", "CI_KEYS")
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Paths:
|
||||||
|
"""Filesystem paths used by the orchestrator."""
|
||||||
|
|
||||||
|
root: Path
|
||||||
|
templates: Path
|
||||||
|
build_vms: Path
|
||||||
|
artifacts: Path
|
||||||
|
keys: Path
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BackendConfig:
|
||||||
|
"""Backend selector. ``type`` is currently always 'workstation'."""
|
||||||
|
|
||||||
|
type: str = "workstation"
|
||||||
|
options: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Config:
|
||||||
|
"""Top-level configuration object."""
|
||||||
|
|
||||||
|
paths: Paths
|
||||||
|
backend: BackendConfig
|
||||||
|
vmrun_path: str | None = None
|
||||||
|
ssh_key_path: Path | None = None
|
||||||
|
guest_cred_target: str = "BuildVMGuest"
|
||||||
|
|
||||||
|
|
||||||
|
def _platform_defaults() -> Paths:
|
||||||
|
root = Path(r"F:\CI") if os.name == "nt" else Path("/var/lib/ci")
|
||||||
|
return Paths(
|
||||||
|
root=root,
|
||||||
|
templates=root / "Templates" if os.name == "nt" else root / "templates",
|
||||||
|
build_vms=root / "BuildVMs" if os.name == "nt" else root / "build-vms",
|
||||||
|
artifacts=root / "Artifacts" if os.name == "nt" else root / "artifacts",
|
||||||
|
keys=root / "keys" if os.name == "nt" else Path("/etc/ci/keys"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_toml(path: Path) -> dict[str, Any]:
|
||||||
|
with path.open("rb") as fh:
|
||||||
|
return tomllib.load(fh)
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_paths(base: Paths, overrides: dict[str, Any]) -> Paths:
|
||||||
|
def pick(key: str, current: Path) -> Path:
|
||||||
|
val = overrides.get(key)
|
||||||
|
return Path(val) if val else current
|
||||||
|
|
||||||
|
return Paths(
|
||||||
|
root=pick("root", base.root),
|
||||||
|
templates=pick("templates", base.templates),
|
||||||
|
build_vms=pick("build_vms", base.build_vms),
|
||||||
|
artifacts=pick("artifacts", base.artifacts),
|
||||||
|
keys=pick("keys", base.keys),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_env(paths: Paths, env: dict[str, str]) -> Paths:
|
||||||
|
mapping = {
|
||||||
|
"CI_ROOT": "root",
|
||||||
|
"CI_TEMPLATES": "templates",
|
||||||
|
"CI_BUILD_VMS": "build_vms",
|
||||||
|
"CI_ARTIFACTS": "artifacts",
|
||||||
|
"CI_KEYS": "keys",
|
||||||
|
}
|
||||||
|
overrides: dict[str, Any] = {}
|
||||||
|
for env_key, attr in mapping.items():
|
||||||
|
val = env.get(env_key)
|
||||||
|
if val:
|
||||||
|
overrides[attr] = val
|
||||||
|
return _merge_paths(paths, overrides)
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(
|
||||||
|
config_path: Path | None = None,
|
||||||
|
env: dict[str, str] | None = None,
|
||||||
|
) -> Config:
|
||||||
|
"""Build a :class:`Config` from defaults, optional TOML file, and env vars.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
config_path:
|
||||||
|
Explicit path to a TOML file. If ``None``, looks at ``$CI_CONFIG`` then
|
||||||
|
``$CI_ROOT/config.toml``.
|
||||||
|
env:
|
||||||
|
Environment dict (defaults to :data:`os.environ`). Injected for tests.
|
||||||
|
"""
|
||||||
|
if env is None:
|
||||||
|
env = dict(os.environ)
|
||||||
|
|
||||||
|
paths = _platform_defaults()
|
||||||
|
|
||||||
|
# Resolve config file
|
||||||
|
toml_path: Path | None = None
|
||||||
|
if config_path is not None:
|
||||||
|
toml_path = config_path
|
||||||
|
elif env.get("CI_CONFIG"):
|
||||||
|
toml_path = Path(env["CI_CONFIG"])
|
||||||
|
else:
|
||||||
|
candidate = paths.root / "config.toml"
|
||||||
|
if candidate.is_file():
|
||||||
|
toml_path = candidate
|
||||||
|
|
||||||
|
backend = BackendConfig()
|
||||||
|
vmrun_path: str | None = env.get("CI_VMRUN_PATH")
|
||||||
|
ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None
|
||||||
|
guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest")
|
||||||
|
|
||||||
|
if toml_path and toml_path.is_file():
|
||||||
|
data = _load_toml(toml_path)
|
||||||
|
paths = _merge_paths(paths, data.get("paths", {}))
|
||||||
|
backend_section = data.get("backend", {})
|
||||||
|
if backend_section:
|
||||||
|
backend = BackendConfig(
|
||||||
|
type=str(backend_section.get("type", "workstation")),
|
||||||
|
options={k: v for k, v in backend_section.items() if k != "type"},
|
||||||
|
)
|
||||||
|
if vmrun_path is None:
|
||||||
|
vmrun_path = data.get("vmrun_path")
|
||||||
|
if ssh_key_path is None and data.get("ssh_key_path"):
|
||||||
|
ssh_key_path = Path(data["ssh_key_path"])
|
||||||
|
guest_cred_target = data.get("guest_cred_target", guest_cred_target)
|
||||||
|
|
||||||
|
paths = _apply_env(paths, env)
|
||||||
|
|
||||||
|
return Config(
|
||||||
|
paths=paths,
|
||||||
|
backend=backend,
|
||||||
|
vmrun_path=vmrun_path,
|
||||||
|
ssh_key_path=ssh_key_path,
|
||||||
|
guest_cred_target=guest_cred_target,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["BackendConfig", "Config", "Paths", "load_config"]
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""Credential store abstraction.
|
||||||
|
|
||||||
|
Wraps the ``keyring`` library so callers do not depend on it directly.
|
||||||
|
Phase B note: the same interface is used on both Windows (Credential
|
||||||
|
Manager) and Linux (Secret Service / file-based vault); see
|
||||||
|
``plans/implementation-plan-A-B.md`` step B3.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Credential:
|
||||||
|
"""Username + secret pair retrieved from a credential store."""
|
||||||
|
|
||||||
|
username: str
|
||||||
|
password: str
|
||||||
|
|
||||||
|
|
||||||
|
class CredentialStore(Protocol):
|
||||||
|
"""Read-only credential lookup."""
|
||||||
|
|
||||||
|
def get(self, target: str) -> Credential:
|
||||||
|
"""Return the credential associated with ``target``.
|
||||||
|
|
||||||
|
Raises :class:`KeyError` if the target is unknown.
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
class KeyringCredentialStore:
|
||||||
|
"""Default :class:`CredentialStore` backed by the ``keyring`` library."""
|
||||||
|
|
||||||
|
def __init__(self, service: str = "ci") -> None:
|
||||||
|
self._service = service
|
||||||
|
|
||||||
|
def get(self, target: str) -> Credential:
|
||||||
|
try:
|
||||||
|
import keyring
|
||||||
|
except ImportError as exc: # pragma: no cover - dep declared
|
||||||
|
raise RuntimeError(
|
||||||
|
"keyring is not installed; run `pip install keyring`."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
cred = keyring.get_credential(target, None)
|
||||||
|
if cred is None:
|
||||||
|
# Fallback: some backends only support get_password and need the
|
||||||
|
# username to be the same as the target.
|
||||||
|
password = keyring.get_password(self._service, target)
|
||||||
|
if password is None:
|
||||||
|
raise KeyError(f"Credential '{target}' not found in keyring.")
|
||||||
|
return Credential(username=target, password=password)
|
||||||
|
return Credential(username=cred.username, password=cred.password)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["Credential", "CredentialStore", "KeyringCredentialStore"]
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
"""Guest transport layer (WinRM for Windows, SSH for Linux)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ci_orchestrator.transport.errors import TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
__all__ = ["SshTransport", "TransportError", "WinRmTransport"]
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
"""Transport error hierarchy."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
class TransportError(RuntimeError):
|
||||||
|
"""Base class for transport-layer (WinRM/SSH) failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class TransportConnectError(TransportError):
|
||||||
|
"""Raised when the underlying connection cannot be established."""
|
||||||
|
|
||||||
|
|
||||||
|
class TransportCommandError(TransportError):
|
||||||
|
"""Raised when a remote command exits non-zero."""
|
||||||
|
|
||||||
|
def __init__(self, returncode: int, stdout: str, stderr: str) -> None:
|
||||||
|
super().__init__(
|
||||||
|
f"Remote command failed (exit={returncode}): {(stderr or stdout).strip()[:500]}"
|
||||||
|
)
|
||||||
|
self.returncode = returncode
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""SSH transport via ``paramiko``.
|
||||||
|
|
||||||
|
Replaces the ``ssh.exe`` / ``scp.exe`` calls used in
|
||||||
|
``scripts/_Transport.psm1``. By default the host-key policy is
|
||||||
|
``AutoAddPolicy`` with an empty ``known_hosts`` file (i.e. permissive),
|
||||||
|
which mirrors the ``StrictHostKeyChecking=no UserKnownHostsFile=NUL``
|
||||||
|
default used for ephemeral CI build VMs.
|
||||||
|
|
||||||
|
See ``AGENTS.md`` error #12: in PowerShell with ``$ErrorActionPreference =
|
||||||
|
'Stop'`` the native ``ssh-keygen -R`` call could raise terminating errors
|
||||||
|
on stderr output. With paramiko there is no equivalent issue.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
import paramiko
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SshResult:
|
||||||
|
"""Result of a remote SSH command."""
|
||||||
|
|
||||||
|
stdout: str
|
||||||
|
stderr: str
|
||||||
|
returncode: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return self.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
class SshTransport:
|
||||||
|
"""Thin wrapper around :class:`paramiko.SSHClient`."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
host: str,
|
||||||
|
*,
|
||||||
|
username: str = "ci_build",
|
||||||
|
key_path: str | Path | None = None,
|
||||||
|
port: int = 22,
|
||||||
|
connect_timeout: float = 10.0,
|
||||||
|
known_hosts: str | Path | None = None,
|
||||||
|
) -> None:
|
||||||
|
self._host = host
|
||||||
|
self._username = username
|
||||||
|
self._key_path = Path(key_path) if key_path else None
|
||||||
|
self._port = port
|
||||||
|
self._connect_timeout = connect_timeout
|
||||||
|
self._known_hosts = Path(known_hosts) if known_hosts else None
|
||||||
|
self._client: paramiko.SSHClient | None = None
|
||||||
|
|
||||||
|
# ----------------------------------------------------------- connection
|
||||||
|
def _connect(self) -> paramiko.SSHClient:
|
||||||
|
if self._client is not None:
|
||||||
|
return self._client
|
||||||
|
try:
|
||||||
|
import paramiko
|
||||||
|
except ImportError as exc: # pragma: no cover - dep declared
|
||||||
|
raise TransportConnectError(
|
||||||
|
"paramiko is not installed; run `pip install paramiko`."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
client = paramiko.SSHClient()
|
||||||
|
if self._known_hosts is not None and self._known_hosts.is_file():
|
||||||
|
client.load_host_keys(str(self._known_hosts))
|
||||||
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
else:
|
||||||
|
# Permissive mode for ephemeral CI VMs (see _Transport.psm1).
|
||||||
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
|
||||||
|
try:
|
||||||
|
client.connect(
|
||||||
|
hostname=self._host,
|
||||||
|
port=self._port,
|
||||||
|
username=self._username,
|
||||||
|
key_filename=str(self._key_path) if self._key_path else None,
|
||||||
|
timeout=self._connect_timeout,
|
||||||
|
allow_agent=False,
|
||||||
|
look_for_keys=False,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"SSH connection to {self._username}@{self._host}:{self._port} failed: {exc}"
|
||||||
|
) from exc
|
||||||
|
self._client = client
|
||||||
|
return client
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
if self._client is not None:
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
self._client.close()
|
||||||
|
self._client = None
|
||||||
|
|
||||||
|
def __enter__(self) -> SshTransport:
|
||||||
|
self._connect()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_exc: object) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- ops
|
||||||
|
def run(self, command: str, *, check: bool = True, timeout: float | None = None) -> SshResult:
|
||||||
|
"""Run ``command`` in the guest shell."""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
_stdin, stdout, stderr = client.exec_command(command, timeout=timeout)
|
||||||
|
out = stdout.read().decode("utf-8", errors="replace")
|
||||||
|
err = stderr.read().decode("utf-8", errors="replace")
|
||||||
|
rc = stdout.channel.recv_exit_status()
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc
|
||||||
|
result = SshResult(stdout=out, stderr=err, returncode=rc)
|
||||||
|
if check and not result.ok:
|
||||||
|
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def run_streaming(
|
||||||
|
self, command: str, *, check: bool = True, timeout: float | None = None
|
||||||
|
) -> SshResult:
|
||||||
|
"""Run ``command`` streaming stdout/stderr live to this process.
|
||||||
|
|
||||||
|
Unlike :meth:`run` (which blocks on a full channel read), output
|
||||||
|
is forwarded chunk-by-chunk as it arrives so long builds show
|
||||||
|
progress instead of one block at the end. Captured output is
|
||||||
|
still returned in :class:`SshResult`.
|
||||||
|
"""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
chan = client.get_transport().open_session() # type: ignore[union-attr]
|
||||||
|
if timeout:
|
||||||
|
chan.settimeout(timeout)
|
||||||
|
chan.exec_command(command)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc
|
||||||
|
|
||||||
|
out_parts: list[str] = []
|
||||||
|
err_parts: list[str] = []
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
progressed = False
|
||||||
|
while chan.recv_ready():
|
||||||
|
d = chan.recv(65536).decode("utf-8", errors="replace")
|
||||||
|
out_parts.append(d)
|
||||||
|
sys.stdout.write(d)
|
||||||
|
sys.stdout.flush()
|
||||||
|
progressed = True
|
||||||
|
while chan.recv_stderr_ready():
|
||||||
|
d = chan.recv_stderr(65536).decode("utf-8", errors="replace")
|
||||||
|
err_parts.append(d)
|
||||||
|
sys.stderr.write(d)
|
||||||
|
sys.stderr.flush()
|
||||||
|
progressed = True
|
||||||
|
if (
|
||||||
|
chan.exit_status_ready()
|
||||||
|
and not chan.recv_ready()
|
||||||
|
and not chan.recv_stderr_ready()
|
||||||
|
):
|
||||||
|
break
|
||||||
|
if not progressed:
|
||||||
|
time.sleep(0.05)
|
||||||
|
rc = chan.recv_exit_status()
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"SSH stream failed: {exc}") from exc
|
||||||
|
finally:
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
chan.close()
|
||||||
|
|
||||||
|
result = SshResult(
|
||||||
|
stdout="".join(out_parts), stderr="".join(err_parts), returncode=rc
|
||||||
|
)
|
||||||
|
if check and not result.ok:
|
||||||
|
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def copy(self, local_path: str | Path, remote_path: str) -> None:
|
||||||
|
"""Upload via SFTP."""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
with client.open_sftp() as sftp:
|
||||||
|
sftp.put(str(local_path), remote_path)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"SFTP put failed: {exc}") from exc
|
||||||
|
|
||||||
|
def fetch(self, remote_path: str, local_path: str | Path) -> None:
|
||||||
|
"""Download via SFTP."""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
with client.open_sftp() as sftp:
|
||||||
|
sftp.get(remote_path, str(local_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"SFTP get failed: {exc}") from exc
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
"""Lightweight readiness probe."""
|
||||||
|
try:
|
||||||
|
result = self.run("true", check=False, timeout=5.0)
|
||||||
|
except (TransportConnectError, TransportCommandError):
|
||||||
|
return False
|
||||||
|
return result.ok
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
"""WinRM transport via ``pypsrp``.
|
||||||
|
|
||||||
|
Replaces the ``New-PSSession`` / ``Invoke-Command`` calls used in the
|
||||||
|
PowerShell scripts. CI build VMs use a self-signed TLS certificate on
|
||||||
|
WinRM port 5986, so SSL validation is disabled by default — appropriate
|
||||||
|
for an isolated lab network (see ``scripts/_Common.psm1``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError
|
||||||
|
|
||||||
|
# WSMan faults that are transient — the guest momentarily cannot launch
|
||||||
|
# the WSMan host process (wsmprovhost), typically right after a
|
||||||
|
# memory-heavy build. Retrying after a short settle usually succeeds.
|
||||||
|
_WSMAN_TRANSIENT_MARKERS = (
|
||||||
|
"1018", # could not launch a host process to process the given request
|
||||||
|
"could not launch a host process",
|
||||||
|
"the wsman provider host server",
|
||||||
|
"1726", # RPC server unavailable (service still coming back)
|
||||||
|
"connection was aborted",
|
||||||
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from pypsrp.client import Client
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class WinRmResult:
|
||||||
|
"""Result of a remote PowerShell invocation."""
|
||||||
|
|
||||||
|
stdout: str
|
||||||
|
stderr: str
|
||||||
|
returncode: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return self.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
class WinRmTransport:
|
||||||
|
"""Thin wrapper around :class:`pypsrp.client.Client`.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
host:
|
||||||
|
IP or DNS name of the guest.
|
||||||
|
username, password:
|
||||||
|
Credentials. Pass via :class:`~ci_orchestrator.credentials.Credential`
|
||||||
|
in caller code; do not embed plaintext.
|
||||||
|
port:
|
||||||
|
WinRM HTTPS port (default 5986).
|
||||||
|
cert_validation:
|
||||||
|
When ``False`` (default), self-signed certs are accepted — required
|
||||||
|
for the ephemeral build VMs in this lab.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
host: str,
|
||||||
|
username: str,
|
||||||
|
password: str,
|
||||||
|
*,
|
||||||
|
port: int = 5986,
|
||||||
|
ssl: bool = True,
|
||||||
|
cert_validation: bool = False,
|
||||||
|
connection_timeout: int = 30,
|
||||||
|
auth: str = "ntlm",
|
||||||
|
) -> None:
|
||||||
|
self._host = host
|
||||||
|
self._username = username
|
||||||
|
self._password = password
|
||||||
|
self._port = port
|
||||||
|
self._ssl = ssl
|
||||||
|
self._cert_validation = cert_validation
|
||||||
|
self._connection_timeout = connection_timeout
|
||||||
|
# Build VMs use local (workgroup) accounts. pypsrp defaults to
|
||||||
|
# 'negotiate', which tries Kerberos with no domain and fails under
|
||||||
|
# SSPI with SEC_E_UNKNOWN_CREDENTIALS. NTLM is required for local
|
||||||
|
# accounts. The guest username must also be host-qualified
|
||||||
|
# (WINBUILD-2025\ci_build), stored that way in the credential vault.
|
||||||
|
self._auth = auth
|
||||||
|
self._client: Client | None = None
|
||||||
|
|
||||||
|
# ----------------------------------------------------------- connection
|
||||||
|
def _connect(self) -> Client:
|
||||||
|
if self._client is not None:
|
||||||
|
return self._client
|
||||||
|
try:
|
||||||
|
from pypsrp.client import Client # local import: optional dep at runtime
|
||||||
|
except ImportError as exc: # pragma: no cover - dep declared in pyproject
|
||||||
|
raise TransportConnectError(
|
||||||
|
"pypsrp is not installed; run `pip install pypsrp`."
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
self._client = Client(
|
||||||
|
self._host,
|
||||||
|
username=self._username,
|
||||||
|
password=self._password,
|
||||||
|
port=self._port,
|
||||||
|
ssl=self._ssl,
|
||||||
|
auth=self._auth,
|
||||||
|
cert_validation=self._cert_validation,
|
||||||
|
connection_timeout=self._connection_timeout,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"WinRM connection to {self._host}:{self._port} failed: {exc}"
|
||||||
|
) from exc
|
||||||
|
return self._client
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
if self._client is not None:
|
||||||
|
with contextlib.suppress(Exception):
|
||||||
|
self._client.wsman.close()
|
||||||
|
self._client = None
|
||||||
|
|
||||||
|
def __enter__(self) -> WinRmTransport:
|
||||||
|
self._connect()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_exc: object) -> None:
|
||||||
|
self.close()
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- ops
|
||||||
|
def run(self, script: str, *, check: bool = True) -> WinRmResult:
|
||||||
|
"""Run a PowerShell script block in the guest.
|
||||||
|
|
||||||
|
Retries transient WSMan host-process-launch failures (e.g. fault
|
||||||
|
1018 right after a memory-heavy build) with a short backoff.
|
||||||
|
"""
|
||||||
|
attempts = 4
|
||||||
|
delay = 5.0
|
||||||
|
last_exc: Exception | None = None
|
||||||
|
for attempt in range(attempts):
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
stdout, stderr, returncode = client.execute_ps(script)
|
||||||
|
break
|
||||||
|
except Exception as exc:
|
||||||
|
last_exc = exc
|
||||||
|
msg = str(exc).lower()
|
||||||
|
transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS)
|
||||||
|
if transient and attempt < attempts - 1:
|
||||||
|
# Drop the cached client so the next try opens a fresh
|
||||||
|
# WSMan shell once the guest can spawn the host again.
|
||||||
|
self.close()
|
||||||
|
time.sleep(delay)
|
||||||
|
delay *= 2
|
||||||
|
continue
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"WinRM execute_ps failed: {exc}"
|
||||||
|
) from exc
|
||||||
|
else: # pragma: no cover - loop always breaks or raises
|
||||||
|
raise TransportConnectError(
|
||||||
|
f"WinRM execute_ps failed: {last_exc}"
|
||||||
|
) from last_exc
|
||||||
|
# pypsrp returns stderr as a PSDataStreams object (newer versions) or a
|
||||||
|
# list of error records (older versions); normalise to a plain string.
|
||||||
|
stderr_any: Any = stderr
|
||||||
|
if isinstance(stderr_any, str):
|
||||||
|
stderr_str = stderr_any
|
||||||
|
elif hasattr(stderr_any, "error"):
|
||||||
|
# PSDataStreams — extract .error records.
|
||||||
|
error_records = getattr(stderr_any, "error", None) or []
|
||||||
|
stderr_str = "\n".join(str(e) for e in error_records)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
stderr_str = "\n".join(str(s) for s in stderr_any)
|
||||||
|
except TypeError:
|
||||||
|
stderr_str = str(stderr_any)
|
||||||
|
result = WinRmResult(stdout=stdout, stderr=stderr_str, returncode=int(returncode))
|
||||||
|
if check and not result.ok:
|
||||||
|
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def copy(self, local_path: str | Path, remote_path: str) -> None:
|
||||||
|
"""Upload a file to the guest via PSRP."""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
client.copy(str(local_path), remote_path)
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"WinRM copy failed: {exc}") from exc
|
||||||
|
|
||||||
|
def fetch(self, remote_path: str, local_path: str | Path) -> None:
|
||||||
|
"""Download a file from the guest via PSRP."""
|
||||||
|
client = self._connect()
|
||||||
|
try:
|
||||||
|
client.fetch(remote_path, str(local_path))
|
||||||
|
except Exception as exc:
|
||||||
|
raise TransportConnectError(f"WinRM fetch failed: {exc}") from exc
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
"""Lightweight readiness probe: run ``$true`` and check exit code."""
|
||||||
|
try:
|
||||||
|
result = self.run("$true | Out-Null", check=False)
|
||||||
|
except (TransportConnectError, TransportCommandError):
|
||||||
|
return False
|
||||||
|
return result.ok
|
||||||
@@ -242,11 +242,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function New-IsoFromFolder {
|
function New-IsoFromFolder {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $SourceFolder,
|
[Parameter(Mandatory)] [string] $SourceFolder,
|
||||||
[Parameter(Mandatory)] [string] $IsoPath,
|
[Parameter(Mandatory)] [string] $IsoPath,
|
||||||
[Parameter(Mandatory)] [string] $VolumeLabel
|
[Parameter(Mandatory)] [string] $VolumeLabel
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return }
|
||||||
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
||||||
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
||||||
$result = $null
|
$result = $null
|
||||||
@@ -296,11 +298,13 @@ function Invoke-Vmrun {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Set-VmxKey {
|
function Set-VmxKey {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $VmxPath,
|
[Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[Parameter(Mandatory)] [string] $Key,
|
[Parameter(Mandatory)] [string] $Key,
|
||||||
[Parameter(Mandatory)] [string] $Value
|
[Parameter(Mandatory)] [string] $Value
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return }
|
||||||
$lines = Get-Content -LiteralPath $VmxPath
|
$lines = Get-Content -LiteralPath $VmxPath
|
||||||
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
||||||
$found = $false
|
$found = $false
|
||||||
|
|||||||
@@ -270,11 +270,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function New-IsoFromFolder {
|
function New-IsoFromFolder {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $SourceFolder,
|
[Parameter(Mandatory)] [string] $SourceFolder,
|
||||||
[Parameter(Mandatory)] [string] $IsoPath,
|
[Parameter(Mandatory)] [string] $IsoPath,
|
||||||
[Parameter(Mandatory)] [string] $VolumeLabel
|
[Parameter(Mandatory)] [string] $VolumeLabel
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return }
|
||||||
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
||||||
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
||||||
$result = $null
|
$result = $null
|
||||||
@@ -305,10 +307,12 @@ function New-WinIsoNoPrompt {
|
|||||||
Result: UEFI boot proceeds straight into Windows Setup, no
|
Result: UEFI boot proceeds straight into Windows Setup, no
|
||||||
"Press any key to boot from CD" prompt.
|
"Press any key to boot from CD" prompt.
|
||||||
#>
|
#>
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $SourceIsoPath,
|
[Parameter(Mandatory)] [string] $SourceIsoPath,
|
||||||
[Parameter(Mandatory)] [string] $OutputIsoPath
|
[Parameter(Mandatory)] [string] $OutputIsoPath
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($OutputIsoPath, 'Rebuild Windows ISO without boot prompt')) { return }
|
||||||
if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force }
|
if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force }
|
||||||
|
|
||||||
$mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly
|
$mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly
|
||||||
@@ -329,7 +333,7 @@ function New-WinIsoNoPrompt {
|
|||||||
$fsi.VolumeName = $label
|
$fsi.VolumeName = $label
|
||||||
$fsi.ChooseImageDefaultsForMediaType(13)
|
$fsi.ChooseImageDefaultsForMediaType(13)
|
||||||
# 5+ GB ISO needs explicit large-volume defaults if available.
|
# 5+ GB ISO needs explicit large-volume defaults if available.
|
||||||
try { $fsi.UDFRevision = 0x102 } catch {}
|
try { $fsi.UDFRevision = 0x102 } catch { Write-Verbose "UDF revision not supported on this IMAPI version." }
|
||||||
|
|
||||||
Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow
|
Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow
|
||||||
$fsi.Root.AddTree($letter, $false)
|
$fsi.Root.AddTree($letter, $false)
|
||||||
@@ -383,11 +387,13 @@ function Invoke-Vmrun {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Set-VmxKey {
|
function Set-VmxKey {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $VmxPath,
|
[Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[Parameter(Mandatory)] [string] $Key,
|
[Parameter(Mandatory)] [string] $Key,
|
||||||
[Parameter(Mandatory)] [string] $Value
|
[Parameter(Mandatory)] [string] $Value
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return }
|
||||||
$lines = Get-Content -LiteralPath $VmxPath
|
$lines = Get-Content -LiteralPath $VmxPath
|
||||||
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
||||||
$found = $false
|
$found = $false
|
||||||
@@ -400,8 +406,10 @@ function Set-VmxKey {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Remove-VmxLines {
|
function Remove-VmxLines {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param([Parameter(Mandatory)] [string] $VmxPath,
|
param([Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[Parameter(Mandatory)] [string] $KeyPrefix)
|
[Parameter(Mandatory)] [string] $KeyPrefix)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, "Remove VMX lines starting with '$KeyPrefix'")) { return }
|
||||||
$pat = "^\s*$([regex]::Escape($KeyPrefix))"
|
$pat = "^\s*$([regex]::Escape($KeyPrefix))"
|
||||||
$lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat }
|
$lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat }
|
||||||
Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII
|
Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII
|
||||||
@@ -428,8 +436,10 @@ function Invoke-VmrunBounded {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Stop-Vm {
|
function Stop-Vm {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param([Parameter(Mandatory)] [string] $VmxPath,
|
param([Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[int] $SoftTimeoutMinutes = 5)
|
[int] $SoftTimeoutMinutes = 5)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, 'Stop VM')) { return }
|
||||||
# Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s.
|
# Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s.
|
||||||
# Guest OS continues shutdown independently once the signal is received.
|
# Guest OS continues shutdown independently once the signal is received.
|
||||||
Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30
|
Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30
|
||||||
@@ -1114,4 +1124,4 @@ Write-Host " .\Validate-DeployState.ps1 -VMIPAddress $guestIP"
|
|||||||
Write-Host " .\Prepare-WinBuild2022.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]"
|
Write-Host " .\Prepare-WinBuild2022.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]"
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green
|
Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green
|
||||||
|
|
||||||
|
|||||||
@@ -270,11 +270,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function New-IsoFromFolder {
|
function New-IsoFromFolder {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $SourceFolder,
|
[Parameter(Mandatory)] [string] $SourceFolder,
|
||||||
[Parameter(Mandatory)] [string] $IsoPath,
|
[Parameter(Mandatory)] [string] $IsoPath,
|
||||||
[Parameter(Mandatory)] [string] $VolumeLabel
|
[Parameter(Mandatory)] [string] $VolumeLabel
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return }
|
||||||
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force }
|
||||||
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
$fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage
|
||||||
$result = $null
|
$result = $null
|
||||||
@@ -305,10 +307,12 @@ function New-WinIsoNoPrompt {
|
|||||||
Result: UEFI boot proceeds straight into Windows Setup, no
|
Result: UEFI boot proceeds straight into Windows Setup, no
|
||||||
"Press any key to boot from CD" prompt.
|
"Press any key to boot from CD" prompt.
|
||||||
#>
|
#>
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $SourceIsoPath,
|
[Parameter(Mandatory)] [string] $SourceIsoPath,
|
||||||
[Parameter(Mandatory)] [string] $OutputIsoPath
|
[Parameter(Mandatory)] [string] $OutputIsoPath
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($OutputIsoPath, 'Rebuild Windows ISO without boot prompt')) { return }
|
||||||
if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force }
|
if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force }
|
||||||
|
|
||||||
$mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly
|
$mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly
|
||||||
@@ -329,7 +333,7 @@ function New-WinIsoNoPrompt {
|
|||||||
$fsi.VolumeName = $label
|
$fsi.VolumeName = $label
|
||||||
$fsi.ChooseImageDefaultsForMediaType(13)
|
$fsi.ChooseImageDefaultsForMediaType(13)
|
||||||
# 5+ GB ISO needs explicit large-volume defaults if available.
|
# 5+ GB ISO needs explicit large-volume defaults if available.
|
||||||
try { $fsi.UDFRevision = 0x102 } catch {}
|
try { $fsi.UDFRevision = 0x102 } catch { Write-Verbose "UDF revision not supported on this IMAPI version." }
|
||||||
|
|
||||||
Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow
|
Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow
|
||||||
$fsi.Root.AddTree($letter, $false)
|
$fsi.Root.AddTree($letter, $false)
|
||||||
@@ -383,11 +387,13 @@ function Invoke-Vmrun {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Set-VmxKey {
|
function Set-VmxKey {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $VmxPath,
|
[Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[Parameter(Mandatory)] [string] $Key,
|
[Parameter(Mandatory)] [string] $Key,
|
||||||
[Parameter(Mandatory)] [string] $Value
|
[Parameter(Mandatory)] [string] $Value
|
||||||
)
|
)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return }
|
||||||
$lines = Get-Content -LiteralPath $VmxPath
|
$lines = Get-Content -LiteralPath $VmxPath
|
||||||
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
$pattern = "^\s*$([regex]::Escape($Key))\s*="
|
||||||
$found = $false
|
$found = $false
|
||||||
@@ -400,8 +406,10 @@ function Set-VmxKey {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Remove-VmxLines {
|
function Remove-VmxLines {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param([Parameter(Mandatory)] [string] $VmxPath,
|
param([Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[Parameter(Mandatory)] [string] $KeyPrefix)
|
[Parameter(Mandatory)] [string] $KeyPrefix)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, "Remove VMX lines starting with '$KeyPrefix'")) { return }
|
||||||
$pat = "^\s*$([regex]::Escape($KeyPrefix))"
|
$pat = "^\s*$([regex]::Escape($KeyPrefix))"
|
||||||
$lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat }
|
$lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat }
|
||||||
Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII
|
Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII
|
||||||
@@ -428,8 +436,10 @@ function Invoke-VmrunBounded {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function Stop-Vm {
|
function Stop-Vm {
|
||||||
|
[CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')]
|
||||||
param([Parameter(Mandatory)] [string] $VmxPath,
|
param([Parameter(Mandatory)] [string] $VmxPath,
|
||||||
[int] $SoftTimeoutMinutes = 5)
|
[int] $SoftTimeoutMinutes = 5)
|
||||||
|
if (-not $PSCmdlet.ShouldProcess($VmxPath, 'Stop VM')) { return }
|
||||||
# Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s.
|
# Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s.
|
||||||
# Guest OS continues shutdown independently once the signal is received.
|
# Guest OS continues shutdown independently once the signal is received.
|
||||||
Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30
|
Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30
|
||||||
@@ -1114,4 +1124,4 @@ Write-Host " .\Validate-DeployState.ps1 -VMIPAddress $guestIP"
|
|||||||
Write-Host " .\Prepare-WinBuild2025.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]"
|
Write-Host " .\Prepare-WinBuild2025.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]"
|
||||||
Write-Host ""
|
Write-Host ""
|
||||||
Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green
|
Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green
|
||||||
|
|
||||||
|
|||||||
@@ -123,6 +123,8 @@
|
|||||||
Set-ExecutionPolicy Bypass -Scope Process -Force
|
Set-ExecutionPolicy Bypass -Scope Process -Force
|
||||||
.\Install-CIToolchain-WinBuild2022.ps1 -BuildPassword 'YourPassword'
|
.\Install-CIToolchain-WinBuild2022.ps1 -BuildPassword 'YourPassword'
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for local user creation. Intentional by design.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
# Local username for the CI build account inside the VM
|
# Local username for the CI build account inside the VM
|
||||||
@@ -865,7 +867,6 @@ else {
|
|||||||
$vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe'
|
$vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe'
|
||||||
$vsInstallerPath = 'C:\CI\vs_BuildTools.exe'
|
$vsInstallerPath = 'C:\CI\vs_BuildTools.exe'
|
||||||
$vsResultPath = 'C:\CI\vs_result.txt'
|
$vsResultPath = 'C:\CI\vs_result.txt'
|
||||||
$vsLogPath = 'C:\CI\vs_log.txt'
|
|
||||||
|
|
||||||
Write-Host "Downloading VS Build Tools installer..."
|
Write-Host "Downloading VS Build Tools installer..."
|
||||||
Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy
|
Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy
|
||||||
|
|||||||
@@ -123,6 +123,8 @@
|
|||||||
Set-ExecutionPolicy Bypass -Scope Process -Force
|
Set-ExecutionPolicy Bypass -Scope Process -Force
|
||||||
.\Install-CIToolchain-WinBuild2025.ps1 -BuildPassword 'YourPassword'
|
.\Install-CIToolchain-WinBuild2025.ps1 -BuildPassword 'YourPassword'
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for local user creation. Intentional by design.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
# Local username for the CI build account inside the VM
|
# Local username for the CI build account inside the VM
|
||||||
@@ -865,7 +867,6 @@ else {
|
|||||||
$vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe'
|
$vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe'
|
||||||
$vsInstallerPath = 'C:\CI\vs_BuildTools.exe'
|
$vsInstallerPath = 'C:\CI\vs_BuildTools.exe'
|
||||||
$vsResultPath = 'C:\CI\vs_result.txt'
|
$vsResultPath = 'C:\CI\vs_result.txt'
|
||||||
$vsLogPath = 'C:\CI\vs_log.txt'
|
|
||||||
|
|
||||||
Write-Host "Downloading VS Build Tools installer..."
|
Write-Host "Downloading VS Build Tools installer..."
|
||||||
Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy
|
Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy
|
||||||
|
|||||||
@@ -113,6 +113,8 @@
|
|||||||
-VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 `
|
-VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 `
|
||||||
-BuildPassword 'StrongCIPass!2026' -StoreCredential
|
-BuildPassword 'StrongCIPass!2026' -StoreCredential
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for WinRM PSCredential. Intentional by design.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
# Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot.
|
# Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot.
|
||||||
@@ -436,7 +438,7 @@ try {
|
|||||||
-Authentication Basic -ErrorAction Stop
|
-Authentication Basic -ErrorAction Stop
|
||||||
Remove-PSSession $probe -ErrorAction SilentlyContinue
|
Remove-PSSession $probe -ErrorAction SilentlyContinue
|
||||||
return $true
|
return $true
|
||||||
} catch { }
|
} catch { Write-Verbose "WinRM probe attempt failed." }
|
||||||
}
|
}
|
||||||
return $false
|
return $false
|
||||||
}
|
}
|
||||||
@@ -479,7 +481,7 @@ try {
|
|||||||
if ($session) {
|
if ($session) {
|
||||||
try {
|
try {
|
||||||
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
||||||
} catch { }
|
} catch { Write-Verbose "Reboot command sent; session may have already dropped." }
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -595,7 +597,7 @@ try {
|
|||||||
try {
|
try {
|
||||||
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
catch { }
|
catch { Write-Verbose "Reboot command sent; session may have already dropped." }
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||||
$session = $null
|
$session = $null
|
||||||
|
|
||||||
@@ -721,5 +723,5 @@ finally {
|
|||||||
}
|
}
|
||||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -113,6 +113,8 @@
|
|||||||
-VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 `
|
-VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 `
|
||||||
-BuildPassword 'StrongCIPass!2026' -StoreCredential
|
-BuildPassword 'StrongCIPass!2026' -StoreCredential
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for WinRM PSCredential. Intentional by design.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
# Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot.
|
# Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot.
|
||||||
@@ -436,7 +438,7 @@ try {
|
|||||||
-Authentication Basic -ErrorAction Stop
|
-Authentication Basic -ErrorAction Stop
|
||||||
Remove-PSSession $probe -ErrorAction SilentlyContinue
|
Remove-PSSession $probe -ErrorAction SilentlyContinue
|
||||||
return $true
|
return $true
|
||||||
} catch { }
|
} catch { Write-Verbose "WinRM probe attempt failed." }
|
||||||
}
|
}
|
||||||
return $false
|
return $false
|
||||||
}
|
}
|
||||||
@@ -479,7 +481,7 @@ try {
|
|||||||
if ($session) {
|
if ($session) {
|
||||||
try {
|
try {
|
||||||
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
||||||
} catch { }
|
} catch { Write-Verbose "Reboot command sent; session may have already dropped." }
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -595,7 +597,7 @@ try {
|
|||||||
try {
|
try {
|
||||||
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue
|
||||||
}
|
}
|
||||||
catch { }
|
catch { Write-Verbose "Reboot command sent; session may have already dropped." }
|
||||||
Remove-PSSession $session -ErrorAction SilentlyContinue
|
Remove-PSSession $session -ErrorAction SilentlyContinue
|
||||||
$session = $null
|
$session = $null
|
||||||
|
|
||||||
@@ -721,5 +723,5 @@ finally {
|
|||||||
}
|
}
|
||||||
Write-Host "[Prepare] Host TrustedHosts restored."
|
Write-Host "[Prepare] Host TrustedHosts restored."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -41,6 +41,8 @@
|
|||||||
# Linux
|
# Linux
|
||||||
.\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux
|
.\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template validation script: plain-text password converted to SecureString for WinRM PSCredential. Intentional by design.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $VMIPAddress,
|
[Parameter(Mandatory)] [string] $VMIPAddress,
|
||||||
|
|||||||
@@ -37,6 +37,8 @@
|
|||||||
.EXAMPLE
|
.EXAMPLE
|
||||||
.\Validate-SetupState.ps1 -VMIPAddress 192.168.79.129 -Remediate
|
.\Validate-SetupState.ps1 -VMIPAddress 192.168.79.129 -Remediate
|
||||||
#>
|
#>
|
||||||
|
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '',
|
||||||
|
Justification = 'Template setup script: plain-text password required to bootstrap WinRM credential chain before Credential Manager is available.')]
|
||||||
[CmdletBinding()]
|
[CmdletBinding()]
|
||||||
param(
|
param(
|
||||||
[Parameter(Mandatory)] [string] $VMIPAddress,
|
[Parameter(Mandatory)] [string] $VMIPAddress,
|
||||||
@@ -273,8 +275,6 @@ try {
|
|||||||
|
|
||||||
Write-Host ''
|
Write-Host ''
|
||||||
$failed = 0
|
$failed = 0
|
||||||
$deployCnt = 0
|
|
||||||
$setupCnt = 0
|
|
||||||
$inSetup = $false
|
$inSetup = $false
|
||||||
|
|
||||||
foreach ($r in $checks) {
|
foreach ($r in $checks) {
|
||||||
|
|||||||
@@ -1,106 +0,0 @@
|
|||||||
#Requires -Version 5.1
|
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Pester v5 tests for scripts/New-BuildVM.ps1
|
|
||||||
#>
|
|
||||||
|
|
||||||
BeforeAll {
|
|
||||||
$script:ScriptPath = Join-Path $PSScriptRoot '..\scripts\New-BuildVM.ps1'
|
|
||||||
$script:CloneBaseDir = Join-Path $env:TEMP "CI-Tests-CloneBase-$PID"
|
|
||||||
New-Item -ItemType Directory -Path $script:CloneBaseDir -Force | Out-Null
|
|
||||||
|
|
||||||
# Fake vmrun.ps1 — exits with $env:FAKE_VMRUN_EXIT (default 0)
|
|
||||||
# On success it also creates the destination VMX so the post-clone Test-Path passes.
|
|
||||||
# Using .ps1 instead of .cmd avoids Windows cmd redirect limitations inside if blocks.
|
|
||||||
$script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-$PID.ps1"
|
|
||||||
Set-Content $script:FakeVmrun -Value @'
|
|
||||||
# Args: -T ws clone <template> <dst_vmx> linked -snapshot <snapshot>
|
|
||||||
$exitCode = if ($env:FAKE_VMRUN_EXIT) { [int]$env:FAKE_VMRUN_EXIT } else { 0 }
|
|
||||||
if ($exitCode -eq 0) {
|
|
||||||
# Create the destination VMX (5th positional arg) so Test-Path in caller passes.
|
|
||||||
# Real vmrun creates the clone dir+file; we must do the same here.
|
|
||||||
$dstVmx = $args[4] # -T ws clone <tmpl> <dst> ...
|
|
||||||
$dstDir = Split-Path $dstVmx -Parent
|
|
||||||
if (-not (Test-Path $dstDir)) { New-Item -ItemType Directory -Path $dstDir -Force | Out-Null }
|
|
||||||
Set-Content $dstVmx ''
|
|
||||||
}
|
|
||||||
exit $exitCode
|
|
||||||
'@
|
|
||||||
|
|
||||||
# Fake template VMX — just needs to exist
|
|
||||||
$script:FakeTemplate = Join-Path $env:TEMP "fake-template-$PID.vmx"
|
|
||||||
Set-Content $script:FakeTemplate -Value 'config.version = "8"'
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterAll {
|
|
||||||
Remove-Item $script:CloneBaseDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
Remove-Item $script:FakeVmrun -Force -ErrorAction SilentlyContinue
|
|
||||||
Remove-Item $script:FakeTemplate -Force -ErrorAction SilentlyContinue
|
|
||||||
$env:FAKE_VMRUN_EXIT = $null
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'New-BuildVM — parameter validation' {
|
|
||||||
It 'throws when TemplatePath does not exist' {
|
|
||||||
{
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-TemplatePath 'C:\DoesNotExist\template.vmx' `
|
|
||||||
-CloneBaseDir $script:CloneBaseDir `
|
|
||||||
-JobId 'test-job-1' `
|
|
||||||
-VmrunPath $script:FakeVmrun
|
|
||||||
} | Should -Throw
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'New-BuildVM — vmrun failure' {
|
|
||||||
It 'throws and cleans up clone dir when vmrun clone exits non-zero' {
|
|
||||||
$env:FAKE_VMRUN_EXIT = '1'
|
|
||||||
{
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-TemplatePath $script:FakeTemplate `
|
|
||||||
-CloneBaseDir $script:CloneBaseDir `
|
|
||||||
-JobId 'test-job-fail' `
|
|
||||||
-VmrunPath $script:FakeVmrun
|
|
||||||
} | Should -Throw -ExpectedMessage '*clone failed*'
|
|
||||||
|
|
||||||
# No leftover clone dir
|
|
||||||
$leftover = Get-ChildItem $script:CloneBaseDir -Directory |
|
|
||||||
Where-Object { $_.Name -like '*test-job-fail*' }
|
|
||||||
$leftover | Should -BeNullOrEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterEach { $env:FAKE_VMRUN_EXIT = $null }
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'New-BuildVM — clone name format' {
|
|
||||||
It 'clone folder is named Clone_{JobId}_{yyyyMMdd_HHmmss}' {
|
|
||||||
$env:FAKE_VMRUN_EXIT = '0'
|
|
||||||
$vmx = & $script:ScriptPath `
|
|
||||||
-TemplatePath $script:FakeTemplate `
|
|
||||||
-CloneBaseDir $script:CloneBaseDir `
|
|
||||||
-JobId 'run-42' `
|
|
||||||
-VmrunPath $script:FakeVmrun
|
|
||||||
|
|
||||||
$vmx | Should -Match 'Clone_run-42_\d{8}_\d{6}'
|
|
||||||
}
|
|
||||||
|
|
||||||
It 'returns a string ending in .vmx' {
|
|
||||||
$env:FAKE_VMRUN_EXIT = '0'
|
|
||||||
$vmx = & $script:ScriptPath `
|
|
||||||
-TemplatePath $script:FakeTemplate `
|
|
||||||
-CloneBaseDir $script:CloneBaseDir `
|
|
||||||
-JobId 'run-99' `
|
|
||||||
-VmrunPath $script:FakeVmrun
|
|
||||||
|
|
||||||
$vmx | Should -Match '\.vmx$'
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterEach {
|
|
||||||
$env:FAKE_VMRUN_EXIT = $null
|
|
||||||
# Clean up clones created by successful tests
|
|
||||||
Get-ChildItem $script:CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
|
||||||
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
#Requires -Version 5.1
|
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Pester v5 tests for scripts/Remove-BuildVM.ps1
|
|
||||||
#>
|
|
||||||
|
|
||||||
BeforeAll {
|
|
||||||
$script:ScriptPath = Join-Path $PSScriptRoot '..\scripts\Remove-BuildVM.ps1'
|
|
||||||
|
|
||||||
# Fake vmrun: accepts any args, always exits 0 (best-effort ops don't need to fail here)
|
|
||||||
$script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-remove-$PID.cmd"
|
|
||||||
Set-Content $script:FakeVmrun -Value '@echo off & exit /b 0'
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterAll {
|
|
||||||
Remove-Item $script:FakeVmrun -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Remove-BuildVM — missing VMX' {
|
|
||||||
It 'returns without error when VMX does not exist' {
|
|
||||||
{ & $script:ScriptPath -VMPath 'C:\DoesNotExist\clone.vmx' -VmrunPath $script:FakeVmrun } |
|
|
||||||
Should -Not -Throw
|
|
||||||
}
|
|
||||||
|
|
||||||
It 'removes partial clone directory even when VMX is missing' {
|
|
||||||
$partialDir = Join-Path $env:TEMP "partial-clone-$PID"
|
|
||||||
New-Item -ItemType Directory -Path $partialDir -Force | Out-Null
|
|
||||||
$fakeMissingVmx = Join-Path $partialDir 'missing.vmx'
|
|
||||||
|
|
||||||
& $script:ScriptPath -VMPath $fakeMissingVmx -VmrunPath $script:FakeVmrun
|
|
||||||
|
|
||||||
Test-Path $partialDir | Should -Be $false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Remove-BuildVM — WhatIf' {
|
|
||||||
It 'does not remove clone directory when -WhatIf is passed' {
|
|
||||||
$cloneDir = Join-Path $env:TEMP "whatif-clone-$PID"
|
|
||||||
$cloneVmx = Join-Path $cloneDir 'whatif-clone.vmx'
|
|
||||||
New-Item -ItemType Directory -Path $cloneDir -Force | Out-Null
|
|
||||||
Set-Content $cloneVmx -Value 'config.version = "8"'
|
|
||||||
|
|
||||||
& $script:ScriptPath -VMPath $cloneVmx -VmrunPath $script:FakeVmrun -WhatIf
|
|
||||||
|
|
||||||
Test-Path $cloneDir | Should -Be $true
|
|
||||||
|
|
||||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Remove-BuildVM — successful destroy' {
|
|
||||||
It 'removes the clone directory when VMX exists and vmrun succeeds' {
|
|
||||||
$cloneDir = Join-Path $env:TEMP "live-clone-$PID"
|
|
||||||
$cloneVmx = Join-Path $cloneDir 'live-clone.vmx'
|
|
||||||
New-Item -ItemType Directory -Path $cloneDir -Force | Out-Null
|
|
||||||
Set-Content $cloneVmx -Value 'config.version = "8"'
|
|
||||||
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-VMPath $cloneVmx `
|
|
||||||
-VmrunPath $script:FakeVmrun `
|
|
||||||
-GracefulTimeoutSeconds 1 # short timeout for test speed
|
|
||||||
|
|
||||||
Test-Path $cloneDir | Should -Be $false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
#Requires -Version 5.1
|
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Pester v5 tests for scripts/Wait-VMReady.ps1
|
|
||||||
#>
|
|
||||||
|
|
||||||
BeforeAll {
|
|
||||||
$script:ScriptPath = Join-Path $PSScriptRoot '..\scripts\Wait-VMReady.ps1'
|
|
||||||
|
|
||||||
# Fake vmrun: getGuestIPAddress exits with %FAKE_VMRUN_EXIT% (default 1 = not ready)
|
|
||||||
$script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-wait-$PID.cmd"
|
|
||||||
Set-Content $script:FakeVmrun -Value @'
|
|
||||||
@echo off
|
|
||||||
if "%FAKE_VMRUN_EXIT%"=="" ( exit /b 1 ) else ( exit /b %FAKE_VMRUN_EXIT% )
|
|
||||||
'@
|
|
||||||
|
|
||||||
# Fake VMX file — just needs to exist
|
|
||||||
$script:FakeVmx = Join-Path $env:TEMP "fake-wait-$PID.vmx"
|
|
||||||
Set-Content $script:FakeVmx -Value 'config.version = "8"'
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterAll {
|
|
||||||
Remove-Item $script:FakeVmrun -Force -ErrorAction SilentlyContinue
|
|
||||||
Remove-Item $script:FakeVmx -Force -ErrorAction SilentlyContinue
|
|
||||||
$env:FAKE_VMRUN_EXIT = $null
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Wait-VMReady — missing vmrun' {
|
|
||||||
It 'throws when VmrunPath does not exist' {
|
|
||||||
{
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-VMPath $script:FakeVmx `
|
|
||||||
-IPAddress '192.168.79.101' `
|
|
||||||
-VmrunPath 'C:\DoesNotExist\vmrun.exe'
|
|
||||||
} | Should -Throw
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Wait-VMReady — timeout' {
|
|
||||||
It 'throws with timeout message when VM never becomes ready' {
|
|
||||||
$env:FAKE_VMRUN_EXIT = '1' # vmrun getGuestIPAddress always fails → VM not running
|
|
||||||
{
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-VMPath $script:FakeVmx `
|
|
||||||
-IPAddress '192.168.79.101' `
|
|
||||||
-VmrunPath $script:FakeVmrun `
|
|
||||||
-TimeoutSeconds 5 `
|
|
||||||
-PollIntervalSeconds 2 `
|
|
||||||
-SkipPing
|
|
||||||
} | Should -Throw -ExpectedMessage '*Timed out*'
|
|
||||||
}
|
|
||||||
|
|
||||||
AfterEach { $env:FAKE_VMRUN_EXIT = $null }
|
|
||||||
}
|
|
||||||
|
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
|
||||||
Describe 'Wait-VMReady — IP validation' {
|
|
||||||
It 'rejects an invalid IP address' {
|
|
||||||
{
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-VMPath $script:FakeVmx `
|
|
||||||
-IPAddress '999.999.999.999' `
|
|
||||||
-VmrunPath $script:FakeVmrun
|
|
||||||
} | Should -Throw
|
|
||||||
}
|
|
||||||
|
|
||||||
It 'accepts a valid IP address without immediate error from parameter binding' {
|
|
||||||
# This will still time out (vmrun fake exits 1), but the IP itself must not
|
|
||||||
# cause a parameter validation error — that would throw before the timeout.
|
|
||||||
$env:FAKE_VMRUN_EXIT = '1'
|
|
||||||
$err = $null
|
|
||||||
try {
|
|
||||||
& $script:ScriptPath `
|
|
||||||
-VMPath $script:FakeVmx `
|
|
||||||
-IPAddress '192.168.79.101' `
|
|
||||||
-VmrunPath $script:FakeVmrun `
|
|
||||||
-TimeoutSeconds 3 `
|
|
||||||
-PollIntervalSeconds 2 `
|
|
||||||
-SkipPing `
|
|
||||||
-ErrorAction Stop
|
|
||||||
} catch {
|
|
||||||
$err = $_
|
|
||||||
}
|
|
||||||
# Error must be the timeout, not a parameter validation error
|
|
||||||
"$err" | Should -Match 'Timed out'
|
|
||||||
"$err" | Should -Not -Match 'Cannot validate'
|
|
||||||
|
|
||||||
$env:FAKE_VMRUN_EXIT = $null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
"""Shared pytest fixtures for ci_orchestrator unit tests."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Ensure src/ is on sys.path so `import ci_orchestrator` works without install.
|
||||||
|
_SRC = Path(__file__).resolve().parents[2] / "src"
|
||||||
|
if str(_SRC) not in sys.path:
|
||||||
|
sys.path.insert(0, str(_SRC))
|
||||||
@@ -0,0 +1,489 @@
|
|||||||
|
"""Regression tests for ``AGENTS.md`` "Errori frequenti da evitare" #9-#12.
|
||||||
|
|
||||||
|
Each test is a guard against re-introducing a class of bug that already
|
||||||
|
caused incidents in the PowerShell stack and that the Python rewrite must
|
||||||
|
keep fixed. If you find yourself relaxing one of these tests, update
|
||||||
|
``AGENTS.md`` first and explain why the constraint no longer applies.
|
||||||
|
|
||||||
|
Cross-reference:
|
||||||
|
|
||||||
|
* ``AGENTS.md`` §"Errori frequenti da evitare", entries 9-12.
|
||||||
|
* ``plans/implementation-plan-A-B.md`` step A5.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
from ci_orchestrator.backends.errors import BackendError, BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
from ci_orchestrator.commands.vm import vm as vm_group
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _completed(returncode: int = 0, stdout: str = "", stderr: str = "") -> Any:
|
||||||
|
return subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=returncode, stdout=stdout, stderr=stderr
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fake_vmrun(tmp_path: Path) -> str:
|
||||||
|
p = tmp_path / "vmrun.exe"
|
||||||
|
p.write_bytes(b"")
|
||||||
|
return str(p)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────── AGENTS.md #9 — powered-on snapshot
|
||||||
|
|
||||||
|
|
||||||
|
class TestError9PoweredOnSnapshot:
|
||||||
|
"""``vmrun clone ... linked -snapshot=<name>`` rejects a snapshot
|
||||||
|
captured with the template VM still powered on (memory ``*.vmem`` files
|
||||||
|
present). The backend MUST surface that error as a typed
|
||||||
|
:class:`BackendError` with the original vmrun message preserved so the
|
||||||
|
operator can act (re-take the snapshot from a fully powered-off VM).
|
||||||
|
"""
|
||||||
|
|
||||||
|
_POWERED_ON_STDERR = (
|
||||||
|
"Error: The virtual machine should not be powered on. "
|
||||||
|
"It is already running.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_clone_linked_propagates_powered_on_error_typed(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch, fake_vmrun: str
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
subprocess,
|
||||||
|
"run",
|
||||||
|
lambda *a, **kw: _completed(255, stderr=self._POWERED_ON_STDERR),
|
||||||
|
)
|
||||||
|
backend = WorkstationVmrunBackend(vmrun_path=fake_vmrun)
|
||||||
|
|
||||||
|
with pytest.raises(BackendOperationFailed) as exc_info:
|
||||||
|
backend.clone_linked(
|
||||||
|
template=r"C:\tpl\WinBuild2025\WinBuild2025.vmx",
|
||||||
|
snapshot="BaseClean",
|
||||||
|
name="job-9",
|
||||||
|
destination=r"C:\bvm\job-9\job-9.vmx",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Must be a BackendError subclass (typed, not a bare RuntimeError).
|
||||||
|
assert isinstance(exc_info.value, BackendError)
|
||||||
|
# Operation name must match so callers can pattern-match.
|
||||||
|
assert exc_info.value.operation == "clone"
|
||||||
|
assert exc_info.value.returncode == 255
|
||||||
|
# Original vmrun message must be preserved verbatim for triage.
|
||||||
|
assert "should not be powered on" in exc_info.value.output
|
||||||
|
|
||||||
|
|
||||||
|
# ───────────────────────────────────────────── AGENTS.md #10 — is_running via vmrun list
|
||||||
|
|
||||||
|
|
||||||
|
class TestError10IsRunningUsesVmrunList:
|
||||||
|
"""``vmrun getGuestIPAddress`` blocks 30-60s when VMware Tools are not
|
||||||
|
yet up; using it as a "VM running?" probe makes ``is_running`` appear
|
||||||
|
hung. The backend MUST use ``vmrun list`` parsing instead.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_is_running_calls_vmrun_list_only(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fake_vmrun: str
|
||||||
|
) -> None:
|
||||||
|
vmx = tmp_path / "job10" / "job10.vmx"
|
||||||
|
vmx.parent.mkdir()
|
||||||
|
vmx.write_text("")
|
||||||
|
|
||||||
|
captured: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake_run(cmd: list[str], **_kwargs: object) -> Any:
|
||||||
|
captured.append(cmd)
|
||||||
|
assert cmd[3] == "list", (
|
||||||
|
f"is_running invoked unexpected vmrun op: {cmd[3]!r} "
|
||||||
|
"(MUST be 'list', see AGENTS.md error #10)"
|
||||||
|
)
|
||||||
|
return _completed(0, stdout=f"Total running VMs: 1\n{vmx}\n")
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||||
|
backend = WorkstationVmrunBackend(vmrun_path=fake_vmrun)
|
||||||
|
|
||||||
|
assert backend.is_running(VmHandle(str(vmx))) is True
|
||||||
|
|
||||||
|
# Exactly one subprocess call, and its operation MUST be 'list'.
|
||||||
|
assert len(captured) == 1
|
||||||
|
ops = [cmd[3] for cmd in captured]
|
||||||
|
assert ops == ["list"]
|
||||||
|
# Hard guard: no getGuestIPAddress invocation, ever.
|
||||||
|
for cmd in captured:
|
||||||
|
assert "getGuestIPAddress" not in cmd
|
||||||
|
|
||||||
|
def test_is_running_false_when_not_listed(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fake_vmrun: str
|
||||||
|
) -> None:
|
||||||
|
vmx = tmp_path / "absent.vmx"
|
||||||
|
vmx.write_text("")
|
||||||
|
|
||||||
|
captured: list[str] = []
|
||||||
|
|
||||||
|
def fake_run(cmd: list[str], **_kwargs: object) -> Any:
|
||||||
|
captured.append(cmd[3])
|
||||||
|
return _completed(0, stdout="Total running VMs: 0\n")
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||||
|
backend = WorkstationVmrunBackend(vmrun_path=fake_vmrun)
|
||||||
|
assert backend.is_running(VmHandle(str(vmx))) is False
|
||||||
|
assert captured == ["list"]
|
||||||
|
assert "getGuestIPAddress" not in captured
|
||||||
|
|
||||||
|
def test_is_running_false_on_vmrun_failure_without_getip_fallback(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, fake_vmrun: str
|
||||||
|
) -> None:
|
||||||
|
"""Even on ``vmrun list`` failure, the backend MUST NOT fall back
|
||||||
|
to ``getGuestIPAddress`` (which would block on Tools)."""
|
||||||
|
vmx = tmp_path / "absent.vmx"
|
||||||
|
vmx.write_text("")
|
||||||
|
|
||||||
|
captured: list[str] = []
|
||||||
|
|
||||||
|
def fake_run(cmd: list[str], **_kwargs: object) -> Any:
|
||||||
|
captured.append(cmd[3])
|
||||||
|
return _completed(1, stderr="vmrun broke")
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||||
|
backend = WorkstationVmrunBackend(vmrun_path=fake_vmrun)
|
||||||
|
assert backend.is_running(VmHandle(str(vmx))) is False
|
||||||
|
assert captured == ["list"]
|
||||||
|
assert "getGuestIPAddress" not in captured
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────── AGENTS.md #11 — machine-id / unique clones
|
||||||
|
|
||||||
|
|
||||||
|
class TestError11UniqueCloneNames:
|
||||||
|
"""Ubuntu cloud images ship with a fixed ``/etc/machine-id``. Two
|
||||||
|
clones that share machine-id collide on the VMware DHCP lease and
|
||||||
|
deadlock on duplicate IPs. The template-side fix lives in
|
||||||
|
:file:`template/Prepare-LinuxBuild2404.ps1` (``truncate -s 0
|
||||||
|
/etc/machine-id`` before snapshot).
|
||||||
|
|
||||||
|
On the orchestrator side, the safety net is that ``vm new`` MUST never
|
||||||
|
reuse a clone identifier — otherwise even a correct template would
|
||||||
|
end up cloning into an existing VMX directory and the duplicate-IP
|
||||||
|
failure mode would re-appear from a different angle. We assert that
|
||||||
|
two ``vm new`` invocations with identical inputs produce distinct
|
||||||
|
destination paths (timestamp-suffixed clone names).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _invoke_vm_new(
|
||||||
|
self,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
fake_vmrun: str,
|
||||||
|
) -> tuple[CliRunner, list[dict[str, str]]]:
|
||||||
|
# Ensure load_config has env keys it expects (no .toml present).
|
||||||
|
for k in ("CI_ROOT", "CI_TEMPLATES", "CI_BUILD_VMS", "CI_ARTIFACTS", "CI_KEYS"):
|
||||||
|
monkeypatch.delenv(k, raising=False)
|
||||||
|
|
||||||
|
template = tmp_path / "tpl" / "tpl.vmx"
|
||||||
|
template.parent.mkdir()
|
||||||
|
template.write_text("")
|
||||||
|
|
||||||
|
clone_base = tmp_path / "clones"
|
||||||
|
|
||||||
|
calls: list[dict[str, str]] = []
|
||||||
|
|
||||||
|
def fake_clone(
|
||||||
|
self: WorkstationVmrunBackend,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
assert destination is not None
|
||||||
|
# Materialise the clone VMX so vm_new's post-condition passes.
|
||||||
|
dest = Path(destination)
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
dest.write_text("")
|
||||||
|
calls.append(
|
||||||
|
{"name": name, "destination": destination, "snapshot": snapshot}
|
||||||
|
)
|
||||||
|
return VmHandle(identifier=destination, name=name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
WorkstationVmrunBackend, "clone_linked", fake_clone, raising=True
|
||||||
|
)
|
||||||
|
|
||||||
|
runner = CliRunner()
|
||||||
|
args = [
|
||||||
|
"new",
|
||||||
|
"--template-path",
|
||||||
|
str(template),
|
||||||
|
"--snapshot-name",
|
||||||
|
"BaseClean",
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(clone_base),
|
||||||
|
"--job-id",
|
||||||
|
"job-11",
|
||||||
|
"--vmrun-path",
|
||||||
|
fake_vmrun,
|
||||||
|
]
|
||||||
|
# Two consecutive runs.
|
||||||
|
for _ in range(2):
|
||||||
|
result = runner.invoke(vm_group, args, catch_exceptions=False)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
return runner, calls
|
||||||
|
|
||||||
|
def test_two_vm_new_invocations_produce_distinct_clones(
|
||||||
|
self,
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
fake_vmrun: str,
|
||||||
|
) -> None:
|
||||||
|
# Drive the timestamp generator so the two invocations land in
|
||||||
|
# different seconds (vm_new uses %Y%m%d_%H%M%S).
|
||||||
|
from ci_orchestrator.commands import vm as vm_module
|
||||||
|
|
||||||
|
ticks = iter(["20260514_120000", "20260514_120001"])
|
||||||
|
|
||||||
|
class _FakeDT:
|
||||||
|
@staticmethod
|
||||||
|
def now(tz: object | None = None) -> Any:
|
||||||
|
class _F:
|
||||||
|
def strftime(self_inner, _fmt: str) -> str:
|
||||||
|
return next(ticks)
|
||||||
|
|
||||||
|
return _F()
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "datetime", _FakeDT)
|
||||||
|
|
||||||
|
_runner, calls = self._invoke_vm_new(monkeypatch, tmp_path, fake_vmrun)
|
||||||
|
|
||||||
|
assert len(calls) == 2
|
||||||
|
names = {c["name"] for c in calls}
|
||||||
|
destinations = {c["destination"] for c in calls}
|
||||||
|
assert len(names) == 2, (
|
||||||
|
f"vm new must never produce duplicate clone names; got {names} "
|
||||||
|
"(see AGENTS.md error #11 — machine-id / DHCP collision)"
|
||||||
|
)
|
||||||
|
assert len(destinations) == 2, (
|
||||||
|
"Two vm new invocations produced the same destination VMX path; "
|
||||||
|
"this would clone over an existing build VM and risks reproducing "
|
||||||
|
"the duplicate-IP failure mode (AGENTS.md error #11)."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────── AGENTS.md #12 — no ssh-keygen / AutoAddPolicy
|
||||||
|
|
||||||
|
|
||||||
|
class TestError12SshTransportNoSshKeygen:
|
||||||
|
"""``& ssh-keygen -R <ip> 2>$null`` does NOT suppress stderr in
|
||||||
|
Windows PowerShell 5.1 with ``$ErrorActionPreference='Stop'``: stderr
|
||||||
|
is converted to a terminating ErrorRecord before ``2>$null`` can
|
||||||
|
discard it. The Python rewrite sidesteps the entire problem by using
|
||||||
|
paramiko with :class:`paramiko.AutoAddPolicy` and an empty
|
||||||
|
``known_hosts`` (mirroring ``StrictHostKeyChecking=no
|
||||||
|
UserKnownHostsFile=NUL``). We assert there is no native shell-out to
|
||||||
|
``ssh-keygen`` and that ``AutoAddPolicy`` is installed by default.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def _install_fake_paramiko(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> tuple[Any, Any]:
|
||||||
|
"""Install a stub paramiko module that records all attribute access."""
|
||||||
|
|
||||||
|
accessed: list[str] = []
|
||||||
|
|
||||||
|
class _Channel:
|
||||||
|
def recv_exit_status(self) -> int:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
class _Stream:
|
||||||
|
channel = _Channel()
|
||||||
|
|
||||||
|
def read(self) -> bytes:
|
||||||
|
return b""
|
||||||
|
|
||||||
|
class _SFTP:
|
||||||
|
def __enter__(self) -> _SFTP:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def put(self, _l: str, _r: str) -> None: # pragma: no cover - unused
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get(self, _r: str, _l: str) -> None: # pragma: no cover - unused
|
||||||
|
pass
|
||||||
|
|
||||||
|
class _Client:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.policy: Any = None
|
||||||
|
self.connect_kwargs: dict[str, Any] = {}
|
||||||
|
self.commands: list[str] = []
|
||||||
|
|
||||||
|
def set_missing_host_key_policy(self, policy: Any) -> None:
|
||||||
|
self.policy = policy
|
||||||
|
|
||||||
|
def load_host_keys(self, _path: str) -> None: # pragma: no cover
|
||||||
|
pass
|
||||||
|
|
||||||
|
def connect(self, **kwargs: Any) -> None:
|
||||||
|
self.connect_kwargs = kwargs
|
||||||
|
|
||||||
|
def exec_command(
|
||||||
|
self, command: str, timeout: float | None = None
|
||||||
|
) -> Any:
|
||||||
|
self.commands.append(command)
|
||||||
|
return None, _Stream(), _Stream()
|
||||||
|
|
||||||
|
def open_sftp(self) -> _SFTP: # pragma: no cover - unused
|
||||||
|
return _SFTP()
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
client = _Client()
|
||||||
|
|
||||||
|
class _ModWrap(types.ModuleType):
|
||||||
|
def __getattr__(self, item: str) -> Any:
|
||||||
|
accessed.append(item)
|
||||||
|
raise AttributeError(item)
|
||||||
|
|
||||||
|
fake = _ModWrap("paramiko")
|
||||||
|
# Pre-define exactly the attributes SshTransport is allowed to use.
|
||||||
|
fake.SSHClient = lambda: client # type: ignore[attr-defined]
|
||||||
|
fake.AutoAddPolicy = lambda: "auto-add" # type: ignore[attr-defined]
|
||||||
|
|
||||||
|
monkeypatch.setitem(sys.modules, "paramiko", fake)
|
||||||
|
return client, accessed
|
||||||
|
|
||||||
|
def test_default_policy_is_auto_add_no_known_hosts(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
client, _accessed = self._install_fake_paramiko(monkeypatch)
|
||||||
|
t = SshTransport("10.0.0.99", username="ci_build", key_path="/tmp/k")
|
||||||
|
t.run("true")
|
||||||
|
# AutoAddPolicy is the marker that we're matching the
|
||||||
|
# `StrictHostKeyChecking=no` permissive default.
|
||||||
|
assert client.policy == "auto-add", (
|
||||||
|
"SshTransport must default to paramiko.AutoAddPolicy "
|
||||||
|
"(see AGENTS.md error #12)."
|
||||||
|
)
|
||||||
|
# Ensure the connect call did NOT pass any known_hosts file path,
|
||||||
|
# which would re-introduce the host-key churn problem.
|
||||||
|
assert "key_filename" in client.connect_kwargs
|
||||||
|
|
||||||
|
def test_no_ssh_keygen_invocation(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""SshTransport must not shell out to ``ssh-keygen`` (or anything
|
||||||
|
else): all transport goes through paramiko."""
|
||||||
|
called: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake_run(cmd: list[str], *_a: object, **_kw: object) -> Any:
|
||||||
|
called.append(cmd)
|
||||||
|
return _completed(0)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", fake_run)
|
||||||
|
|
||||||
|
# Also patch Popen as a belt-and-braces guard.
|
||||||
|
def fake_popen(*a: object, **_kw: object) -> Any: # pragma: no cover
|
||||||
|
called.append(list(a))
|
||||||
|
raise AssertionError(
|
||||||
|
"SshTransport must not Popen any external process "
|
||||||
|
"(AGENTS.md error #12)."
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "Popen", fake_popen)
|
||||||
|
|
||||||
|
client, _accessed = self._install_fake_paramiko(monkeypatch)
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
t.run("uname -a")
|
||||||
|
t.close()
|
||||||
|
|
||||||
|
assert called == [], (
|
||||||
|
"SshTransport spawned a subprocess; it must rely solely on "
|
||||||
|
"paramiko (no ssh-keygen, no ssh.exe, no scp.exe). Calls: "
|
||||||
|
f"{called!r}"
|
||||||
|
)
|
||||||
|
assert client.commands == ["uname -a"]
|
||||||
|
|
||||||
|
def test_known_hosts_optional_param_is_off_by_default(self) -> None:
|
||||||
|
"""The ``known_hosts`` constructor parameter must default to None
|
||||||
|
so ephemeral CI VMs never hit a stale host-key error that would
|
||||||
|
previously have driven the broken ``ssh-keygen -R`` workaround."""
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
# Internal attribute is part of the documented contract for this
|
||||||
|
# regression: flipping the default would silently break ephemeral
|
||||||
|
# VM workflows. If you need to change it, update AGENTS.md #12.
|
||||||
|
assert t._known_hosts is None
|
||||||
|
|
||||||
|
def test_paramiko_missing_raises_typed_error(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""When paramiko is not installed, the transport must raise a
|
||||||
|
typed :class:`TransportConnectError` — not a bare ``ImportError``
|
||||||
|
leaking through. Symmetric to the PowerShell guard that wrapped
|
||||||
|
``ssh.exe`` failures."""
|
||||||
|
from ci_orchestrator.transport.errors import TransportConnectError
|
||||||
|
|
||||||
|
monkeypatch.setitem(sys.modules, "paramiko", None)
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
with pytest.raises(TransportConnectError):
|
||||||
|
t.run("true")
|
||||||
|
|
||||||
|
def test_exec_command_exception_is_wrapped(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""Channel-level paramiko errors must surface as
|
||||||
|
:class:`TransportConnectError`, never as raw ``Exception``."""
|
||||||
|
from ci_orchestrator.transport.errors import TransportConnectError
|
||||||
|
|
||||||
|
client, _ = self._install_fake_paramiko(monkeypatch)
|
||||||
|
|
||||||
|
def boom(*_a: object, **_kw: object) -> Any:
|
||||||
|
raise OSError("channel closed")
|
||||||
|
|
||||||
|
client.exec_command = boom # type: ignore[method-assign]
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
with pytest.raises(TransportConnectError):
|
||||||
|
t.run("uname")
|
||||||
|
|
||||||
|
def test_sftp_failures_are_wrapped(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.transport.errors import TransportConnectError
|
||||||
|
|
||||||
|
client, _ = self._install_fake_paramiko(monkeypatch)
|
||||||
|
|
||||||
|
def boom_sftp() -> Any:
|
||||||
|
raise OSError("sftp denied")
|
||||||
|
|
||||||
|
client.open_sftp = boom_sftp # type: ignore[method-assign]
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
with pytest.raises(TransportConnectError):
|
||||||
|
t.copy("local.bin", "/remote/local.bin")
|
||||||
|
with pytest.raises(TransportConnectError):
|
||||||
|
t.fetch("/remote/out.bin", "out.bin")
|
||||||
|
|
||||||
|
def test_is_ready_returns_false_on_typed_error(
|
||||||
|
self, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
client, _ = self._install_fake_paramiko(monkeypatch)
|
||||||
|
|
||||||
|
def boom(*_a: object, **_kw: object) -> Any:
|
||||||
|
raise OSError("dead")
|
||||||
|
|
||||||
|
client.exec_command = boom # type: ignore[method-assign]
|
||||||
|
t = SshTransport("10.0.0.99", key_path="/tmp/k")
|
||||||
|
assert t.is_ready() is False
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""Smoke tests for the click CLI entry point.
|
||||||
|
|
||||||
|
These do not hit any real VM — backend, transport, credentials and
|
||||||
|
``time.sleep`` are all monkeypatched.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.__main__ as cli_module
|
||||||
|
import ci_orchestrator.commands.wait as wait_module
|
||||||
|
from ci_orchestrator import __version__
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
running: bool = True,
|
||||||
|
ip: str | None = "10.0.0.42",
|
||||||
|
) -> None:
|
||||||
|
self._running = running
|
||||||
|
self._ip = ip
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
return self._running
|
||||||
|
|
||||||
|
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return self._ip
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTransport:
|
||||||
|
def __init__(self, *_a: Any, **_kw: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self) -> _FakeTransport:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeStore:
|
||||||
|
def get(self, _target: str) -> Credential:
|
||||||
|
return Credential("user", "pwd")
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_common(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend) -> None:
|
||||||
|
monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend)
|
||||||
|
monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore())
|
||||||
|
monkeypatch.setattr(wait_module, "WinRmTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(wait_module, "SshTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_help_lists_wait_ready() -> None:
|
||||||
|
result = CliRunner().invoke(cli_module.cli, ["--help"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "wait-ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_version() -> None:
|
||||||
|
result = CliRunner().invoke(cli_module.cli, ["--version"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert __version__ in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_windows_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch_common(monkeypatch, _FakeBackend(running=True, ip="10.0.0.42"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli_module.cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--guest-os", "windows", "--timeout", "1"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "WinRM ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_linux_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch_common(monkeypatch, _FakeBackend(running=True, ip="10.0.0.42"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli_module.cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--guest-os", "linux", "--timeout", "1"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "SSH ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_timeout_when_not_running(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch_common(monkeypatch, _FakeBackend(running=False))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli_module.cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--guest-os",
|
||||||
|
"windows",
|
||||||
|
"--timeout",
|
||||||
|
"0.01",
|
||||||
|
"--poll-interval",
|
||||||
|
"0.001",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "timeout waiting for VM" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_timeout_no_ip(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch_common(monkeypatch, _FakeBackend(running=True, ip=None))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli_module.cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--guest-os",
|
||||||
|
"windows",
|
||||||
|
"--timeout",
|
||||||
|
"0.01",
|
||||||
|
"--poll-interval",
|
||||||
|
"0.001",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 3
|
||||||
|
assert "timeout waiting for guest IP" in result.output
|
||||||
@@ -0,0 +1,324 @@
|
|||||||
|
"""Tests for ``ci_orchestrator artifacts collect``."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import tarfile
|
||||||
|
import zipfile
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.artifacts as artifacts_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResult:
|
||||||
|
def __init__(self, stdout: str = "OK", returncode: int = 0) -> None:
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = ""
|
||||||
|
self.returncode = returncode
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return self.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTransport:
|
||||||
|
instances: ClassVar[list[_FakeTransport]] = []
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
self.runs: list[str] = []
|
||||||
|
self.fetched: list[tuple[str, str]] = []
|
||||||
|
self.fetch_payloads: dict[str, bytes] = {}
|
||||||
|
self.run_overrides: dict[str, _FakeResult] = {}
|
||||||
|
type(self).instances.append(self)
|
||||||
|
|
||||||
|
def __enter__(self) -> _FakeTransport:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def run(self, script: str, *, check: bool = True) -> _FakeResult:
|
||||||
|
self.runs.append(script)
|
||||||
|
for key, override in self.run_overrides.items():
|
||||||
|
if key in script:
|
||||||
|
if check and not override.ok:
|
||||||
|
raise TransportCommandError(
|
||||||
|
override.returncode, override.stdout, override.stderr
|
||||||
|
)
|
||||||
|
return override
|
||||||
|
return _FakeResult()
|
||||||
|
|
||||||
|
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||||
|
self.fetched.append((remote_path, str(local_path)))
|
||||||
|
payload = self.fetch_payloads.get(remote_path, b"data")
|
||||||
|
Path(local_path).write_bytes(payload)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_instances() -> None:
|
||||||
|
_FakeTransport.instances = []
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
class _Store:
|
||||||
|
def get(self, _t: str) -> Credential:
|
||||||
|
return Credential("ci", "pwd")
|
||||||
|
|
||||||
|
monkeypatch.setattr(artifacts_module, "WinRmTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(artifacts_module, "SshTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(artifacts_module, "KeyringCredentialStore", lambda: _Store())
|
||||||
|
|
||||||
|
|
||||||
|
# ── Windows happy path ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_payload(tmp_path: Path) -> bytes:
|
||||||
|
"""Build a real zip whose contents the fake transport will 'fetch'."""
|
||||||
|
src = tmp_path / "zsrc"
|
||||||
|
src.mkdir()
|
||||||
|
(src / "binary.bin").write_bytes(b"\x01\x02\x03")
|
||||||
|
(src / "log.txt").write_text("hello", encoding="utf-8")
|
||||||
|
zip_local = tmp_path / "transfer.zip"
|
||||||
|
with zipfile.ZipFile(zip_local, "w") as zf:
|
||||||
|
for f in src.iterdir():
|
||||||
|
zf.write(f, arcname=f.name)
|
||||||
|
return zip_local.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def _win_transport_with_zip(payload: bytes) -> type:
|
||||||
|
class _WinTransport(_FakeTransport):
|
||||||
|
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||||
|
self.fetched.append((remote_path, str(local_path)))
|
||||||
|
Path(local_path).write_bytes(payload)
|
||||||
|
|
||||||
|
return _WinTransport
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_windows_happy_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
payload = _zip_payload(tmp_path)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
artifacts_module, "WinRmTransport", _win_transport_with_zip(payload)
|
||||||
|
)
|
||||||
|
out_dir = tmp_path / "artifacts"
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"C:\\CI\\output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(out_dir),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
# Raw files extracted into host_dir (no zip-in-zip).
|
||||||
|
assert (out_dir / "binary.bin").read_bytes() == b"\x01\x02\x03"
|
||||||
|
assert (out_dir / "log.txt").read_text(encoding="utf-8") == "hello"
|
||||||
|
cmds = _FakeTransport.instances[0].runs
|
||||||
|
assert any("Compress-Archive" in c for c in cmds)
|
||||||
|
# Temp transport zip on the guest is cleaned up.
|
||||||
|
assert any("Remove-Item" in c and "ci-artifacts-transfer.zip" in c for c in cmds)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_windows_artifact_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
|
||||||
|
class _Missing(_FakeTransport):
|
||||||
|
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||||
|
super().__init__(*a, **kw)
|
||||||
|
self.run_overrides["-and (Get-ChildItem"] = _FakeResult(
|
||||||
|
stdout="MISSING\r\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(artifacts_module, "WinRmTransport", _Missing)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"C:\\CI\\output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(tmp_path / "out"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "artifact not found or empty" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_windows_include_logs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
payload = _zip_payload(tmp_path)
|
||||||
|
|
||||||
|
class _WithLogs(_win_transport_with_zip(payload)): # type: ignore[misc]
|
||||||
|
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||||
|
super().__init__(*a, **kw)
|
||||||
|
self.run_overrides["C:\\CI\\build"] = _FakeResult(
|
||||||
|
stdout="C:\\CI\\build\\one.log\r\nC:\\CI\\build\\sub\\two.log\r\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(artifacts_module, "WinRmTransport", _WithLogs)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"C:\\CI\\output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(tmp_path / "out"),
|
||||||
|
"--include-logs",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
fetched_remotes = [r for r, _ in _FakeTransport.instances[0].fetched]
|
||||||
|
assert "C:\\CI\\build\\one.log" in fetched_remotes
|
||||||
|
assert "C:\\CI\\build\\sub\\two.log" in fetched_remotes
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_writes_manifest_when_job_id_set(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
payload = _zip_payload(tmp_path)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
artifacts_module, "WinRmTransport", _win_transport_with_zip(payload)
|
||||||
|
)
|
||||||
|
out = tmp_path / "out"
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"C:\\CI\\output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(out),
|
||||||
|
"--job-id",
|
||||||
|
"run-42",
|
||||||
|
"--commit",
|
||||||
|
"abc1234",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
manifest_path = out / "manifest.json"
|
||||||
|
assert manifest_path.is_file()
|
||||||
|
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
assert data["jobId"] == "run-42"
|
||||||
|
assert data["commit"] == "abc1234"
|
||||||
|
names = [f["name"] for f in data["files"]]
|
||||||
|
assert "binary.bin" in names and "log.txt" in names
|
||||||
|
|
||||||
|
|
||||||
|
# ── Linux happy path ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_linux_extracts_tarball(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
|
||||||
|
# Build a real tarball that the fake transport will "fetch".
|
||||||
|
payload_dir = tmp_path / "payload"
|
||||||
|
payload_dir.mkdir()
|
||||||
|
(payload_dir / "binary.bin").write_bytes(b"\x01\x02\x03")
|
||||||
|
(payload_dir / "log.txt").write_text("hello", encoding="utf-8")
|
||||||
|
tar_local = tmp_path / "src.tar.gz"
|
||||||
|
with tarfile.open(tar_local, "w:gz") as tf:
|
||||||
|
for f in payload_dir.iterdir():
|
||||||
|
tf.add(f, arcname=f.name)
|
||||||
|
payload_bytes = tar_local.read_bytes()
|
||||||
|
|
||||||
|
class _LinuxTransport(_FakeTransport):
|
||||||
|
def __init__(self, *a: Any, **kw: Any) -> None:
|
||||||
|
super().__init__(*a, **kw)
|
||||||
|
# Any fetched path returns the same tarball payload.
|
||||||
|
self.fetch_payloads["__default__"] = payload_bytes
|
||||||
|
|
||||||
|
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||||
|
self.fetched.append((remote_path, str(local_path)))
|
||||||
|
Path(local_path).write_bytes(payload_bytes)
|
||||||
|
|
||||||
|
monkeypatch.setattr(artifacts_module, "SshTransport", _LinuxTransport)
|
||||||
|
|
||||||
|
out = tmp_path / "out"
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"/opt/ci/output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(out),
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert (out / "binary.bin").read_bytes() == b"\x01\x02\x03"
|
||||||
|
assert (out / "log.txt").read_text(encoding="utf-8") == "hello"
|
||||||
|
cmds = _FakeTransport.instances[0].runs
|
||||||
|
assert any("tar -czf" in c for c in cmds)
|
||||||
|
# Cleanup of the temp tarball on the guest was attempted.
|
||||||
|
assert any("rm -f" in c for c in cmds)
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_linux_no_files_raises(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
|
||||||
|
# Empty tarball => extraction yields nothing.
|
||||||
|
empty_tar = tmp_path / "empty.tar.gz"
|
||||||
|
with tarfile.open(empty_tar, "w:gz"):
|
||||||
|
pass
|
||||||
|
payload = empty_tar.read_bytes()
|
||||||
|
|
||||||
|
class _Empty(_FakeTransport):
|
||||||
|
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||||
|
self.fetched.append((remote_path, str(local_path)))
|
||||||
|
Path(local_path).write_bytes(payload)
|
||||||
|
|
||||||
|
monkeypatch.setattr(artifacts_module, "SshTransport", _Empty)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"artifacts",
|
||||||
|
"collect",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--guest-artifact-path",
|
||||||
|
"/opt/ci/output",
|
||||||
|
"--host-artifact-dir",
|
||||||
|
str(tmp_path / "out"),
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "no files were transferred" in result.output
|
||||||
@@ -0,0 +1,372 @@
|
|||||||
|
"""Tests for ``ci_orchestrator build run``."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, ClassVar
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.build as build_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
from ci_orchestrator.transport.errors import TransportCommandError
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResult:
|
||||||
|
def __init__(self, stdout: str = "", stderr: str = "", returncode: int = 0) -> None:
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
|
self.returncode = returncode
|
||||||
|
|
||||||
|
@property
|
||||||
|
def ok(self) -> bool:
|
||||||
|
return self.returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTransport:
|
||||||
|
"""Records every ``run/copy/fetch`` call."""
|
||||||
|
|
||||||
|
instances: ClassVar[list[_FakeTransport]] = []
|
||||||
|
|
||||||
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
|
self.args = args
|
||||||
|
self.kwargs = kwargs
|
||||||
|
self.runs: list[tuple[str, bool]] = []
|
||||||
|
self.copies: list[tuple[str, str]] = []
|
||||||
|
self.fetches: list[tuple[str, str]] = []
|
||||||
|
self.run_results: dict[int, _FakeResult] = {}
|
||||||
|
self.default_result = _FakeResult(stdout="ok")
|
||||||
|
type(self).instances.append(self)
|
||||||
|
|
||||||
|
def __enter__(self) -> _FakeTransport:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def run(self, script: str, *, check: bool = True) -> _FakeResult:
|
||||||
|
self.runs.append((script, check))
|
||||||
|
result = self.run_results.get(len(self.runs) - 1, self.default_result)
|
||||||
|
if check and not result.ok:
|
||||||
|
raise TransportCommandError(result.returncode, result.stdout, result.stderr)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def run_streaming(
|
||||||
|
self, script: str, *, check: bool = True, timeout: float | None = None
|
||||||
|
) -> _FakeResult:
|
||||||
|
# Tests don't exercise live streaming; delegate to run() so
|
||||||
|
# capture + check semantics (and _BuildFails overrides) hold.
|
||||||
|
return self.run(script, check=check)
|
||||||
|
|
||||||
|
def copy(self, local_path: str, remote_path: str) -> None:
|
||||||
|
self.copies.append((str(local_path), remote_path))
|
||||||
|
|
||||||
|
def fetch(self, remote_path: str, local_path: str) -> None:
|
||||||
|
self.fetches.append((remote_path, str(local_path)))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_instances() -> None:
|
||||||
|
_FakeTransport.instances = []
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch: pytest.MonkeyPatch) -> type[_FakeTransport]:
|
||||||
|
class _Store:
|
||||||
|
def get(self, _t: str) -> Credential:
|
||||||
|
return Credential("ci", "pwd")
|
||||||
|
|
||||||
|
monkeypatch.setattr(build_module, "WinRmTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(build_module, "SshTransport", _FakeTransport)
|
||||||
|
monkeypatch.setattr(build_module, "KeyringCredentialStore", lambda: _Store())
|
||||||
|
return _FakeTransport
|
||||||
|
|
||||||
|
|
||||||
|
# ── parameter validation ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_requires_source(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["build", "run", "--ip-address", "10.0.0.1"]
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "either --host-source-dir or --clone-url" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_rejects_both_modes(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
src = tmp_path / "src"
|
||||||
|
src.mkdir()
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.1",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(src),
|
||||||
|
"--clone-url",
|
||||||
|
"https://example/repo.git",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "not both" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_rejects_missing_source_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.1",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(tmp_path / "ghost"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "does not exist" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_rejects_bad_extra_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
src = tmp_path / "src"
|
||||||
|
src.mkdir()
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.1",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(src),
|
||||||
|
"--build-command",
|
||||||
|
"make",
|
||||||
|
"--extra-env",
|
||||||
|
"NO_EQUALS",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "KEY=VALUE" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── happy path: Linux ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_linux_clone_url_happy_path(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--clone-url",
|
||||||
|
"https://example/repo.git",
|
||||||
|
"--clone-branch",
|
||||||
|
"feature/x",
|
||||||
|
"--clone-commit",
|
||||||
|
"deadbeef",
|
||||||
|
"--clone-submodules",
|
||||||
|
"--build-command",
|
||||||
|
"make all",
|
||||||
|
"--guest-linux-work-dir",
|
||||||
|
"/work",
|
||||||
|
"--guest-linux-output-dir",
|
||||||
|
"/out",
|
||||||
|
"--guest-artifact-source",
|
||||||
|
"build",
|
||||||
|
"--extra-env",
|
||||||
|
"FOO=bar baz",
|
||||||
|
"--extra-env",
|
||||||
|
"TOKEN=secret",
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
instance = _FakeTransport.instances[0]
|
||||||
|
cmds = [c[0] for c in instance.runs]
|
||||||
|
assert any("rm -rf /work && mkdir -p /work" in c for c in cmds)
|
||||||
|
assert any("git clone --depth 1 --branch feature/x" in c for c in cmds)
|
||||||
|
assert any("--recurse-submodules" in c for c in cmds)
|
||||||
|
assert any("checkout deadbeef" in c for c in cmds)
|
||||||
|
# extra-env exported before build
|
||||||
|
assert any("export FOO='bar baz'" in c and "export TOKEN=secret" in c for c in cmds)
|
||||||
|
assert any("cd /work && make all" in c for c in cmds)
|
||||||
|
# artifact stage uses the requested source dir
|
||||||
|
assert any("/work/build" in c for c in cmds)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_linux_host_source_dir(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
src = tmp_path / "src"
|
||||||
|
src.mkdir()
|
||||||
|
(src / "hello.txt").write_text("hi", encoding="utf-8")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(src),
|
||||||
|
"--build-command",
|
||||||
|
"true",
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
instance = _FakeTransport.instances[0]
|
||||||
|
# A single tarball was uploaded.
|
||||||
|
assert len(instance.copies) == 1
|
||||||
|
local, remote = instance.copies[0]
|
||||||
|
assert local.endswith(".tar.gz")
|
||||||
|
assert remote.startswith("/tmp/")
|
||||||
|
# Extraction command was invoked.
|
||||||
|
assert any("tar -xzf" in c[0] for c in instance.runs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_linux_skip_artifact(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--clone-url",
|
||||||
|
"https://example/repo.git",
|
||||||
|
"--build-command",
|
||||||
|
"make",
|
||||||
|
"--skip-artifact",
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||||
|
# No staging command should run.
|
||||||
|
assert not any("/opt/ci/output" in c for c in cmds)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_linux_build_failure_exits_nonzero(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
# Make the build command (4th run: rm/git clone/checkout-skipped/build)
|
||||||
|
# return non-zero. Index of build run varies; easiest: make any non-zero
|
||||||
|
# result for the run whose script contains 'cd '.
|
||||||
|
src = tmp_path / "src"
|
||||||
|
src.mkdir()
|
||||||
|
|
||||||
|
class _BuildFails(_FakeTransport):
|
||||||
|
def run(self, script: str, *, check: bool = True) -> _FakeResult:
|
||||||
|
if "cd /opt/ci/build && make" in script:
|
||||||
|
self.runs.append((script, check))
|
||||||
|
return _FakeResult(stdout="boom\n", stderr="err\n", returncode=2)
|
||||||
|
return super().run(script, check=check)
|
||||||
|
|
||||||
|
monkeypatch.setattr(build_module, "SshTransport", _BuildFails)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.5",
|
||||||
|
"--guest-os",
|
||||||
|
"linux",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(src),
|
||||||
|
"--ssh-key-path",
|
||||||
|
str(tmp_path / "key"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "build command failed (exit 2)" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── happy path: Windows ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_windows_clone_url_default_dotnet(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--guest-os",
|
||||||
|
"windows",
|
||||||
|
"--clone-url",
|
||||||
|
"https://example/repo.git",
|
||||||
|
"--use-shared-cache",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
cmds = [c[0] for c in _FakeTransport.instances[0].runs]
|
||||||
|
assert any("git clone --depth 1" in c for c in cmds)
|
||||||
|
assert any("dotnet restore" in c for c in cmds)
|
||||||
|
assert any("NUGET_PACKAGES" in c for c in cmds)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_run_windows_host_source_dir(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
_patch(monkeypatch)
|
||||||
|
src = tmp_path / "src"
|
||||||
|
src.mkdir()
|
||||||
|
(src / "a.cs").write_text("//", encoding="utf-8")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"build",
|
||||||
|
"run",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.7",
|
||||||
|
"--host-source-dir",
|
||||||
|
str(src),
|
||||||
|
"--build-command",
|
||||||
|
"echo built > dist\\out.txt",
|
||||||
|
"--guest-artifact-source",
|
||||||
|
"dist",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
instance = _FakeTransport.instances[0]
|
||||||
|
assert len(instance.copies) == 1
|
||||||
|
assert instance.copies[0][1] == "C:\\CI\\src-transfer.zip"
|
||||||
|
cmds = [c[0] for c in instance.runs]
|
||||||
|
assert any("Expand-Archive" in c for c in cmds)
|
||||||
|
# Staging copies the raw source into the collect dir (no zip-in-zip).
|
||||||
|
assert any("Copy-Item" in c and "'dist'" in c for c in cmds)
|
||||||
|
assert not any("Compress-Archive" in c and "'dist'" in c for c in cmds)
|
||||||
@@ -0,0 +1,533 @@
|
|||||||
|
"""Tests for ``ci_orchestrator job`` (Phase A4)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import click
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.job as job_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────── fakes
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
"""Records every backend call and creates the VMX file on clone."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
clone_fail: bool = False,
|
||||||
|
start_fail: bool = False,
|
||||||
|
delete_fail: bool = False,
|
||||||
|
ip: str | None = "10.99.0.42",
|
||||||
|
running_after: int = 0,
|
||||||
|
clone_guest_os: str = "ubuntu-64",
|
||||||
|
) -> None:
|
||||||
|
self.clone_fail = clone_fail
|
||||||
|
self.start_fail = start_fail
|
||||||
|
self.delete_fail = delete_fail
|
||||||
|
self.ip = ip
|
||||||
|
self.running_after = running_after
|
||||||
|
self.clone_guest_os = clone_guest_os
|
||||||
|
self.calls: list[str] = []
|
||||||
|
self._is_running_count = 0
|
||||||
|
|
||||||
|
def clone_linked(
|
||||||
|
self,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
self.calls.append("clone")
|
||||||
|
if self.clone_fail:
|
||||||
|
raise BackendOperationFailed("clone", 1, "boom")
|
||||||
|
assert destination is not None
|
||||||
|
dst = Path(destination)
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
dst.write_text(
|
||||||
|
f'config.version = "8"\nguestOS = "{self.clone_guest_os}"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return VmHandle(identifier=str(dst), name=name)
|
||||||
|
|
||||||
|
def start(self, handle: VmHandle, headless: bool = True) -> None:
|
||||||
|
self.calls.append("start")
|
||||||
|
if self.start_fail:
|
||||||
|
raise BackendOperationFailed("start", 1, "no")
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
self.calls.append(f"stop:{'hard' if hard else 'soft'}")
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
self.calls.append("delete")
|
||||||
|
if self.delete_fail:
|
||||||
|
raise BackendOperationFailed("deleteVM", 1, "locked")
|
||||||
|
|
||||||
|
def is_running(self, handle: VmHandle) -> bool:
|
||||||
|
self._is_running_count += 1
|
||||||
|
return self._is_running_count > self.running_after
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return self.ip
|
||||||
|
|
||||||
|
def list_snapshots(self, handle: VmHandle) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_common(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
backend: _FakeBackend,
|
||||||
|
*,
|
||||||
|
transport_ready: bool = True,
|
||||||
|
linux_build_raises: bool = False,
|
||||||
|
) -> dict[str, list[Any]]:
|
||||||
|
"""Patch all in-process collaborators of the ``job`` command."""
|
||||||
|
calls: dict[str, list[Any]] = {
|
||||||
|
"linux_build": [],
|
||||||
|
"windows_build": [],
|
||||||
|
"linux_collect": [],
|
||||||
|
"windows_collect": [],
|
||||||
|
"manifest": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "load_backend", lambda _cfg: backend)
|
||||||
|
|
||||||
|
class _ProbeOk:
|
||||||
|
def __enter__(self) -> _ProbeOk:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return transport_ready
|
||||||
|
|
||||||
|
class _Store:
|
||||||
|
def get(self, _t: str) -> Any:
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
|
||||||
|
return Credential("ci", "pwd")
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "WinRmTransport", lambda *a, **k: _ProbeOk())
|
||||||
|
monkeypatch.setattr(job_module, "SshTransport", lambda *a, **k: _ProbeOk())
|
||||||
|
monkeypatch.setattr(job_module, "KeyringCredentialStore", lambda: _Store())
|
||||||
|
|
||||||
|
def _linux_build(**kwargs: Any) -> None:
|
||||||
|
calls["linux_build"].append(kwargs)
|
||||||
|
if linux_build_raises:
|
||||||
|
raise RuntimeError("build exploded")
|
||||||
|
|
||||||
|
def _windows_build(**kwargs: Any) -> None:
|
||||||
|
calls["windows_build"].append(kwargs)
|
||||||
|
|
||||||
|
def _linux_collect(**kwargs: Any) -> None:
|
||||||
|
calls["linux_collect"].append(kwargs)
|
||||||
|
# Simulate a transferred file so manifest writing has something to do.
|
||||||
|
Path(kwargs["host_dir"], "out.txt").write_text("ok", encoding="utf-8")
|
||||||
|
|
||||||
|
def _windows_collect(**kwargs: Any) -> None:
|
||||||
|
calls["windows_collect"].append(kwargs)
|
||||||
|
|
||||||
|
def _manifest(host_dir: Path, job_id: str, commit: str) -> int:
|
||||||
|
calls["manifest"].append((str(host_dir), job_id, commit))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "_linux_build", _linux_build)
|
||||||
|
monkeypatch.setattr(job_module, "_windows_build", _windows_build)
|
||||||
|
monkeypatch.setattr(job_module, "_linux_collect", _linux_collect)
|
||||||
|
monkeypatch.setattr(job_module, "_windows_collect", _windows_collect)
|
||||||
|
monkeypatch.setattr(job_module, "_write_manifest", _manifest)
|
||||||
|
# Make poll loops instant.
|
||||||
|
monkeypatch.setattr(job_module.time, "sleep", lambda _s: None)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def template_vmx(tmp_path: Path) -> Path:
|
||||||
|
"""Linux-flavoured VMX so auto-detect picks the SSH transport."""
|
||||||
|
vmx = tmp_path / "templates" / "LinuxBuild2404.vmx"
|
||||||
|
vmx.parent.mkdir(parents=True)
|
||||||
|
vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8")
|
||||||
|
return vmx
|
||||||
|
|
||||||
|
|
||||||
|
def _common_args(
|
||||||
|
template: Path,
|
||||||
|
*,
|
||||||
|
clone_base: Path,
|
||||||
|
artifact_base: Path,
|
||||||
|
extra: list[str] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
return [
|
||||||
|
"job",
|
||||||
|
"--job-id",
|
||||||
|
"ci-test-1",
|
||||||
|
"--repo-url",
|
||||||
|
"http://gitea.local/org/repo.git",
|
||||||
|
"--branch",
|
||||||
|
"main",
|
||||||
|
"--template-path",
|
||||||
|
str(template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(clone_base),
|
||||||
|
"--artifact-base-dir",
|
||||||
|
str(artifact_base),
|
||||||
|
"--ready-timeout",
|
||||||
|
"30",
|
||||||
|
"--poll-interval",
|
||||||
|
"1",
|
||||||
|
*(extra or []),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────── tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_happy_path_linux(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=1)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
|
||||||
|
base_clone = tmp_path / "build-vms"
|
||||||
|
base_artifact = tmp_path / "artifacts"
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, _common_args(template_vmx, clone_base=base_clone, artifact_base=base_artifact)
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
# Backend received the full lifecycle including delete in finally.
|
||||||
|
assert backend.calls[0] == "clone"
|
||||||
|
assert "start" in backend.calls
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
# Linux build branch was taken (auto-detected from VMX guestOS).
|
||||||
|
assert len(calls["linux_build"]) == 1
|
||||||
|
assert calls["linux_build"][0]["build_command"] == ""
|
||||||
|
# Artifacts were collected into the per-job directory and manifest written.
|
||||||
|
assert (base_artifact / "ci-test-1" / "out.txt").is_file()
|
||||||
|
assert calls["manifest"] and calls["manifest"][0][1] == "ci-test-1"
|
||||||
|
# Clone directory removed.
|
||||||
|
assert not any((base_clone).iterdir()) or all(not p.exists() for p in base_clone.iterdir())
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_default_transport_in_guest_with_submodules(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=1)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx, clone_base=tmp_path / "v", artifact_base=tmp_path / "a"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
kw = calls["linux_build"][0]
|
||||||
|
# Default: in-guest clone (clone_url set, no host source) + submodules.
|
||||||
|
assert kw["host_source_dir"] is None
|
||||||
|
assert kw["clone_url"] and "repo.git" in kw["clone_url"]
|
||||||
|
assert kw["clone_submodules"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_host_clone_transport(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=1)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
root = tmp_path / "hostclone"
|
||||||
|
src = root / "src"
|
||||||
|
src.mkdir(parents=True)
|
||||||
|
|
||||||
|
def _fake_host_clone(
|
||||||
|
repo_url: str, branch: str, commit: str, submodules: bool
|
||||||
|
) -> tuple[Path, Path]:
|
||||||
|
assert submodules is True
|
||||||
|
return root, src
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "_host_clone", _fake_host_clone)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "v",
|
||||||
|
artifact_base=tmp_path / "a",
|
||||||
|
extra=["--host-clone"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
kw = calls["linux_build"][0]
|
||||||
|
assert kw["host_source_dir"] == str(src)
|
||||||
|
assert kw["clone_url"] is None
|
||||||
|
assert kw["clone_submodules"] is True
|
||||||
|
# Host-side temp clone removed after the build.
|
||||||
|
assert not root.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_no_submodules_flag(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=1)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "v",
|
||||||
|
artifact_base=tmp_path / "a",
|
||||||
|
extra=["--no-submodules"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert calls["linux_build"][0]["clone_submodules"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_skip_artifact_branch(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=0)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=["--skip-artifact"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert calls["linux_collect"] == []
|
||||||
|
assert calls["windows_collect"] == []
|
||||||
|
assert calls["manifest"] == []
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_cleanup_on_build_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
"""If the build raises, the VM must still be destroyed (try/finally)."""
|
||||||
|
backend = _FakeBackend(running_after=0)
|
||||||
|
_patch_common(monkeypatch, backend, linux_build_raises=True)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "build exploded" in result.output
|
||||||
|
assert "delete" in backend.calls
|
||||||
|
# Stop attempt happened before delete.
|
||||||
|
assert any(c.startswith("stop:") for c in backend.calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_cleanup_on_clone_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(clone_fail=True)
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "clone failed" in result.output
|
||||||
|
# Backend.start/delete must not have been called: handle is None.
|
||||||
|
assert backend.calls == ["clone"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_rejects_missing_template(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend()
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
tmp_path / "ghost.vmx",
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "Template VMX not found" in result.output
|
||||||
|
assert backend.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_windows_branch_with_overrides(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Windows-flavoured VMX selects the WinRM build path; CPU/RAM override applied."""
|
||||||
|
vmx = tmp_path / "templates" / "WinBuild2025.vmx"
|
||||||
|
vmx.parent.mkdir(parents=True)
|
||||||
|
vmx.write_text('guestOS = "windows10srv-64"\n', encoding="utf-8")
|
||||||
|
|
||||||
|
backend = _FakeBackend(running_after=0, clone_guest_os="windows10srv-64")
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
extra = [
|
||||||
|
"--guest-cpu",
|
||||||
|
"4",
|
||||||
|
"--guest-memory-mb",
|
||||||
|
"8192",
|
||||||
|
"--extra-env-json",
|
||||||
|
'{"BUILD_TAG":"abc"}',
|
||||||
|
]
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=extra,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert calls["windows_build"], "Windows build branch not taken"
|
||||||
|
assert calls["windows_build"][0]["extra_env"] == {"BUILD_TAG": "abc"}
|
||||||
|
# CPU/RAM override patched the clone VMX.
|
||||||
|
clone_vmx = next((tmp_path / "vms").rglob("*.vmx"), None)
|
||||||
|
# After successful job the clone dir is deleted, so the file is gone.
|
||||||
|
# Check that the Windows collect helper got the collect dir instead.
|
||||||
|
assert calls["windows_collect"][0]["guest_path"] == "C:\\CI\\output"
|
||||||
|
assert clone_vmx is None # cleaned up
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_invalid_extra_env_json(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend()
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=["--extra-env-json", '{"bad-key":"x"}'],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "not a valid env var name" in result.output
|
||||||
|
# Failure is parameter-time; clone never started.
|
||||||
|
assert backend.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────── helper-level tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_vmx_overrides_replaces_existing_lines(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text(
|
||||||
|
'config.version = "8"\nnumvcpus = "1"\nmemsize = "2048"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
job_module._apply_vmx_overrides(vmx, cpu=8, memory_mb=16384)
|
||||||
|
text = vmx.read_text(encoding="utf-8")
|
||||||
|
assert 'numvcpus = "8"' in text
|
||||||
|
assert 'memsize = "16384"' in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_vmx_overrides_appends_when_missing(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text('config.version = "8"\n', encoding="utf-8")
|
||||||
|
job_module._apply_vmx_overrides(vmx, cpu=2, memory_mb=4096)
|
||||||
|
text = vmx.read_text(encoding="utf-8")
|
||||||
|
assert 'numvcpus = "2"' in text
|
||||||
|
assert 'memsize = "4096"' in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_guest_os_detects_linux(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8")
|
||||||
|
assert job_module._read_guest_os_from_vmx(vmx) == "linux"
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_guest_os_defaults_to_windows(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text("# nothing useful\n", encoding="utf-8")
|
||||||
|
assert job_module._read_guest_os_from_vmx(vmx) == "windows"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_empty_inputs() -> None:
|
||||||
|
assert job_module._parse_extra_env_json("") == {}
|
||||||
|
assert job_module._parse_extra_env_json("{}") == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_rejects_non_object() -> None:
|
||||||
|
with pytest.raises(click.exceptions.UsageError):
|
||||||
|
job_module._parse_extra_env_json("[]")
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_guest_os_handles_oserror(tmp_path: Path) -> None:
|
||||||
|
"""Unreadable VMX (e.g. a directory) returns the safe ``windows`` default."""
|
||||||
|
p = tmp_path / "is_a_dir.vmx"
|
||||||
|
p.mkdir()
|
||||||
|
assert job_module._read_guest_os_from_vmx(p) == "windows"
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_vmx_overrides_noop_when_both_zero(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text('numvcpus = "1"\n', encoding="utf-8")
|
||||||
|
job_module._apply_vmx_overrides(vmx, cpu=0, memory_mb=0)
|
||||||
|
assert vmx.read_text(encoding="utf-8") == 'numvcpus = "1"\n'
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_invalid_json_raises() -> None:
|
||||||
|
with pytest.raises(click.exceptions.UsageError):
|
||||||
|
job_module._parse_extra_env_json("{not json")
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_coerces_value_types() -> None:
|
||||||
|
out = job_module._parse_extra_env_json('{"A":null,"B":42,"C":"x"}')
|
||||||
|
assert out == {"A": "", "B": "42", "C": "x"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_rejects_non_string_key() -> None:
|
||||||
|
# JSON keys are always strings, so coerce via raw call to guarantee branch.
|
||||||
|
with pytest.raises(click.exceptions.UsageError):
|
||||||
|
job_module._parse_extra_env_json('{"1bad":"x"}')
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_wait_running_swallows_backend_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendError
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
class _Boom:
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
raise BackendError("nope")
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module.time, "sleep", lambda _s: None)
|
||||||
|
seq = iter([0.0, 1.0])
|
||||||
|
monkeypatch.setattr(job_module, "_now", lambda: next(seq))
|
||||||
|
assert (
|
||||||
|
job_module._wait_running(_Boom(), VmHandle("x"), deadline=0.5, poll=0.001) # type: ignore[arg-type]
|
||||||
|
is False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_wait_for_ip_returns_none_on_backend_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendError
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
class _Boom:
|
||||||
|
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
raise BackendError("ip nope")
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module.time, "sleep", lambda _s: None)
|
||||||
|
seq = iter([0.0, 1.0])
|
||||||
|
monkeypatch.setattr(job_module, "_now", lambda: next(seq))
|
||||||
|
assert (
|
||||||
|
job_module._wait_for_ip(_Boom(), VmHandle("x"), deadline=0.5, poll=0.001) # type: ignore[arg-type]
|
||||||
|
is None
|
||||||
|
)
|
||||||
@@ -0,0 +1,661 @@
|
|||||||
|
"""Tests for ``ci_orchestrator monitor disk`` and ``monitor runner``."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.monitor as mon
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _no_event_log_or_webhook(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_write_windows_event", lambda *_a, **_kw: True)
|
||||||
|
monkeypatch.setattr(mon, "_post_webhook", lambda *_a, **_kw: True)
|
||||||
|
|
||||||
|
|
||||||
|
# ── monitor disk ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_usage(free_gb: float, total_gb: float = 1000.0) -> shutil._ntuple_diskusage:
|
||||||
|
gb = 1024 ** 3
|
||||||
|
return shutil._ntuple_diskusage(int(total_gb * gb), int((total_gb - free_gb) * gb), int(free_gb * gb))
|
||||||
|
|
||||||
|
|
||||||
|
def test_disk_ok_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||||
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "OK" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_disk_alert_exits_one(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
||||||
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "below" in result.output or "WARNING" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_disk_drive_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def _raise(_p: Any) -> Any:
|
||||||
|
raise FileNotFoundError("nope")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.shutil, "disk_usage", _raise)
|
||||||
|
result = CliRunner().invoke(cli, ["monitor", "disk", "--drive-letter", "Z"])
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "not found" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_disk_json_output_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
payload = json.loads(result.output.strip())
|
||||||
|
assert payload["status"] == "ok"
|
||||||
|
assert payload["free_gb"] == 200.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_disk_webhook_invoked_on_alert(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
calls: list[tuple[str, str]] = []
|
||||||
|
monkeypatch.setattr(mon, "_post_webhook", lambda url, content, **_kw: calls.append((url, content)) or True)
|
||||||
|
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor",
|
||||||
|
"disk",
|
||||||
|
"--min-free-gb",
|
||||||
|
"50",
|
||||||
|
"--webhook-url",
|
||||||
|
"https://example.invalid/hook",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert calls and calls[0][0] == "https://example.invalid/hook"
|
||||||
|
|
||||||
|
|
||||||
|
# ── monitor runner ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_running_exits_zero(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["monitor", "runner", "--state-dir", str(tmp_path)],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "Running" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_not_installed_exits_two(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: None)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "not found" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_maintenance_flag_skips(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
flag = tmp_path / "runner-maintenance.flag"
|
||||||
|
flag.write_text("")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "maintenance mode" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_restart_attempted_when_stopped(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
statuses = iter(["Stopped", "Running"])
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
||||||
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "restarted" in result.output
|
||||||
|
assert (tmp_path / "runner-restart-log.json").is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_rate_limit_exits_one(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
# Pre-fill the cooldown log with recent restarts up to the limit.
|
||||||
|
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
||||||
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
||||||
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor",
|
||||||
|
"runner",
|
||||||
|
"--state-dir",
|
||||||
|
str(tmp_path),
|
||||||
|
"--max-restarts",
|
||||||
|
"3",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "limit" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _post_webhook ─────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_webhook_empty_url_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
assert mon._post_webhook("", "msg") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_webhook_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
class _Resp:
|
||||||
|
status = 204
|
||||||
|
|
||||||
|
def __enter__(self) -> _Resp:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_a: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.urllib.request, "urlopen", lambda *_a, **_kw: _Resp())
|
||||||
|
assert mon._post_webhook("https://example.invalid/h", "hi") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_webhook_http_error_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
def _raise(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise mon.urllib.error.URLError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _raise)
|
||||||
|
assert mon._post_webhook("https://example.invalid/h", "hi") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_webhook_non_2xx_returns_false(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
class _Resp:
|
||||||
|
status = 500
|
||||||
|
|
||||||
|
def __enter__(self) -> _Resp:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_a: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.urllib.request, "urlopen", lambda *_a, **_kw: _Resp())
|
||||||
|
assert mon._post_webhook("https://example.invalid/h", "hi") is False
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _write_windows_event ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_windows_event_non_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
# Note: autouse fixture monkeypatches the symbol; call the real impl.
|
||||||
|
monkeypatch.undo()
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_windows_event_eventcreate_missing(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
||||||
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_windows_event_subprocess_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
||||||
|
|
||||||
|
class _R:
|
||||||
|
returncode = 0
|
||||||
|
stderr = ""
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _R())
|
||||||
|
assert mon._write_windows_event("src", 1, "Information", "msg") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_windows_event_subprocess_nonzero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
||||||
|
|
||||||
|
class _R:
|
||||||
|
returncode = 5
|
||||||
|
stderr = "nope"
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _R())
|
||||||
|
assert mon._write_windows_event("src", 1, "Error", "msg") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_windows_event_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.undo()
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\eventcreate.exe")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
||||||
|
assert mon._write_windows_event("src", 1, "Warning", "msg") is False
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _resolve_drive_path ───────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_drive_path_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
assert mon._resolve_drive_path("F") == "F:\\"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_drive_path_non_windows(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
assert mon._resolve_drive_path("/mnt/data") == "/mnt/data"
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _service_status ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class _RunResult:
|
||||||
|
def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None:
|
||||||
|
self.returncode = returncode
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_windows_running(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 4 RUNNING"))
|
||||||
|
assert mon._service_status("svc") == "Running"
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_windows_stopped(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 1 STOPPED"))
|
||||||
|
assert mon._service_status("svc") == "Stopped"
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_windows_unknown_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "STATE : 7 PAUSED"))
|
||||||
|
assert mon._service_status("svc") == "Unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_windows_sc_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(1, ""))
|
||||||
|
assert mon._service_status("svc") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_windows_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
||||||
|
assert mon._service_status("svc") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_linux_active(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, "active\n"))
|
||||||
|
assert mon._service_status("svc") == "Running"
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_linux_inactive(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(3, "inactive\n"))
|
||||||
|
assert mon._service_status("svc") == "Stopped"
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_linux_unknown(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(4, "activating\n"))
|
||||||
|
assert mon._service_status("svc") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_linux_no_systemctl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
||||||
|
assert mon._service_status("svc") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_service_status_linux_subprocess_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
||||||
|
assert mon._service_status("svc") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _restart_service ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_service_windows_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, ""))
|
||||||
|
monkeypatch.setattr(mon.time, "sleep", lambda _s: None)
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
assert mon._restart_service("svc") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_service_windows_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "nt")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: r"C:\Windows\System32\sc.exe")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
||||||
|
assert mon._restart_service("svc") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_service_linux_success(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", lambda *_a, **_kw: _RunResult(0, ""))
|
||||||
|
monkeypatch.setattr(mon.time, "sleep", lambda _s: None)
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
assert mon._restart_service("svc") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_service_linux_no_systemctl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: None)
|
||||||
|
assert mon._restart_service("svc") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_restart_service_linux_oserror(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.os, "name", "posix")
|
||||||
|
monkeypatch.setattr(mon.shutil, "which", lambda _n: "/usr/bin/systemctl")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise subprocess.TimeoutExpired(cmd="systemctl", timeout=30.0)
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
monkeypatch.setattr(mon.subprocess, "run", _boom)
|
||||||
|
assert mon._restart_service("svc") is False
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _load_restart_log / _save_restart_log ─────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_missing(tmp_path: Path) -> None:
|
||||||
|
assert mon._load_restart_log(tmp_path / "no.json") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_empty(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "empty.json"
|
||||||
|
p.write_text(" ")
|
||||||
|
assert mon._load_restart_log(p) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_invalid_json(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "bad.json"
|
||||||
|
p.write_text("{not json")
|
||||||
|
assert mon._load_restart_log(p) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_non_list(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "obj.json"
|
||||||
|
p.write_text(json.dumps({"a": 1}))
|
||||||
|
assert mon._load_restart_log(p) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_valid(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "ok.json"
|
||||||
|
p.write_text(json.dumps(["2026-05-14T10:00:00+00:00"]))
|
||||||
|
assert mon._load_restart_log(p) == ["2026-05-14T10:00:00+00:00"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_restart_log_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "bad.json"
|
||||||
|
p.write_text("[]")
|
||||||
|
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "read_text", _boom)
|
||||||
|
assert mon._load_restart_log(p) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_restart_log_writes_file(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "sub" / "log.json"
|
||||||
|
mon._save_restart_log(p, ["a", "b"])
|
||||||
|
assert json.loads(p.read_text()) == ["a", "b"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_restart_log_oserror_swallowed(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
def _boom(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise OSError("readonly")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "mkdir", _boom)
|
||||||
|
# Should not raise.
|
||||||
|
mon._save_restart_log(tmp_path / "x" / "log.json", ["a"])
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers: _gitea_runners_online ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_urlopen(payload: Any) -> Any:
|
||||||
|
class _Resp:
|
||||||
|
def __enter__(self) -> _Resp:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_a: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def read(self) -> bytes:
|
||||||
|
return json.dumps(payload).encode("utf-8")
|
||||||
|
|
||||||
|
return lambda *_a, **_kw: _Resp()
|
||||||
|
|
||||||
|
|
||||||
|
def test_gitea_runners_online_counts(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
mon.urllib.request,
|
||||||
|
"urlopen",
|
||||||
|
_make_urlopen([{"online": True}, {"online": False}, {"online": True}]),
|
||||||
|
)
|
||||||
|
assert mon._gitea_runners_online("https://g/", "tok") == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_gitea_runners_online_non_list(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _make_urlopen({"x": 1}))
|
||||||
|
assert mon._gitea_runners_online("https://g/", "tok") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_gitea_runners_online_url_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
def _raise(*_a: Any, **_kw: Any) -> Any:
|
||||||
|
raise mon.urllib.error.URLError("nope")
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon.urllib.request, "urlopen", _raise)
|
||||||
|
assert mon._gitea_runners_online("https://g/", "tok") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── monitor_runner: gitea-online branches & restart-fail path ──────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _stub_keyring(monkeypatch: pytest.MonkeyPatch, password: str = "tok") -> None:
|
||||||
|
class _Cred:
|
||||||
|
username = "u"
|
||||||
|
|
||||||
|
def __init__(self, pw: str) -> None:
|
||||||
|
self.password = pw
|
||||||
|
|
||||||
|
class _Store:
|
||||||
|
def get(self, _target: str) -> Any:
|
||||||
|
return _Cred(password)
|
||||||
|
|
||||||
|
import ci_orchestrator.credentials as creds
|
||||||
|
|
||||||
|
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_running_with_gitea_online(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 2)
|
||||||
|
_stub_keyring(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--gitea-url", "https://g/",
|
||||||
|
"--gitea-credential-target", "tgt",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "2 runner(s) online" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_running_with_gitea_zero_online(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: 0)
|
||||||
|
_stub_keyring(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--gitea-url", "https://g/",
|
||||||
|
"--gitea-credential-target", "tgt",
|
||||||
|
"--webhook-url", "https://hook/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "0 online runners" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_running_with_gitea_api_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
monkeypatch.setattr(mon, "_gitea_runners_online", lambda *_a, **_kw: None)
|
||||||
|
_stub_keyring(monkeypatch)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--gitea-url", "https://g/",
|
||||||
|
"--gitea-credential-target", "tgt",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "skipped/failed" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_running_with_gitea_keyring_keyerror(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||||
|
|
||||||
|
class _Store:
|
||||||
|
def get(self, _target: str) -> Any:
|
||||||
|
raise KeyError("missing")
|
||||||
|
|
||||||
|
import ci_orchestrator.credentials as creds
|
||||||
|
|
||||||
|
monkeypatch.setattr(creds, "KeyringCredentialStore", _Store)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--gitea-url", "https://g/",
|
||||||
|
"--gitea-credential-target", "tgt",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "could not load Gitea PAT" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_restart_fails_exits_one(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
statuses = iter(["Stopped", "Stopped"])
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
||||||
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: False)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--webhook-url", "https://hook/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "restarted" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_rate_limit_with_webhook(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
||||||
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"monitor", "runner",
|
||||||
|
"--state-dir", str(tmp_path),
|
||||||
|
"--max-restarts", "3",
|
||||||
|
"--webhook-url", "https://hook/",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "limit" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_pruned_old_entries(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Old timestamps and invalid entries must be pruned."""
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
old = (datetime.now(tz=UTC) - timedelta(hours=2)).isoformat()
|
||||||
|
naive = datetime.now(tz=UTC).replace(tzinfo=None).isoformat()
|
||||||
|
bad = "not-a-timestamp"
|
||||||
|
(tmp_path / "runner-restart-log.json").write_text(json.dumps([old, naive, bad]))
|
||||||
|
statuses = iter(["Stopped", "Running"])
|
||||||
|
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
||||||
|
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
"""Tests for ``ci_orchestrator report job``."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
|
||||||
|
|
||||||
|
def _write_jsonl(path: Path, events: list[dict[str, object]]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text("\n".join(json.dumps(e) for e in events) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _success_events(job_id: str = "run-1") -> list[dict[str, object]]:
|
||||||
|
return [
|
||||||
|
{"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T10:00:00Z"},
|
||||||
|
{"jobId": job_id, "phase": "phase4.wait-ready", "status": "success", "ts": "2026-05-14T10:01:00Z"},
|
||||||
|
{
|
||||||
|
"jobId": job_id,
|
||||||
|
"phase": "job",
|
||||||
|
"status": "success",
|
||||||
|
"ts": "2026-05-14T10:05:00Z",
|
||||||
|
"data": {"elapsedSec": 305},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _failure_events(job_id: str = "run-2") -> list[dict[str, object]]:
|
||||||
|
return [
|
||||||
|
{"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T11:00:00Z"},
|
||||||
|
{
|
||||||
|
"jobId": job_id,
|
||||||
|
"phase": "job",
|
||||||
|
"status": "failure",
|
||||||
|
"ts": "2026-05-14T11:00:30Z",
|
||||||
|
"data": {"elapsedSec": 30, "error": "Clone failed: source not found"},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_summary_table(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||||
|
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||||
|
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "run-1" in result.output
|
||||||
|
assert "run-2" in result.output
|
||||||
|
assert "success" in result.output
|
||||||
|
assert "FAILED" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_failed_filter(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||||
|
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--failed"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "run-1" not in result.output
|
||||||
|
assert "run-2" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_detail_view(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["report", "job", "--log-dir", str(tmp_path), "--job-id", "run-2"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "Job: run-2" in result.output
|
||||||
|
assert "Clone failed" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_missing_log_dir(tmp_path: Path) -> None:
|
||||||
|
missing = tmp_path / "nope"
|
||||||
|
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(missing)])
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "not found" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_no_logs(tmp_path: Path) -> None:
|
||||||
|
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "No invoke-ci.jsonl" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_json_output(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--json"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
payload = json.loads(result.output.strip())
|
||||||
|
assert isinstance(payload, list)
|
||||||
|
assert payload[0]["jobId"] == "run-1"
|
||||||
|
assert payload[0]["status"] == "success"
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_unknown_id(tmp_path: Path) -> None:
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--job-id", "ghost"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "No JSONL log" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.report as report_module # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_oserror_returns_empty(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "d" # directory: read_text raises OSError
|
||||||
|
p.mkdir()
|
||||||
|
assert report_module._read_events(p) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_events_skips_blank_invalid_and_non_dict(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "x.jsonl"
|
||||||
|
p.write_text(
|
||||||
|
"\n" # blank
|
||||||
|
"{not json\n" # invalid JSON
|
||||||
|
"[1, 2]\n" # not a dict
|
||||||
|
'{"phase": "job", "status": "start", "ts": "t"}\n'
|
||||||
|
'{"phase": "job", "status": "success", "data": {"elapsedSec": 1.5, "error": "e"}}\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
events = report_module._read_events(p)
|
||||||
|
assert len(events) == 2
|
||||||
|
assert events[1].elapsed_sec == 1
|
||||||
|
assert events[1].error == "e"
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_hms_negative_returns_question_mark() -> None:
|
||||||
|
assert report_module._format_hms(-1) == "?"
|
||||||
|
|
||||||
|
|
||||||
|
def test_format_hms_with_hours() -> None:
|
||||||
|
assert report_module._format_hms(3725) == "1h02m05s"
|
||||||
|
assert report_module._format_hms(45) == "0m45s"
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarise_in_progress_when_no_terminal_event() -> None:
|
||||||
|
events = [
|
||||||
|
report_module._Event(
|
||||||
|
phase="job", status="start", ts="t", elapsed_sec=None, error=None,
|
||||||
|
raw={"jobId": "jx"},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
row = report_module._summarise(events, "default")
|
||||||
|
assert row.status == "in-progress"
|
||||||
|
assert row.job_id == "jx"
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarise_uses_default_id_when_jobid_missing() -> None:
|
||||||
|
events = [
|
||||||
|
report_module._Event(
|
||||||
|
phase="job", status="start", ts="t", elapsed_sec=None, error=None,
|
||||||
|
raw={},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
row = report_module._summarise(events, "fallback")
|
||||||
|
assert row.job_id == "fallback"
|
||||||
|
|
||||||
|
|
||||||
|
def test_summarise_truncates_long_error() -> None:
|
||||||
|
long_err = "x" * 100
|
||||||
|
events = [
|
||||||
|
report_module._Event(
|
||||||
|
phase="job", status="failure", ts="t", elapsed_sec=10, error=long_err,
|
||||||
|
raw={"jobId": "j"},
|
||||||
|
)
|
||||||
|
]
|
||||||
|
row = report_module._summarise(events, "d")
|
||||||
|
assert row.status == "FAILED"
|
||||||
|
assert row.error.endswith("...")
|
||||||
|
assert len(row.error) == 60
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_detail_json_emits_raw_events(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "r1" / "invoke-ci.jsonl", _success_events("r1"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--job-id", "r1", "--json"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
payload = json.loads(result.output.strip())
|
||||||
|
assert isinstance(payload, list)
|
||||||
|
assert payload[0]["phase"] == "job"
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_detail_empty_jsonl(tmp_path: Path) -> None:
|
||||||
|
p = tmp_path / "empty" / "invoke-ci.jsonl"
|
||||||
|
p.parent.mkdir(parents=True)
|
||||||
|
p.write_text("")
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--job-id", "empty"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 1
|
||||||
|
assert "empty or unreadable" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_report_job_failed_filter_no_results(tmp_path: Path) -> None:
|
||||||
|
_write_jsonl(tmp_path / "r1" / "invoke-ci.jsonl", _success_events("r1"))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["report", "job", "--log-dir", str(tmp_path), "--failed"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "No matching jobs" in result.output
|
||||||
@@ -0,0 +1,499 @@
|
|||||||
|
"""Tests for ``ci_orchestrator vm remove`` and ``vm cleanup``.
|
||||||
|
|
||||||
|
Migrated from Pester ``tests/Remove-BuildVM.Tests.ps1`` (now removed).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.vm as vm_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
running_after_stop: bool = False,
|
||||||
|
delete_fails_first: int = 0,
|
||||||
|
) -> None:
|
||||||
|
self.calls: list[tuple[str, Any]] = []
|
||||||
|
self._running = True
|
||||||
|
self._running_after_stop = running_after_stop
|
||||||
|
self._delete_fails_remaining = delete_fails_first
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
self.calls.append(("stop", hard))
|
||||||
|
if not hard or not self._running_after_stop:
|
||||||
|
self._running = False
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
return self._running
|
||||||
|
|
||||||
|
def delete(self, _h: VmHandle) -> None:
|
||||||
|
self.calls.append(("delete", None))
|
||||||
|
if self._delete_fails_remaining > 0:
|
||||||
|
self._delete_fails_remaining -= 1
|
||||||
|
raise BackendOperationFailed("deleteVM", 1, "transient")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setattr(time, "sleep", lambda _s: None)
|
||||||
|
monkeypatch.setattr(vm_module.time, "sleep", lambda _s: None)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_clone_dir(tmp_path: Path, name: str = "clone") -> tuple[Path, Path]:
|
||||||
|
clone_dir = tmp_path / name
|
||||||
|
clone_dir.mkdir()
|
||||||
|
vmx = clone_dir / f"{name}.vmx"
|
||||||
|
vmx.write_text('config.version = "8"', encoding="utf-8")
|
||||||
|
return clone_dir, vmx
|
||||||
|
|
||||||
|
|
||||||
|
# ── vm remove ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_missing_vmx_returns_without_error(tmp_path: Path) -> None:
|
||||||
|
"""Pester migration: returns without error when VMX does not exist."""
|
||||||
|
missing = tmp_path / "ghost" / "ghost.vmx"
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(missing)])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "nothing to remove" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_partial_clone_dir_is_removed(tmp_path: Path) -> None:
|
||||||
|
"""Pester migration: partial clone dir without VMX is still removed."""
|
||||||
|
clone_dir, vmx = _make_clone_dir(tmp_path, "partial")
|
||||||
|
vmx.unlink() # leave the directory but no VMX
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert not clone_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_happy_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
"""Pester migration: removes the clone dir when VMX exists and stop+delete succeed."""
|
||||||
|
clone_dir, vmx = _make_clone_dir(tmp_path, "live")
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["vm", "remove", "--vmx", str(vmx), "--graceful-timeout", "1"]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert ("delete", None) in backend.calls
|
||||||
|
assert not clone_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_force_skips_soft_stop(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
_, vmx = _make_clone_dir(tmp_path, "force")
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx), "--force"])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
# No soft stop was issued.
|
||||||
|
soft_stops = [c for c in backend.calls if c == ("stop", False)]
|
||||||
|
assert soft_stops == []
|
||||||
|
hard_stops = [c for c in backend.calls if c == ("stop", True)]
|
||||||
|
assert hard_stops != []
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_delete_retry_succeeds(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
clone_dir, vmx = _make_clone_dir(tmp_path, "retry")
|
||||||
|
backend = _FakeBackend(delete_fails_first=2)
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
delete_calls = [c for c in backend.calls if c[0] == "delete"]
|
||||||
|
assert len(delete_calls) == 3 # 2 failures + 1 success
|
||||||
|
assert not clone_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ── vm cleanup ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _age_dir(path: Path, hours: float) -> None:
|
||||||
|
ts = (datetime.now() - timedelta(hours=hours)).timestamp()
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.utime(path, (ts, ts))
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_dry_run_lists_orphans(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
base.mkdir()
|
||||||
|
fresh, _ = _make_clone_dir(base, "fresh")
|
||||||
|
old, _ = _make_clone_dir(base, "old")
|
||||||
|
_age_dir(old, hours=10)
|
||||||
|
_age_dir(fresh, hours=0)
|
||||||
|
# Make sure backend isn't constructed (vmrun absent on test host).
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"cleanup",
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--max-age-hours",
|
||||||
|
"4",
|
||||||
|
"--what-if",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "would destroy" in result.output
|
||||||
|
assert old.exists() # dry run did NOT delete
|
||||||
|
assert fresh.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_destroys_old_dirs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
base.mkdir()
|
||||||
|
old, _ = _make_clone_dir(base, "old")
|
||||||
|
_age_dir(old, hours=10)
|
||||||
|
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"cleanup",
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--max-age-hours",
|
||||||
|
"4",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert not old.exists()
|
||||||
|
# Backend was used to stop+delete.
|
||||||
|
assert any(c[0] == "delete" for c in backend.calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_handles_missing_base_dir(tmp_path: Path) -> None:
|
||||||
|
missing = tmp_path / "does-not-exist"
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["vm", "cleanup", "--clone-base-dir", str(missing)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "nothing to do" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_removes_stale_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
base = tmp_path / "bv"
|
||||||
|
base.mkdir()
|
||||||
|
lock = tmp_path / "vm-start.lock"
|
||||||
|
lock.write_text("x")
|
||||||
|
_age_dir(lock, hours=1)
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"cleanup",
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--lock-file",
|
||||||
|
str(lock),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert not lock.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers + uncovered branches ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_backend_with_vmrun_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
"""Override path returns a WorkstationVmrunBackend honouring vmrun_path."""
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
class _StubConfig:
|
||||||
|
pass
|
||||||
|
|
||||||
|
fake_vmrun = tmp_path / "vmrun.exe"
|
||||||
|
fake_vmrun.write_text("x")
|
||||||
|
monkeypatch.setattr(vm_module, "load_config", lambda: _StubConfig())
|
||||||
|
backend = vm_module._make_backend(str(fake_vmrun))
|
||||||
|
assert isinstance(backend, WorkstationVmrunBackend)
|
||||||
|
|
||||||
|
|
||||||
|
def test_make_backend_default_uses_load_backend(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
sentinel = object()
|
||||||
|
monkeypatch.setattr(vm_module, "load_config", lambda: "cfg")
|
||||||
|
monkeypatch.setattr(vm_module, "load_backend", lambda _c: sentinel)
|
||||||
|
assert vm_module._make_backend(None) is sentinel
|
||||||
|
|
||||||
|
|
||||||
|
def test_try_remove_dir_returns_true_when_missing(tmp_path: Path) -> None:
|
||||||
|
assert vm_module._try_remove_dir(tmp_path / "ghost") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_try_remove_dir_real_removal(tmp_path: Path) -> None:
|
||||||
|
d = tmp_path / "x"
|
||||||
|
(d / "sub").mkdir(parents=True)
|
||||||
|
(d / "sub" / "f.txt").write_text("a")
|
||||||
|
assert vm_module._try_remove_dir(d) is True
|
||||||
|
assert not d.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_with_fallback_soft_error_then_running_with_hard_error() -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: list[tuple[str, bool]] = []
|
||||||
|
|
||||||
|
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||||
|
self.calls.append(("stop", hard))
|
||||||
|
raise BackendOperationFailed("stop", 1, "boom")
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
b = _B()
|
||||||
|
# Should swallow both errors.
|
||||||
|
vm_module._stop_with_fallback(b, VmHandle(identifier="x.vmx"))
|
||||||
|
assert ("stop", False) in b.calls
|
||||||
|
assert ("stop", True) in b.calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendNotAvailable
|
||||||
|
|
||||||
|
clone_dir, vmx = _make_clone_dir(tmp_path, "noback")
|
||||||
|
|
||||||
|
def _raise(_v: Any) -> Any:
|
||||||
|
raise BackendNotAvailable("vmrun missing")
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", _raise)
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "vmrun unavailable" in result.output
|
||||||
|
assert not clone_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_force_hard_stop_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
|
||||||
|
_, vmx = _make_clone_dir(tmp_path, "forcefail")
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||||
|
raise BackendOperationFailed("stop", 1, "no")
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def delete(self, _h: VmHandle) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: _B())
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx), "--force"])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "hard stop failed" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_deadline_loop_with_running_then_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Cover the deadline loop that breaks on BackendError."""
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
|
||||||
|
_, vmx = _make_clone_dir(tmp_path, "loop")
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._calls = 0
|
||||||
|
|
||||||
|
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
self._calls += 1
|
||||||
|
if self._calls == 1:
|
||||||
|
return True
|
||||||
|
raise BackendOperationFailed("isrunning", 1, "lost")
|
||||||
|
|
||||||
|
def delete(self, _h: VmHandle) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Make monotonic strictly advance so the loop runs at least twice but exits.
|
||||||
|
base_t = [0.0]
|
||||||
|
|
||||||
|
def _mono() -> float:
|
||||||
|
base_t[0] += 1.0
|
||||||
|
return base_t[0]
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module.time, "monotonic", _mono)
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: _B())
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_delete_fails_all_retries(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Delete that fails on every retry triggers fallback dir-removal log."""
|
||||||
|
clone_dir, vmx = _make_clone_dir(tmp_path, "neverdeletes")
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def delete(self, _h: VmHandle) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
raise BackendOperationFailed("delete", 1, "always")
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: _B())
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "deleteVM failed after retries" in result.output
|
||||||
|
assert not clone_dir.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_remove_clone_dir_removal_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""When _try_remove_dir reports failure, the manual-cleanup notice fires."""
|
||||||
|
_, vmx = _make_clone_dir(tmp_path, "stubborn")
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
monkeypatch.setattr(vm_module, "_try_remove_dir", lambda _p: False)
|
||||||
|
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "manual cleanup needed" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_orphans_non_dir_base(tmp_path: Path) -> None:
|
||||||
|
f = tmp_path / "file.txt"
|
||||||
|
f.write_text("x")
|
||||||
|
assert vm_module._list_orphans(f, 4) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_orphans_skips_non_dir_and_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
base = tmp_path / "b"
|
||||||
|
base.mkdir()
|
||||||
|
(base / "a-file").write_text("not a dir")
|
||||||
|
bad_dir = base / "bad"
|
||||||
|
bad_dir.mkdir()
|
||||||
|
_age_dir(bad_dir, hours=10)
|
||||||
|
|
||||||
|
real_stat = Path.stat
|
||||||
|
|
||||||
|
def _stat(self: Path, *a: Any, **kw: Any) -> Any:
|
||||||
|
if self == bad_dir:
|
||||||
|
raise OSError("nope")
|
||||||
|
return real_stat(self, *a, **kw)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "stat", _stat)
|
||||||
|
assert vm_module._list_orphans(base, 4) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_vmx_returns_none(tmp_path: Path) -> None:
|
||||||
|
d = tmp_path / "novmx"
|
||||||
|
d.mkdir()
|
||||||
|
(d / "readme.txt").write_text("x")
|
||||||
|
assert vm_module._find_vmx(d) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_orphans_dir_without_vmx_logs_and_removes(tmp_path: Path) -> None:
|
||||||
|
base = tmp_path / "b"
|
||||||
|
base.mkdir()
|
||||||
|
no_vmx = base / "no-vmx"
|
||||||
|
no_vmx.mkdir()
|
||||||
|
_age_dir(no_vmx, hours=10)
|
||||||
|
runner_count = vm_module.cleanup_orphans(
|
||||||
|
clone_base=base, max_age_hours=4, backend=None
|
||||||
|
)
|
||||||
|
assert runner_count == 1
|
||||||
|
assert not no_vmx.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_orphans_backend_stop_error_swallowed(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
|
||||||
|
base = tmp_path / "b"
|
||||||
|
base.mkdir()
|
||||||
|
old, _ = _make_clone_dir(base, "old1")
|
||||||
|
_age_dir(old, hours=10)
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def stop(self, _h: VmHandle, hard: bool = False) -> None:
|
||||||
|
raise BackendOperationFailed("stop", 1, "no")
|
||||||
|
|
||||||
|
def delete(self, _h: VmHandle) -> None:
|
||||||
|
raise BackendOperationFailed("delete", 1, "no")
|
||||||
|
|
||||||
|
count = vm_module.cleanup_orphans(
|
||||||
|
clone_base=base, max_age_hours=4, backend=_B()
|
||||||
|
)
|
||||||
|
assert count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_orphans_dir_removal_failure_logs(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
base = tmp_path / "b"
|
||||||
|
base.mkdir()
|
||||||
|
old, _ = _make_clone_dir(base, "stuck")
|
||||||
|
_age_dir(old, hours=10)
|
||||||
|
monkeypatch.setattr(vm_module, "_try_remove_dir", lambda _p: False)
|
||||||
|
count = vm_module.cleanup_orphans(
|
||||||
|
clone_base=base, max_age_hours=4, backend=None
|
||||||
|
)
|
||||||
|
assert count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_orphans_lock_unlink_oserror(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
base = tmp_path / "b"
|
||||||
|
base.mkdir()
|
||||||
|
lock = tmp_path / "stale.lock"
|
||||||
|
lock.write_text("x")
|
||||||
|
_age_dir(lock, hours=1)
|
||||||
|
|
||||||
|
real_unlink = Path.unlink
|
||||||
|
|
||||||
|
def _boom(self: Path, *a: Any, **kw: Any) -> Any:
|
||||||
|
if self == lock:
|
||||||
|
raise OSError("locked")
|
||||||
|
return real_unlink(self, *a, **kw)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "unlink", _boom)
|
||||||
|
vm_module.cleanup_orphans(
|
||||||
|
clone_base=base, max_age_hours=4, backend=None, lock_file=lock
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_cleanup_backend_unavailable(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendNotAvailable
|
||||||
|
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
base.mkdir()
|
||||||
|
|
||||||
|
def _raise(_v: Any) -> Any:
|
||||||
|
raise BackendNotAvailable("no vmrun")
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", _raise)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, ["vm", "cleanup", "--clone-base-dir", str(base)]
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0
|
||||||
|
assert "vmrun not available" in result.output
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
"""Tests for ``ci_orchestrator vm new``.
|
||||||
|
|
||||||
|
Migrated from Pester ``tests/New-BuildVM.Tests.ps1``. Preserves:
|
||||||
|
|
||||||
|
* parameter validation: missing template VMX rejected
|
||||||
|
* failure path: backend clone failure cleans up the partial clone dir
|
||||||
|
* clone naming: ``Clone_{JobId}_{yyyyMMdd_HHmmss}``
|
||||||
|
* output: stdout final line is the clone VMX path
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.vm as vm_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
"""Records calls and creates the destination VMX on success."""
|
||||||
|
|
||||||
|
def __init__(self, *, fail: bool = False) -> None:
|
||||||
|
self.fail = fail
|
||||||
|
self.calls: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
def clone_linked(
|
||||||
|
self,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
self.calls.append(
|
||||||
|
{
|
||||||
|
"template": template,
|
||||||
|
"snapshot": snapshot,
|
||||||
|
"name": name,
|
||||||
|
"destination": destination,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if self.fail:
|
||||||
|
raise BackendOperationFailed("clone", 1, "Error: source not found")
|
||||||
|
# Real vmrun creates dest dir + VMX file. Mirror that here so the
|
||||||
|
# post-clone existence check passes.
|
||||||
|
assert destination is not None
|
||||||
|
dst = Path(destination)
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
dst.write_text("config.version = \"8\"", encoding="utf-8")
|
||||||
|
return VmHandle(identifier=str(dst), name=name)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fake_template(tmp_path: Path) -> Path:
|
||||||
|
p = tmp_path / "WinBuild2025.vmx"
|
||||||
|
p.write_text("config.version = \"8\"", encoding="utf-8")
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_rejects_missing_template(tmp_path: Path) -> None:
|
||||||
|
"""Pester migration: throws when TemplatePath does not exist."""
|
||||||
|
missing = tmp_path / "ghost.vmx"
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template",
|
||||||
|
str(missing),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"test-1",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "Template VMX not found" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_clone_failure_cleans_up_partial_dir(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Pester migration: vmrun failure removes the partially-created clone dir."""
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
backend = _FakeBackend(fail=True)
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template",
|
||||||
|
str(fake_template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"test-fail",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "clone failed" in result.output
|
||||||
|
leftovers = [p for p in base.iterdir() if p.is_dir() and "test-fail" in p.name]
|
||||||
|
assert leftovers == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_clone_name_format_and_stdout(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Pester migration: clone folder is ``Clone_{JobId}_{yyyyMMdd_HHmmss}``
|
||||||
|
and the final stdout line is the clone VMX path."""
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template",
|
||||||
|
str(fake_template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"run-42",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
final = result.output.strip().splitlines()[-1]
|
||||||
|
assert final.endswith(".vmx")
|
||||||
|
assert "Clone_run-42_" in final
|
||||||
|
# Format check: timestamp is 8 digits + underscore + 6 digits.
|
||||||
|
import re
|
||||||
|
|
||||||
|
assert re.search(r"Clone_run-42_\d{8}_\d{6}\.vmx$", final)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_passes_snapshot_default(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template",
|
||||||
|
str(fake_template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"run-99",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert backend.calls[0]["snapshot"] == "BaseClean"
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_pascal_case_aliases(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""The PS-shim translation produces ``--template-path`` / ``--snapshot-name``."""
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template-path",
|
||||||
|
str(fake_template),
|
||||||
|
"--snapshot-name",
|
||||||
|
"Custom",
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"alias",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert backend.calls[0]["snapshot"] == "Custom"
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_backend_unavailable(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
from ci_orchestrator.backends.errors import BackendNotAvailable
|
||||||
|
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
|
||||||
|
def _raise(_v: Any) -> Any:
|
||||||
|
raise BackendNotAvailable("no vmrun")
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", _raise)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm", "new",
|
||||||
|
"--template", str(fake_template),
|
||||||
|
"--clone-base-dir", str(base),
|
||||||
|
"--job-id", "novm",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "vmrun unavailable" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_partial_clone_dir_cleaned(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Backend creates the partial dir then fails -> dir is removed."""
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def clone_linked(
|
||||||
|
self, template: str, snapshot: str, name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> Any:
|
||||||
|
assert destination is not None
|
||||||
|
Path(destination).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
(Path(destination).parent / "stale.txt").write_text("x")
|
||||||
|
raise BackendOperationFailed("clone", 1, "boom")
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: _B())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm", "new",
|
||||||
|
"--template", str(fake_template),
|
||||||
|
"--clone-base-dir", str(base),
|
||||||
|
"--job-id", "partial",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "clone failed" in result.output
|
||||||
|
assert not any(p.is_dir() and "partial" in p.name for p in base.iterdir())
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_backend_success_but_missing_vmx(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Backend reports success but VMX file isn't there -> ClickException."""
|
||||||
|
base = tmp_path / "build-vms"
|
||||||
|
|
||||||
|
class _B:
|
||||||
|
def clone_linked(
|
||||||
|
self, template: str, snapshot: str, name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
return VmHandle(identifier=str(destination), name=name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: _B())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm", "new",
|
||||||
|
"--template", str(fake_template),
|
||||||
|
"--clone-base-dir", str(base),
|
||||||
|
"--job-id", "ghost",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "clone VMX is missing" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_vm_new_creates_base_dir(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, fake_template: Path, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
base = tmp_path / "missing-base"
|
||||||
|
backend = _FakeBackend()
|
||||||
|
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"vm",
|
||||||
|
"new",
|
||||||
|
"--template",
|
||||||
|
str(fake_template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(base),
|
||||||
|
"--job-id",
|
||||||
|
"bd",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert base.is_dir()
|
||||||
@@ -0,0 +1,238 @@
|
|||||||
|
"""Tests for ``ci_orchestrator wait-ready``.
|
||||||
|
|
||||||
|
Migrated from Pester ``tests/Wait-VMReady.Tests.ps1`` (now removed).
|
||||||
|
Preserves the negative cases:
|
||||||
|
|
||||||
|
* missing/invalid VMX → command surfaces a backend error path
|
||||||
|
* never-becomes-ready → exit 2 with timeout message
|
||||||
|
* IP override skips the get_ip polling step
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.wait as wait_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
def __init__(self, *, running: bool = True, ip: str | None = "10.0.0.42") -> None:
|
||||||
|
self._running = running
|
||||||
|
self._ip = ip
|
||||||
|
self.running_calls = 0
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
self.running_calls += 1
|
||||||
|
return self._running
|
||||||
|
|
||||||
|
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return self._ip
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTransport:
|
||||||
|
def __init__(self, *_a: Any, **_kw: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self) -> _FakeTransport:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class _BrokenTransport(_FakeTransport):
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeStore:
|
||||||
|
def get(self, _target: str) -> Credential:
|
||||||
|
return Credential("user", "pwd")
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend, transport: type = _FakeTransport) -> None:
|
||||||
|
monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend)
|
||||||
|
monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore())
|
||||||
|
monkeypatch.setattr(wait_module, "WinRmTransport", transport)
|
||||||
|
monkeypatch.setattr(wait_module, "SshTransport", transport)
|
||||||
|
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||||
|
|
||||||
|
|
||||||
|
# ── happy paths ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_windows_via_transport_alias(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""``--transport WinRM`` is a PS-shim alias of ``--guest-os windows``."""
|
||||||
|
_patch(monkeypatch, _FakeBackend())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--transport", "WinRM", "--timeout", "1"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "WinRM ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_skip_ping_is_no_op(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--skip-ping", "--timeout", "1"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_ip_override_skips_get_ip(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""When --ip-address is provided, the backend is not polled for an IP."""
|
||||||
|
backend = _FakeBackend(running=True, ip=None) # would time out at IP step
|
||||||
|
_patch(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.55",
|
||||||
|
"--timeout",
|
||||||
|
"1",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "10.0.0.55" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_linux_via_ssh_alias(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--transport", "SSH", "--timeout", "1"],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert "SSH ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── failure paths (Pester migration) ───────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_timeout_when_vm_never_running(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend(running=False))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--timeout",
|
||||||
|
"0.01",
|
||||||
|
"--poll-interval",
|
||||||
|
"0.001",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 2
|
||||||
|
assert "timeout waiting for VM" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_timeout_no_ip(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend(running=True, ip=None))
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--timeout",
|
||||||
|
"0.01",
|
||||||
|
"--poll-interval",
|
||||||
|
"0.001",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 3
|
||||||
|
assert "timeout waiting for guest IP" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_timeout_transport_never_ready(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend(), transport=_BrokenTransport)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
[
|
||||||
|
"wait-ready",
|
||||||
|
"--vmx",
|
||||||
|
"x.vmx",
|
||||||
|
"--ip-address",
|
||||||
|
"10.0.0.55",
|
||||||
|
"--timeout",
|
||||||
|
"0.01",
|
||||||
|
"--poll-interval",
|
||||||
|
"0.001",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert result.exit_code == 4
|
||||||
|
assert "transport to be ready" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_ready_invalid_transport_value(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
_patch(monkeypatch, _FakeBackend())
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
["wait-ready", "--vmx", "x.vmx", "--transport", "bogus"],
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "Unknown" in result.output or "bogus" in result.output
|
||||||
|
|
||||||
|
|
||||||
|
# ── helpers (wait_module._wait_running / _wait_for_ip / branches) ───────────────────
|
||||||
|
|
||||||
|
|
||||||
|
from ci_orchestrator.backends.errors import BackendError # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class _ErrorBackend:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
def is_running(self, _h: VmHandle) -> bool:
|
||||||
|
self.calls += 1
|
||||||
|
raise BackendError("probe boom")
|
||||||
|
|
||||||
|
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
raise BackendError("ip boom")
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_running_swallows_backend_error_and_returns_false(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||||
|
monkeypatch.setattr(wait_module.time, "monotonic", lambda: 0.0)
|
||||||
|
backend = _ErrorBackend()
|
||||||
|
# deadline already passed after first iteration
|
||||||
|
seq = iter([0.0, 1.0])
|
||||||
|
monkeypatch.setattr(wait_module.time, "monotonic", lambda: next(seq))
|
||||||
|
ok = wait_module._wait_running(backend, VmHandle("x"), deadline=0.5, poll=0.001) # type: ignore[arg-type]
|
||||||
|
assert ok is False
|
||||||
|
assert backend.calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_wait_for_ip_returns_none_on_backend_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||||
|
seq = iter([0.0, 1.0])
|
||||||
|
monkeypatch.setattr(wait_module.time, "monotonic", lambda: next(seq))
|
||||||
|
backend = _ErrorBackend()
|
||||||
|
ip = wait_module._wait_for_ip(backend, VmHandle("x"), deadline=0.5, poll=0.001) # type: ignore[arg-type]
|
||||||
|
assert ip is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalise_guest_os_rejects_unknown() -> None:
|
||||||
|
import click
|
||||||
|
with pytest.raises(click.BadParameter):
|
||||||
|
wait_module._normalise_guest_os("freebsd")
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
"""Tests for ci_orchestrator.config."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from ci_orchestrator.config import load_config
|
||||||
|
|
||||||
|
|
||||||
|
def test_defaults_are_os_aware() -> None:
|
||||||
|
cfg = load_config(env={})
|
||||||
|
if os.name == "nt":
|
||||||
|
assert str(cfg.paths.root) == r"F:\CI"
|
||||||
|
else:
|
||||||
|
assert str(cfg.paths.root) == "/var/lib/ci"
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_overrides_paths(tmp_path: Path) -> None:
|
||||||
|
cfg = load_config(
|
||||||
|
env={
|
||||||
|
"CI_ROOT": str(tmp_path / "root"),
|
||||||
|
"CI_TEMPLATES": str(tmp_path / "tpl"),
|
||||||
|
"CI_BUILD_VMS": str(tmp_path / "bvm"),
|
||||||
|
"CI_ARTIFACTS": str(tmp_path / "art"),
|
||||||
|
"CI_KEYS": str(tmp_path / "keys"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
assert cfg.paths.root == tmp_path / "root"
|
||||||
|
assert cfg.paths.templates == tmp_path / "tpl"
|
||||||
|
assert cfg.paths.build_vms == tmp_path / "bvm"
|
||||||
|
assert cfg.paths.artifacts == tmp_path / "art"
|
||||||
|
assert cfg.paths.keys == tmp_path / "keys"
|
||||||
|
|
||||||
|
|
||||||
|
def test_toml_overrides_defaults(tmp_path: Path) -> None:
|
||||||
|
toml = tmp_path / "config.toml"
|
||||||
|
toml.write_text(
|
||||||
|
'[paths]\n'
|
||||||
|
f'root = "{(tmp_path / "altroot").as_posix()}"\n'
|
||||||
|
'[backend]\n'
|
||||||
|
'type = "workstation"\n'
|
||||||
|
'extra = "value"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
cfg = load_config(config_path=toml, env={})
|
||||||
|
assert cfg.paths.root == tmp_path / "altroot"
|
||||||
|
assert cfg.backend.type == "workstation"
|
||||||
|
assert cfg.backend.options == {"extra": "value"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_wins_over_toml(tmp_path: Path) -> None:
|
||||||
|
toml = tmp_path / "config.toml"
|
||||||
|
toml.write_text(
|
||||||
|
f'[paths]\nroot = "{(tmp_path / "fromtoml").as_posix()}"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
cfg = load_config(
|
||||||
|
config_path=toml,
|
||||||
|
env={"CI_ROOT": str(tmp_path / "fromenv")},
|
||||||
|
)
|
||||||
|
assert cfg.paths.root == tmp_path / "fromenv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ci_config_env_loads_toml(tmp_path: Path) -> None:
|
||||||
|
toml = tmp_path / "alt.toml"
|
||||||
|
toml.write_text(
|
||||||
|
f'[paths]\nroot = "{(tmp_path / "rootenv").as_posix()}"\n', encoding="utf-8"
|
||||||
|
)
|
||||||
|
cfg = load_config(env={"CI_CONFIG": str(toml)})
|
||||||
|
assert cfg.paths.root == tmp_path / "rootenv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_config_toml_in_root_is_loaded(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||||
|
) -> None:
|
||||||
|
"""When no config_path/env is given, ``<root>/config.toml`` is autoloaded."""
|
||||||
|
import ci_orchestrator.config as cfg_mod
|
||||||
|
from ci_orchestrator.config import Paths
|
||||||
|
|
||||||
|
fake = Paths(
|
||||||
|
root=tmp_path,
|
||||||
|
templates=tmp_path / "t",
|
||||||
|
build_vms=tmp_path / "b",
|
||||||
|
artifacts=tmp_path / "a",
|
||||||
|
keys=tmp_path / "k",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(cfg_mod, "_platform_defaults", lambda: fake)
|
||||||
|
(tmp_path / "config.toml").write_text(
|
||||||
|
'[backend]\ntype = "workstation"\noption = "auto"\n', encoding="utf-8"
|
||||||
|
)
|
||||||
|
cfg = load_config(env={})
|
||||||
|
assert cfg.backend.options == {"option": "auto"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_toml_provides_ssh_key_path_and_guest_cred_target(tmp_path: Path) -> None:
|
||||||
|
toml = tmp_path / "config.toml"
|
||||||
|
key = tmp_path / "id_ed25519"
|
||||||
|
key.write_text("k")
|
||||||
|
toml.write_text(
|
||||||
|
f'ssh_key_path = "{key.as_posix()}"\n'
|
||||||
|
'guest_cred_target = "MyTarget"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
cfg = load_config(config_path=toml, env={})
|
||||||
|
assert cfg.ssh_key_path == key
|
||||||
|
assert cfg.guest_cred_target == "MyTarget"
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user