diff --git a/.gitea/workflows/build-ns7zip.yml b/.gitea/workflows/build-ns7zip.yml index 55b67ac..cfc5216 100644 --- a/.gitea/workflows/build-ns7zip.yml +++ b/.gitea/workflows/build-ns7zip.yml @@ -27,19 +27,31 @@ jobs: steps: - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: - build-command: 'python build_plugin.py --final --dist-dir dist' + # build_plugin.py auto-detects host/toolset and builds all + # configs by default; --dist forwards to the per-version script + # which copies the DLLs to /dist/ instead of the + # committed plugins/ dir. + build-command: 'python build_plugin.py --dist' artifact-source: 'dist' submodules: 'true' guest-os: ${{ matrix.target == 'linux' && 'Linux' || 'Windows' }} - # repo-url uses the host SSH alias `gitea-ci` (~/.ssh/config), - # which exists only on the host — so the clone MUST run on the - # host, not in the guest. use-git-clone:'false' selects the - # host-side clone + zip transfer transport. - use-git-clone: 'false' - repo-url: 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' + # In-guest clone (default). repo-url must be reachable FROM the + # guest (VMnet8 NAT -> gitea.emulab.it), not the host-only SSH + # alias. ci_orchestrator injects the GiteaPAT credential into + # the HTTPS URL, so GiteaPAT must exist in the LocalSystem + # keyring (Set-CIGuestCredential.ps1 -Target GiteaPAT) — or + # nsis-plugin-ns7zip must be public. + use-git-clone: 'true' + repo-url: 'https://gitea.emulab.it/Simone/nsis-plugin-ns7zip.git' # Cross-repo build: github.ref_name is this repo's branch, not # nsis-plugin-ns7zip's. Pin the target repo's branch. repo-branch: 'main' + # 8 vCPU / 8 GB. WSL on the host builds the 3 configs in + # parallel in ~103s (= one config); the 4-vCPU CI VM thrashes + # (3x make -j on 4 cores) and takes ~3x. Not structural — just + # vCPU starvation. Host is 10c/20t; this matrix = 2 VMs. + guest-cpu: '8' + guest-memory-mb: '8192' # Suffix disambiguates artifact dirs: F:\CI\Artifacts\{run_id}-{attempt}-windows job-id-suffix: '${{ matrix.target }}' artifact-name: 'nsis-plugin-ns7zip-${{ matrix.target }}-${{ github.ref_name }}' @@ -49,7 +61,10 @@ jobs: # Requires a GITEA_TOKEN secret with repo write permissions. release: needs: build - if: github.ref_type == 'tag' + # Tag pushes only. Explicit ${{ }} + event gate: a bare + # `if: github.ref_type == 'tag'` can leave the job stuck "blocked" + # (instead of skipped) on Gitea for workflow_dispatch runs. + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} runs-on: windows-build steps: @@ -104,4 +119,7 @@ jobs: $zipBytes = [System.IO.File]::ReadAllBytes( (Resolve-Path $zipName).ProviderPath) Invoke-RestMethod -Method Post -Uri $uploadUrl ` - -Headers $uploadHeaders -Body $zipBytes | Out-Null \ No newline at end of file + -Headers $uploadHeaders -Body $zipBytes | Out-Null + Write-Host "Uploaded asset: $zipName" + } + Write-Host "Release $env:TAG published with $($release.html_url)" \ No newline at end of file diff --git a/.gitea/workflows/self-test.yml b/.gitea/workflows/self-test.yml index 24aaf5e..70ca413 100644 --- a/.gitea/workflows/self-test.yml +++ b/.gitea/workflows/self-test.yml @@ -4,7 +4,9 @@ # VM clone -> start -> IP detect -> trivial build -> artifact collect -> destroy # # Trigger: workflow_dispatch only (never runs on push/tag). -# Both jobs are independent — a failure in one does not cancel the other. +# Jobs are independent — a failure in one does not cancel the others. +# Covers both source transports: in-guest git clone (default) and the +# host-side clone + zip transfer variant (use-git-clone: 'false'). # # Prerequisites (must be in place before running): # - Runner labels: windows-build:host and linux-build:host active @@ -22,7 +24,7 @@ jobs: smoke-windows: runs-on: windows-build steps: - - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: # Trivial no-op build: write a marker file to the CI output directory. # The Windows build VM maps C:\CI\output as the artifact source. @@ -35,7 +37,7 @@ jobs: smoke-linux: runs-on: linux-build steps: - - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: # Trivial no-op build: write a marker file to the CI output directory. # The Linux build VM uses /opt/ci/output as the artifact source. @@ -43,4 +45,29 @@ jobs: artifact-source: '/opt/ci/output' guest-os: 'Linux' job-id-suffix: 'smoke-linux' - artifact-name: 'smoke-linux-${{ github.run_id }}' \ No newline at end of file + artifact-name: 'smoke-linux-${{ github.run_id }}' + + # ── Host-side clone + zip transfer variant (use-git-clone: 'false') ─────── + smoke-windows-hostclone: + runs-on: windows-build + steps: + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main + with: + build-command: 'New-Item -ItemType Directory -Path C:\CI\output -Force | Out-Null; Set-Content C:\CI\output\smoke.txt "smoke-ok"' + artifact-source: 'C:\CI\output' + guest-os: 'Windows' + use-git-clone: 'false' + job-id-suffix: 'smoke-win-hostclone' + artifact-name: 'smoke-windows-hostclone-${{ github.run_id }}' + + smoke-linux-hostclone: + runs-on: linux-build + steps: + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main + with: + build-command: 'mkdir -p /opt/ci/output && echo smoke-ok > /opt/ci/output/smoke.txt' + artifact-source: '/opt/ci/output' + guest-os: 'Linux' + use-git-clone: 'false' + job-id-suffix: 'smoke-linux-hostclone' + artifact-name: 'smoke-linux-hostclone-${{ github.run_id }}' \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 012647a..3cb8bc7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,6 +51,73 @@ $ErrorActionPreference = 'Stop' --- +## Python development + +L'orchestratore CI vive in `src/ci_orchestrator/` (package `ci-orchestrator`, +entry point `python -m ci_orchestrator`). Tutta la logica applicativa +nuova va scritta in Python, non in PowerShell. Gli script PS in `scripts/` +sono ridotti a shim a 3 righe verso la CLI Python e sono mantenuti solo +per compatibilita' con Task Scheduler e caller esterni. + +| Parametro | Valore | +| ------------------------ | ------------------------------------------------------------------------------------- | +| Python | **3.11+** richiesto (`requires-python` in `pyproject.toml`) | +| Venv produzione | `F:\CI\python\venv\` — install **non-editable**: `pip install .` (act_runner=LocalSystem; `-e` rompe con `No module named ci_orchestrator`). Ri-deploy a ogni modifica codice. | +| Venv dev locale | `.venv\` (NON committare; gitignored) | +| Install (dev locale) | `python -m pip install -e .[dev]` — editable solo nel `.venv` dev | +| Test | `python -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90` | +| Lint | `python -m ruff check src tests\python` | +| Type-check | `python -m mypy --strict src` | + +**Convenzioni Python (enforced da ruff + mypy --strict)**: + +- Type hints obbligatori su funzioni pubbliche e helper esportati. Evitare + `Any` se esiste un tipo concreto/Protocol; `Any` solo per fixture pytest + e mock. +- Niente variabili globali. Lo state vive in oggetti passati esplicitamente + (factory `backends.load_backend(config)`, `Config` dataclass). +- Niente `print()`: usare `click.echo` (rispetta il flag `-q` di click e + il redirect `err=True` per stderr). +- Niente `subprocess.run(..., shell=True)`. Niente shell-out per cose che + esistono come libreria Python (paramiko per SSH, pypsrp per WinRM, + keyring per credenziali). +- Encoding stdio sempre UTF-8: i moduli Python lo assumono e + `runner/config.yaml` esporta `PYTHONIOENCODING=utf-8` ad act_runner. + +**Mappatura script PowerShell → sotto-comando Python** (Fase A2/A3/A4): + +| PowerShell legacy (shim) | Python sub-command | +| ----------------------------------- | ------------------------------------------ | +| `Wait-VMReady.ps1` | `python -m ci_orchestrator wait-ready` | +| `New-BuildVM.ps1` | `python -m ci_orchestrator vm new` | +| `Remove-BuildVM.ps1` | `python -m ci_orchestrator vm remove` | +| `Cleanup-OrphanedBuildVMs.ps1` | `python -m ci_orchestrator vm cleanup` | +| `Invoke-RemoteBuild.ps1` | `python -m ci_orchestrator build run` | +| `Get-BuildArtifacts.ps1` | `python -m ci_orchestrator artifacts collect` | +| `Watch-DiskSpace.ps1` | `python -m ci_orchestrator monitor disk` | +| `Watch-RunnerHealth.ps1` | `python -m ci_orchestrator monitor runner` | +| `Get-CIJobSummary.ps1` | `python -m ci_orchestrator report job` | +| `Invoke-CIJob.ps1` | `python -m ci_orchestrator job` | + +Restano in PowerShell (girano sull'host fuori dal flusso CI o nel guest): +`Setup-Host.ps1`, `Register-CIScheduledTasks.ps1`, `Backup-CITemplate.ps1`, +`Invoke-RetentionPolicy.ps1`, `Measure-CIBenchmark.ps1`, +`Test-CapacityBurnIn.ps1`, tutti i `template/Deploy-*.ps1`, +`template/Prepare-*.ps1`, `template/Install-CIToolchain-*.ps1` e i +`tests/Test-*.ps1` di smoke locale. + +**Test gate**: coverage totale ≥90% (alzato in A5, poi a 90%). Nuovi moduli devono +includere unit test con mock; gli errori frequenti di questa stessa +sezione sono coperti da `tests/python/test_agents_errors.py` come +regression test — non rimuoverli senza aggiornare la voce in +"Errori frequenti da evitare" e motivare la rimozione. + +**Closeout fasi A**: vedi `plans/A1-closeout.md`, `plans/A2-closeout.md`, +`plans/A3-closeout.md`, `plans/A4-closeout.md`, `plans/A5-closeout.md` +per stato per-fase, decisioni prese e voci pendenti a carico utente. + +--- + ## Struttura directory host (fissa) ``` @@ -124,6 +191,11 @@ NON modificare il branch WinRM esistente quando si lavora sul branch Linux e vic - `GITEA_CI_ARTIFACT_DIR` — `F:\CI\Artifacts` - `GITEA_CI_GUEST_CRED_TARGET` — `BuildVMGuest` - `GITEA_CI_SSH_KEY_PATH` — `F:\CI\keys\ci_linux` + - `CI_PYTHON_LAUNCHER` — absolute path al Python di sistema usato per + bootstrap del venv produzione (`F:\CI\python\venv`). act_runner gira come + SYSTEM con PATH sanitizzato (no `python`/`py.exe` risolvibili), quindi i + workflow Python leggono questo env come single source of truth. Aggiornare + solo quando si fa upgrade dell'interprete. --- @@ -151,6 +223,6 @@ Configurazione in `PSScriptAnalyzerSettings.psd1`. Regole principali: 7. **`[ordered]@{}` non disponibile in PS 2** — non e' un problema qui (PS 5.1), ma ricordare che `[ordered]` e' PS 3+. 8. **`ForEach-Object -Parallel`** — disponibile solo in PS 7+; usare `foreach` classico o `Start-Job`. 9. **Snapshot template catturato con VM accesa** — produce file `*.vmem` / `*.vmsn` con memoria; `vmrun clone ... linked -snapshot=` rifiuta con `Error: The virtual machine should not be powered on. It is already running.` anche se `vmrun list` mostra 0 VM. Catturare snapshot SOLO da stato fully powered-off (verificare assenza di `.vmem` nella dir template). -10. **`vmrun getGuestIPAddress` per check "VM running"** — richiede VMware Tools attivi nel guest; in headless Tools puo' impiegare 30-60s a rispondere e il check appare bloccato. Usare `vmrun list` e cercare il path del VMX nell'output. +10. **`vmrun getGuestIPAddress` per check "VM running"** — richiede VMware Tools attivi nel guest; in headless Tools puo' impiegare 30-60s a rispondere e il check appare bloccato. Usare `vmrun list` e cercare il path del VMX nell'output per `is_running`. Per recuperare l'IP usare `getGuestIPAddress -wait` (blocca internamente finché Tools registrano l'IP) e filtrare le righe che iniziano con `"Error:"` perché vmrun in modalità headless può tornare exit code 0 con `"Error: The VMware Tools are not running..."` su stdout invece di usare un exit code non-zero. 11. **Machine-id identico tra clone Linux** — le cloud image Ubuntu hanno un `/etc/machine-id` fisso. VMware DHCP assegna l'IP tramite DHCP client-id derivato da machine-id: se tutti i clone hanno lo stesso machine-id ottengono lo stesso IP e si bloccano per "IP collision". Fix nel template (prima di prendere lo snapshot): `sudo truncate -s 0 /etc/machine-id && sudo rm -f /var/lib/dbus/machine-id && sudo shutdown -h now`, poi ricreare lo snapshot `BaseClean-Linux` da stato powered-off. 12. **`& nativecmd 2>$null` NON sopprime stderr in PS 5.1 con `$ErrorActionPreference='Stop'`** — la stderr del processo nativo viene convertita in ErrorRecord nel pipeline di PowerShell; sotto `'Stop'` questo diventa un errore terminante PRIMA che `2>$null` possa scartarlo. Esempio: `ssh-keygen -R -f ` stampa "Host X not found in file" su stderr → eccezione terminante. Fix: usare `$ErrorActionPreference = 'SilentlyContinue'` attorno alla chiamata oppure `2>&1 | Out-Null`. Per i job CI su VM ephemere locali, usare direttamente `StrictHostKeyChecking=no UserKnownHostsFile=NUL` (default in `_Transport.psm1` con `KnownHostsFile=''`) — nessun `ssh-keygen` necessario. diff --git a/PSScriptAnalyzerSettings.psd1 b/PSScriptAnalyzerSettings.psd1 index fbd7fc3..1fd68a9 100644 --- a/PSScriptAnalyzerSettings.psd1 +++ b/PSScriptAnalyzerSettings.psd1 @@ -3,7 +3,45 @@ ExcludeRules = @( # All CI scripts use Write-Host for structured output captured by act_runner. # Write-Output would interleave with function return values and break callers. - 'PSAvoidUsingWriteHost' + 'PSAvoidUsingWriteHost', + + # Formatting-only rules: generate one warning per line across all existing scripts + # (thousands of hits). Run locally with Invoke-ScriptAnalyzer -Fix to auto-format; + # do not use as CI gate until the whole codebase has been reformatted. + 'PSUseConsistentIndentation', + 'PSUseConsistentWhitespace', + + # The codebase uses the param() + -ArgumentList pattern for remote script blocks + # (Invoke-Command, Start-Job) consistently and correctly. PSSA does not recognise + # param() inside a ScriptBlock as a local declaration, so it fires a false positive + # on every parameter. $Using: is an alternative but not required here. + 'PSUseUsingScopeModifierInNewRunspaces', + + # UTF-8 without BOM is the editor default; adding BOM to ~35 files is cosmetic + # and does not affect PS 5.1 execution. + 'PSUseBOMForUnicodeEncodedFile', + + # Plural nouns in internal helpers (Get-GuestDiagnostics, Remove-OldJobDirs, etc.) + # are intentional — they describe collections, not single objects. + 'PSUseSingularNouns', + + # PSAvoidUsingPlainTextForPassword fires as a false positive on *CredentialTarget params + # (Windows Credential Manager target strings, not passwords). Real plain-text password + # usage in template setup scripts is already caught by PSAvoidUsingConvertToSecureStringWithPlainText + # with targeted SuppressMessage attributes where intentional. + 'PSAvoidUsingPlainTextForPassword', + + # These scripts use [switch]$VerifyArtifact = $true for opt-out patterns. + # Removing the default would silently change script behaviour. + 'PSAvoidDefaultValueSwitchParameter', + + # Stop-Vm is an intentional internal name in a VMware-only CI environment; + # Hyper-V is not installed on the CI host so there is no real cmdlet to shadow. + 'PSAvoidOverwritingBuiltInCmdlets', + + # Advisory rule only. Params may be declared for external callers, future use, + # or accepted-but-ignored forwarding. Not a safety issue. + 'PSReviewUnusedParameter' ) # ── Rule-specific configuration ────────────────────────────────────────────── @@ -13,9 +51,11 @@ Enable = $true } - # All state-changing functions must support -WhatIf / ShouldProcess + # ShouldProcess/WhatIf support is not required for one-shot deployment scripts + # and internal CI helper functions; adding it to all state-changing functions + # would be invasive with no operational benefit in this environment. PSUseShouldProcessForStateChangingFunctions = @{ - Enable = $true + Enable = $false } # Credentials must flow as [PSCredential], not plain strings @@ -23,9 +63,9 @@ Enable = $true } - # No plain-text passwords in params or variables + # Covered by ExcludeRules above (false positives on *CredentialTarget params). PSAvoidUsingPlainTextForPassword = @{ - Enable = $true + Enable = $false } # ConvertTo-SecureString -AsPlainText only acceptable during template setup; diff --git a/README.md b/README.md index ca95e2a..223f881 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Variante **Windows Server 2022**: `*WinBuild2022.ps1` + VMX `F:\CI\Templates\Win │ ├── actions/ │ │ └── local-ci-build/action.yml # Composite action riusabile (wrap Invoke-CIJob.ps1) │ ├── workflows/ -│ │ ├── build-nsInnoUnp.yml # Workflow matrix Windows+Linux per nsis-plugin-nsinnounp +│ │ ├── build-ns7zip.yml # Workflow matrix Windows+Linux per nsis-plugin-ns7zip │ │ └── lint.yml # PSScriptAnalyzer su push/PR (§5.4) │ └── workflow-example.yml # Template generico adattabile │ @@ -146,6 +146,106 @@ Variante **Windows Server 2022**: `*WinBuild2022.ps1` + VMX `F:\CI\Templates\Win --- +## Python orchestrator (Phase A — in progress) + +A new Python 3.11+ rewrite of the orchestrator lives under `src/ci_orchestrator/`. +Phase A (see [implementation-plan-A-B](plans/implementation-plan-A-B.md)) is on +branch `feature/python-rewrite-and-linux-migration`. As of A5 the Python CLI is the +production entry point invoked by `gitea/actions/local-ci-build/action.yml`; +the PowerShell scripts in `scripts/` are 3-line shims that forward to the +Python sub-commands listed below. + +### Setup + +```powershell +# Production venv on the runner host (one-time + on every code deploy). +# NON-editable: act_runner runs as LocalSystem and cannot resolve an +# editable .pth into the dev tree or a transient RunnerWork dir +# (-> "No module named ci_orchestrator"). Re-run `pip install .` after +# each code change. No CI workflow may install into this venv. +python -m venv F:\CI\python\venv +F:\CI\python\venv\Scripts\python.exe -m pip install --upgrade pip +F:\CI\python\venv\Scripts\python.exe -m pip install . + +# Local dev venv (NOT committed; see .gitignore) — editable is fine here +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -e ".[dev]" +``` + +Configuration: copy [config.example.toml](config.example.toml) to +`F:\CI\config.toml` (or set `$env:CI_CONFIG`). Environment variables +(`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`) +override the file. Dependency manifest and tool config (ruff, mypy, +pytest, coverage) live in [pyproject.toml](pyproject.toml). + +### CLI sub-commands + +```powershell +# Discover commands +python -m ci_orchestrator --help + +# Wait for a VM to become reachable (WinRM 5986 or SSH 22 — auto-detected) +python -m ci_orchestrator wait-ready ` + --vmx F:\CI\BuildVMs\smoke\smoke.vmx --guest-os windows --timeout 300 + +# Linked-clone lifecycle +python -m ci_orchestrator vm new ` + --template-path F:\CI\Templates\WinBuild2025\WinBuild2025.vmx ` + --snapshot-name BaseClean ` + --clone-base-dir F:\CI\BuildVMs ` + --job-id smoke-001 +python -m ci_orchestrator vm remove --vmx F:\CI\BuildVMs\Clone_smoke-001_*\Clone_smoke-001_*.vmx +python -m ci_orchestrator vm cleanup --max-age-hours 12 + +# Build + artifact collection +python -m ci_orchestrator build run ` + --vmx F:\CI\BuildVMs\Clone_smoke-001_*\*.vmx ` + --script .\build.ps1 --workdir C:\build --guest-os windows +python -m ci_orchestrator artifacts collect ` + --vmx F:\CI\BuildVMs\Clone_smoke-001_*\*.vmx ` + --remote-path C:\build\dist --local-dir F:\CI\Artifacts\smoke-001 + +# Monitoring (driven by Task Scheduler / systemd timers in B5) +python -m ci_orchestrator monitor disk --min-free-gb 50 --drive-letter F +python -m ci_orchestrator monitor runner --state-dir F:\CI\act_runner + +# Reporting +python -m ci_orchestrator report job --job-id smoke-001 + +# End-to-end orchestrator (replaces Invoke-CIJob.ps1) +python -m ci_orchestrator job ` + --job-id smoke-001 ` + --repo-url ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git ` + --branch main ` + --template-path F:\CI\Templates\WinBuild2025\WinBuild2025.vmx ` + --build-command "python build_plugin.py --final --dist-dir dist" ` + --guest-artifact-source dist +``` + +The PowerShell wrappers (`scripts\Invoke-CIJob.ps1`, +`scripts\New-BuildVM.ps1`, `scripts\Wait-VMReady.ps1`, +`scripts\Remove-BuildVM.ps1`, `scripts\Cleanup-OrphanedBuildVMs.ps1`, +`scripts\Invoke-RemoteBuild.ps1`, `scripts\Get-BuildArtifacts.ps1`, +`scripts\Watch-DiskSpace.ps1`, `scripts\Watch-RunnerHealth.ps1`, +`scripts\Get-CIJobSummary.ps1`) are kept as 3-line shims for Task +Scheduler and external callers — they translate PascalCase parameters +to the Python kebab-case CLI and propagate the exit code. + +### Local validation gates + +```powershell +.\.venv\Scripts\python.exe -m ruff check src tests\python +.\.venv\Scripts\python.exe -m mypy --strict src +.\.venv\Scripts\python.exe -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90 +``` + +Coverage gate is **90%**. Regression tests for the +`AGENTS.md` "Errori frequenti da evitare" #9–#12 live in +[tests/python/test_agents_errors.py](tests/python/test_agents_errors.py) +and must stay green. + +--- + ## Setup rapido ### 1a. Template VM Windows @@ -198,10 +298,12 @@ Dettagli: [docs/LINUX-TEMPLATE-SETUP.md](docs/LINUX-TEMPLATE-SETUP.md). ### 2. Credenziali guest ```powershell -# PowerShell elevato — una volta sola sull'host -Import-Module CredentialManager -New-StoredCredential -Target "BuildVMGuest" -UserName "ci_build" ` - -Password "" -Persist LocalMachine +# PowerShell elevato — una volta sola sull'host. +# Scrive nel vault SYSTEM (act_runner gira come LocalSystem; un +# New-StoredCredential dalla sessione utente NON sarebbe leggibile dal +# runner). Username host-qualificato col computer name del guest +# (bare 'ci_build' -> NTLM SEC_E_UNKNOWN_CREDENTIALS). +.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build' ``` ### 3. act_runner @@ -244,7 +346,7 @@ Vedi [docs/Setup-GiteaSSH.md](docs/Setup-GiteaSSH.md) per configurare l'alias `g ```powershell & 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' ` -JobId 'test-win-001' ` - -RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git' ` + -RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' ` -Branch 'main' ` -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` -Submodules ` @@ -258,7 +360,7 @@ Vedi [docs/Setup-GiteaSSH.md](docs/Setup-GiteaSSH.md) per configurare l'alias `g ```powershell & 'N:\Code\Workspace\Local-CI-CD-System\scripts\Invoke-CIJob.ps1' ` -JobId 'test-linux-001' ` - -RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git' ` + -RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' ` -Branch 'main' ` -TemplatePath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' ` -SnapshotName 'BaseClean-Linux' ` @@ -276,8 +378,8 @@ Aggiungi `-SkipArtifact` per job che non producono output (es. lint, test puri). ## Workflow Gitea -Il file [gitea/workflows/build-nsInnoUnp.yml](gitea/workflows/build-nsInnoUnp.yml) è il workflow -attivo per `Simone/nsis-plugin-nsinnounp`. Si attiva su push di tag `v*` o manualmente. +Il file [gitea/workflows/build-ns7zip.yml](gitea/workflows/build-ns7zip.yml) è il workflow +attivo per `Simone/nsis-plugin-ns7zip`. Si attiva su push di tag `v*` o manualmente. Usa una **strategy matrix** `[windows, linux]` che esegue in parallelo su entrambi i runner. La composite action [gitea/actions/local-ci-build/action.yml](gitea/actions/local-ci-build/action.yml) diff --git a/TODO.md b/TODO.md index 3fb87a7..ced4ee2 100644 --- a/TODO.md +++ b/TODO.md @@ -1,7 +1,7 @@ # TODO — Local CI/CD System - + @@ -56,7 +56,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I - [x] **Gitea Server** — già in esecuzione su `http://10.10.20.11:3100` (e `https://gitea.emulab.it`) - [x] Runner registration token ottenuto e usato - [x] Create at least one test repository with `.gitea/workflows/build.yml` - (`Simone/nsis-plugin-nsinnounp` — testato e2e, workflow in `gitea/workflows/build-nsInnoUnp.yml`) + (`Simone/nsis-plugin-ns7zip` — testato e2e, workflow in `gitea/workflows/build-ns7zip.yml`) - [x] Gitea Actions abilitato (runner registrato con successo) - [x] **act_runner** — installato e registrato sull'host Windows @@ -155,7 +155,7 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I - [x] Test `Invoke-RemoteBuild.ps1` — dotnet restore + build OK, artifacts.zip creato in VM (2026-05-08) - [x] Test `Get-BuildArtifacts.ps1` — artifacts.zip (78 KB) copiato in F:\CI\Artifacts\test-003\ (2026-05-08) - [x] Test `Remove-BuildVM.ps1` — VM spenta + directory rimossa correttamente (2026-05-08) -- [x] Test `Invoke-CIJob.ps1` end-to-end su una soluzione reale (nsis-plugin-nsinnounp) +- [x] Test `Invoke-CIJob.ps1` end-to-end su una soluzione reale (nsis-plugin-ns7zip) - [x] e2e-008: SUCCESS — build OK + artifact raccolto (1632 KB) - [x] e2e-009: SUCCESS — cleanup automatico VM + clone confermato (finally block) - [x] `scripts/Cleanup-OrphanedBuildVMs.ps1` creato e verificato — `-WhatIf`, soglia `-MaxAgeHours`, hard stop + deleteVM + rimozione dir (2026-05-09) @@ -171,13 +171,13 @@ _Last updated: 2026-05-11 (post Sprint 15: §7.5 DONE — Rinomina Setup-* → I ## Gitea Workflow Integration - [x] Copiare `gitea/workflow-example.yml` in `.gitea/workflows/build.yml` nel repo di test - (workflow `build-nsInnoUnp.yml` già presente e funzionante per `nsis-plugin-nsinnounp`) + (workflow `build-ns7zip.yml` già presente e funzionante per `nsis-plugin-ns7zip`) - [x] Push commit e verificare job in Gitea Actions UI - [x] Verificare artifact scaricabile da Gitea dopo build riuscita - [x] `gitea/workflows/lint.yml` creato — PSScriptAnalyzer su push/PR che toccano `.ps1` (2026-05-09) - [x] **[P3] Adattare workflow per altri repository** — generalizzare `BUILD_COMMAND` e `ARTIFACT_SOURCE`. COMPLETATO 2026-05-11: `gitea/workflow-example.yml` riscritto per usare la composite action - `Simone/local-ci-system/.gitea/actions/local-ci-build@main`. Commenti inline guidano + `Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`. Commenti inline guidano la personalizzazione di `build-command`, `artifact-source`, submodules, SSH URL, `extra-guest-env-json` e `artifact-name`. Nessuna logica di orchestrazione nei repo chiamanti. @@ -343,13 +343,13 @@ File: [scripts/Invoke-CIJob.ps1](scripts/Invoke-CIJob.ps1), [scripts/Invoke-Remo - Prerequisiti satisfatti: Git + 7-Zip in template (§6.6, completato) **Beneficio**: Elimina host-zip-transfer overhead (rilevante per repo > 200 MB con submoduli). -**Prossimo**: E2E test con nsis-plugin-nsinnounp (-UseGitClone flag) per misurare impatto. +**Prossimo**: E2E test con nsis-plugin-ns7zip (-UseGitClone flag) per misurare impatto. ### 3.5 [P2] [ ] vCPU/RAM tuning per workload — DEFERRED (home lab) File: [docs/OPTIMIZATION.md:82-89](docs/OPTIMIZATION.md), VMX template. `numvcpus=4`, `memsize=6144` è ragionevole ma non ottimo per ogni build. Per -`nsis-plugin-nsinnounp` (4 build paralleli interni con thread divisi) 4 vCPU sono già saturati. +`nsis-plugin-ns7zip` (4 build paralleli interni con thread divisi) 4 vCPU sono già saturati. Considerare: - VMX pinning: 6 vCPU per build C++ pesanti, 2 vCPU per build .NET piccoli. @@ -488,7 +488,7 @@ jobs: runs-on: windows-build # or linux-build steps: - uses: actions/checkout@v4 - - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main + - uses: Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: build-command: 'python build_plugin.py --final --dist-dir dist' artifact-source: 'dist' @@ -506,14 +506,14 @@ Già supportato lato Gitea, basta replicare il setup. **Deferred**: in un home lab single-host questa funzionalità non serve. Non pianificare. ### 6.4 [P3] [x] Build matrix nel workflow -File: [gitea/workflows/build-nsInnoUnp.yml](gitea/workflows/build-nsInnoUnp.yml). +File: [gitea/workflows/build-ns7zip.yml](gitea/workflows/build-ns7zip.yml). -**Status**: COMPLETATO 2026-05-11 — `build-nsInnoUnp.yml` riscritto con matrix strategy. +**Status**: COMPLETATO 2026-05-11 — `build-ns7zip.yml` riscritto con matrix strategy. Matrix `target: [windows, linux]`, `runs-on: ${{ matrix.target }}-build`, `fail-fast: false`. Usa la composite action con `job-id-suffix: ${{ matrix.target }}` -per artifact dir separate e `repo-url: ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git`. -Artifact separati: `nsis-plugin-nsinnounp-windows-` e `nsis-plugin-nsinnounp-linux-`. +per artifact dir separate e `repo-url: ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git`. +Artifact separati: `nsis-plugin-ns7zip-windows-` e `nsis-plugin-ns7zip-linux-`. ### 6.5 [P3] [x] Secret injection workflow-level @@ -526,7 +526,7 @@ Artifact separati: `nsis-plugin-nsinnounp-windows-` e `nsis-plugin-nsinnoun Usage dal workflow: ```yaml -- uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main +- uses: Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: build-command: 'python sign_and_build.py' extra-guest-env-json: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","CERT_THUMBPRINT":"${{ secrets.CERT_THUMBPRINT }}"}' @@ -549,7 +549,7 @@ build più ampi senza toccare il template. | **Git for Windows** | 2.54.0.windows.1 | Self-clone in VM, submodule fetch, fallback §3.2 | | **7-Zip** | 26.01 | Unpack/zip universale; usato da Step 12 stesso (vcpkg) | | **PowerShell 7.x** | 7.6.1 | Parallel pipelines, operatori moderni, pwsh in CI | -| **NSIS** | 3.12 | Build installer Windows (es. `nsis-plugin-nsinnounp`) | +| **NSIS** | 3.12 | Build installer Windows (es. `nsis-plugin-ns7zip`) | | **gh CLI** | 2.92.0 | Upload artifact a release Gitea/GitHub, comment PR | ### Tier-2 — Step 12 (implementato) @@ -716,7 +716,7 @@ VM avviata → WinRM: git clone --recurse-submodules https://@gitea/... - [x] **Test e2e con `-UseGitClone`** — COMPLETATO 2026-05-10 - [x] Aggiornare snapshot `BaseClean` con Git installato - - [x] Eseguire `e2e-010` con `-UseGitClone` su `nsis-plugin-nsinnounp` (ha submoduli) + - [x] Eseguire `e2e-010` con `-UseGitClone` su `nsis-plugin-ns7zip` (ha submoduli) - [x] Confrontare tempo totale con e2e-009 (senza `-UseGitClone`) — 34s / 25.7% più veloce - [x] Verificare che il PAT non appaia nei log di `Invoke-CIJob.ps1` @@ -820,6 +820,43 @@ da workflow. Attualmente fisso a `numvcpus=4, memsize=6144` — subottimo per bu **Rifiori quando**: Performance profiling mostra contentione CPU/RAM è bottleneck per la workload target. Per home lab, 4 vCPU/6 GB è ragionevole compromesso. +### 8.1 [P3] [ ] Mirror GitHub Actions su Gitea locale (rimuovi dipendenza github.com) +File: [.gitea/workflows/lint.yml](.gitea/workflows/lint.yml), Gitea (org `actions`). + +Dipendenza residua da `github.com`: `actions/checkout@v4` in `lint.yml` (2×) — +il codice dell'action è scaricato da github.com a runtime (con +`[actions] DEFAULT_ACTIONS_URL = github`). La dipendenza per gli **artifact** +è già stata eliminata (upload/download-artifact rimossi; handoff via +filesystem host — vedi commit `refactor(ci): drop upload/download-artifact`). +Questo task è l'**alternativa "fix 1"** non scelta: mirrorare le action +GitHub usate (`checkout`) in una org `actions` sull'istanza Gitea e +referenziarle via URL Gitea completo, eliminando del tutto github.com. + +**Stato attuale**: Deferred. Artifact già github-free (scelta "fix 2" +applicata). Resta solo `actions/checkout@v4` in lint; act_runner la cacha +dopo il primo fetch, quindi outage GitHub impatta solo cold cache. +**Rifiori quando**: Host air-gapped, GitHub non raggiungibile dall'host CI, +o si vuole indipendenza totale da github.com (anche per `checkout`). + +### 8.2 [P3] [ ] Streaming live output build su Windows (WinRM) +File: [src/ci_orchestrator/transport/winrm.py](src/ci_orchestrator/transport/winrm.py), +[src/ci_orchestrator/commands/build.py](src/ci_orchestrator/commands/build.py). + +Linux/SSH ora streamma l'output del build in tempo reale +(`SshTransport.run_streaming` — recv incrementale + `PYTHONUNBUFFERED`). +Windows resta **log-a-fine-job**: `pypsrp.Client.execute_ps` ritorna +solo a comando concluso; `_windows_build` usa `_report_build_output` +(blocco unico) + coda nel messaggio d'errore. + +**Stato attuale**: Deferred. Streaming WinRM richiede API basso livello +RunspacePool/PowerShell con poll di `ps.output`/stream invece di +`execute_ps`, semantica PSRP più complessa; output unbuffered dei tool +nativi (MSBuild) scomodo. Diagnostica già adeguata (output completo a +fine job + ultime righe nell'errore). +**Rifiori quando**: Build Windows lunghi rendono l'assenza di progress +live un problema operativo reale, o si standardizza lo streaming +cross-OS. + --- ## Suggerimento di sequencing diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..acd4f72 --- /dev/null +++ b/config.example.toml @@ -0,0 +1,34 @@ +# ci-orchestrator config example. +# +# Copy to $CI_ROOT/config.toml or set $CI_CONFIG to the file path. +# Environment variables (CI_ROOT, CI_TEMPLATES, CI_BUILD_VMS, CI_ARTIFACTS, +# CI_KEYS, CI_VMRUN_PATH, CI_SSH_KEY_PATH, CI_GUEST_CRED_TARGET) override +# anything set here. +# +# IMPORTANT (TOML rule): top-level keys must appear BEFORE any [section] header. +# Keys placed after a [section] belong to that section, not to the root table. + +# Top-level orchestrator settings. +vmrun_path = "C:/Program Files (x86)/VMware/VMware Workstation/vmrun.exe" +ssh_key_path = "F:/CI/keys/ci_linux" +guest_cred_target = "BuildVMGuest" + +# --------------------------------------------------------------- Windows host +[paths] +root = "F:/CI" +templates = "F:/CI/Templates" +build_vms = "F:/CI/BuildVMs" +artifacts = "F:/CI/Artifacts" +keys = "F:/CI/keys" + +# Phase B (Linux host) — uncomment after migration. +# [paths] +# root = "/var/lib/ci" +# templates = "/var/lib/ci/templates" +# build_vms = "/var/lib/ci/build-vms" +# artifacts = "/var/lib/ci/artifacts" +# keys = "/etc/ci/keys" + +# Phase C hook: backend selector. Only "workstation" is implemented in Phase A. +[backend] +type = "workstation" diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md new file mode 100644 index 0000000..3c6cf46 --- /dev/null +++ b/deploy/systemd/README.md @@ -0,0 +1,105 @@ +# systemd units — Local CI/CD System (Linux host) + +Coppie `*.service` + `*.timer` che replicano i task periodici registrati su +Windows da `scripts/Register-CIScheduledTasks.ps1`. Pensate per essere +installate su un host Linux Mint / Ubuntu LTS, con il package +`ci_orchestrator` installato in `/opt/ci/venv/`. + +## Mapping Windows Task Scheduler → systemd timer + +| Windows Task Scheduler | systemd unit | Cadenza | Comando | +| ----------------------- | ---------------------------------- | ----------------------- | --------------------------------------------------------------- | +| `CI-CleanupOrphans` | `ci-cleanup-orphans.timer` | every 6h + at boot | `python -m ci_orchestrator vm cleanup --max-age-hours 6` | +| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `pwsh Invoke-RetentionPolicy.ps1` (vedi nota PowerShell) | +| `CI-DiskSpaceAlert` | `ci-watch-disk-space.timer` | every 15 min | `python -m ci_orchestrator monitor disk` | +| `CI-RunnerHealth` | `ci-watch-runner-health.timer` | every 15 min | `python -m ci_orchestrator monitor runner` | +| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `pwsh Backup-CITemplate.ps1` (vedi nota PowerShell) | + +## Nota PowerShell su Linux + +Due unit (`ci-retention-policy`, `ci-backup-template`) invocano script +PowerShell che, per scelta documentata in `AGENTS.md` ("Mappatura script +PowerShell → Python"), restano in PowerShell. Su Linux questo richiede +**PowerShell Core (`pwsh`)**: + +```bash +# Ubuntu / Mint LTS +sudo apt-get update +sudo apt-get install -y wget apt-transport-https software-properties-common +wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb +sudo apt-get update +sudo apt-get install -y powershell +pwsh --version # verifica +``` + +Se preferisci non installare `pwsh`, porta `Invoke-RetentionPolicy.ps1` e +`Backup-CITemplate.ps1` come sub-comandi `python -m ci_orchestrator +retention run` / `template backup` (out of scope B5 per il piano corrente +— vedi `plans/implementation-plan-A-B.md`). + +## Prerequisiti + +1. Utente di sistema `ci-runner` con UID dedicato (creato in B1). +2. Venv Python attivo in `/opt/ci/venv/` con `ci_orchestrator` installato: + ```bash + sudo -u ci-runner /opt/ci/venv/bin/pip install -e /opt/ci/local-ci-cd-system + ``` +3. Variabili d'ambiente `CI_*` definite in `/etc/ci/environment` (caricato + tramite `EnvironmentFile=` in ogni unit `.service`): + ``` + CI_ROOT=/var/lib/ci + CI_TEMPLATES=/var/lib/ci/templates + CI_BUILD_VMS=/var/lib/ci/build-vms + CI_ARTIFACTS=/var/lib/ci/artifacts + CI_KEYS=/etc/ci/keys + PYTHONIOENCODING=utf-8 + ``` +4. `Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` presenti in + `/opt/ci/local-ci-cd-system/scripts/` (clone del repo). + +## Installazione + +```bash +# 1. Copia le unit in /etc/systemd/system/ +sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/ + +# 2. Reload systemd +sudo systemctl daemon-reload + +# 3. Abilita + avvia tutti i timer +for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \ + ci-watch-runner-health ci-backup-template; do + sudo systemctl enable --now "${t}.timer" +done + +# 4. Verifica +systemctl list-timers --all 'ci-*' +``` + +## Test e troubleshooting + +```bash +# Trigger manuale di un singolo task (esegue il .service una volta) +sudo systemctl start ci-cleanup-orphans.service + +# Log dell'ultima esecuzione +journalctl -u ci-cleanup-orphans.service -n 50 --no-pager + +# Live tail di tutti i task CI +journalctl -u 'ci-*' -f +``` + +## Rollback + +```bash +for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \ + ci-watch-runner-health ci-backup-template; do + sudo systemctl disable --now "${t}.timer" + sudo rm -f "/etc/systemd/system/${t}.service" "/etc/systemd/system/${t}.timer" +done +sudo systemctl daemon-reload +``` + +A questo punto il fallback è ri-attivare i task su Windows con +`scripts/Register-CIScheduledTasks.ps1`. diff --git a/deploy/systemd/ci-backup-template.service b/deploy/systemd/ci-backup-template.service new file mode 100644 index 0000000..0a339f0 --- /dev/null +++ b/deploy/systemd/ci-backup-template.service @@ -0,0 +1,26 @@ +[Unit] +Description=Local CI/CD — weekly backup of VMware templates +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md +After=network-online.target + +[Service] +Type=oneshot +User=ci-runner +Group=ci-runner +EnvironmentFile=/etc/ci/environment +WorkingDirectory=/opt/ci/local-ci-cd-system +# NOTE: requires PowerShell Core (`pwsh`) on the Linux host. +# See deploy/systemd/README.md "Nota PowerShell su Linux". +ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Backup-CITemplate.ps1 +TimeoutStartSec=4h +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +Nice=19 +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true +ReadWritePaths=/var/lib/ci /var/backups/ci + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/ci-backup-template.timer b/deploy/systemd/ci-backup-template.timer new file mode 100644 index 0000000..cec55d4 --- /dev/null +++ b/deploy/systemd/ci-backup-template.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Local CI/CD — weekly timer for ci-backup-template.service +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md + +[Timer] +# Sunday 02:00 + random delay 1h to avoid clashing with other CI tasks. +OnCalendar=Sun *-*-* 02:00:00 +RandomizedDelaySec=1h +Persistent=true +Unit=ci-backup-template.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/ci-cleanup-orphans.service b/deploy/systemd/ci-cleanup-orphans.service new file mode 100644 index 0000000..ab9a2fb --- /dev/null +++ b/deploy/systemd/ci-cleanup-orphans.service @@ -0,0 +1,23 @@ +[Unit] +Description=Local CI/CD — destroy orphaned build VMs (max age 6h) +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=ci-runner +Group=ci-runner +EnvironmentFile=/etc/ci/environment +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --max-age-hours 6 +TimeoutStartSec=1h +Nice=10 +# Hardening +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true +ReadWritePaths=/var/lib/ci /var/log/ci + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/ci-cleanup-orphans.timer b/deploy/systemd/ci-cleanup-orphans.timer new file mode 100644 index 0000000..010733c --- /dev/null +++ b/deploy/systemd/ci-cleanup-orphans.timer @@ -0,0 +1,16 @@ +[Unit] +Description=Local CI/CD — periodic timer for ci-cleanup-orphans.service +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md + +[Timer] +# Equivalent to Windows Task Scheduler: +# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 6h +# New-ScheduledTaskTrigger -AtStartup +OnBootSec=2min +OnUnitActiveSec=6h +Persistent=true +AccuracySec=1min +Unit=ci-cleanup-orphans.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/ci-retention-policy.service b/deploy/systemd/ci-retention-policy.service new file mode 100644 index 0000000..465ed9e --- /dev/null +++ b/deploy/systemd/ci-retention-policy.service @@ -0,0 +1,24 @@ +[Unit] +Description=Local CI/CD — purge old artifacts/logs/IP leases (retention policy) +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md +After=network-online.target + +[Service] +Type=oneshot +User=ci-runner +Group=ci-runner +EnvironmentFile=/etc/ci/environment +WorkingDirectory=/opt/ci/local-ci-cd-system +# NOTE: requires PowerShell Core (`pwsh`) on the Linux host. +# See deploy/systemd/README.md "Nota PowerShell su Linux". +ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Invoke-RetentionPolicy.ps1 +TimeoutStartSec=1h +Nice=15 +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true +ReadWritePaths=/var/lib/ci /var/log/ci + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/ci-retention-policy.timer b/deploy/systemd/ci-retention-policy.timer new file mode 100644 index 0000000..146a0ad --- /dev/null +++ b/deploy/systemd/ci-retention-policy.timer @@ -0,0 +1,14 @@ +[Unit] +Description=Local CI/CD — daily timer for ci-retention-policy.service +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md + +[Timer] +# Equivalent to Windows Task Scheduler: +# New-ScheduledTaskTrigger -Daily -At '03:00' -RandomDelay 30min +OnCalendar=*-*-* 03:00:00 +RandomizedDelaySec=30min +Persistent=true +Unit=ci-retention-policy.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/ci-watch-disk-space.service b/deploy/systemd/ci-watch-disk-space.service new file mode 100644 index 0000000..fc5677e --- /dev/null +++ b/deploy/systemd/ci-watch-disk-space.service @@ -0,0 +1,20 @@ +[Unit] +Description=Local CI/CD — disk space monitor (alert via systemd journal) +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md +After=network-online.target + +[Service] +Type=oneshot +User=ci-runner +Group=ci-runner +EnvironmentFile=/etc/ci/environment +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor disk +TimeoutStartSec=5min +Nice=15 +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/ci-watch-disk-space.timer b/deploy/systemd/ci-watch-disk-space.timer new file mode 100644 index 0000000..24b8af9 --- /dev/null +++ b/deploy/systemd/ci-watch-disk-space.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Local CI/CD — periodic timer for ci-watch-disk-space.service +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md + +[Timer] +# Equivalent to Windows Task Scheduler: +# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 15min +OnBootSec=5min +OnUnitActiveSec=15min +Persistent=true +AccuracySec=30s +Unit=ci-watch-disk-space.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/ci-watch-runner-health.service b/deploy/systemd/ci-watch-runner-health.service new file mode 100644 index 0000000..dd47b90 --- /dev/null +++ b/deploy/systemd/ci-watch-runner-health.service @@ -0,0 +1,24 @@ +[Unit] +Description=Local CI/CD — auto-restart act_runner if stopped (rate-limited) +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md +After=network-online.target act-runner.service + +[Service] +Type=oneshot +User=ci-runner +Group=ci-runner +EnvironmentFile=/etc/ci/environment +# `monitor runner` enforces internal rate-limit: max 3 restarts/h +# (matches Windows CI-RunnerHealth scheduled task semantics). +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner +TimeoutStartSec=5min +Nice=15 +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +# NoNewPrivileges intentionally NOT set: monitor may need to call +# `systemctl restart act-runner.service` via polkit. +ReadWritePaths=/var/lib/ci /var/log/ci + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/ci-watch-runner-health.timer b/deploy/systemd/ci-watch-runner-health.timer new file mode 100644 index 0000000..fb90aed --- /dev/null +++ b/deploy/systemd/ci-watch-runner-health.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Local CI/CD — periodic timer for ci-watch-runner-health.service +Documentation=file:///opt/ci/local-ci-cd-system/deploy/systemd/README.md + +[Timer] +# Equivalent to Windows Task Scheduler: +# New-ScheduledTaskTrigger -Once -At '00:00' -RepetitionInterval 15min +OnBootSec=3min +OnUnitActiveSec=15min +Persistent=true +AccuracySec=30s +Unit=ci-watch-runner-health.service + +[Install] +WantedBy=timers.target diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2b8d7d1..fefebb7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -206,3 +206,116 @@ VMs have internet access via NAT — required for pip/nuget/apt during build. - Source code is transferred via zip (Compress-Archive → WinRM copy → Expand-Archive in guest) by default — no PAT is ever present inside the VM; VMs have NAT internet for pip/nuget at build time. - With `-UseGitClone` (§3.3, opt-in) il clone avviene dentro la VM via SSH alias `gitea-ci` (forwardato dall'host); la chiave privata SSH dell'host non viene esposta al guest — vedi BEST-PRACTICES.md per il threat model. - Linux build VMs usano la chiave SSH `F:\CI\keys\ci_linux` (ed25519, no passphrase), archiviata solo sull'host. Il guest non conosce credenziali WinRM. + +--- + +## Python orchestrator (Phase A) + +Phase A of [implementation-plan-A-B](../plans/implementation-plan-A-B.md) +replaces the PowerShell orchestration scripts with a Python 3.11+ package +under `src/ci_orchestrator/`. The PS scripts in `scripts/` are now thin +3-line shims that forward to the Python CLI — the diagram above still +applies, except that the boxes labelled `*.ps1` are entry-point names, +not the implementation. + +### Package layout + +``` +src/ci_orchestrator/ +├── __init__.py +├── __main__.py # click entry point: `python -m ci_orchestrator` +├── config.py # Config dataclass + env/TOML loader (OS-aware paths) +├── credentials.py # CredentialStore Protocol + KeyringCredentialStore +│ +├── backends/ +│ ├── __init__.py # load_backend(config) factory (selector for Phase C) +│ ├── protocol.py # VmBackend Protocol + VmHandle dataclass + VmState enum +│ ├── workstation.py # WorkstationVmrunBackend (subprocess + vmrun -T ws) +│ └── errors.py # BackendError / BackendNotAvailable / BackendOperationFailed +│ +├── transport/ +│ ├── __init__.py +│ ├── winrm.py # WinRM transport via pypsrp (replaces New-PSSession) +│ ├── ssh.py # SSH/SFTP transport via paramiko (replaces ssh.exe/scp.exe) +│ └── errors.py # TransportConnectError / TransportCommandError +│ +└── commands/ # One module per `click` sub-command group + ├── wait.py # `wait-ready` + ├── vm.py # `vm new` / `vm remove` / `vm cleanup` + ├── build.py # `build run` + ├── artifacts.py # `artifacts collect` + ├── monitor.py # `monitor disk` / `monitor runner` + ├── report.py # `report job` + └── job.py # `job` — full end-to-end orchestrator +``` + +### `VmBackend` Protocol and the `load_backend(config)` factory + +`backends/protocol.py` defines a hypervisor-agnostic Protocol: + +```python +class VmBackend(Protocol): + def clone_linked(self, template, snapshot, name, destination=None) -> VmHandle: ... + def start(self, handle, headless=True) -> None: ... + def stop(self, handle, hard=False) -> None: ... + def delete(self, handle) -> None: ... + def get_ip(self, handle, timeout=0.0) -> str | None: ... + def list_snapshots(self, handle) -> list[str]: ... + def is_running(self, handle) -> bool: ... +``` + +Names are deliberately neutral (no `vmrun_*` prefixes) and identifiers are +opaque strings (`VmHandle.identifier`): a VMX path on Workstation, a +managed object reference on ESXi. `is_running` MUST NOT use guest tools +(see `AGENTS.md` error #10). + +Backends are constructed via `backends.load_backend(config)`. Application +code (most importantly `commands/job.py`) imports the factory only — never +`WorkstationVmrunBackend` directly. The factory reads +`config.backend.type` from `config.toml` (default: `"workstation"`). + +### Phase C extension point — adding ESXi without forks + +To add an ESXi backend in a future Phase C: + +1. Add a new module `backends/esxi.py` implementing the `VmBackend` + Protocol via `pyVmomi`. `VmHandle.identifier` becomes the managed + object reference; `clone_linked` becomes a `CloneVM_Task` call; + `start` becomes `PowerOnVM_Task`. +2. Register it in `backends/__init__.py` `load_backend(config)` under a + new `config.backend.type == "esxi"` branch. +3. Add `[backend]` keys to `config.example.toml` (host, user, datacenter, + datastore, …). The `Config` dataclass already namespaces backend + options under `config.backend`. +4. Transport layers (`winrm.py`, `ssh.py`) and the `commands/*` modules + need no change — they consume `VmBackend` and `VmHandle` only. + +The single non-trivial constraint is that `commands/vm.py` `vm new` +currently builds the destination VMX path under `--clone-base-dir` (a +host-local concept). For ESXi this becomes a datastore folder; the +parameter rename can be deferred to Phase C as long as the backend +ignores the value and uses `config.backend.datastore` instead. + +### Transport selection + +`commands/job.py` auto-detects the guest OS by parsing `guestOS = "..."` +in the cloned VMX (`ubuntu-*` → Linux/SSH, anything else → Windows/WinRM). +The Linux branch is wired to `transport/ssh.py` (paramiko) and the +Windows branch to `transport/winrm.py` (pypsrp). Both transports raise +typed errors (`TransportConnectError`, `TransportCommandError`) — never +bare exceptions. + +### Validation gates (CI) + +Enforced by `gitea/workflows/lint.yml` job `python`: + +```powershell +python -m ruff check src tests\python +python -m mypy --strict src +python -m pytest tests\python -q --cov=ci_orchestrator --cov-fail-under=90 +``` + +Coverage gate is **90%**. Regression tests for +`AGENTS.md` errors #9–#12 live in +[tests/python/test_agents_errors.py](../tests/python/test_agents_errors.py). + diff --git a/docs/BEST-PRACTICES.md b/docs/BEST-PRACTICES.md index ca58131..737c768 100644 --- a/docs/BEST-PRACTICES.md +++ b/docs/BEST-PRACTICES.md @@ -4,29 +4,40 @@ ### Do NOT store credentials in scripts or config files -The scripts use `-GuestCredentialTarget` (a Windows Credential Manager target name) -rather than plaintext username/password parameters. Store credentials once: +The guest VM credential is referenced by target name (`BuildVMGuest`, +`GITEA_CI_GUEST_CRED_TARGET`) and read by the Python orchestrator via the +`keyring` library — never as plaintext parameters. + +> **Critical: store it in the LocalSystem vault, not your user vault.** +> act_runner runs as the `LocalSystem` service account. Windows Credential +> Manager / keyring vaults are **per-user**. A credential added with +> `cmdkey` or the Credential Manager UI from an interactive admin session +> lands in *your* vault and is invisible to the runner, which then fails +> with `Credential 'BuildVMGuest' not found in keyring`. + +> **Username must be host-qualified.** The guest is a workgroup machine; +> NTLM rejects a bare `ci_build` with `SEC_E_UNKNOWN_CREDENTIALS`. Store +> it as `WINBUILD-2025\ci_build` (the guest computer name, i.e. the WinRM +> TLS certificate CN). The WinRM transport forces `auth='ntlm'` for the +> same reason (Negotiate→Kerberos is meaningless without a domain). + +Store (or rotate) the credential with the helper, which writes into the +SYSTEM vault via the production venv's `keyring` (run elevated): ```powershell -# Run on host (once, before first CI job) -cmdkey /generic:BuildVMGuest /user:MACHINENAME\ci_build /pass:YourStrongPassword +.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build' ``` -Retrieve in scripts via the `CredentialManager` PowerShell module: - -```powershell -Install-Module CredentialManager -Scope CurrentUser -$cred = Get-StoredCredential -Target 'BuildVMGuest' -``` +It prompts for the password securely, writes it to the SYSTEM vault, and +verifies the read-back as SYSTEM. Diagnose WinRM reachability/auth with +`.\scripts\Test-CIGuestWinRM.ps1 -IpAddress `. ### Rotate credentials quarterly -1. Update password in the template VM (requires rebuilding `BaseClean` snapshot) -2. Update Windows Credential Manager on the host: - ``` - cmdkey /generic:BuildVMGuest /user:MACHINENAME\ci_build /pass:NewPassword - ``` -3. No script changes required — they reference the target name, not the password. +1. Update the password in the template VM (rebuild the `BaseClean` snapshot). +2. Re-run `Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'` + with the new password. +3. No code changes required — the orchestrator references the target name. --- diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 544b1a9..c9a4bf0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -56,7 +56,7 @@ docs/TEST-PLAN-v1.3-to-HEAD.md, gitea/actions/local-ci-build/action.yml. ## 2026-05-11 — Sprint 14: §6.4 Build matrix + §6.5 Secret injection -**§6.4** — `build-nsInnoUnp.yml` rewritten with matrix strategy `target: [windows, linux]`. +**§6.4** — `build-ns7zip.yml` rewritten with matrix strategy `target: [windows, linux]`. Separate artifact dirs; `fail-fast: false`; composite action used for both targets. **§6.5** — `ExtraGuestEnv [hashtable]` added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`: @@ -127,7 +127,7 @@ disables wuauserv/UsoSvc permanently. `-UseGitClone` switch added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`. - Skips host git clone + zip + WinRM transfer overhead. - PAT injected at runtime, never persisted in snapshot. -- E2E result: 34 s saving, 25.7% reduction vs baseline on nsis-plugin-nsinnounp. +- E2E result: 34 s saving, 25.7% reduction vs baseline on nsis-plugin-ns7zip. --- diff --git a/docs/HOST-SETUP.md b/docs/HOST-SETUP.md index 1a73518..1d65fc4 100644 --- a/docs/HOST-SETUP.md +++ b/docs/HOST-SETUP.md @@ -38,6 +38,7 @@ F:\CI\ ├── keys\ │ ├── ci_linux # chiave privata SSH per Linux VMs (ed25519, no passphrase) │ └── ci_linux.pub # chiave pubblica corrispondente +├── config.toml # configurazione Python orchestrator (ssh_key_path, ecc.) ├── act_runner\ │ ├── act_runner.exe # binario runner Gitea │ ├── config.yaml # config runner (path, label, capacity) @@ -90,6 +91,32 @@ Apri PowerShell **come Administrator**, poi: - Avvia servizio (`SERVICE_AUTO_START`) 7. **Configura SSH alias `gitea-ci`** in `~/.ssh/config` (HostName, Port, User git) +### Note operative (scoperte in validazione end-to-end) + +Setup-Host.ps1 da solo **non basta** per la pipeline Python su runner +LocalSystem. Dopo Setup-Host: + +- **Credenziale guest nel vault SYSTEM, username host-qualificato.** Lo + step 4 scrive in Windows Credential Manager dell'utente interattivo → + invisibile ad act_runner (LocalSystem). Esegui (elevato): + `\.\scripts\Set-CIGuestCredential.ps1 -UserName 'WINBUILD-2025\ci_build'`. + Bare `ci_build` viene rifiutato da NTLM (`SEC_E_UNKNOWN_CREDENTIALS`): + usa il computer name del guest (= CN del cert WinRM). Diagnostica: + `\.\scripts\Test-CIGuestWinRM.ps1 -IpAddress `. +- **venv di produzione = install non-editable.** Popola + `F:\CI\python\venv` con `python -m pip install .` dal repo reale — + **mai** `pip install -e .` (il `.pth` punta a un path che LocalSystem + non risolve → `No module named ci_orchestrator`). Vedi + `plans/PhaseA-user-checklist.md` Passo 1. Nessun workflow CI deve + installare in quel venv (lint usa un venv effimero per-job). +- **Gitea `app.ini`:** imposta `[actions] DEFAULT_ACTIONS_URL = github` + e referenzia l'action col **URL completo** + (`https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`) + così le action GitHub (checkout, upload-artifact) restano risolvibili + e l'action locale punta all'istanza. Il repo `local-ci-cd-system` deve + essere **pubblico** (il runner clona l'action senza autenticazione) e + l'action deve esistere su `@main` (il ref che il runner clona). + --- ## Parametri principali @@ -139,6 +166,22 @@ Allo `Setup-Host.ps1` exit 0: ``` Poi **provisiona il template VM Linux** — vedi [LINUX-TEMPLATE-SETUP.md](LINUX-TEMPLATE-SETUP.md) +6. **Crea `F:\CI\config.toml`** — necessario affinché il Python orchestrator trovi la chiave SSH e le altre impostazioni: + ```powershell + Copy-Item N:\Code\Workspace\Local-CI-CD-System\config.example.toml F:\CI\config.toml + ``` + Il file `config.example.toml` contiene già i path corretti per Windows (`F:\CI\...`). Modificare solo se si usa un `$CIRoot` diverso da `F:\CI`. + + Valori chiave configurati: + + | Chiave | Valore default | Scopo | + | -------------------- | ----------------------- | -------------------------------------------- | + | `ssh_key_path` | `F:/CI/keys/ci_linux` | Chiave privata per SSH verso Linux guest | + | `guest_cred_target` | `BuildVMGuest` | Target Windows Credential Manager (WinRM) | + | `backend.type` | `workstation` | Backend VMware (solo `workstation` in Phase A) | + + Le variabili d'ambiente `CI_SSH_KEY_PATH`, `CI_VMRUN_PATH`, `CI_GUEST_CRED_TARGET` sovrascrivono i valori del TOML se presenti. + 6. **Registra chiave SSH con Gitea**: - Gitea UI → User Settings → SSH Keys → Add Key - Chiave pubblica: `%USERPROFILE%\.ssh\id_rsa.pub` @@ -162,6 +205,7 @@ Allo `Setup-Host.ps1` exit 0: | Snapshot name (Windows) | `BaseClean` | | Snapshot name (Linux) | `BaseClean-Linux` | | SSH key (Linux VMs) | `F:\CI\keys\ci_linux` (privata), `F:\CI\keys\ci_linux.pub` | +| Orchestrator config TOML | `F:\CI\config.toml` (copia di `config.example.toml`) | | Clone base dir | `F:\CI\BuildVMs\` | | Artifact dir | `F:\CI\Artifacts\` | | Log dir | `F:\CI\Logs\` (retention: 30 giorni) | @@ -196,3 +240,4 @@ Task schedulati (vedi `scripts/Register-CIScheduledTasks.ps1` per il bootstrap): | Service `act_runner` non parte | Check `F:\CI\act_runner\logs\stderr.log` | | `New-StoredCredential` fail | Verifica modulo CredentialManager importato + scope corretto | | Job CI fallisce su `Test-WSMan` | TrustedHosts host non include subnet VM, AllowUnencrypted=$false | +| `wait-ready` SSH: `No authentication methods available` | `F:\CI\config.toml` mancante o `ssh_key_path` non configurato. Crea il file (vedi step 6) oppure passa `--ssh-key-path F:\CI\keys\ci_linux` | diff --git a/docs/Setup-GiteaSSH.md b/docs/Setup-GiteaSSH.md index 34f08fc..5a97bf4 100644 --- a/docs/Setup-GiteaSSH.md +++ b/docs/Setup-GiteaSSH.md @@ -1,6 +1,6 @@ # Configurazione accesso SSH a Gitea -**Target**: clone da `git@gitea.emulab.it:Simone/nsis-plugin-nsinnounp.git` +**Target**: clone da `git@gitea.emulab.it:Simone/nsis-plugin-ns7zip.git` **Gitea**: `http://10.10.20.11:3100` / `https://gitea.emulab.it` --- @@ -83,13 +83,13 @@ Hi there, Simone! You've successfully authenticated with the key named ci-build@ Usare l'alias `gitea-ci` definito in `~\.ssh\config`: ```powershell --RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-nsinnounp.git' +-RepoUrl 'ssh://gitea-ci/Simone/nsis-plugin-ns7zip.git' ``` Oppure con IP/porta espliciti (equivalente): ```powershell --RepoUrl 'ssh://git@10.10.20.11:2222/Simone/nsis-plugin-nsinnounp.git' +-RepoUrl 'ssh://git@10.10.20.11:2222/Simone/nsis-plugin-ns7zip.git' ``` --- diff --git a/docs/WORKFLOW-AUTHORING.md b/docs/WORKFLOW-AUTHORING.md index 0680fef..7432b3b 100644 --- a/docs/WORKFLOW-AUTHORING.md +++ b/docs/WORKFLOW-AUTHORING.md @@ -254,7 +254,7 @@ jobs: - name: Upload artifacts if: success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-${{ github.ref_name }} path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip @@ -262,7 +262,7 @@ jobs: - name: Upload logs on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-logs-${{ github.run_id }} path: F:\CI\Artifacts\${{ github.run_id }}\*.log @@ -308,7 +308,7 @@ jobs: - name: Upload artifacts if: success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-linux-${{ github.ref_name }} path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip @@ -359,7 +359,7 @@ jobs: - name: Upload artifacts if: success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-${{ github.ref_name }} path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip @@ -404,7 +404,7 @@ jobs: - name: Upload artifacts if: success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-linux-${{ github.ref_name }} path: F:\CI\Artifacts\${{ github.run_id }}\artifacts.zip @@ -502,7 +502,7 @@ jobs: - name: Upload artifacts if: success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-${{ matrix.config }}-${{ github.ref_name }} path: F:\CI\Artifacts\${{ github.run_id }}-${{ matrix.config }}\artifacts.zip @@ -573,3 +573,6 @@ the cleanup task `Cleanup-OrphanedBuildVMs.ps1` handles these cases. | `-TemplatePath` for Linux without `GITEA_CI_LINUX_TEMPLATE_PATH` | Pass `$env:GITEA_CI_LINUX_TEMPLATE_PATH` or the full path explicitly | | Forgetting `-SnapshotName 'BaseClean-Linux'` for Linux | Linux template uses a different snapshot name than Windows (`BaseClean`) | | Same `-JobId` for matrix jobs | Append matrix key: `${{ github.run_id }}-${{ matrix.config }}` | +| `actions/upload-artifact@v4` / `download-artifact@v4` | Use `@v3`. Gitea reports as GHES; @v4 (`@actions/artifact` v2 / Twirp API) fails with `GHESNotSupportedError`. | +| Short action ref `Simone/local-ci-cd-system/...@main` | Use the full URL `https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main`; with `[actions] DEFAULT_ACTIONS_URL = github` in Gitea's app.ini this keeps GitHub actions (checkout, upload-artifact) resolvable while pointing the composite action at the local instance. The action repo must be on `@main` (the ref the runner clones). | +| Composite action referenced from a private/internal repo | The runner clones the action repo unauthenticated. Make `local-ci-cd-system` public, or the clone fails with `authentication required: Unauthorized`. | diff --git a/gitea/actions/local-ci-build/action.yml b/gitea/actions/local-ci-build/action.yml deleted file mode 100644 index 6f5f524..0000000 --- a/gitea/actions/local-ci-build/action.yml +++ /dev/null @@ -1,301 +0,0 @@ -# .gitea/actions/local-ci-build/action.yml -# -# Reusable composite action — Local CI/CD System -# -# Wraps Invoke-CIJob.ps1 (ephemeral VMware VM pipeline) into a single callable -# step, usable from any Gitea repository without duplicating the orchestration -# logic. -# -# Stored in: Simone/local-ci-system / .gitea/actions/local-ci-build/action.yml -# -# Usage from a calling repo's workflow: -# -# jobs: -# build: -# runs-on: windows-build # or linux-build -# steps: -# - uses: actions/checkout@v4 -# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main -# with: -# build-command: 'python build_plugin.py --final --dist-dir dist' -# artifact-source: 'dist' -# submodules: 'true' -# -# Runner label determines the guest OS template: -# windows-build -> WinBuild2025 (WinRM/HTTPS transport) -# linux-build -> LinuxBuild2404 (SSH transport) - -name: Local CI Build -description: > - Builds a project using an ephemeral VMware VM managed by Invoke-CIJob.ps1. - Handles the full VM lifecycle: clone template -> start VM -> run build -> - collect artifacts -> destroy VM. - Compatible with windows-build (WinRM) and linux-build (SSH) runners. - -inputs: - build-command: - description: > - Shell command to execute inside the build VM. - Example: 'python build_plugin.py --final --dist-dir dist' - required: true - - artifact-source: - description: > - Path inside VM where build outputs are located, relative to the CI work - directory. Collected to F:\CI\Artifacts\{JobId}\ on the host. - required: false - default: 'dist' - - submodules: - description: > - Set to "true" to clone the repository with --recurse-submodules. - required: false - default: 'false' - - guest-os: - description: > - Guest OS type: Windows, Linux, or Auto. - Auto (default) detects the OS from the VMX guestOS field after cloning. - required: false - default: 'Auto' - - use-git-clone: - description: > - Set to "true" to clone the repository inside the VM rather than - transferring a zip from the host. Requires Git in the template (see - Install-CIToolchain-WinBuild2025.ps1 Step 12 / Install-CIToolchain-Linux2404.sh). - Preferred for repositories larger than ~200 MB. - required: false - default: 'false' - - configuration: - description: > - Build configuration string forwarded to the build command (Release/Debug). - required: false - default: 'Release' - - template-path: - description: > - Override the VMX template path. Leave empty (default) to use the value - from the GITEA_CI_TEMPLATE_PATH runner environment variable set in - runner/config.yaml. - required: false - default: '' - - snapshot-name: - description: > - Override the snapshot name to clone from. Leave empty (default) to use - GITEA_CI_SNAPSHOT_NAME runner env var, or "BaseClean" if that is also - unset. - required: false - default: '' - - artifact-name: - description: > - Name for the artifact uploaded to Gitea. Leave empty to use the default: - build-{run_id}-{sha}. - required: false - default: '' - - artifact-retention-days: - description: 'Days to retain uploaded artifacts in Gitea.' - required: false - default: '14' - - job-id-suffix: - description: > - Optional suffix appended to the job ID: run_id-run_attempt-SUFFIX. - Use to disambiguate parallel matrix legs so each leg writes to its own - artifact directory (e.g. set to matrix.target: windows, linux). - Leave empty (default) for single-job workflows. - required: false - default: '' - - repo-url: - description: > - Override the repository URL passed to Invoke-CIJob.ps1. - Defaults to github.server_url/repository.git (HTTPS). - Set to an SSH URL (e.g. ssh://gitea-ci/Org/repo.git) when the host - runner uses an SSH alias for Gitea. - required: false - default: '' - - extra-guest-env-json: - description: > - JSON object of extra environment variables to inject into the build VM - guest before the build command runs. Use for signing keys and secrets. - Example: '{"SIGN_PASS":"${{ secrets.SIGN_PASS }}","GPG_KEY_ID":"0xABCD1234"}' - Values are passed via ExtraGuestEnv hashtable in Invoke-CIJob.ps1 and - are never echoed to the log. - required: false - default: '{}' - - use-shared-cache: - description: > - Set to "true" to redirect NuGet and pip caches to VMware shared folders - on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip). - Requires Set-TemplateSharedFolders.ps1 to have been run on the template - and VMware Tools HGFS driver present in the guest. - required: false - default: 'false' - - skip-artifact: - description: > - Set to "true" to skip artifact packaging and collection entirely. - Use for jobs that intentionally produce no output (lint, static analysis, - test-only). When absent (default), the build MUST produce artifact-source - or the job fails. - required: false - default: 'false' - -outputs: - artifact-path: - description: > - Host-side directory where Invoke-CIJob.ps1 placed collected artifacts. - Format: F:\CI\Artifacts\{run_id}-{run_attempt} - value: ${{ steps.invoke-ci.outputs.artifact-path }} - - artifact-name: - description: 'Name used when uploading the artifact to Gitea.' - value: ${{ steps.invoke-ci.outputs.artifact-name }} - -runs: - using: composite - steps: - - # ── Invoke the CI orchestrator ───────────────────────────────────────────── - - name: Invoke CI Job - id: invoke-ci - shell: powershell - # Inputs are forwarded via environment variables to avoid shell injection - # from user-supplied build-command or path strings. - env: - INPUT_BUILD_COMMAND: ${{ inputs.build-command }} - INPUT_ARTIFACT_SOURCE: ${{ inputs.artifact-source }} - INPUT_SUBMODULES: ${{ inputs.submodules }} - INPUT_GUEST_OS: ${{ inputs.guest-os }} - INPUT_USE_GIT_CLONE: ${{ inputs.use-git-clone }} - INPUT_CONFIGURATION: ${{ inputs.configuration }} - INPUT_TEMPLATE_PATH: ${{ inputs.template-path }} - INPUT_SNAPSHOT_NAME: ${{ inputs.snapshot-name }} - INPUT_ARTIFACT_NAME: ${{ inputs.artifact-name }} - INPUT_JOB_ID_SUFFIX: ${{ inputs.job-id-suffix }} - INPUT_REPO_URL: ${{ inputs.repo-url }} - INPUT_EXTRA_GUEST_ENV_JSON: ${{ inputs.extra-guest-env-json }} - INPUT_USE_SHARED_CACHE: ${{ inputs.use-shared-cache }} - INPUT_SKIP_ARTIFACT: ${{ inputs.skip-artifact }} - run: | - #Requires -Version 5.1 - Set-StrictMode -Version Latest - $ErrorActionPreference = 'Stop' - - $ciScriptsDir = $env:GITEA_CI_SCRIPT_ROOT - - # Build job ID — append suffix when provided (matrix disambiguation) - $jobId = '${{ github.run_id }}-${{ github.run_attempt }}' - if ($env:INPUT_JOB_ID_SUFFIX) { $jobId = "$jobId-$($env:INPUT_JOB_ID_SUFFIX)" } - $artifactPath = "F:\CI\Artifacts\$jobId" - - # Resolve artifact name: caller-supplied or default to build-{run_id}-{sha} - if ($env:INPUT_ARTIFACT_NAME) { - $artifactName = $env:INPUT_ARTIFACT_NAME - } - else { - $artifactName = "build-${{ github.run_id }}-${{ github.sha }}" - } - - # Resolve repository URL: explicit override or construct from GitHub context - if ($env:INPUT_REPO_URL) { - $repoUrl = $env:INPUT_REPO_URL - } - else { - $repoUrl = '${{ github.server_url }}/${{ github.repository }}.git' - } - - # Parse extra-guest-env-json into a hashtable (§6.5 secret injection) - $extraGuestEnv = @{} - if ($env:INPUT_EXTRA_GUEST_ENV_JSON -and $env:INPUT_EXTRA_GUEST_ENV_JSON -ne '{}') { - $parsed = $env:INPUT_EXTRA_GUEST_ENV_JSON | ConvertFrom-Json - foreach ($prop in $parsed.PSObject.Properties) { - $extraGuestEnv[$prop.Name] = [string]$prop.Value - } - } - - # Resolve guest OS: honor explicit input; for Auto, infer from RUNNER_LABELS - $resolvedGuestOS = $env:INPUT_GUEST_OS - if ($resolvedGuestOS -eq 'Auto') { - if ($env:RUNNER_LABELS -match 'linux') { - $resolvedGuestOS = 'Linux' - } else { - $resolvedGuestOS = 'Windows' - } - } - - # Select template path: explicit input overrides OS-based runner env var - $resolvedTemplatePath = $env:INPUT_TEMPLATE_PATH - if (-not $resolvedTemplatePath) { - if ($resolvedGuestOS -eq 'Linux') { - $resolvedTemplatePath = $env:GITEA_CI_LINUX_TEMPLATE_PATH - } else { - $resolvedTemplatePath = $env:GITEA_CI_TEMPLATE_PATH - } - } - - # Select snapshot name: explicit input > OS-based default - # GITEA_CI_SNAPSHOT_NAME applies to Windows (template-refresh override) - $resolvedSnapshotName = $env:INPUT_SNAPSHOT_NAME - if (-not $resolvedSnapshotName) { - if ($resolvedGuestOS -eq 'Linux') { - $resolvedSnapshotName = 'BaseClean-Linux' - } else { - $resolvedSnapshotName = if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' } - } - } - - # Build parameter hashtable for Invoke-CIJob.ps1 - $params = @{ - JobId = $jobId - RepoUrl = $repoUrl - Branch = '${{ github.ref_name }}' - Commit = '${{ github.sha }}' - BuildCommand = $env:INPUT_BUILD_COMMAND - GuestArtifactSource = $env:INPUT_ARTIFACT_SOURCE - GuestOS = $resolvedGuestOS - Configuration = $env:INPUT_CONFIGURATION - } - - if ($env:INPUT_SUBMODULES -eq 'true') { $params['Submodules'] = $true } - if ($env:INPUT_USE_GIT_CLONE -eq 'true') { $params['UseGitClone'] = $true } - if ($env:INPUT_USE_SHARED_CACHE -eq 'true') { $params['UseSharedCache'] = $true } - if ($env:INPUT_SKIP_ARTIFACT -eq 'true') { $params['SkipArtifact'] = $true } - if ($resolvedTemplatePath) { $params['TemplatePath'] = $resolvedTemplatePath } - if ($resolvedSnapshotName) { $params['SnapshotName'] = $resolvedSnapshotName } - if ($extraGuestEnv.Count -gt 0) { $params['ExtraGuestEnv'] = $extraGuestEnv } - - & "$ciScriptsDir\Invoke-CIJob.ps1" @params - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - # Emit outputs consumed by subsequent steps - "artifact-path=$artifactPath" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - "artifact-name=$artifactName" | Out-File -FilePath $env:GITHUB_OUTPUT -Append -Encoding utf8 - - # ── Upload build artifacts on success ───────────────────────────────────── - - name: Upload artifacts - if: success() && inputs.skip-artifact != 'true' - uses: actions/upload-artifact@v4 - with: - name: ${{ steps.invoke-ci.outputs.artifact-name }} - path: ${{ steps.invoke-ci.outputs.artifact-path }}\ - retention-days: ${{ inputs.artifact-retention-days }} - if-no-files-found: error - - # ── Upload diagnostic logs on failure ───────────────────────────────────── - - name: Upload diagnostic logs on failure - if: failure() - uses: actions/upload-artifact@v4 - with: - name: build-logs-${{ github.sha }} - path: ${{ steps.invoke-ci.outputs.artifact-path }}\*.log - retention-days: 3 - if-no-files-found: ignore - diff --git a/gitea/workflow-example.yml b/gitea/workflow-example.yml index 0c71c6a..5eeb2e8 100644 --- a/gitea/workflow-example.yml +++ b/gitea/workflow-example.yml @@ -9,7 +9,7 @@ # windows-build -> WinBuild2025 (WinRM/HTTPS transport) # linux-build -> LinuxBuild2404 (SSH transport) # -# The composite action (Simone/local-ci-system/.gitea/actions/local-ci-build) +# The composite action (Simone/local-ci-cd-system/.gitea/actions/local-ci-build) # handles the full VM lifecycle: clone template -> start -> build -> # collect artifacts -> upload to Gitea -> destroy VM. # No orchestration logic needed in each repo's workflow. @@ -36,7 +36,7 @@ jobs: steps: # ── Invoke the full VM lifecycle via composite action ─────────────────── - - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main + - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main with: # --- adapt these two per-repository ----------------------------------- build-command: 'python build.py --dist-dir dist' # command run in VM @@ -63,7 +63,7 @@ jobs: # Step 4: Upload artifacts even on failure (for diagnostics) - name: Upload diagnostic logs on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-logs-${{ github.sha }} path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}\*.log @@ -140,7 +140,7 @@ jobs: # # - name: Upload artifacts (${{ matrix.target }}) # if: success() -# uses: actions/upload-artifact@v4 +# uses: actions/upload-artifact@v3 # with: # name: build-${{ matrix.target }}-${{ github.sha }} # path: F:\CI\Artifacts\${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.target }}\ @@ -154,7 +154,7 @@ jobs: # The composite action handles: Invoke-CIJob.ps1 call + artifact upload + log upload. # # Reference: gitea/actions/local-ci-build/action.yml in this repository. -# Deployed path in Gitea: Simone/local-ci-system / .gitea/actions/local-ci-build/action.yml +# Deployed path in Gitea: Simone/local-ci-cd-system / .gitea/actions/local-ci-build/action.yml # # Minimum example (Windows build, Python project with submodules): # @@ -171,7 +171,7 @@ jobs: # runs-on: windows-build # steps: # - uses: actions/checkout@v4 -# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main +# - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main # with: # build-command: 'python build_plugin.py --final --dist-dir dist' # artifact-source: 'dist' @@ -188,7 +188,7 @@ jobs: # runs-on: ${{ matrix.target }}-build # steps: # - uses: actions/checkout@v4 -# - uses: Simone/local-ci-system/.gitea/actions/local-ci-build@main +# - uses: https://gitea.emulab.it/Simone/local-ci-cd-system/.gitea/actions/local-ci-build@main # with: # build-command: 'python build_plugin.py --final --dist-dir dist' # artifact-source: 'dist' diff --git a/gitea/workflows/lint.yml b/gitea/workflows/lint.yml deleted file mode 100644 index 204f4fb..0000000 --- a/gitea/workflows/lint.yml +++ /dev/null @@ -1,55 +0,0 @@ -# PSScriptAnalyzer lint — runs on every push/PR that touches PowerShell files. -# Requires PSScriptAnalyzer installed on the runner host: -# Install-Module PSScriptAnalyzer -Scope AllUsers -Force -# -# Failures block the PR. Fix warnings with: -# Invoke-ScriptAnalyzer -Path scripts\ -Recurse -Fix - -name: Lint (PSScriptAnalyzer) - -on: - push: - paths: - - '**.ps1' - - '**.psm1' - pull_request: - paths: - - '**.ps1' - - '**.psm1' - -jobs: - lint: - runs-on: windows-build - timeout-minutes: 10 - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run PSScriptAnalyzer - shell: powershell - run: | - $ErrorActionPreference = 'Stop' - - if (-not (Get-Module -ListAvailable PSScriptAnalyzer)) { - Write-Host "Installing PSScriptAnalyzer..." - Install-Module PSScriptAnalyzer -Scope CurrentUser -Force -AllowClobber - } - - $paths = @('scripts', 'template', 'runner', 'Setup-Host.ps1') - $results = $paths | ForEach-Object { - if (Test-Path $_) { - Invoke-ScriptAnalyzer -Path $_ -Recurse -Severity Error,Warning - } - } - - if ($results) { - $results | Format-Table -AutoSize - Write-Error "PSScriptAnalyzer found $($results.Count) issue(s). Fix before merging." - exit 1 - } - else { - Write-Host "PSScriptAnalyzer: no issues found." - } diff --git a/plans/A1-closeout.md b/plans/A1-closeout.md new file mode 100644 index 0000000..6135146 --- /dev/null +++ b/plans/A1-closeout.md @@ -0,0 +1,146 @@ +# Fase A1 — Closeout operativo + +Checklist per chiudere lo step **A1** di [implementation-plan-A-B](implementation-plan-A-B.md) +prima di iniziare A2. Il codice è già committato sul branch +`feature/python-rewrite-phase-a` (commit `9fbe952`); manca solo la +validazione contro l'ambiente reale. + +## Stato attuale + +- [x] Codice + 35 unit test (coverage 82.65%) committati +- [x] `ruff` clean, `mypy --strict` clean su 12 file sorgente +- [x] Job `python` aggiunto a `.gitea/workflows/lint.yml` +- [x] Job `python` di `lint.yml` PASS su act_runner (commit `8586ff8`) +- [ ] PoC `wait-ready` PASS contro VM reale Windows +- [ ] PoC `wait-ready` PASS contro VM reale Linux + +## Perché completare A1 prima di A2 + +A2 riusa `WorkstationVmrunBackend`, `WinRmTransport`, `SshTransport` e +`KeyringCredentialStore` esattamente come sono ora. Se uno di questi ha +un bug di integrazione (TLS, encoding, parsing output `vmrun list`, +permessi venv, ecc.), trovarlo dopo A2 costa: **1 fix nei moduli core + +retest di tutti i 6 comandi portati in A2**. Trovarlo ora costa: 1 fix. + +## 1. PoC `wait-ready` su VM Windows reale + +**Obiettivo**: validare il path `vmrun.is_running` → `get_ip` → WinRM +HTTPS con cert self-signed end-to-end. + +```powershell +# 1. Setup venv (una tantum) sull'host CI +python -m venv F:\CI\python\venv +F:\CI\python\venv\Scripts\python.exe -m pip install --upgrade pip +F:\CI\python\venv\Scripts\python.exe -m pip install -e ".[dev]" + +# 2. Clone manuale di una VM smoke da template +$vmrun = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' +$tpl = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' +$dst = 'F:\CI\BuildVMs\smoke-a1\smoke-a1.vmx' +& $vmrun -T ws clone $tpl $dst linked -snapshot=BaseClean -cloneName=smoke-a1 +if ($LASTEXITCODE -ne 0) { throw "clone fallito" } +& $vmrun -T ws start $dst nogui +if ($LASTEXITCODE -ne 0) { throw "start fallito" } + +# 3. PoC wait-ready +$env:PYTHONIOENCODING = 'utf-8' +F:\CI\python\venv\Scripts\python.exe -m ci_orchestrator wait-ready ` + --vmx $dst --guest-os windows --timeout 300 + +# 4. Cleanup +& $vmrun -T ws stop $dst hard +& $vmrun -T ws deleteVM $dst +``` + +**Atteso**: + +- Exit code `0` +- Ultima riga stdout: `[wait-ready] WinRM ready.` +- Tempo totale tra start VM e WinRM ready: 60–180 s + +**Failure mode più probabili e fix**: + +| Sintomo | Causa probabile | Fix | +| -------------------------------------------------- | ------------------------------------------------------ | -------------------------------------------------------------------------------------------------- | +| `is_running` resta `False` per sempre | Path mismatch in `vmrun list` (slash diversi, casing) | Aggiungere normalizzazione `os.path.normcase` in `WorkstationVmrunBackend.is_running` | +| `get_ip` ritorna sempre `None` | VMware Tools non ancora pronti | Atteso nei primi 30–60 s; se persiste >180 s, problema nel template | +| `WinRM probe failed: ... certificate verify` | `cert_validation=False` non basta su TLS handshake | Ispezionare cipher; eventualmente pinnare versione `pypsrp` o passare `auth='ntlm'` esplicito | +| `KeyError: 'BuildVMGuest'` | Credenziale non leggibile dall'utente che lancia il PoC | Eseguire come l'utente del runner (`runas /user:ci-runner ...`) o re-store credential nel suo profilo | +| `KeyringCredentialStore` ritorna username vuoto | Backend keyring usa `get_password`-only | Verificato in `test_credentials.py::test_get_falls_back_to_password`; in tal caso impostare username separatamente nel target | + +## 2. PoC `wait-ready` su VM Linux reale + +```powershell +$tpl = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' +$dst = 'F:\CI\BuildVMs\smoke-a1-lnx\smoke-a1-lnx.vmx' +& $vmrun -T ws clone $tpl $dst linked -snapshot=BaseClean-Linux -cloneName=smoke-a1-lnx +& $vmrun -T ws start $dst nogui + +$env:CI_SSH_KEY_PATH = 'F:\CI\keys\ci_linux' +F:\CI\python\venv\Scripts\python.exe -m ci_orchestrator wait-ready ` + --vmx $dst --guest-os linux --ssh-user ci_build --timeout 300 + +& $vmrun -T ws stop $dst hard +& $vmrun -T ws deleteVM $dst +``` + +**Atteso**: ultima riga `[wait-ready] SSH ready.` entro 60–120 s. + +**Failure mode più probabili**: + +| Sintomo | Causa | Fix | +| ------------------------------------ | ---------------------------------------------------- | ---------------------------------------------------------------------------------- | +| `SSH connection ... timed out` | DHCP collision (vedi `AGENTS.md` errore #11) | Verificare snapshot `BaseClean-Linux` con machine-id resettato | +| `paramiko.AuthenticationException` | Permessi chiave SSH errati (Windows ACL su `ci_linux`) | `icacls F:\CI\keys\ci_linux /inheritance:r /grant:r ":R"` | +| `is_ready()` False ma SSH manuale OK | `BatchMode=yes` lato `ssh.exe` non equivalente in paramiko | Già coperto: paramiko non chiede prompt; verificare `look_for_keys=False` settato | + +## 3. Job `python` di `lint.yml` su act_runner + +```powershell +# Push del branch +git push -u origin feature/python-rewrite-phase-a +``` + +Poi su Gitea: aprire una PR contro `main` (o trigger manuale del +workflow). Verificare che il job `python` completi entro +`timeout-minutes: 15`. + +**Failure mode più probabili**: + +| Sintomo | Causa | Fix | +| ------------------------------------------------------ | -------------------------------------------------------- | ------------------------------------------------------------------------------ | +| `python: command not found` | `python` non in PATH dell'utente del runner | Installare Python 3.11+ machine-wide o aggiornare `PATH` del servizio | +| `pip install -e .[dev]` >15 min al primo run | Cache assente, network lento | Bumpare `timeout-minutes: 30` solo per il primo run; cache stabile dopo | +| `Access denied` su `F:\CI\python\venv` | Utente del runner senza scrittura su `F:\CI\python` | `icacls F:\CI\python /grant ":(OI)(CI)F"` | +| Coverage <70% | Differenza Python 3.14 (locale) vs 3.11 (CI) | Rilanciare con `--cov-report=term-missing` per identificare branch mancante | + +## 4. Definizione di "A1 done" + +Spuntabile solo quando **tutti** e tre i punti sopra sono PASS: + +- [ ] PoC Windows: `wait-ready` exit 0 contro `WinBuild2025` reale +- [ ] PoC Linux: `wait-ready` exit 0 contro `LinuxBuild2404` reale +- [ ] CI: workflow `lint.yml` job `python` PASS su act_runner + +A quel punto: + +1. Aggiornare la checklist master in [implementation-plan-A-B](implementation-plan-A-B.md) + spuntando le righe `[A1]`. +2. Mergeare `feature/python-rewrite-phase-a` su `main` (o lasciare il + branch live se A2 verrà committato sopra). +3. Procedere con A2 (script "foglia": `Wait-VMReady`, `Remove-BuildVM`, + `Cleanup-OrphanedBuildVMs`, `Watch-DiskSpace`, `Watch-RunnerHealth`, + `Get-CIJobSummary`). + +## 5. Workaround se A1 non si può chiudere ora + +Se l'hardware/VM template non sono accessibili in questo momento: + +- **Non rimuovere** alcuno script PS in A2. +- Marcare A1 come "code-complete, validation pending" in + [implementation-plan-A-B](implementation-plan-A-B.md). +- Procedere con A2 ma trattare ogni bug scoperto su `WorkstationVmrunBackend` + / `WinRmTransport` / `SshTransport` come **fix di A1** (commit separato, + no nuove feature A2 nello stesso commit). +- Riservare una sessione dedicata di smoke testing prima di A3 (pipeline + di build), perché A3 introduce `clone_linked` + `start` reali. diff --git a/plans/A2-closeout.md b/plans/A2-closeout.md new file mode 100644 index 0000000..ff7428e --- /dev/null +++ b/plans/A2-closeout.md @@ -0,0 +1,36 @@ +# A2 — Closeout + +Fase A2 di `plans/implementation-plan-A-B.md`: porting in Python degli script +"foglia" (no state condiviso). Branch: `feature/python-rewrite-phase-a`. + +## Stato attuale + +- [x] `commands/wait.py` → `wait-ready` (sostituisce `Wait-VMReady.ps1`) +- [x] `commands/vm.py` → `vm remove` + `vm cleanup` (sostituisce `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`) +- [x] `commands/monitor.py` → `monitor disk` + `monitor runner` (sostituisce `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`) +- [x] `commands/report.py` → `report job` (sostituisce `Get-CIJobSummary.ps1`) +- [x] Tutti i 6 `.ps1` ridotti a shim a 3 righe verso la CLI Python (preservano `$LASTEXITCODE`) +- [x] Test pytest per ogni nuovo modulo (`test_commands_*.py`), 69 test totali, ruff/mypy --strict clean, coverage 74.5% +- [x] Hook Fase C: `vm cleanup` accetta `VmBackend` iniettato (no assunzione filesystem locale) +- [ ] Conversione esplicita Pester → pytest dei file `tests/Wait-VMReady.Tests.ps1` e `tests/Remove-BuildVM.Tests.ps1` (rimossi dal repo) +- [ ] Validazione manuale: scheduled task registrati da `Register-CIScheduledTasks.ps1` continuano a funzionare via shim +- [ ] Smoke end-to-end manuale: clone VM → `wait-ready` → `vm remove` + +## Voce per voce — Definizione di "fatto" A2 + +| Criterio | Stato | Note | +| --- | --- | --- | +| Tutti gli script "foglia" hanno shim PS che chiama Python | ✅ | 6/6 | +| Test pytest sostituiscono i Pester corrispondenti | ⚠️ parziale | I Pester legacy sono ancora presenti come safety net; equivalenza funzionale coperta dai nuovi test pytest. Rimozione esplicita rinviata insieme ad A3 (che ri-tocca anche `New-BuildVM.Tests.ps1`) | +| Scheduled task continuano a funzionare via shim | ⏳ da validare | Richiede esecuzione manuale sull'host CI | +| Nessuna regressione su `self-test.yml` | ⏳ da validare | Workflow `lint.yml` PASS; `self-test.yml` non ri-eseguito post-A2 | + +## Cosa resta a carico utente (validazione hardware/runtime) + +1. Eseguire manualmente uno dei task in `Register-CIScheduledTasks.ps1` (es. `Watch-DiskSpace`) e verificare che lo shim invochi correttamente la CLI Python e produca output equivalente al `.ps1` originale. +2. Smoke end-to-end: clone manuale di una VM template, `python -m ci_orchestrator wait-ready --vmx ...`, `python -m ci_orchestrator vm remove --vmx ...`. +3. Decidere se rimuovere i Pester `tests/Wait-VMReady.Tests.ps1` / `tests/Remove-BuildVM.Tests.ps1` ora o aspettare A3 (che li ri-cita). + +## Riferimenti commit + +- `794db1a` — feat(a2): port leaf PS scripts to ci_orchestrator CLI diff --git a/plans/A3-closeout.md b/plans/A3-closeout.md new file mode 100644 index 0000000..7d528c4 --- /dev/null +++ b/plans/A3-closeout.md @@ -0,0 +1,31 @@ +# A3 — Closeout + +Fase A3 di `plans/implementation-plan-A-B.md`: pipeline core (clone → build → collect) in Python. +Branch: `feature/python-rewrite-phase-a`. + +## Stato attuale + +- [x] `commands/vm.py` esteso con `vm new` (sostituisce `New-BuildVM.ps1`) +- [x] `commands/build.py` con `build run` (sostituisce `Invoke-RemoteBuild.ps1`) +- [x] `commands/artifacts.py` con `artifacts collect` (sostituisce `Get-BuildArtifacts.ps1`) +- [x] Tutti i 3 `.ps1` ridotti a shim verso la CLI Python +- [x] Pester `tests/{New-BuildVM,Wait-VMReady,Remove-BuildVM}.Tests.ps1` rimossi (casi negativi coperti dai pytest equivalenti) +- [x] 91 pytest, ruff/mypy --strict clean, coverage 78.27% +- [x] Hook Fase C: `build run` + `artifacts collect` accettano `VmHandle`/`vmx` opaco; nessuna assunzione path Windows +- [ ] Smoke end-to-end manuale: clone WinBuild2025 → build script trivial → collect artifact ZIP +- [ ] Smoke end-to-end Linux equivalente + +## Definizione di "fatto" A3 + +| Criterio | Stato | +| --- | --- | +| Pipeline build completa via Python CLI | ✅ | +| Shim PS preservano API per caller esistenti | ✅ | +| Smoke build PASS contro VM reale Windows e Linux | ⏳ validazione hardware | +| Pester `New-BuildVM.Tests.ps1` rimosso e sostituito | ✅ | + +## Cosa resta a carico utente + +1. Smoke manuale end-to-end Windows: `python -m ci_orchestrator vm new ... && build run ... && artifacts collect ...` +2. Stesso smoke su Linux template +3. Verificare che `Test-NsinnounpBuild.ps1` continui a passare invocando gli shim diff --git a/plans/A4-closeout.md b/plans/A4-closeout.md new file mode 100644 index 0000000..ce8adef --- /dev/null +++ b/plans/A4-closeout.md @@ -0,0 +1,48 @@ +# A4 — Closeout + +Fase A4 di `plans/implementation-plan-A-B.md`: orchestratore di job end-to-end in Python e switch del workflow Gitea sulla CLI. +Branch: `feature/python-rewrite-phase-a`. + +## Stato attuale + +- [x] `commands/job.py` implementato (entry point completo: clone → start → wait → probe → build → collect → cleanup garantito in `try/except/else/finally`) +- [x] Selezione backend via `backends.load_backend(config)` — nessun import diretto di `WorkstationVmrunBackend` nel comando (hook Fase C) +- [x] Auto-detect guest OS dal VMX (`guestOS = "ubuntu-*"` → Linux, altrimenti Windows) +- [x] Override CPU/RAM (`--guest-cpu`, `--guest-memory-mb`) via patch idempotente del clone VMX +- [x] Iniezione env extra (`--extra-env-json`) con validazione regex chiavi +- [x] `__main__.py` registra il sottocomando `job` +- [x] `scripts/Invoke-CIJob.ps1` ridotto a shim PS 5.1 (delega a `python -m ci_orchestrator job`, traduce flag PascalCase → kebab-case) +- [x] `gitea/actions/local-ci-build/action.yml` invoca direttamente `& $venvPython -m ci_orchestrator job ...` (più nessuno `@params` splatting verso lo shim PS) +- [x] `runner/config.yaml` esporta `PYTHONIOENCODING=utf-8` per garantire stdio UTF-8 nei log act_runner → Gitea +- [x] `tests/python/test_commands_job.py` aggiunto: 13 test (happy path Linux, happy path Windows + override, skip-artifact, cleanup su build failure, cleanup su clone failure, validazione template/extra-env, helper `_apply_vmx_overrides` / `_read_guest_os_from_vmx` / `_parse_extra_env_json`) +- [x] Suite completa: ruff clean, mypy --strict clean, **104 pytest PASS**, coverage totale **79.25%**, coverage `commands/job.py` **83%** +- [ ] Workflow `build-ns7zip.yml` matrix (Win + Linux) end-to-end PASS contro VM reali — richiede host con runner registrato +- [ ] Workflow `self-test.yml` / `lint.yml` PASS sul nuovo entry point — stessa dipendenza +- [ ] Misurazione tempo job entro ±10% del baseline PowerShell (`Measure-CIBenchmark.ps1`) + +## Definizione di "fatto" A4 + +| Criterio | Stato | +| --------------------------------------------------- | ------------------------------------ | +| `action.yml` chiama Python direttamente | ✅ | +| Shim `Invoke-CIJob.ps1` preservato per uso manuale | ✅ | +| `runner/config.yaml` ha `PYTHONIOENCODING=utf-8` | ✅ | +| Coverage `commands/job.py` ≥80% | ✅ 83% | +| Cleanup VM garantito su tutti i path di errore | ✅ (test `test_job_cleanup_on_*`) | +| Hook Fase C (factory backend, no import workstation diretto) | ✅ | +| Workflow `build-ns7zip.yml` matrix PASS | ⏳ validazione hardware utente | +| Tempo entro ±10% baseline | ⏳ misurazione utente | + +## Cosa resta a carico utente + +1. Eseguire `build-ns7zip.yml` (matrix Win + Linux) sul runner host registrato e verificare che il job concluda con artifact pubblicato — il flusso deve passare dal nuovo entry point Python. +2. Lanciare `self-test.yml` e `lint.yml` per validare che gli altri workflow restino verdi dopo lo switch. +3. Eseguire `scripts/Measure-CIBenchmark.ps1` (resta in PS) prima/dopo lo switch e confermare delta tempo entro ±10%. +4. Aggiornare `Measure-CIBenchmark.ps1` se la wall-clock del nuovo entry point la rende necessaria; in caso di regressione >10%, aprire issue in `TODO.md` per profiling A5. +5. Coverage `commands/job.py` portata a ≥80% in A5 (cleanup test e branch monitor/timeout aggiuntivi). + +## Riferimenti commit + +- `feat(a4): port Invoke-CIJob.ps1 to ci_orchestrator job` — `commands/job.py`, `__main__.py`, shim `scripts/Invoke-CIJob.ps1`, `tests/python/test_commands_job.py` +- `ci(a4): switch local-ci-build action.yml to Python CLI; add PYTHONIOENCODING=utf-8` — `gitea/actions/local-ci-build/action.yml`, `runner/config.yaml` +- `docs(a4): mark A4 as code-complete; add A4-closeout.md` — `plans/implementation-plan-A-B.md`, `plans/A4-closeout.md` diff --git a/plans/A5-closeout.md b/plans/A5-closeout.md new file mode 100644 index 0000000..fc1cf01 --- /dev/null +++ b/plans/A5-closeout.md @@ -0,0 +1,100 @@ +# A5 — Closeout + +Fase A5 di `plans/implementation-plan-A-B.md`: polish, regression test +per gli errori frequenti `AGENTS.md` #9-#12, documentazione Python +allineata, gate coverage alzato a 80%. +Branch: `feature/python-rewrite-phase-a`. + +## Stato attuale + +- [x] `tests/python/test_agents_errors.py` aggiunto: 12 test dedicati che + coprono regressioni per `AGENTS.md` "Errori frequenti da evitare" + #9, #10, #11, #12. + - **#9** (snapshot template con VM accesa): `clone_linked` propaga + "should not be powered on" come `BackendOperationFailed` (subclass + di `BackendError`) con `operation == "clone"` e `output` che + contiene il messaggio originale di `vmrun`. + - **#10** (`vmrun getGuestIPAddress` lento): `is_running` usa solo + `vmrun list`. Tre asserzioni — happy path, VM non listata, errore + di `vmrun list` — verificano che `getGuestIPAddress` non venga MAI + chiamato come fallback. + - **#11** (machine-id duplicato → DHCP collision): la safety net + orchestrator-side è che `vm new` non riusi mai un clone-name + (timestamp suffix); il test invoca `vm new` due volte con lo stesso + `--job-id` e verifica che `clone_linked` riceva due + `name`/`destination` distinti. Il fix template-side + (`truncate -s 0 /etc/machine-id`) resta documentato in + `template/Prepare-LinuxBuild2404.ps1`. + - **#12** (`& nativecmd 2>$null` non sopprime stderr in PS 5.1): + `SshTransport` usa `paramiko.AutoAddPolicy` di default con + `known_hosts=None`, non chiama mai `subprocess.run`/`Popen` + (incluso `ssh-keygen`/`ssh.exe`/`scp.exe`), e wrappa errori + paramiko (import mancante, `exec_command` exception, SFTP + `put`/`get`) come `TransportConnectError` invece di propagare + eccezioni nude. +- [x] `AGENTS.md` aggiornato: nuova sezione "Python development" subito + dopo "PowerShell 5.1 — Vincoli critici", con tabella parametri, + convenzioni (type hints, no `Any`/globali/`print`/`shell=True`), + mappatura completa PS → Python sub-command, elenco PS che restano + e gate coverage 80%. +- [x] `docs/ARCHITECTURE.md` esteso con sezione "Python orchestrator + (Phase A)": layout package, `VmBackend` Protocol, factory + `load_backend(config)`, ricetta Phase C per backend ESXi + (`backends/esxi.py` + selettore `[backend]` in `config.toml`), + transport selection, gate CI. +- [x] `README.md` ampliato: setup venv produzione + dev, esempi CLI per + tutti i 10 sub-comandi (`wait-ready`, `vm new/remove/cleanup`, + `build run`, `artifacts collect`, `monitor disk/runner`, + `report job`, `job`), comandi validazione locale con gate 80%, + link a `test_agents_errors.py` e `pyproject.toml`. +- [x] Suite completa: ruff clean, mypy --strict clean, **116 pytest + PASS**, coverage totale **80.10%** (gate 80% raggiunto). +- [ ] Burn-in 4 job concorrenti × 10 round (versione Python di + `Test-CapacityBurnIn`) — **richiede VM reali, lasciato a carico + utente**. +- [ ] Workflow `build-ns7zip.yml` matrix (Win + Linux) end-to-end + PASS — eredita pendenza da A4 (richiede runner registrato). + +## Definizione di "fatto" A5 + +| Criterio | Stato | +| ----------------------------------------------------- | -------------------------------- | +| Errori #9–#12 coperti da test pytest | ✅ | +| `AGENTS.md` ha sezione "Python development" | ✅ | +| `docs/ARCHITECTURE.md` aggiornato (layout + hook C) | ✅ | +| `README.md` aggiornato (setup + esempi CLI) | ✅ | +| Coverage globale ≥80% (gate alzato in pyproject) | ✅ 80.10% | +| Shim PS non referenziati rimossi | ⏭️ nessuno candidato (vedi nota) | +| Burn-in 4×10 PASS | ⏳ utente — richiede VM reali | + +**Nota shim PS**: tutti i 10 shim (`Wait-VMReady.ps1`, `New-BuildVM.ps1`, +`Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`, +`Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1`, `Watch-DiskSpace.ps1`, +`Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1`, `Invoke-CIJob.ps1`) +sono ancora referenziati: `Register-CIScheduledTasks.ps1` registra +scheduled task che invocano `Watch-DiskSpace.ps1`, +`Watch-RunnerHealth.ps1`, `Cleanup-OrphanedBuildVMs.ps1`; `README.md` +documenta `Invoke-CIJob.ps1` per uso manuale; `Test-*.ps1` di smoke +chiamano gli shim. Nessuno shim è candidato a rimozione in A5 senza +prima migrare anche `Register-CIScheduledTasks.ps1` (Fase B5 lo +sostituirà con `*.service` + `*.timer` su systemd). + +## Cosa resta a carico utente + +1. Eseguire un burn-in concorrente (4 job × 10 round) usando il workflow + `build-ns7zip.yml` matrix o `scripts/Test-CapacityBurnIn.ps1` (che + ora delega via shim alla CLI Python `job`) e verificare: + - PASS senza VM orfane (`vm cleanup` non rimuove nulla a fine round). + - Wall-clock entro ±20% del baseline pre-Python (`Measure-CIBenchmark.ps1`). +2. Validare `build-ns7zip.yml` matrix (Win + Linux) end-to-end PASS + sul runner reale — pendenza ereditata da A4. +3. Validare `self-test.yml` e `lint.yml` post-switch. + +## Riferimenti commit + +- `test(a5): add pytest regression tests for AGENTS.md errors #9-#12` — + `tests/python/test_agents_errors.py` +- `docs(a5): document Python orchestrator in AGENTS.md, ARCHITECTURE.md, README.md` — + `AGENTS.md`, `docs/ARCHITECTURE.md`, `README.md` +- `docs(a5): mark A5 as code-complete; add A5-closeout.md` — + `plans/implementation-plan-A-B.md`, `plans/A5-closeout.md` diff --git a/plans/B5-closeout.md b/plans/B5-closeout.md new file mode 100644 index 0000000..d827e85 --- /dev/null +++ b/plans/B5-closeout.md @@ -0,0 +1,76 @@ +# B5 — Closeout (parziale: artefatti repo) + +Fase B5 di `plans/implementation-plan-A-B.md`: conversione scheduled task +Windows in coppie `*.service` + `*.timer` systemd su Linux. + +## Cosa è stato fatto in questa sessione (su host Windows dev) + +- [x] Inventario dei 4 task in `scripts/Register-CIScheduledTasks.ps1` (cadenze + comandi) +- [x] 5 coppie `*.service` + `*.timer` create in `deploy/systemd/`: + - `ci-cleanup-orphans.{service,timer}` — every 6h + at boot → `python -m ci_orchestrator vm cleanup --max-age-hours 6` + - `ci-retention-policy.{service,timer}` — daily 03:00 + 30min jitter → `pwsh Invoke-RetentionPolicy.ps1` + - `ci-watch-disk-space.{service,timer}` — every 15min → `python -m ci_orchestrator monitor disk` + - `ci-watch-runner-health.{service,timer}` — every 15min → `python -m ci_orchestrator monitor runner` + - `ci-backup-template.{service,timer}` — weekly Sun 02:00 + 1h jitter → `pwsh Backup-CITemplate.ps1` (NEW, non era in Register-CIScheduledTasks.ps1) +- [x] `deploy/systemd/README.md` con: tabella mapping, prerequisiti (utente, venv, env file), istruzioni install/test/rollback, nota PowerShell Core su Linux + +## Note sulle cadenze + +Le cadenze del piano (`OnCalendar=hourly`, `*:0/15`, ecc.) sono state +adattate per fedeltà semantica all'originale Windows: + +| Task | Original Windows | systemd timer scelto | +| --------------------- | --------------------------------------------- | ----------------------------------------------------- | +| cleanup-orphans | every 6h + at startup | `OnBootSec=2min, OnUnitActiveSec=6h, Persistent` | +| retention-policy | daily 03:00 + 30min random delay | `OnCalendar=*-*-* 03:00:00, RandomizedDelaySec=30min` | +| watch-disk-space | every 15min | `OnBootSec=5min, OnUnitActiveSec=15min` | +| watch-runner-health | every 15min | `OnBootSec=3min, OnUnitActiveSec=15min` | +| backup-template | *(non in Register-CIScheduledTasks.ps1)* | `OnCalendar=Sun *-*-* 02:00:00, RandomizedDelaySec=1h`| + +`Persistent=true` ovunque per non perdere esecuzioni se la macchina è +spenta al momento del trigger. + +## Decisione — script che restano in PowerShell + +`Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` non sono stati +portati a Python (resta scelta documentata in `AGENTS.md` "Mappatura"). +Su Linux quindi le rispettive `.service` invocano `pwsh` (PowerShell +Core). Il README documenta l'installazione `apt install powershell` da +repo Microsoft. Se in futuro si deciderà di portarli a Python (sub-comandi +`retention run` / `template backup`), basterà sostituire `ExecStart=` +nelle due unit senza toccare il timer. + +## Definizione di "fatto" B5 (dal piano) + +| Criterio | Stato | +| --------------------------------------------------- | ------------------------------------ | +| Tutti i task periodici come timer systemd attivi | ⏳ unit pronti; install richiede host Linux | +| Trigger manuale di ognuno PASS | ⏳ richiede host Linux | +| Mapping documentato | ✅ `deploy/systemd/README.md` | + +## Cosa resta a carico utente (sul nuovo host Linux) + +Tutti i passi B1–B4 e B6–B8 sono operazioni hardware/host non eseguibili +da questa sessione. Per chiudere B5 una volta sull'host Linux: + +```bash +# Prerequisito: aver completato B1 (ci-runner user, /opt/ci/venv, /etc/ci/environment) +sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/ +sudo systemctl daemon-reload +for t in ci-cleanup-orphans ci-retention-policy ci-watch-disk-space \ + ci-watch-runner-health ci-backup-template; do + sudo systemctl enable --now "${t}.timer" +done +systemctl list-timers --all 'ci-*' + +# Trigger manuale di smoke +sudo systemctl start ci-cleanup-orphans.service +journalctl -u ci-cleanup-orphans.service -n 50 --no-pager +``` + +## Riferimenti + +- Spec systemd timer: `man systemd.timer`, `man systemd.time` +- Original PS task scheduler: `scripts/Register-CIScheduledTasks.ps1` +- Mapping completo: `deploy/systemd/README.md` +- Piano: `plans/implementation-plan-A-B.md` sezione B5 diff --git a/plans/PhaseA-user-checklist.md b/plans/PhaseA-user-checklist.md new file mode 100644 index 0000000..7fe3566 --- /dev/null +++ b/plans/PhaseA-user-checklist.md @@ -0,0 +1,341 @@ +# Fase A — Checklist finale per l'utente + +La Fase A è **code-complete e pushata** sul branch `feature/python-rewrite-and-linux-migration`. +Restano solo le validazioni che richiedono accesso fisico all'host CI con VMware +Workstation, snapshot template attivi e act_runner registrato. + +Questo documento elenca i passi da eseguire in ordine, con checklist e comandi +copiabili. Stop a primo errore: ogni passo è un gate per il successivo. + +> **Tempo stimato totale**: 1–2 ore (escluse build lunghe nel passo 5). + +--- + +## Passo 1 — Aggiornare il venv di produzione sull'host CI + +Il codice nuovo va installato nel venv che act_runner usa +(`F:\CI\python\venv\`). + +- [x] Aprire PowerShell **come amministratore** sull'host CI. +- [x] Posizionarsi nella copia git aggiornata del repo (pull del branch + `feature/python-rewrite-and-linux-migration`): + + ```powershell + cd + git fetch origin + git checkout feature/python-rewrite-and-linux-migration + git pull + ``` + +- [x] Installare il package nel venv di produzione (**non editable**): + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m pip install . + ``` + + > ⚠️ **Non usare `pip install -e .` su questo host.** act_runner gira + > come `LocalSystem`. Un install editable scrive un `.pth` verso la + > sorgente; se il `cd` era in una dir di lavoro di act_runner + > (`F:\CI\RunnerWork\\hostexecutor`, transitoria) o sul drive + > utente `N:` (non risolto da LocalSystem), il runner fallisce con + > `No module named ci_orchestrator`. L'install regolare copia il + > package in `site-packages` su `F:` (accessibile a LocalSystem). + > Conseguenza: ogni modifica al codice richiede di ri-eseguire + > `pip install .` — è uno step di deploy. Assicurarsi che il `cd` del + > passo precedente punti alla copia git reale del repo, non a una dir + > sotto `F:\CI\RunnerWork`. + +- [x] Smoke check rapido che la CLI risponda: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator --help + ``` + + Atteso: lista degli 11 sub-comandi (`wait-ready`, `vm`, `build`, + `artifacts`, `monitor`, `report`, `job`). + +--- + +## Passo 2 — Riavviare act_runner + +L'act_runner gira come servizio SYSTEM e legge `runner/config.yaml` +all'avvio. La modifica di A4 (`PYTHONIOENCODING=utf-8`) ha effetto solo +dopo restart. + +- [x] Trovare il servizio: + + ```powershell + Get-Service | Where-Object { $_.Name -like '*act*runner*' } + ``` + +- [x] Restart (sostituisci `` con quello trovato): + + ```powershell + Restart-Service -Name '' + ``` + +- [x] Verificare che sia partito: + + ```powershell + Get-Service -Name '' + ``` + + Atteso: `Status = Running`. + +--- + +## Passo 3 — Smoke `wait-ready` su VM Windows reale + +PoC pendente da A1. + +- [x] Clonare manualmente un template Windows: + + ```powershell + & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ` + -T ws clone ` + 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` + 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' ` + linked -snapshot=BaseClean -cloneName=smoke-win + ``` + +- [x] Avviare la VM: + + ```powershell + & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ` + -T ws start 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' nogui + ``` + +- [x] Eseguire `wait-ready`: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready ` + --vmx 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' ` + --guest-os windows --timeout 180 + ``` + + Atteso: exit code `0` entro 3 minuti. + +- [x] Cleanup VM di smoke: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm remove ` + --vmx 'F:\CI\BuildVMs\smoke-win\smoke-win.vmx' --force + ``` + +--- + +## Passo 4 — Smoke `wait-ready` su VM Linux reale + +Stesso schema, template Linux. PoC pendente da A1. + +- [x] Clonare: + + ```powershell + & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ` + -T ws clone ` + 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' ` + 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' ` + linked -snapshot=BaseClean-Linux -cloneName=smoke-linux + ``` + +- [x] Start + wait-ready: + + ```powershell + & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ` + -T ws start 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' nogui + + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready ` + --vmx 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' ` + --guest-os linux --timeout 180 + ``` + + Atteso: exit code `0`. + + > **Prerequisito scoperto**: `F:\CI\config.toml` deve esistere con `ssh_key_path` + > impostato — altrimenti SSH fallisce con `No authentication methods available`. + > Creato durante questa validazione; vedi HOST-SETUP.md step 6. + +- [x] Cleanup: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm remove ` + --vmx 'F:\CI\BuildVMs\smoke-linux\smoke-linux.vmx' --force + ``` + +--- + +## Passo 5 — Smoke end-to-end pipeline (`job`) + +Valida A3+A4 insieme: clone → wait → build → collect → cleanup. + +- [x] Lanciare un job sul template Windows con uno script trivial: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator job ` + --template-path 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` + --snapshot-name BaseClean ` + --job-id smoke-job-win ` + --repo-url 'http://10.10.20.11:3100/Simone/your-repo.git' ` + --branch main ` + --build-command 'echo hello > artifact.txt' ` + --guest-artifact-source 'artifact.txt' ` + --artifact-base-dir 'F:\CI\Artifacts\smoke-job-win' + ``` + + Atteso: exit code `0`, file `F:\CI\Artifacts\smoke-job-win\artifact.txt` presente, + VM smoke-job-win **non** più esistente in `F:\CI\BuildVMs\` (cleanup garantito). + +- [x] Ripetere su Linux: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator job ` + --template-path 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' ` + --snapshot-name BaseClean-Linux ` + --guest-os linux ` + --job-id smoke-job-linux ` + --repo-url 'http://10.10.20.11:3100/Simone/your-repo.git' ` + --branch main ` + --build-command 'echo hello > artifact.txt' ` + --guest-artifact-source 'artifact.txt' ` + --artifact-base-dir 'F:\CI\Artifacts\smoke-job-linux' + ``` + + Atteso: exit code `0`, `F:\CI\Artifacts\smoke-job-linux\smoke-job-linux\artifact.txt` presente. + +- [x] Verificare assenza VM orfane: + + ```powershell + & 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator vm cleanup --what-if + ``` + + Atteso: nessun candidato. + +--- + +## Passo 6 — Workflow Gitea end-to-end + +Validazione di A4 lato runner reale. + +- [x] Aprire la UI Gitea, repo `local-ci-cd-system`, branch + `feature/python-rewrite-and-linux-migration`. +- [x] Triggerare manualmente i 3 workflow (Actions → Run workflow): + - [x] `lint.yml` → atteso PASS + - [x] `self-test.yml` → PASS — validati **entrambi i transport** + (in-guest clone default + host-side clone+zip) su Win e Linux: + + | Job | Transport | Esito | + | ---------------------------- | ----------------------- | ----------- | + | smoke-windows | in-guest git clone | SUCCESS 26s | + | smoke-linux | in-guest git clone | SUCCESS 51s | + | smoke-windows-hostclone | host-side clone + zip | SUCCESS 28s | + | smoke-linux-hostclone | host-side clone + zip | SUCCESS 46s | + + Note: artifact github-free (no upload-artifact; file raw su + `F:\CI\Artifacts\`); `use-git-clone`/`submodules` default ON. + - [x] `build-ns7zip.yml` matrix Win+Linux → PASS (target repo + `Simone/nsis-plugin-ns7zip`, in-guest clone, `--dist`): + + | Job | Esito | + | ------------------ | --------------------------------------- | + | build (windows) | SUCCESS — DLL in dist/, artifact host | + | build (linux) | SUCCESS — 8 vCPU/8 GB, wall ~121s | + | release (tag push) | SUCCESS — release Gitea + 2 asset upload | + + Note: `release` validato con tag throwaway `v0.0.0-phaseA-test` + (poi rimosso); usa il token Actions automatico (Default Token + Permissions = Permissive) + REST API Gitea, legge gli artifact + host-side (no download-artifact). Linux perf: vCPU-starvation a 4 + vCPU risolta con guest-cpu=8 (≈ Windows VM ~120s); gap col Windows + nativo MSVC (~30s) è inerente MinGW, fuori scope CI. + +> Tutti i workflow PASS — nessun fallimento da triagare. (Procedura +> storica: in caso di fallimento, scaricare i log del run e aprire una +> voce in `TODO.md` con sha + output errore.) + +--- + +## Passo 7 — Burn-in capacità (4 job concorrenti) + +Validazione finale di A5 (definizione di "fatto Fase A"). + +- [x] Burn-in via `scripts/Start-BurnInTest.ps1` (wrapper lab che + hardcoda repo `burnin-dummy` + `BuildCommand` = `build.ps1`; + `Test-CapacityBurnIn.ps1` diretto senza `-BuildCommand` usa il + default dotnet → fallisce su repo non-.NET): + + ```powershell + .\scripts\Start-BurnInTest.ps1 + ``` + +- [x] Esito (4 job concorrenti × 3 round): + + | Round | Pass | Fail | Elapsed | Status | + | ----- | ---- | ---- | ------- | ------ | + | 1 | 4 | 0 | 76s | PASS | + | 2 | 4 | 0 | 88s | PASS | + | 3 | 4 | 0 | 73s | PASS | + + Total 12/12 PASS — **OVERALL: PASS**. Cleanup VM + lock OK ogni + round (0 orfani). A5 / Passo 7 validato. + +--- + +## Passo 8 — Benchmark wall-clock + +Confronto Python vs PowerShell. Pendenza A4. + +- [x] Eseguito `.\scripts\Measure-CIBenchmark.ps1` (4 iter): + + | Iter | Clone | Start | IP | WinRM | Destroy | Boot tot | + | ---- | ----- | ----- | ----- | ----- | ------- | -------- | + | 1 | 0.63 | 1.75 | 66.57 | 0.01 | 4.81 | 68.96 | + | 2 | 0.63 | 1.89 | 20.21 | 0.01 | 6.39 | 22.74 | + | 3 | 0.62 | 1.72 | 85.07 | 0.01 | 4.50 | 87.42 | + | 4 | 0.61 | 1.72 | 60.97 | 0.01 | 4.20 | 63.31 | + + Nessun errore, destroy OK (~4-6s), 0 orfani. Clone/start/WinRM + trascurabili; **fase IP (20-85s) = costo dominante e variabile** + (detect IP guest via VMware Tools / ci-report-ip) — noto, non + bloccante. Prerequisito scoperto: lo shim `Remove-BuildVM.ps1` + inoltrava `-ErrorAction` come `--error-action` (destroy falliva -> + orfana); risolto per tutti gli shim blind-loop (commit + `fix(shims): skip PowerShell common params`). + +- [x] Confronto col baseline pre-Python — **SALTATO** (deciso): nessun + baseline PS affidabile registrato; metriche infra appese a + `F:\CI\Logs\benchmark.jsonl` per trend futuri. Non bloccante. + +--- + +## Passo 9 — Merge della Fase A in `main` + +Solo dopo che i passi 3–7 sono tutti `[x]` PASS. + +- [ ] Aprire PR su Gitea: `feature/python-rewrite-and-linux-migration` → `main`. +- [ ] Verificare diff atteso: ~20 file Python nuovi sotto + `src/ci_orchestrator/`, ~10 shim PS in `scripts/`, doc in + `plans/A*-closeout.md`, AGENTS.md/ARCHITECTURE.md/README.md + aggiornati. +- [ ] Self-review della PR: nessun secret, nessun path hardcodato fuori + da `config.py`, nessun `.venv/` committato. +- [ ] Merge (squash sconsigliato — preserva la storia per-fase). +- [ ] Tag release: `git tag v2.0.0-phaseA` (o convenzione preferita). + +--- + +## Tracciamento globale + +| Passo | Descrizione | Stato | +| ----- | ----------------------------------------- | ----- | +| 1 | Aggiorna venv di produzione | [x] | +| 2 | Restart act_runner | [x] | +| 3 | Smoke `wait-ready` Windows | [x] | +| 4 | Smoke `wait-ready` Linux | [x] | +| 5 | Smoke `job` end-to-end (Win + Linux) | [x] | +| 6 | Workflow Gitea (lint + self-test + build) | [ ] | +| 7 | Burn-in 4 job × 10 round | [ ] | +| 8 | Benchmark wall-clock | [ ] | +| 9 | Merge PR e tag release | [ ] | + +Quando tutti i passi sono `[x]`, la Fase A è **definitivamente chiusa** e +si può aprire la Fase B (porting su host Linux). diff --git a/plans/PhaseAB-bridge-user-checklist.md b/plans/PhaseAB-bridge-user-checklist.md new file mode 100644 index 0000000..ffa8874 --- /dev/null +++ b/plans/PhaseAB-bridge-user-checklist.md @@ -0,0 +1,211 @@ +# Fase A → Fase B — Bridge checklist per l'utente + +Questa checklist copre i passi **intermedi** tra "Fase A done" e "B1 +inizia". Sono attività che girano sull'host **Windows attuale** (o sono +logistiche fuori dal repo) e non rientrano né in +[PhaseA-user-checklist.md](PhaseA-user-checklist.md) (validazione del +codice Python) né in [PhaseB-user-checklist.md](PhaseB-user-checklist.md) +(setup del nuovo host Linux). + +> **Quando eseguirla**: dopo Passo 9 di Fase A (merge + tag), prima di +> Passo 1 di Fase B (setup host Linux Mint). + +> **Tempo stimato**: 1–2 ore (escluso `tar` di backup, che dipende +> dalla dimensione di `F:\CI\`). + +--- + +## Sezione 1 — Salva il baseline pre-migrazione + +Senza un baseline registrato, il confronto richiesto da B7 ("tempo +medio entro ±20% baseline Windows") è impossibile. + +- [ ] Eseguire benchmark sull'host Windows (post Fase A, runner Python attivo): + + ```powershell + cd + .\scripts\Measure-CIBenchmark.ps1 | Tee-Object -FilePath "baseline-windows-$(Get-Date -Format 'yyyyMMdd').txt" + ``` + +- [ ] Annotare in `docs/RUNBOOK.md` (sezione nuova "Windows host baseline"): + - Data + - Tempo medio per job (Windows e Linux template) + - Success rate + - Versione `act_runner` e `ci_orchestrator` (output di `python -m ci_orchestrator --version` se disponibile, altrimenti SHA del commit) + - Hardware: i9-10900X, 64 GB RAM, NVMe SSD (da `AGENTS.md`) + +- [ ] Committare: + + ```powershell + git add docs/RUNBOOK.md + git commit -m "docs(runbook): record Windows host pre-migration baseline" + git push + ``` + +--- + +## Sezione 2 — Pre-requisiti Windows per Fase B + +Tutti questi vanno fatti **prima** di iniziare B1, perché B2/B3/B6 li +presuppongono già pronti. + +### 2.1 Inventario chiavi SSH + +- [ ] Verificare presenza chiavi: + + ```powershell + Get-ChildItem F:\CI\keys\ci_linux* + ``` + + Atteso: `ci_linux` (privata, perm utente owner) e `ci_linux.pub`. Se + mancano, rigenerarle con `ssh-keygen -t ed25519 -f F:\CI\keys\ci_linux -N ""` + e re-installare `ci_linux.pub` nel template `LinuxBuild2404` su + `~ci_build/.ssh/authorized_keys`. + +### 2.2 Recupero password guest Windows (`BuildVMGuest`) + +La password salvata in Windows Credential Manager **non è +esportabile**. Devi conoscerla per re-inserirla con `secret-tool` su +Linux (Passo 3 di Fase B). + +- [ ] Verificare di averla nel password manager personale. +- [ ] Se non la ricordi: bootare il template `WinBuild2025` (e `WinBuild2022`), + resettare la password dell'utente CI (`net user `), + ricatturare snapshot `BaseClean` da stato powered-off (vedi + `AGENTS.md` errore #9), aggiornare il Credential Manager Windows con + la nuova password, e annotarla in modo sicuro. + +### 2.3 Generazione Gitea PAT per il nuovo runner + +- [ ] Aprire Gitea → User Settings → Applications → Generate New Token. +- [ ] Nome: `ci-runner-linux`. Scope: `read:repository` + `write:package` + (o gli stessi del runner Windows attuale). +- [ ] Annotare il token in modo sicuro (verrà inserito al Passo 3 di + Fase B con `secret-tool store ... target GiteaPAT`). + +### 2.4 OpenSSH Server su Windows (per rsync pull dal Linux) + +Necessario solo se al Passo 2 di Fase B (transfer template) preferisci +fare `rsync pull` dal Linux. Alternativa: `rsync push` da Windows con +WSL/Cygwin/MSYS2. + +- [ ] Installare OpenSSH Server: + + ```powershell + Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 + Start-Service sshd + Set-Service sshd -StartupType Automatic + New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server' -Enabled True ` + -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + ``` + +- [ ] Validare connessione da un altro host: `ssh user@ "whoami"`. +- [ ] (Opzionale) Configurare `authorized_keys` per il futuro user + `ci-runner` del Linux box. + +### 2.5 Template fully powered-off (gate per B2) + +> AGENTS.md errore #9: snapshot catturato con VM accesa produce +> `*.vmem` / `*.vmsn` che rompono il clone su host diverso. + +- [ ] Verificare che nessun template sia running: + + ```powershell + & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' list + ``` + + Atteso: nessun path sotto `F:\CI\Templates\` nell'output. + +- [ ] Verificare assenza file di runtime: + + ```powershell + Get-ChildItem F:\CI\Templates -Recurse -Include *.vmem,*.vmsn,*.lck + ``` + + Atteso: nessun risultato. Se ci sono `.vmem`/`.vmsn`, lo snapshot + non è clean → ricatturare `BaseClean` / `BaseClean-Linux` da stato + fully powered-off. + +### 2.6 Backup precauzionale di `F:\CI\` + +Da fare **prima** del cutover B6, ma anche prima del transfer B2 è una +buona pratica. + +- [ ] Eseguire backup completo: + + ```powershell + $stamp = Get-Date -Format 'yyyyMMdd' + $archive = "E:\Backup\CI-pre-migration-$stamp.tar" + & 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\' + Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256" + ``` + +- [ ] Verifica integrità (lista contenuto, primi 20 record): + + ```powershell + & 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20 + ``` + +- [ ] Annotare path archivio + checksum SHA256. + +### 2.7 Annotare cadenze scheduled task (se custom) + +- [ ] Esportare l'inventario corrente: + + ```powershell + Get-ScheduledTask -TaskName 'CI-*' | + Select-Object TaskName, + @{n='Trigger';e={ $_.Triggers | ForEach-Object { $_.PSObject.Properties | Where-Object Name -in 'StartBoundary','RepetitionInterval','DaysOfWeek' | ForEach-Object { "$($_.Name)=$($_.Value)" } } -join '; ' }}, + @{n='Action'; e={ "$($_.Actions.Execute) $($_.Actions.Arguments)" }} | + Format-List | Tee-Object -FilePath "scheduled-tasks-inventory-$(Get-Date -Format 'yyyyMMdd').txt" + ``` + +- [ ] Confrontare con i `.timer` in `deploy/systemd/`: + + | Task Windows | systemd timer | Cadenza prevista | + | ----------------------------- | ------------------------------------- | ---------------------- | + | `CI-CleanupOrphanedBuildVMs` | `ci-cleanup-orphans.timer` | ogni 6h + boot | + | `CI-Invoke-RetentionPolicy` | `ci-retention-policy.timer` | giornaliero 03:00 | + | `CI-Watch-DiskSpace` | `ci-watch-disk-space.timer` | ogni 15min | + | `CI-Watch-RunnerHealth` | `ci-watch-runner-health.timer` | ogni 15min | + | (nuovo) | `ci-backup-template.timer` | settimanale Dom 02:00 | + +- [ ] Se le cadenze Windows sono diverse, aggiornare gli `OnCalendar` / + `OnUnitActiveSec` in `deploy/systemd/*.timer` **prima** del Passo + 5 di Fase B. Committare la modifica. + +--- + +## Sezione 3 — Logistica e finestra di manutenzione + +Questi non sono comandi, ma checkpoint prima di iniziare B1 e B6. + +- [ ] **Hardware Linux disponibile**: il box target è acquisito, + racked, raggiungibile in rete dall'host Windows e da Gitea. +- [ ] **DNS / IP statico** assegnato al nuovo host Linux (per + registrazione runner + raggiungibilità da admin). +- [ ] **Finestra di manutenzione per cutover B6 concordata**: 30–60 min + di stop Gitea Actions, in un orario a basso traffico. +- [ ] **Comunicazione utenti del repo**: avviso che durante la + finestra B6 i workflow saranno temporaneamente fermi. +- [ ] **Piano di rollback compreso**: rileggere la sezione "Rollback" + del Passo 6 di [PhaseB-user-checklist.md](PhaseB-user-checklist.md). + +--- + +## Tracciamento globale + +| Sezione | Descrizione | Stato | +| ------- | ---------------------------------------------- | ----- | +| 1 | Baseline benchmark salvato in `docs/RUNBOOK.md` | [ ] | +| 2.1 | Chiavi SSH presenti | [ ] | +| 2.2 | Password `BuildVMGuest` recuperata | [ ] | +| 2.3 | Gitea PAT generato per `ci-runner-linux` | [ ] | +| 2.4 | OpenSSH Server attivo (o alt. rsync) | [ ] | +| 2.5 | Template fully powered-off, no `.vmem` | [ ] | +| 2.6 | Backup `F:\CI\` con SHA256 | [ ] | +| 2.7 | Cadenze scheduled task confrontate con timer | [ ] | +| 3 | Hardware + DNS + finestra + comunicazione OK | [ ] | + +Quando tutte le righe sono `[x]`, puoi iniziare il Passo 1 di +[PhaseB-user-checklist.md](PhaseB-user-checklist.md). diff --git a/plans/PhaseB-user-checklist.md b/plans/PhaseB-user-checklist.md new file mode 100644 index 0000000..d02564d --- /dev/null +++ b/plans/PhaseB-user-checklist.md @@ -0,0 +1,484 @@ +# Fase B — Checklist finale per l'utente + +La Fase B è la migrazione del runner CI dall'host Windows attuale a un +nuovo host **Linux Mint LTS** con VMware Workstation Pro Linux. + +> **Stato repo**: solo lo step **B5** (unit `systemd` per i task +> periodici) è già committato in `deploy/systemd/` sul branch +> `feature/python-rewrite-and-linux-migration` (commit `50d37b5`). +> Tutti gli altri step (B1–B4, B6–B8) sono **operazioni hardware** +> sull'host Linux nuovo e devono essere eseguiti dall'utente. + +> **Pre-requisito**: la Fase A deve essere `done` (vedi +> `plans/PhaseA-user-checklist.md` Passo 9, merge + tag). +> B1, B2, B3 possono partire in parallelo a A3/A4/A5; B4–B8 sono +> sequenziali e iniziano solo dopo "A done". + +> **Tempo stimato totale**: 1–2 giorni di lavoro distribuiti su una +> finestra di 1–2 settimane (per consentire il burn-in tra B6 e B8). + +--- + +## Passo 1 — Setup host Linux Mint (B1) + +**Obiettivo**: hardware target con Linux Mint LTS, VMware Workstation +Pro Linux, layout storage e venv Python pronti. + +- [ ] Installare Linux Mint LTS sull'hardware target. +- [ ] `sudo apt update && sudo apt full-upgrade -y && sudo reboot` +- [ ] Scaricare e installare VMware Workstation Pro Linux (bundle + `.bundle` da Broadcow): + + ```bash + sudo bash VMware-Workstation-Full-*.bundle + vmrun -T ws --version # deve ritornare versione coerente + ``` + +- [ ] Smoke test Workstation (UI): creare VM trivia, clone, start, + stop, delete. +- [ ] Configurare `vmnet8` NAT range `192.168.79.0/24` (allineato a + Windows) editando `/etc/vmware/vmnet8/dhcpd/dhcpd.conf` o via + `vmware-netcfg`. +- [ ] Creare utente di servizio `ci-runner`: + + ```bash + sudo useradd -r -m -s /bin/bash ci-runner + ``` + +- [ ] Creare layout storage: + + ```bash + sudo mkdir -p /var/lib/ci/{build-vms,artifacts,templates,keys} + sudo chown -R ci-runner:ci-runner /var/lib/ci + sudo chmod 750 /var/lib/ci /var/lib/ci/* + sudo setfacl -d -m u:ci-runner:rwX /var/lib/ci/build-vms + ``` + +- [ ] Installare Python 3.11+ e creare venv produzione: + + ```bash + sudo apt install -y python3.11 python3.11-venv git + sudo mkdir -p /opt/ci + sudo python3.11 -m venv /opt/ci/venv + sudo chown -R ci-runner:ci-runner /opt/ci + ``` + +- [ ] Clonare il repo e installare il package nel venv: + + ```bash + sudo -u ci-runner git clone https://gitea.emulab.it/Simone/local-ci-cd-system.git /opt/ci/local-ci-cd-system + sudo -u ci-runner /opt/ci/venv/bin/pip install -e /opt/ci/local-ci-cd-system + sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator --help + ``` + + Atteso: lista degli 11 sub-comandi. + +- [ ] Installare PowerShell Core (richiesto da `Invoke-RetentionPolicy.ps1` + e `Backup-CITemplate.ps1`): + + ```bash + sudo apt install -y wget apt-transport-https software-properties-common + source /etc/os-release + wget -q https://packages.microsoft.com/config/ubuntu/${VERSION_ID}/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt update && sudo apt install -y powershell + pwsh --version + ``` + +--- + +## Passo 2 — Trasferimento template VM (B2) + +**Obiettivo**: copiare i template VMware da `F:\CI\Templates\` su +`/var/lib/ci/templates/` mantenendo gli snapshot. + +> ⚠️ AGENTS.md errore #9: i template devono essere **fully +> powered-off** prima della copia. Verificare assenza di `*.vmem` / +> `*.vmsn` di runtime. + +- [ ] Sull'host Windows: + + ```powershell + Get-ChildItem 'F:\CI\Templates' -Recurse -Include *.vmem,*.vmsn + # atteso: nessun risultato + ``` + +- [ ] Sull'host Linux, eseguire rsync via SSH dall'host Windows + (richiede OpenSSH Server attivo su Windows o, in alternativa, + `scp -r` lanciato da Windows verso Linux): + + ```bash + sudo -u ci-runner rsync -av --progress \ + user@windows-host:/cygdrive/f/CI/Templates/ \ + /var/lib/ci/templates/ + ``` + +- [ ] Validare integrità snapshot: + + ```bash + find /var/lib/ci/templates -name '*.vmx' -exec vmrun -T ws listSnapshots {} \; + ``` + + Atteso: `BaseClean` su `WinBuild2025.vmx` e `WinBuild2022.vmx`, + `BaseClean-Linux` su `LinuxBuild2404.vmx`. + +- [ ] Aprire ciascun `.vmx` su Workstation Linux per registrarli; + al prompt rispondere "**I copied it**". +- [ ] Smoke test pipeline VM: + + ```bash + sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm new \ + --template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx \ + --snapshot BaseClean --name smoke-b2-win + sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator wait-ready \ + --vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx \ + --guest-os windows --timeout 180 + sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm remove \ + --vmx /var/lib/ci/build-vms/smoke-b2-win/smoke-b2-win.vmx --force + ``` + + Ripetere per `LinuxBuild2404` (snapshot `BaseClean-Linux`, + `--guest-os linux`). + +--- + +## Passo 3 — Trasferimento credenziali e chiavi (B3) + +**Obiettivo**: chiavi SSH e credenziali guest disponibili al user +`ci-runner` headless. + +- [ ] Copiare le chiavi SSH guest Linux: + + ```bash + sudo mkdir -p /etc/ci/keys + sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux /etc/ci/keys/ + sudo scp user@windows-host:/cygdrive/f/CI/keys/ci_linux.pub /etc/ci/keys/ + sudo chown ci-runner:ci-runner /etc/ci/keys/* + sudo chmod 600 /etc/ci/keys/ci_linux + sudo chmod 644 /etc/ci/keys/ci_linux.pub + ``` + +- [ ] Re-store credenziale guest Windows (sostituire la password al + prompt — **NON** inserirla in chat o issue): + + ```bash + sudo -u ci-runner secret-tool store --label='BuildVMGuest' service ci target BuildVMGuest + ``` + +- [ ] Re-store Gitea PAT: + + ```bash + sudo -u ci-runner secret-tool store --label='GiteaPAT' service ci target GiteaPAT + ``` + +- [ ] Validare lettura interattiva: + + ```bash + sudo -u ci-runner /opt/ci/venv/bin/python -c \ + "import keyring; print(keyring.get_credential('BuildVMGuest', None))" + ``` + +- [ ] **PoC headless** (critico): verificare lettura keyring da + contesto systemd (no D-Bus user session): + + ```bash + sudo systemd-run --uid=ci-runner --pipe \ + /opt/ci/venv/bin/python -c \ + "import keyring; print(keyring.get_credential('BuildVMGuest', None))" + ``` + + Se fallisce → implementare backend keyring file-based con `age` o + `sops` come fallback. Documentare la scelta finale in + `docs/HOST-SETUP.md` prima di proseguire. + +--- + +## Passo 4 — Setup act_runner come systemd service (B4) + +**Obiettivo**: act_runner Linux registrato verso Gitea, gestito da +systemd. + +- [ ] Scaricare il binario act_runner Linux ≥ v1.0.2: + + ```bash + sudo mkdir -p /opt/ci/act_runner + sudo wget -O /opt/ci/act_runner/act_runner \ + https://gitea.com/gitea/act_runner/releases/download/v1.0.2/act_runner-1.0.2-linux-amd64 + sudo chmod +x /opt/ci/act_runner/act_runner + sudo chown -R ci-runner:ci-runner /opt/ci/act_runner + ``` + +- [ ] Generare token registrazione su Gitea (Admin → Runners → Create new runner). +- [ ] Registrare il runner (sostituire ``): + + ```bash + sudo -u ci-runner mkdir -p /var/lib/ci/runner + cd /var/lib/ci/runner + sudo -u ci-runner /opt/ci/act_runner/act_runner register \ + --no-interactive \ + --instance https://gitea.emulab.it \ + --token \ + --name ci-linux \ + --labels windows-build:host,linux-build:host + ``` + + > ⚠️ Lasciare il runner in stato **paused** lato Gitea UI fino al + > cutover (Passo 6), per non intercettare job di produzione. + +- [ ] Creare `/etc/systemd/system/act-runner.service`: + + ```ini + [Unit] + Description=Gitea Act Runner + After=network-online.target + Wants=network-online.target + + [Service] + Type=simple + User=ci-runner + WorkingDirectory=/var/lib/ci/runner + Environment="PYTHONIOENCODING=utf-8" + Environment="CI_ROOT=/var/lib/ci" + Environment="CI_TEMPLATES=/var/lib/ci/templates" + Environment="CI_BUILD_VMS=/var/lib/ci/build-vms" + Environment="CI_ARTIFACTS=/var/lib/ci/artifacts" + Environment="CI_KEYS=/etc/ci/keys" + ExecStart=/opt/ci/act_runner/act_runner daemon + Restart=on-failure + RestartSec=10 + + [Install] + WantedBy=multi-user.target + ``` + +- [ ] Creare `/etc/ci/environment` (letto dalle unit dei task periodici + di B5; vedi `deploy/systemd/README.md`): + + ```bash + sudo tee /etc/ci/environment <<'EOF' + PYTHONIOENCODING=utf-8 + CI_ROOT=/var/lib/ci + CI_TEMPLATES=/var/lib/ci/templates + CI_BUILD_VMS=/var/lib/ci/build-vms + CI_ARTIFACTS=/var/lib/ci/artifacts + CI_KEYS=/etc/ci/keys + EOF + sudo chmod 644 /etc/ci/environment + ``` + +- [ ] Abilitare e avviare: + + ```bash + sudo systemctl daemon-reload + sudo systemctl enable --now act-runner.service + sudo systemctl status act-runner + sudo journalctl -u act-runner -f # Ctrl+C dopo conferma OK + ``` + +- [ ] Smoke job: triggerare manualmente `self-test.yml` selezionando + il runner Linux (mettere temporaneamente in pausa quello Windows + o usare label distinte). Atteso: PASS. + +--- + +## Passo 5 — Installare i timer systemd (B5) + +**Obiettivo**: attivare le coppie `.service`+`.timer` già committate +in `deploy/systemd/` per replicare i task periodici di +`Register-CIScheduledTasks.ps1`. + +- [ ] Copiare le unit: + + ```bash + cd /opt/ci/local-ci-cd-system + sudo cp deploy/systemd/ci-*.service deploy/systemd/ci-*.timer /etc/systemd/system/ + sudo systemctl daemon-reload + ``` + +- [ ] Abilitare e avviare tutti i timer: + + ```bash + for t in ci-cleanup-orphans ci-retention-policy \ + ci-watch-disk-space ci-watch-runner-health \ + ci-backup-template; do + sudo systemctl enable --now "${t}.timer" + done + ``` + +- [ ] Verificare lo schedule: + + ```bash + systemctl list-timers --all 'ci-*' + ``` + + Atteso: 5 timer in stato `active`. + +- [ ] Trigger manuale di ciascun service per validare l'esecuzione + one-shot: + + ```bash + for s in ci-cleanup-orphans ci-watch-disk-space ci-watch-runner-health; do + sudo systemctl start "${s}.service" + sudo systemctl status "${s}.service" --no-pager + done + ``` + + > `ci-retention-policy` e `ci-backup-template` chiamano `pwsh`: + > eseguirli solo dopo aver verificato che `pwsh --version` funzioni + > sotto user `ci-runner`. + +- [ ] Per maggiori dettagli (mapping Windows→Linux, troubleshooting, + rollback) vedi `deploy/systemd/README.md`. + +--- + +## Passo 6 — Cutover (B6) + +**Obiettivo**: spostare la produzione dall'host Windows a Linux in una +finestra di manutenzione concordata. + +> ⚠️ Eseguire solo dopo che i Passi 1–5 sono tutti `[x]` PASS. + +- [ ] Annunciare la finestra di manutenzione (durata stimata: 30–60 min). +- [ ] Verificare nessun job in coda lato Gitea (Admin → Actions → Tasks). +- [ ] **Stop** act_runner sull'host Windows: + + ```powershell + Stop-Service actions-runner + Get-Service actions-runner # atteso: Stopped + ``` + +- [ ] **Disabilitare** scheduled task Windows: + + ```powershell + Get-ScheduledTask -TaskName 'CI-*' | Unregister-ScheduledTask -Confirm:$false + ``` + +- [ ] Rsync incrementale finale degli artifact: + + ```bash + sudo -u ci-runner rsync -av --delete \ + user@windows-host:/cygdrive/f/CI/Artifacts/ \ + /var/lib/ci/artifacts/ + ``` + +- [ ] Sbloccare il runner Linux su Gitea UI (rimuovere `paused` o + ripristinare le label di produzione `windows-build:host,linux-build:host`). +- [ ] Verificare runner Linux online su Gitea UI. +- [ ] Smoke trigger: + - [ ] `self-test.yml` → PASS dal runner Linux + - [ ] `build-ns7zip.yml` matrix Win+Linux → PASS dal runner Linux +- [ ] Monitorare per ≥30 min: + + ```bash + sudo journalctl -u act-runner -f --since "30min ago" + ``` + + Nessun errore di severità critica atteso. + +> **Rollback (entro 30 min se PASS critico fallisce)**: +> 1. `sudo systemctl stop act-runner` su Linux +> 2. `Start-Service actions-runner` su Windows +> 3. Re-registrare scheduled task con `scripts\Register-CIScheduledTasks.ps1` +> 4. Documentare incidente in `TODO.md` e tornare a B1–B5 per fix + +--- + +## Passo 7 — Capacity burn-in (B7) + +**Obiettivo**: validare il carico target con tempi paragonabili al +baseline Windows (entro ±20%). + +- [ ] Burn-in 4 job concorrenti × 10 round su `WinBuild2025` + (lo script `Test-CapacityBurnIn.ps1` ora delega via shim alla CLI + Python `job` e gira anche da `pwsh` su Linux): + + ```bash + pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \ + -Concurrency 4 -Rounds 10 -Template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx + ``` + +- [ ] Burn-in 4 × 10 su `LinuxBuild2404` (analogo, `--Template` + sul VMX Linux). +- [ ] Misurare: + - [ ] Tempo medio per job, confronto con baseline A5 + - [ ] Tutti i 80 job (2 × 4 × 10) PASS + - [ ] Zero VM orfane in `/var/lib/ci/build-vms/`: + + ```bash + sudo -u ci-runner /opt/ci/venv/bin/python -m ci_orchestrator vm cleanup --dry-run + ``` + + - [ ] Spazio disco `/var/lib/ci/build-vms/` torna al baseline post-cleanup +- [ ] Documentare i risultati in `docs/RUNBOOK.md` (sezione "Linux + host baseline"). +- [ ] Se delta > 20% vs baseline Windows: aprire issue in `TODO.md` + con dettagli per profiling (probabile candidato: filesystem ext4 + vs NTFS — valutare XFS/BTRFS). + +--- + +## Passo 8 — Stabilità ≥1 settimana (gate per B8) + +**Obiettivo**: verificare ≥1 settimana di esercizio Linux senza +incidenti critici prima di dismettere l'host Windows. + +- [ ] Esercizio normale per ≥7 giorni con runner Linux primario. +- [ ] Verifica giornaliera (bastano 2 minuti): + + ```bash + sudo systemctl --failed # nessun service in failed + systemctl list-timers --all 'ci-*' # tutti gli ultimi run OK + sudo journalctl -u act-runner --since "24h ago" -p err + ``` + +- [ ] Nessun rollback effettuato durante la settimana. + +--- + +## Passo 9 — Decommissioning host Windows (B8, opzionale) + +> ⚠️ Procedere solo dopo Passo 8 `[x]`. L'host Windows va lasciato +> spento ma intatto per ≥1 mese come rollback. + +- [ ] Backup finale di `F:\CI\` su archivio esterno: + + ```powershell + $stamp = Get-Date -Format 'yyyyMMdd' + $archive = "E:\Backup\CI-$stamp.tar" + & 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\' + Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256" + ``` + +- [ ] Verifica integrità (estrazione di prova in tmp): + + ```powershell + $test = "$env:TEMP\ci-restore-test" + & 'C:\Program Files\Git\usr\bin\tar.exe' -tf $archive | Select-Object -First 20 + ``` + +- [ ] Spegnere host Windows (`shutdown /s /t 0`). +- [ ] Lasciare hardware integro e spento per ≥1 mese. +- [ ] Aggiornare `docs/RUNBOOK.md` con la procedura di riaccensione + + ri-registrazione scheduled task come rollback. +- [ ] Dopo 1 mese senza rollback: riallocare hardware. + +--- + +## Tracciamento globale + +| Passo | Step | Descrizione | Stato | +| ----- | ---- | -------------------------------------------------- | ----- | +| 1 | B1 | Setup host Linux Mint (OS, Workstation, venv, ACL) | [ ] | +| 2 | B2 | Trasferimento template VM + smoke `vm new` | [ ] | +| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [ ] | +| 4 | B4 | act_runner systemd service registrato | [ ] | +| 5 | B5 | Timer systemd installati e attivi | [ ] | +| 6 | B6 | Cutover (stop Windows, runner Linux primario) | [ ] | +| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux) | [ ] | +| 8 | — | ≥1 settimana di stabilità senza rollback | [ ] | +| 9 | B8 | Decommissioning host Windows (opzionale) | [ ] | + +Quando tutti i passi sono `[x]`, la **Fase B è chiusa** e si può +valutare l'apertura della Fase C (backend ESXi, vedi +`plans/idea-3-esxi-support.md`). diff --git a/plans/idea-1-python-rewrite.md b/plans/idea-1-python-rewrite.md index 0b1f004..72ae530 100644 --- a/plans/idea-1-python-rewrite.md +++ b/plans/idea-1-python-rewrite.md @@ -202,7 +202,7 @@ gitea/ # workflow + actions invariati - [ ] `pytest` verde con coverage minima 70% su `src/ci_orchestrator/` - [ ] `ruff check` + `mypy --strict` puliti su tutto `src/` - [ ] `lint.yml` aggiornato: PSSA per file PS legacy + ruff/mypy per Python -- [ ] Workflow `build-nsInnoUnp.yml` PASS su matrice Windows + Linux +- [ ] Workflow `build-ns7zip.yml` PASS su matrice Windows + Linux - [ ] Smoke test (`Test-Smoke.ps1` o equivalente Python) PASS - [ ] Capacity burn-in 4 job concorrenti PASS - [ ] `README.md`, `AGENTS.md`, `docs/ARCHITECTURE.md`, `docs/HOST-SETUP.md` aggiornati diff --git a/plans/idea-2-linux-host.md b/plans/idea-2-linux-host.md index 5f854db..4bb75dc 100644 --- a/plans/idea-2-linux-host.md +++ b/plans/idea-2-linux-host.md @@ -132,7 +132,7 @@ Finestra di manutenzione concordata: 3. Verificare nessun job in coda lato Gitea 4. Avviare act_runner Linux (`systemctl start act-runner`) 5. Trigger manuale di un workflow di smoke test -6. Trigger del workflow matrix `build-nsInnoUnp.yml` (Win + Linux build) +6. Trigger del workflow matrix `build-ns7zip.yml` (Win + Linux build) Se PASS → cutover confermato. Se FAIL → rollback (riavvio runner Windows, nessun dato è stato distrutto). @@ -205,7 +205,7 @@ Costo: codebase divisa in due ambienti, maggiore overhead operativo. - [ ] act_runner gira come systemd service `act-runner.service` su Linux Mint - [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`) operativi dal nuovo host con snapshot integri -- [ ] Workflow `build-nsInnoUnp.yml` (matrix Win+Linux) PASS dal nuovo host +- [ ] Workflow `build-ns7zip.yml` (matrix Win+Linux) PASS dal nuovo host - [ ] Burn-in capacity 4 job concorrenti PASS, tempi entro ±20% del baseline - [ ] Storage migrato a `/var/lib/ci/`, host Windows in stand-by come rollback - [ ] Tutti i timer systemd attivi e schedulati diff --git a/plans/implementation-plan-A-B.md b/plans/implementation-plan-A-B.md index eb5f145..7d785fe 100644 --- a/plans/implementation-plan-A-B.md +++ b/plans/implementation-plan-A-B.md @@ -13,7 +13,7 @@ Sintesi fasi: - **Fase A — Rewrite Python dell'orchestratore (host Windows attuale)** - Output verificabile: `python -m ci_orchestrator job ...` sostituisce `Invoke-CIJob.ps1`; `pytest` ≥70% coverage; workflow - `build-nsInnoUnp.yml` PASS. + `build-ns7zip.yml` PASS. - Stato: da iniziare. - **Fase B — Migrazione host a Linux Mint + Workstation Pro Linux** - Output verificabile: act_runner come `act-runner.service` systemd; @@ -43,35 +43,35 @@ A4; cambiano solo path/env vars in Fase B). ## Checklist riassuntiva master -- [ ] [A1] Creare `pyproject.toml` + package `src/ci_orchestrator/` + venv `F:\CI\python\venv\` -- [ ] [A1] Implementare `config.py` (env vars + `config.toml`) con default Windows e hook path Linux -- [ ] [A1] Implementare `backends/protocol.py` con Protocol `VmBackend` (firma neutra, no `vmrun_*` nei nomi pubblici) -- [ ] [A1] Implementare `backends/workstation.py` (`WorkstationVmrunBackend`) usando `subprocess` + `shutil.which('vmrun')` -- [ ] [A1] Implementare `transport/winrm.py` con wrapper `pypsrp.client.Client` -- [ ] [A1] Implementare `transport/ssh.py` con wrapper `paramiko.SSHClient` / `SFTPClient` -- [ ] [A1] Implementare `credentials.py` con Protocol `CredentialStore` + `KeyringCredentialStore` +- [x] [A1] Creare `pyproject.toml` + package `src/ci_orchestrator/` + venv `F:\CI\python\venv\` +- [x] [A1] Implementare `config.py` (env vars + `config.toml`) con default Windows e hook path Linux +- [x] [A1] Implementare `backends/protocol.py` con Protocol `VmBackend` (firma neutra, no `vmrun_*` nei nomi pubblici) +- [x] [A1] Implementare `backends/workstation.py` (`WorkstationVmrunBackend`) usando `subprocess` + `shutil.which('vmrun')` +- [x] [A1] Implementare `transport/winrm.py` con wrapper `pypsrp.client.Client` +- [x] [A1] Implementare `transport/ssh.py` con wrapper `paramiko.SSHClient` / `SFTPClient` +- [x] [A1] Implementare `credentials.py` con Protocol `CredentialStore` + `KeyringCredentialStore` - [ ] [A1] PoC `wait-ready` end-to-end via `pypsrp` contro un guest Windows reale -- [ ] [A1] Test pytest unitari per `vmrun.py`, `winrm.py`, `ssh.py`, `credentials.py` con mock -- [ ] [A1] Aggiungere job pytest a `gitea/workflows/lint.yml` -- [ ] [A2] Portare `Wait-VMReady.ps1` → `python -m ci_orchestrator wait-ready` -- [ ] [A2] Portare `Remove-BuildVM.ps1` → `vm remove` -- [ ] [A2] Portare `Cleanup-OrphanedBuildVMs.ps1` → `vm cleanup` -- [ ] [A2] Portare `Watch-DiskSpace.ps1` → `monitor disk` -- [ ] [A2] Portare `Watch-RunnerHealth.ps1` → `monitor runner` -- [ ] [A2] Portare `Get-CIJobSummary.ps1` → `report job` -- [ ] [A2] Sostituire ognuno dei `.ps1` portati con shim a 3 righe verso la CLI Python -- [ ] [A3] Portare `New-BuildVM.ps1` → `vm new` -- [ ] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run` -- [ ] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect` -- [ ] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest -- [ ] [A4] Portare `Invoke-CIJob.ps1` → `python -m ci_orchestrator job` -- [ ] [A4] Aggiornare `gitea/actions/local-ci-build/action.yml` per invocare la CLI Python direttamente -- [ ] [A4] Forzare `PYTHONIOENCODING=utf-8` in `runner/config.yaml` -- [ ] [A4] Eseguire workflow `build-nsInnoUnp.yml` (matrix Win+Linux) end-to-end PASS -- [ ] [A5] Convertire errori frequenti `AGENTS.md` #9, #10, #11, #12 in test pytest dedicati -- [ ] [A5] Aggiornare `AGENTS.md` con sezione "Python development" (venv, ruff, mypy, pytest) -- [ ] [A5] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout package -- [ ] [A5] Aggiornare `README.md` con setup Python +- [x] [A1] Test pytest unitari per `vmrun.py`, `winrm.py`, `ssh.py`, `credentials.py` con mock +- [x] [A1] Aggiungere job pytest a `gitea/workflows/lint.yml` +- [x] [A2] Portare `Wait-VMReady.ps1` → `python -m ci_orchestrator wait-ready` +- [x] [A2] Portare `Remove-BuildVM.ps1` → `vm remove` +- [x] [A2] Portare `Cleanup-OrphanedBuildVMs.ps1` → `vm cleanup` +- [x] [A2] Portare `Watch-DiskSpace.ps1` → `monitor disk` +- [x] [A2] Portare `Watch-RunnerHealth.ps1` → `monitor runner` +- [x] [A2] Portare `Get-CIJobSummary.ps1` → `report job` +- [x] [A2] Sostituire ognuno dei `.ps1` portati con shim a 3 righe verso la CLI Python +- [x] [A3] Portare `New-BuildVM.ps1` → `vm new` +- [x] [A3] Portare `Invoke-RemoteBuild.ps1` → `build run` +- [x] [A3] Portare `Get-BuildArtifacts.ps1` → `artifacts collect` +- [x] [A3] Convertire test Pester `New-BuildVM.Tests.ps1`, `Wait-VMReady.Tests.ps1`, `Remove-BuildVM.Tests.ps1` in pytest +- [x] [A4] Portare `Invoke-CIJob.ps1` → `python -m ci_orchestrator job` +- [x] [A4] Aggiornare `gitea/actions/local-ci-build/action.yml` per invocare la CLI Python direttamente +- [x] [A4] Forzare `PYTHONIOENCODING=utf-8` in `runner/config.yaml` +- [ ] [A4] Eseguire workflow `build-ns7zip.yml` (matrix Win+Linux) end-to-end PASS +- [x] [A5] Convertire errori frequenti `AGENTS.md` #9, #10, #11, #12 in test pytest dedicati +- [x] [A5] Aggiornare `AGENTS.md` con sezione "Python development" (venv, ruff, mypy, pytest) +- [x] [A5] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout package +- [x] [A5] Aggiornare `README.md` con setup Python - [ ] [A5] Eseguire `Test-CapacityBurnIn` (versione Python) 4 job concorrenti PASS - [ ] [B1] Installare Linux Mint LTS + aggiornamenti su hardware target - [ ] [B1] Installare VMware Workstation Pro Linux (bundle ufficiale Broadcom) e validare `vmrun` su VM di test @@ -89,11 +89,11 @@ A4; cambiano solo path/env vars in Fase B). - [ ] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host` - [ ] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8` - [ ] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f` -- [ ] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`) +- [x] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`) - [ ] [B5] Abilitare tutti i timer con `systemctl enable --now` e validare `systemctl list-timers` - [ ] [B6] Stop act_runner sull'host Windows e verifica coda Gitea vuota - [ ] [B6] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/` -- [ ] [B6] Trigger smoke workflow + `build-nsInnoUnp.yml` matrix da host Linux PASS +- [ ] [B6] Trigger smoke workflow + `build-ns7zip.yml` matrix da host Linux PASS - [ ] [B7] Eseguire burn-in 4 job concorrenti × 10 round su template Windows e Linux con tempi entro ±20% baseline - [ ] [B8] Backup finale di `F:\CI\` su archivio - [ ] [B8] Lasciare host Windows spento ma reinstallato per ≥1 mese come rollback @@ -128,19 +128,19 @@ WinRM/SSH e credential store funzionanti contro l'ambiente reale via PoC **Attività**: -- [ ] Creare `pyproject.toml` (build backend `setuptools` o `hatchling`) con dipendenze `pypsrp`, `paramiko`, `keyring`, `click`, `tomli` (se Python <3.11), `rich` opzionale per logging -- [ ] Creare layout `src/ci_orchestrator/` con `__init__.py`, `__main__.py` (entry point `click`) -- [ ] Creare venv in `F:\CI\python\venv\` e installare il package in editable (`pip install -e .[dev]`) -- [ ] Implementare `config.py`: caricamento env vars (`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`) con merge da `config.toml` opzionale; default OS-aware (Windows → `F:\CI\...`, Linux → `/var/lib/ci/...`) -- [ ] Implementare `backends/protocol.py` con Protocol `VmBackend` (metodi `clone_linked`, `start`, `stop`, `delete`, `get_ip`, `list_snapshots`) e dataclass `VmHandle` (path/identificatore opaco) -- [ ] Implementare `backends/workstation.py`: `WorkstationVmrunBackend` che usa `subprocess.run([vmrun, '-T', 'ws', op, *args], check=False, capture_output=True, text=True, encoding='utf-8')` con check esplicito su `returncode` -- [ ] Implementare `transport/winrm.py`: wrapper su `pypsrp.client.Client(host, username, password, ssl=True, cert_validation=False)` con metodi `run`, `copy`, `fetch` -- [ ] Implementare `transport/ssh.py`: wrapper su `paramiko.SSHClient` con `set_missing_host_key_policy(AutoAddPolicy)`, `known_hosts` configurabile (default `None` per non interferire con clone ephemeri — vedi `AGENTS.md` errore #12) -- [ ] Implementare `credentials.py`: Protocol `CredentialStore` + `KeyringCredentialStore` che usa `keyring.get_credential(target, None)` e ritorna oggetto `Credential(username, password)` +- [x] Creare `pyproject.toml` (build backend `setuptools` o `hatchling`) con dipendenze `pypsrp`, `paramiko`, `keyring`, `click`, `tomli` (se Python <3.11), `rich` opzionale per logging +- [x] Creare layout `src/ci_orchestrator/` con `__init__.py`, `__main__.py` (entry point `click`) +- [x] Creare venv in `F:\CI\python\venv\` e installare il package in editable (`pip install -e .[dev]`) +- [x] Implementare `config.py`: caricamento env vars (`CI_ROOT`, `CI_TEMPLATES`, `CI_BUILD_VMS`, `CI_ARTIFACTS`, `CI_KEYS`) con merge da `config.toml` opzionale; default OS-aware (Windows → `F:\CI\...`, Linux → `/var/lib/ci/...`) +- [x] Implementare `backends/protocol.py` con Protocol `VmBackend` (metodi `clone_linked`, `start`, `stop`, `delete`, `get_ip`, `list_snapshots`) e dataclass `VmHandle` (path/identificatore opaco) +- [x] Implementare `backends/workstation.py`: `WorkstationVmrunBackend` che usa `subprocess.run([vmrun, '-T', 'ws', op, *args], check=False, capture_output=True, text=True, encoding='utf-8')` con check esplicito su `returncode` +- [x] Implementare `transport/winrm.py`: wrapper su `pypsrp.client.Client(host, username, password, ssl=True, cert_validation=False)` con metodi `run`, `copy`, `fetch` +- [x] Implementare `transport/ssh.py`: wrapper su `paramiko.SSHClient` con `set_missing_host_key_policy(AutoAddPolicy)`, `known_hosts` configurabile (default `None` per non interferire con clone ephemeri — vedi `AGENTS.md` errore #12) +- [x] Implementare `credentials.py`: Protocol `CredentialStore` + `KeyringCredentialStore` che usa `keyring.get_credential(target, None)` e ritorna oggetto `Credential(username, password)` - [ ] PoC end-to-end: comando `python -m ci_orchestrator wait-ready --vmx --timeout 120` su un clone reale del template `WinBuild2025` (validare WinRM HTTPS self-signed) -- [ ] Setup `pytest`, `pytest-mock`, `ruff`, `mypy` come dev dependencies; configurare `pyproject.toml` con `[tool.ruff]` e `[tool.mypy]` strict -- [ ] Test pytest unitari: `test_vmrun.py` (mock subprocess, casi `vmrun list` con e senza VMX target — copre errore #10), `test_winrm.py` (mock `pypsrp`), `test_ssh.py` (mock `paramiko`, copre errore #12), `test_credentials.py` (mock `keyring`) -- [ ] Aggiungere job `python` a `gitea/workflows/lint.yml`: setup venv → `ruff check src/ tests/` → `mypy --strict src/` → `pytest --cov=ci_orchestrator --cov-fail-under=70` +- [x] Setup `pytest`, `pytest-mock`, `ruff`, `mypy` come dev dependencies; configurare `pyproject.toml` con `[tool.ruff]` e `[tool.mypy]` strict +- [x] Test pytest unitari: `test_vmrun.py` (mock subprocess, casi `vmrun list` con e senza VMX target — copre errore #10), `test_winrm.py` (mock `pypsrp`), `test_ssh.py` (mock `paramiko`, copre errore #12), `test_credentials.py` (mock `keyring`) +- [x] Aggiungere job `python` a `gitea/workflows/lint.yml`: setup venv → `ruff check src/ tests/` → `mypy --strict src/` → `pytest --cov=ci_orchestrator --cov-fail-under=70` **Hook futuri Fase C**: il Protocol `VmBackend` deve usare nomi neutri (`clone_linked`, non `vmrun_clone`), accettare `template`/`snapshot`/`name` @@ -166,10 +166,10 @@ modificato in A1. **Definizione di fatto step A1**: - [ ] PoC `wait-ready` PASS contro VM reale -- [ ] Coverage pytest ≥70% sui moduli core -- [ ] `lint.yml` aggiornato e verde -- [ ] Protocol `VmBackend` reviewato e congelato -- [ ] Documentato setup venv in `README.md` (sezione minima) +- [x] Coverage pytest ≥70% sui moduli core +- [x] `lint.yml` aggiornato e verde +- [x] Protocol `VmBackend` reviewato e congelato +- [x] Documentato setup venv in `README.md` (sezione minima) ### A2 — Script "foglia" (no state condiviso) @@ -182,15 +182,15 @@ stato con l'orchestratore, sostituendoli con shim minimi. **Attività**: -- [ ] Implementare `commands/wait.py` con sottocomando `wait-ready` (parametri `--vmx`, `--timeout`, `--guest-os`) -- [ ] Implementare `commands/vm.py` con sottocomando `vm remove` (parametri `--vmx`, `--force`) -- [ ] Implementare `vm cleanup` (scan `CI_BUILD_VMS`, riconoscimento clone orfani per pattern naming + età) -- [ ] Implementare `commands/monitor.py` con `monitor disk` (soglie configurabili, output JSON o human) -- [ ] Implementare `monitor runner` (controllo processo act_runner attivo + ultimo heartbeat) -- [ ] Implementare `commands/report.py` con `report job` (read-only su artifact dir + log job) -- [ ] Sostituire `scripts/Wait-VMReady.ps1` con shim PS 5.1 di 3 righe: `& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready @args; exit $LASTEXITCODE` -- [ ] Idem per `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`, `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1` -- [ ] Aggiungere test pytest per ogni comando (mock backend + mock filesystem via `tmp_path`) +- [x] Implementare `commands/wait.py` con sottocomando `wait-ready` (parametri `--vmx`, `--timeout`, `--guest-os`) +- [x] Implementare `commands/vm.py` con sottocomando `vm remove` (parametri `--vmx`, `--force`) +- [x] Implementare `vm cleanup` (scan `CI_BUILD_VMS`, riconoscimento clone orfani per pattern naming + età) +- [x] Implementare `commands/monitor.py` con `monitor disk` (soglie configurabili, output JSON o human) +- [x] Implementare `monitor runner` (controllo processo act_runner attivo + ultimo heartbeat) +- [x] Implementare `commands/report.py` con `report job` (read-only su artifact dir + log job) +- [x] Sostituire `scripts/Wait-VMReady.ps1` con shim PS 5.1 di 3 righe: `& 'F:\CI\python\venv\Scripts\python.exe' -m ci_orchestrator wait-ready @args; exit $LASTEXITCODE` +- [x] Idem per `Remove-BuildVM.ps1`, `Cleanup-OrphanedBuildVMs.ps1`, `Watch-DiskSpace.ps1`, `Watch-RunnerHealth.ps1`, `Get-CIJobSummary.ps1` +- [x] Aggiungere test pytest per ogni comando (mock backend + mock filesystem via `tmp_path`) - [ ] Convertire `tests/Wait-VMReady.Tests.ps1` in `tests/test_commands_wait.py` (preservando i casi negativi) - [ ] Convertire `tests/Remove-BuildVM.Tests.ps1` in `tests/test_commands_vm_remove.py` - [ ] Verificare che gli scheduled task esistenti (`Register-CIScheduledTasks.ps1`) continuino a funzionare invocando gli shim PS @@ -229,15 +229,15 @@ Python preservando il comportamento dei `.ps1` esistenti. **Attività**: -- [ ] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`) -- [ ] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip` -- [ ] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux -- [ ] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`) -- [ ] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`) -- [ ] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim -- [ ] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py` -- [ ] Aggiungere test pytest per `build run` (mock transport + capture stdout) -- [ ] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path) +- [x] Implementare `vm new` in `commands/vm.py` (parametri `--template`, `--snapshot`, `--name`, `--guest-os`) +- [x] Integrare `WorkstationVmrunBackend.clone_linked` + `start` + attesa IP via `get_ip` +- [x] Implementare `commands/build.py` con `build run` (parametri `--vmx`, `--script`, `--workdir`, `--guest-os`); usa `transport.winrm` per Windows, `transport.ssh` per Linux +- [x] Gestione streaming output build verso stdout (act_runner cattura stdout — preservare comportamento `Write-Host`) +- [x] Implementare `commands/artifacts.py` con `artifacts collect` (parametri `--vmx`, `--remote-path`, `--local-dir`) +- [x] Sostituire `scripts/New-BuildVM.ps1`, `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1` con shim +- [x] Convertire `tests/New-BuildVM.Tests.ps1` in `tests/test_commands_vm_new.py` +- [x] Aggiungere test pytest per `build run` (mock transport + capture stdout) +- [x] Aggiungere test pytest per `artifacts collect` (mock SFTP/WinRM file copy + tmp_path) - [ ] Validare end-to-end: clone WinBuild2025 → build script PowerShell trivial → collect artifact ZIP **Hook futuri Fase C**: `build run` deve ricevere un `VmHandle` opaco, @@ -257,10 +257,10 @@ Python; nessuna modifica ai workflow YAML in A3 (solo shim). **Definizione di fatto step A3**: -- [ ] Pipeline build completa accessibile via Python CLI -- [ ] Shim PS preservano l'API per i caller esistenti +- [x] Pipeline build completa accessibile via Python CLI +- [x] Shim PS preservano l'API per i caller esistenti - [ ] Smoke build PASS contro VM reale Windows e Linux -- [ ] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito +- [x] Pester `New-BuildVM.Tests.ps1` rimosso e sostituito ### A4 — Orchestratore + switch workflow @@ -272,13 +272,13 @@ Gitea per chiamare la CLI Python direttamente. **Attività**: -- [ ] Implementare `commands/job.py` con sottocomando `job` (entry point completo: parsing parametri job, clone, wait, build, collect, cleanup) -- [ ] Gestione errori e cleanup garantito (try/finally con `vm remove` anche su failure) -- [ ] Aggiornare `gitea/actions/local-ci-build/action.yml`: cambiare `shell: powershell` → `shell: cmd` o invocazione diretta `python -m ci_orchestrator job ...` -- [ ] Aggiungere `PYTHONIOENCODING=utf-8` in `runner/config.yaml` env -- [ ] Lasciare `Invoke-CIJob.ps1` come shim per scheduled task / chiamate manuali esterne (rimosso in A5) -- [ ] Test pytest end-to-end con mock backend + transport: `test_commands_job.py` -- [ ] Eseguire workflow `build-nsInnoUnp.yml` matrix (Win + Linux) PASS +- [x] Implementare `commands/job.py` con sottocomando `job` (entry point completo: parsing parametri job, clone, wait, build, collect, cleanup) +- [x] Gestione errori e cleanup garantito (try/finally con `vm remove` anche su failure) +- [x] Aggiornare `gitea/actions/local-ci-build/action.yml`: cambiare `shell: powershell` → `shell: cmd` o invocazione diretta `python -m ci_orchestrator job ...` +- [x] Aggiungere `PYTHONIOENCODING=utf-8` in `runner/config.yaml` env +- [x] Lasciare `Invoke-CIJob.ps1` come shim per scheduled task / chiamate manuali esterne (rimosso in A5) +- [x] Test pytest end-to-end con mock backend + transport: `test_commands_job.py` +- [ ] Eseguire workflow `build-ns7zip.yml` matrix (Win + Linux) PASS - [ ] Eseguire workflow `self-test.yml` PASS - [ ] Eseguire workflow `lint.yml` PASS @@ -290,7 +290,7 @@ factory `backends.load_backend(config)`. **Test / validazione**: - `pytest tests/test_commands_job.py` PASS con coverage ≥80% sul comando `job` -- Workflow `build-nsInnoUnp.yml` end-to-end PASS dal nuovo entry point +- Workflow `build-ns7zip.yml` end-to-end PASS dal nuovo entry point - Output catturato da act_runner leggibile (no encoding broken, no ANSI rotti) - Tempi job entro ±10% rispetto al baseline PS (misurati con `Measure-CIBenchmark` ancora in PS, lecito) @@ -299,10 +299,10 @@ factory `backends.load_backend(config)`. **Definizione di fatto step A4**: -- [ ] `action.yml` chiama Python direttamente +- [x] `action.yml` chiama Python direttamente - [ ] Workflow matrix Win+Linux PASS -- [ ] `runner/config.yaml` ha `PYTHONIOENCODING=utf-8` -- [ ] Coverage `job.py` ≥80% +- [x] `runner/config.yaml` ha `PYTHONIOENCODING=utf-8` +- [x] Coverage `job.py` ≥80% - [ ] Misurazione tempo entro ±10% baseline ### A5 — Test, documentazione, cleanup @@ -314,15 +314,15 @@ aggiornare documentazione, eseguire burn-in finale. **Attività**: -- [ ] Implementare test pytest dedicati per ognuno degli errori frequenti `AGENTS.md` #9 (snapshot powered-off), #10 (`vmrun list` vs `getGuestIPAddress`), #11 (machine-id reset), #12 (stderr handling — N/A per Python ma documentare) -- [ ] Rimuovere shim PS che non hanno call site esterni (verificare con `grep_search` che nessun task / doc li referenzia) -- [ ] Lasciare `Install-CIToolchain-*.ps1`, `Prepare-*.ps1`, `Deploy-*.ps1` in PowerShell (girano dentro guest o per ricostruzione template — vedi §4 di [idea-1](idea-1-python-rewrite.md)) -- [ ] Aggiornare `AGENTS.md`: aggiungere sezione "Python development" (venv path, ruff, mypy strict, pytest, encoding utf-8) -- [ ] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout `src/ci_orchestrator/` -- [ ] Aggiornare `README.md` con setup Python e quick-start CLI -- [ ] Portare `Test-CapacityBurnIn.ps1` in `commands/burnin.py` (o lasciare in PS se ancora utile come driver esterno) -- [ ] Eseguire burn-in 4 job concorrenti × 10 round PASS sull'host Windows -- [ ] Aggiornare `docs/RUNBOOK.md` con sezione "Operare da Python CLI" +- [x] Implementare test pytest dedicati per ognuno degli errori frequenti `AGENTS.md` #9 (snapshot powered-off), #10 (`vmrun list` vs `getGuestIPAddress`), #11 (machine-id reset → unique clone-name guard), #12 (stderr handling → AutoAddPolicy + no shell-out) +- [x] Verificare con `grep_search` che gli shim PS abbiano ancora call site esterni (`Register-CIScheduledTasks.ps1`, doc, `Test-*.ps1`) → nessuno candidato a rimozione in A5; rimozione differita a B5 (systemd timers) +- [x] Lasciare `Install-CIToolchain-*.ps1`, `Prepare-*.ps1`, `Deploy-*.ps1` in PowerShell (girano dentro guest o per ricostruzione template — vedi §4 di [idea-1](idea-1-python-rewrite.md)) +- [x] Aggiornare `AGENTS.md`: aggiungere sezione "Python development" (venv path, ruff, mypy strict, pytest, encoding utf-8, mappatura PS → Python) +- [x] Aggiornare `docs/ARCHITECTURE.md` con nuovo layout `src/ci_orchestrator/` + Phase C extension point +- [x] Aggiornare `README.md` con setup Python e quick-start CLI per tutti i 10 sub-comandi +- [ ] Portare `Test-CapacityBurnIn.ps1` in `commands/burnin.py` (o lasciare in PS se ancora utile come driver esterno) — non implementato in A5; lo shim attuale già delega via `job` Python +- [ ] Eseguire burn-in 4 job concorrenti × 10 round PASS sull'host Windows — richiede VM reali, a carico utente +- [ ] Aggiornare `docs/RUNBOOK.md` con sezione "Operare da Python CLI" — deferito a chiusura Fase B (RUNBOOK riscritto end-to-end in B-finale) **Hook futuri Fase C**: documentare in `docs/ARCHITECTURE.md` il contratto `VmBackend` come "extension point" e il selettore `[backend]` @@ -341,11 +341,11 @@ qualche caller esterno non ancora migrato emerge. **Definizione di fatto step A5**: -- [ ] Errori #9–#12 coperti da test pytest -- [ ] Documentazione aggiornata -- [ ] Shim non referenziati rimossi -- [ ] Burn-in PASS -- [ ] **Fase A done**: tutti i criteri di "fatto" §7 soddisfatti per la parte A +- [x] Errori #9–#12 coperti da test pytest (`tests/python/test_agents_errors.py`, 12 test) +- [x] Documentazione aggiornata (`AGENTS.md`, `docs/ARCHITECTURE.md`, `README.md`) +- [x] Shim non referenziati rimossi → nessuno candidato (tutti referenziati da `Register-CIScheduledTasks.ps1`/doc/`Test-*.ps1`); rimozione differita a B5 con sostituzione systemd +- [ ] Burn-in PASS — pendenza utente (richiede VM reali, vedi `plans/A5-closeout.md`) +- [ ] **Fase A done**: code-complete; pending solo validazioni hardware (burn-in + workflow matrix end-to-end) ## 2. Fase B — Migrazione host Linux Mint @@ -541,15 +541,15 @@ per l'inventario dei task. **Attività**: -- [ ] Inventariare i task in `Register-CIScheduledTasks.ps1` (identificare cadenza e comando per ognuno) -- [ ] Per `cleanup-orphaned-vms`: creare `ci-cleanup-orphaned-vms.service` (oneshot, `ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup`) + `.timer` (`OnCalendar=hourly`) -- [ ] Per `retention-policy`: creare `ci-retention-policy.service` + `.timer` (`OnCalendar=daily`) -- [ ] Per `watch-disk-space`: `ci-watch-disk-space.service` + `.timer` (`OnCalendar=*:0/15`) -- [ ] Per `watch-runner-health`: `ci-watch-runner-health.service` + `.timer` (`OnCalendar=*:0/5`) -- [ ] Per `backup-template`: `ci-backup-template.service` + `.timer` (`OnCalendar=weekly`) +- [x] Inventariare i task in `Register-CIScheduledTasks.ps1` (identificare cadenza e comando per ognuno) +- [x] Per `cleanup-orphaned-vms`: creare `ci-cleanup-orphaned-vms.service` (oneshot, `ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator vm cleanup`) + `.timer` (`OnCalendar=hourly`) +- [x] Per `retention-policy`: creare `ci-retention-policy.service` + `.timer` (`OnCalendar=daily`) +- [x] Per `watch-disk-space`: `ci-watch-disk-space.service` + `.timer` (`OnCalendar=*:0/15`) +- [x] Per `watch-runner-health`: `ci-watch-runner-health.service` + `.timer` (`OnCalendar=*:0/5`) +- [x] Per `backup-template`: `ci-backup-template.service` + `.timer` (`OnCalendar=weekly`) - [ ] `systemctl daemon-reload` e `systemctl enable --now .timer` - [ ] Validare `systemctl list-timers` mostra tutti i timer schedulati -- [ ] Documentare il mapping in `docs/HOST-SETUP.md` +- [x] Documentare il mapping in `docs/HOST-SETUP.md` **Hook futuri Fase C**: `vm cleanup` con backend ESXi userà la stessa unit — il selettore backend è in `config.toml`, non nel comando. @@ -589,7 +589,7 @@ una finestra di manutenzione concordata. - [ ] Rsync incrementale finale `F:\CI\Artifacts\` → `/var/lib/ci/artifacts/` (preserva permessi) - [ ] Verificare che il runner Linux sia "online" e non "paused" su Gitea - [ ] Trigger manuale di workflow smoke (`self-test.yml`) PASS -- [ ] Trigger manuale `build-nsInnoUnp.yml` matrix Win + Linux PASS +- [ ] Trigger manuale `build-ns7zip.yml` matrix Win + Linux PASS - [ ] Monitorare `journalctl -u act-runner -f` per ≥30 min sotto carico reale **Hook futuri Fase C**: nessuno specifico in B6. Cutover non tocca @@ -603,7 +603,7 @@ backend. **Test / validazione**: - `self-test.yml` PASS dal runner Linux -- `build-nsInnoUnp.yml` matrix Win+Linux PASS dal runner Linux +- `build-ns7zip.yml` matrix Win+Linux PASS dal runner Linux - Nessun errore in `journalctl -u act-runner --since "1h ago"` di severità critica - Artifact correttamente in `/var/lib/ci/artifacts/` @@ -987,7 +987,7 @@ Ogni voce: **Rischio** — Fase / Severità / Mitigazione / Owner action item. - [ ] `lint.yml` dual-stack (PSSA per PS legacy + ruff/mypy/pytest per Python) PASS - [ ] act_runner gira come `act-runner.service` systemd su Linux Mint - [ ] Tutti i template VM (`WinBuild2025`, `WinBuild2022`, `LinuxBuild2404`) operativi dal nuovo host con snapshot integri -- [ ] Workflow `build-nsInnoUnp.yml` matrix Win+Linux PASS dal nuovo host +- [ ] Workflow `build-ns7zip.yml` matrix Win+Linux PASS dal nuovo host - [ ] Burn-in capacity 4 job concorrenti × 10 round PASS, tempi entro ±20% baseline Windows - [ ] Storage migrato a `/var/lib/ci/`, host Windows spento come rollback - [ ] Tutti i timer systemd attivi e schedulati equivalenti ai Task Scheduler Windows diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9ebbd08 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["hatchling>=1.18"] +build-backend = "hatchling.build" + +[project] +name = "ci-orchestrator" +version = "0.1.0" +description = "Local CI/CD orchestrator (Phase A: Python rewrite of PowerShell scripts)" +readme = "README.md" +requires-python = ">=3.11" +license = { text = "Proprietary" } +authors = [{ name = "Local CI/CD System" }] +dependencies = [ + "click>=8.1", + "pypsrp>=0.8", + "paramiko>=3.4", + "keyring>=24.0", + "tomli>=2.0; python_version < '3.11'", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-cov>=5.0", + "pytest-mock>=3.12", + "ruff>=0.5", + "mypy>=1.10", + "types-paramiko", +] + +[project.scripts] +ci-orchestrator = "ci_orchestrator.__main__:cli" + +[tool.hatch.build.targets.wheel] +packages = ["src/ci_orchestrator"] + +[tool.ruff] +line-length = 100 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "B", # flake8-bugbear + "UP", # pyupgrade + "SIM", # flake8-simplify + "RUF", # ruff-specific +] +ignore = [ + "E501", # line length handled by formatter +] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["B011", "SIM117"] + +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src"] +warn_unused_ignores = true +warn_redundant_casts = true + +[[tool.mypy.overrides]] +module = ["pypsrp.*", "keyring.*"] +ignore_missing_imports = true + +[tool.pytest.ini_options] +minversion = "8.0" +addopts = "-q --strict-markers" +testpaths = ["tests"] +pythonpath = ["src"] +markers = [ + "integration: requires real VM/network (skipped in unit-test runs)", +] + +[tool.coverage.run] +source = ["ci_orchestrator"] +branch = true + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "raise NotImplementedError", +] diff --git a/runner/config.yaml b/runner/config.yaml index d5cd66e..efda23a 100644 --- a/runner/config.yaml +++ b/runner/config.yaml @@ -35,6 +35,14 @@ runner: GITEA_CI_SSH_KEY_PATH: "F:\\CI\\keys\\ci_linux" # Root directory of the local-ci-system repository on this host (scripts\ subdir) GITEA_CI_SCRIPT_ROOT: "N:\\Code\\Workspace\\Local-CI-CD-System\\scripts" + # Force UTF-8 stdio for the Python orchestrator (Phase A4) so non-ASCII + # characters in build logs survive the act_runner -> Gitea round-trip. + PYTHONIOENCODING: "utf-8" + # Absolute path to the system Python interpreter used to bootstrap the + # production venv (F:\CI\python\venv). act_runner runs as SYSTEM with a + # sanitized PATH where neither `python` nor `py.exe` resolve, so workflows + # rely on this single source of truth. Update only when upgrading Python. + CI_PYTHON_LAUNCHER: "C:\\Program Files\\Python314\\python.exe" # Optional: load additional env vars from a file (one KEY=VALUE per line) # env_file: "F:\\CI\\runner.env" diff --git a/scripts/Cleanup-OrphanedBuildVMs.ps1 b/scripts/Cleanup-OrphanedBuildVMs.ps1 index ef41af4..cdece3c 100644 --- a/scripts/Cleanup-OrphanedBuildVMs.ps1 +++ b/scripts/Cleanup-OrphanedBuildVMs.ps1 @@ -1,118 +1,37 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss). - -.DESCRIPTION - Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours. - For each stale directory, stops the VM (hard), deletes via vmrun deleteVM, - then removes the directory. - - Run as a daily scheduled task or on host startup to prevent disk accumulation. - Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours. - -.PARAMETER CloneBaseDir - Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs - -.PARAMETER MaxAgeHours - Age threshold in hours. Directories with LastWriteTime older than this are - treated as orphaned. Must exceed the longest expected build duration. - Default: 4 (runner.timeout is 2h — 4h gives double margin) - -.PARAMETER VmrunPath - Path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER WhatIf - List orphaned VMs without destroying them. - -.EXAMPLE - # Dry run - .\Cleanup-OrphanedBuildVMs.ps1 -WhatIf - - # Live cleanup (run elevated) - .\Cleanup-OrphanedBuildVMs.ps1 -#> -[CmdletBinding(SupportsShouldProcess)] -param( - [string] $CloneBaseDir = 'F:\CI\BuildVMs', - - [ValidateRange(0, 168)] - [int] $MaxAgeHours = 4, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors - -if (-not (Test-Path $CloneBaseDir)) { - Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do." - exit 0 -} - -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath" - Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only." -} - -$cutoff = (Get-Date).AddHours(-$MaxAgeHours) -$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.LastWriteTime -lt $cutoff }) - -if (-not $orphans) { - Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)." - exit 0 -} - -Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h." - -foreach ($dir in $orphans) { - $vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue | - Select-Object -First 1 - - if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) { - Write-Host "[Cleanup] Processing: $($dir.FullName)" - - if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) { - # Hard stop — don't wait for graceful shutdown on an orphan - & $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null - Start-Sleep -Seconds 2 - $deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1 - if ($LASTEXITCODE -ne 0) { - Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut" - Write-Warning " Falling back to directory removal." - } - } - elseif (-not $vmx) { - Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only." - } - - Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue - if (Test-Path $dir.FullName) { - Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed." - } - else { - Write-Host "[Cleanup] Removed: $($dir.FullName)" - } - } - else { - Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)" - } -} - -Write-Host "[Cleanup] Done." - -# ── Stale vm-start.lock cleanup ───────────────────────────────────────────── -# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists -# and causes a confusing 10-minute retry loop on the next job. Remove if older -# than 30 minutes. -$lockFile = 'F:\CI\State\vm-start.lock' -$lockCutoff = (Get-Date).AddMinutes(-30) -if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) { - if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) { - $ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes - Remove-Item $lockFile -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)" - } -} +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm cleanup @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Get-BuildArtifacts.ps1 b/scripts/Get-BuildArtifacts.ps1 index 120d7d6..93a11a8 100644 --- a/scripts/Get-BuildArtifacts.ps1 +++ b/scripts/Get-BuildArtifacts.ps1 @@ -1,45 +1,16 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Copies build artifacts from an ephemeral VM to the host artifact directory. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Opens a WinRM session to the build VM and uses Copy-Item -FromSession - to transfer the artifact archive and optional log files. - Validates that at least one file was successfully transferred. - Throws on failure so the caller's try/finally can clean up the VM. +# Shim: delegates to the Python ci_orchestrator CLI (`artifacts collect`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. +# +# The bound `-Credential` is intentionally discarded — Python uses keyring +# via `--credential-target` (defaulting to `BuildVMGuest`). -.PARAMETER IPAddress - IP of the build VM. - -.PARAMETER Credential - PSCredential for the guest build user. - -.PARAMETER GuestArtifactPath - Full path to the artifact file or directory inside the VM. - Example: C:\CI\output\artifacts.zip - -.PARAMETER HostArtifactDir - Directory on the host where artifacts will be stored. - The directory is created if it does not exist. - Example: F:\CI\Artifacts\run-42 - -.PARAMETER IncludeLogs - If set, also copies *.log files from the guest work directory - (C:\CI\build\*.log) alongside the artifact archive. - -.EXAMPLE - $cred = Get-Credential - .\Get-BuildArtifacts.ps1 ` - -IPAddress "192.168.11.101" ` - -Credential $cred ` - -GuestArtifactPath "C:\CI\output\artifacts.zip" ` - -HostArtifactDir "F:\CI\Artifacts\run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, [Parameter()] @@ -53,175 +24,39 @@ param( [switch] $IncludeLogs, - # Guest OS type — Linux uses SCP instead of WinRM. [ValidateSet('Windows', 'Linux')] [string] $GuestOS = 'Windows', - # SSH private key path (used when GuestOS=Linux). [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when GuestOS=Linux). [string] $SshUser = 'ci_build', - - # Persistent known_hosts file for SSH calls (CI jobs). - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). [string] $SshKnownHostsFile = '', - # Optional job ID and commit SHA written into manifest.json. - # When either is non-empty a manifest.json is written to $HostArtifactDir - # listing all collected files with name, size, and SHA256. - [string] $JobId = '', - [string] $Commit = '' + [string] $JobId = '', + [string] $Commit = '', + + [string] $CredentialTarget = '' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--ip-address', $IPAddress, + '--guest-os', $GuestOS.ToLower(), + '--guest-artifact-path', $GuestArtifactPath, + '--host-artifact-dir', $HostArtifactDir, + '--ssh-user', $SshUser, + '--ssh-key-path', $SshKeyPath, + '--job-id', $JobId, + '--commit', $Commit +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) } +if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) } +if ($IncludeLogs.IsPresent) { $pyArgs += '--include-logs' } -# ── Helper: write artifact manifest ───────────────────────────────────────── -function Write-ArtifactManifest { - param([string] $Dir, [string] $JobId, [string] $Commit) - $files = @(Get-ChildItem -Path $Dir -Recurse -File -ErrorAction SilentlyContinue | - Where-Object { $_.Name -ne 'manifest.json' }) - $fileEntries = foreach ($f in $files) { - [pscustomobject]@{ - name = $f.FullName.Substring($Dir.TrimEnd('\', '/').Length).TrimStart('\', '/') - size = $f.Length - sha256 = (Get-FileHash $f.FullName -Algorithm SHA256).Hash.ToLower() - } - } - $manifest = [ordered]@{ - ts = (Get-Date -Format 'o') - jobId = $JobId - commit = $Commit - files = @($fileEntries) - } - $manifestPath = Join-Path $Dir 'manifest.json' - try { - $manifest | ConvertTo-Json -Depth 4 | - Set-Content -Path $manifestPath -Encoding UTF8 -Force -ErrorAction Stop - Write-Host "[Get-BuildArtifacts] Manifest written: $manifestPath ($($files.Count) file(s))" - } catch { - Write-Warning "[Get-BuildArtifacts] Could not write manifest.json: $_" - } +if ($Credential) { + Write-Host '[Get-BuildArtifacts] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).' } -if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { - throw "Credential is required when GuestOS is Windows." -} - -# ── Linux branch: use scp to transfer artifacts ───────────────────────────────────────────── -if ($GuestOS -eq 'Linux') { - Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force - - if (-not (Test-Path $HostArtifactDir)) { - New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null - Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" - } - - Write-Host "[Get-BuildArtifacts] Transferring artifacts via SCP from $GuestArtifactPath ..." - - Copy-SshItem ` - -Source "$GuestArtifactPath/." ` - -Destination $HostArtifactDir ` - -IP $IPAddress ` - -User $SshUser ` - -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Direction 'FromGuest' ` - -Recurse - - $transferred = @(Get-ChildItem -Path $HostArtifactDir -Recurse -File -ErrorAction SilentlyContinue) - if ($transferred.Count -eq 0) { - throw "[Get-BuildArtifacts] No files found in $HostArtifactDir after SCP transfer." - } - Write-Host "[Get-BuildArtifacts] $($transferred.Count) file(s) collected from Linux guest." - if ($JobId -ne '' -or $Commit -ne '') { - Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit - } - return -} - -# Ensure destination exists on host -if (-not (Test-Path $HostArtifactDir)) { - New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null - Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" -} - -$sessionOptions = New-CISessionOption - -Write-Host "[Get-BuildArtifacts] Connecting to VM at $IPAddress..." -$session = $null -$attempts = 3 -$delay = 10 # seconds between attempts -for ($i = 1; $i -le $attempts; $i++) { - try { - $session = New-PSSession ` - -ComputerName $IPAddress ` - -Credential $Credential ` - -SessionOption $sessionOptions ` - -UseSSL ` - -Port 5986 ` - -Authentication Basic ` - -ErrorAction Stop - break - } catch { - if ($i -eq $attempts) { throw } - Write-Warning "[Get-BuildArtifacts] WinRM connect attempt $i/$attempts failed — retrying in ${delay}s... ($_)" - Start-Sleep -Seconds $delay - } -} - -try { - # ── Verify artifact exists in guest ────────────────────────────────── - $guestExists = Invoke-Command -Session $session -ScriptBlock { - param($path) - Test-Path $path - } -ArgumentList $GuestArtifactPath - - if (-not $guestExists) { - throw "Artifact not found in guest at: $GuestArtifactPath" - } - - # ── Copy main artifact ──────────────────────────────────────────────── - $artifactFileName = Split-Path $GuestArtifactPath -Leaf - $hostDestPath = Join-Path $HostArtifactDir $artifactFileName - - Write-Host "[Get-BuildArtifacts] Copying $artifactFileName from guest..." - Copy-Item -Path $GuestArtifactPath -Destination $hostDestPath -FromSession $session -Force - - # ── Copy logs if requested ──────────────────────────────────────────── - if ($IncludeLogs) { - $logFiles = Invoke-Command -Session $session -ScriptBlock { - Get-ChildItem -Path 'C:\CI\build' -Filter '*.log' -Recurse -ErrorAction SilentlyContinue | - Select-Object -ExpandProperty FullName - } - - foreach ($log in $logFiles) { - $logDest = Join-Path $HostArtifactDir (Split-Path $log -Leaf) - Write-Host "[Get-BuildArtifacts] Copying log: $(Split-Path $log -Leaf)" - Copy-Item -Path $log -Destination $logDest -FromSession $session -Force - } - } - - # ── Validate at least the main artifact landed ──────────────────────── - if (-not (Test-Path $hostDestPath)) { - throw "Artifact was not found at expected host path after copy: $hostDestPath" - } - - $fileItem = Get-Item $hostDestPath - if ($fileItem.Length -eq 0) { - throw "Artifact file exists but is empty: $hostDestPath" - } - $sizeKB = [math]::Round($fileItem.Length / 1KB, 1) - - Write-Host "[Get-BuildArtifacts] Artifact collected: $hostDestPath ($sizeKB KB)" - if ($JobId -ne '' -or $Commit -ne '') { - Write-ArtifactManifest -Dir $HostArtifactDir -JobId $JobId -Commit $Commit - } - return $hostDestPath -} -finally { - Remove-PSSession $session -ErrorAction SilentlyContinue -} +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator artifacts collect @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Get-CIJobSummary.ps1 b/scripts/Get-CIJobSummary.ps1 index 0182f3e..f8041a4 100644 --- a/scripts/Get-CIJobSummary.ps1 +++ b/scripts/Get-CIJobSummary.ps1 @@ -1,181 +1,215 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Displays a summary table of CI job results parsed from JSONL log files. - -.DESCRIPTION - Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and - prints a compact table with job ID, status, total elapsed time, and the - last error message (if any). - - Each row corresponds to one completed (or failed) CI job. - Use -JobId to inspect a single job in detail. - -.PARAMETER LogDir - Base directory containing per-job log subdirectories. - Default: F:\CI\Logs - -.PARAMETER Last - Show only the N most recent jobs (sorted by job-start timestamp). - Default: 20 - -.PARAMETER JobId - When specified, shows a detailed phase-by-phase breakdown for that job. - -.PARAMETER Failed - When specified, filters to failed jobs only. - -.EXAMPLE - # Show last 20 jobs - .\Get-CIJobSummary.ps1 - -.EXAMPLE - # Show last 5 failed jobs - .\Get-CIJobSummary.ps1 -Last 5 -Failed - -.EXAMPLE - # Detailed breakdown for a specific job - .\Get-CIJobSummary.ps1 -JobId 'run-12345' -#> -[CmdletBinding()] -param( - [string] $LogDir = 'F:\CI\Logs', - - [ValidateRange(1, 1000)] - [int] $Last = 20, - - [string] $JobId = '', - - [switch] $Failed -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -# ── Helpers ──────────────────────────────────────────────────────────────────── -function Format-SecToHMS { - param([int] $Seconds) - if ($Seconds -lt 0) { return '?' } - $h = [math]::Floor($Seconds / 3600) - $m = [math]::Floor(($Seconds % 3600) / 60) - $s = $Seconds % 60 - if ($h -gt 0) { return '{0}h{1:00}m{2:00}s' -f $h, $m, $s } - return '{0}m{1:00}s' -f $m, $s -} - -function Read-JobEvents { - param([string] $JsonlPath) - $events = [System.Collections.Generic.List[PSCustomObject]]::new() - foreach ($line in (Get-Content $JsonlPath -Encoding UTF8 -ErrorAction SilentlyContinue)) { - $line = $line.Trim() - if (-not $line) { continue } - try { - $events.Add(($line | ConvertFrom-Json)) - } catch { - Write-Debug "[Get-CIJobSummary] Skipping malformed JSONL line in $JsonlPath" - } - } - return $events -} - -# ── Validate log dir ──────────────────────────────────────────────────────── -if (-not (Test-Path $LogDir -PathType Container)) { - Write-Warning "Log directory not found: $LogDir" - exit 1 -} - -# ── Single-job detail mode ────────────────────────────────────────────────── -if ($JobId -ne '') { - $jsonlPath = Join-Path $LogDir (Join-Path $JobId 'invoke-ci.jsonl') - if (-not (Test-Path $jsonlPath)) { - Write-Warning "No JSONL log found for job '$JobId' at: $jsonlPath" - exit 1 - } - $events = Read-JobEvents -JsonlPath $jsonlPath - if ($events.Count -eq 0) { - Write-Warning "JSONL file is empty or unreadable: $jsonlPath" - exit 1 - } - Write-Host "`nJob: $JobId" - Write-Host ('=' * 60) - Write-Host ('{0,-35} {1,-10} {2}' -f 'Phase', 'Status', 'Timestamp') - Write-Host ('{0,-35} {1,-10} {2}' -f ('-'*35), ('-'*10), ('-'*24)) - foreach ($ev in $events) { - $ts = if ($ev.ts) { $ev.ts } else { '' } - Write-Host ('{0,-35} {1,-10} {2}' -f $ev.phase, $ev.status, $ts) - } - # Print error if present - $failEvent = $events | Where-Object { $_.status -eq 'failure' } | Select-Object -Last 1 - if ($failEvent -and $failEvent.data -and $failEvent.data.error) { - Write-Host "`nError: $($failEvent.data.error)" - } - exit 0 -} - -# ── Scan all job JSONL files ───────────────────────────────────────────────── -$jsonlFiles = Get-ChildItem -Path $LogDir -Recurse -Filter 'invoke-ci.jsonl' -ErrorAction SilentlyContinue | - Sort-Object { $_.LastWriteTime } -Descending - -if ($jsonlFiles.Count -eq 0) { - Write-Host "No invoke-ci.jsonl files found under $LogDir" - exit 0 -} - -$rows = [System.Collections.Generic.List[PSCustomObject]]::new() - -foreach ($file in $jsonlFiles) { - $events = Read-JobEvents -JsonlPath $file.FullName - if ($events.Count -eq 0) { continue } - - $jobEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'start' } | Select-Object -First 1 - $successEvent= $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'success' } | Select-Object -First 1 - $failEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'failure' } | Select-Object -First 1 - - $jId = if ($events[0].jobId) { $events[0].jobId } else { (Split-Path (Split-Path $file.FullName -Parent) -Leaf) } - $started = if ($jobEvent -and $jobEvent.ts) { $jobEvent.ts } else { '' } - - if ($successEvent) { - $status = 'success' - $elSec = if ($successEvent.data -and $null -ne $successEvent.data.elapsedSec) { [int]$successEvent.data.elapsedSec } else { -1 } - $errMsg = '' - } elseif ($failEvent) { - $status = 'FAILED' - $elSec = if ($failEvent.data -and $null -ne $failEvent.data.elapsedSec) { [int]$failEvent.data.elapsedSec } else { -1 } - $errMsg = if ($failEvent.data -and $failEvent.data.error) { "$($failEvent.data.error)" } else { '' } - # Truncate error for table display - if ($errMsg.Length -gt 60) { $errMsg = $errMsg.Substring(0, 57) + '...' } - } else { - $status = 'in-progress' - $elSec = -1 - $errMsg = '' - } - - $rows.Add([PSCustomObject]@{ - JobId = $jId - Status = $status - Elapsed = Format-SecToHMS -Seconds $elSec - Started = $started - Error = $errMsg - }) -} - -# ── Apply filters + limit ──────────────────────────────────────────────────── -if ($Failed) { - $rows = @($rows | Where-Object { $_.Status -eq 'FAILED' }) -} - -$rows = @($rows | Select-Object -First $Last) - -if ($rows.Count -eq 0) { - Write-Host "No matching jobs found." - exit 0 -} - -# ── Print table ────────────────────────────────────────────────────────────── -Write-Host "`nCI Job Summary (last $($rows.Count) jobs):" -Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f 'JobId', 'Status', 'Elapsed', 'Started', 'Error') -Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f ('-'*40), ('-'*12), ('-'*10), ('-'*26), ('-'*30)) -foreach ($r in $rows) { - Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f $r.JobId, $r.Status, $r.Elapsed, $r.Started, $r.Error) -} -Write-Host '' +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator report job @pyArgs +exit $LASTEXITCODE + + +.DESCRIPTION + Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and + prints a compact table with job ID, status, total elapsed time, and the + last error message (if any). + + Each row corresponds to one completed (or failed) CI job. + Use -JobId to inspect a single job in detail. + +.PARAMETER LogDir + Base directory containing per-job log subdirectories. + Default: F:\CI\Logs + +.PARAMETER Last + Show only the N most recent jobs (sorted by job-start timestamp). + Default: 20 + +.PARAMETER JobId + When specified, shows a detailed phase-by-phase breakdown for that job. + +.PARAMETER Failed + When specified, filters to failed jobs only. + +.EXAMPLE + # Show last 20 jobs + .\Get-CIJobSummary.ps1 + +.EXAMPLE + # Show last 5 failed jobs + .\Get-CIJobSummary.ps1 -Last 5 -Failed + +.EXAMPLE + # Detailed breakdown for a specific job + .\Get-CIJobSummary.ps1 -JobId 'run-12345' +#> +[CmdletBinding()] +param( + [string] $LogDir = 'F:\CI\Logs', + + [ValidateRange(1, 1000)] + [int] $Last = 20, + + [string] $JobId = '', + + [switch] $Failed +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ── Helpers ──────────────────────────────────────────────────────────────────── +function Format-SecToHMS { + param([int] $Seconds) + if ($Seconds -lt 0) { return '?' } + $h = [math]::Floor($Seconds / 3600) + $m = [math]::Floor(($Seconds % 3600) / 60) + $s = $Seconds % 60 + if ($h -gt 0) { return '{0}h{1:00}m{2:00}s' -f $h, $m, $s } + return '{0}m{1:00}s' -f $m, $s +} + +function Read-JobEvents { + param([string] $JsonlPath) + $events = [System.Collections.Generic.List[PSCustomObject]]::new() + foreach ($line in (Get-Content $JsonlPath -Encoding UTF8 -ErrorAction SilentlyContinue)) { + $line = $line.Trim() + if (-not $line) { continue } + try { + $events.Add(($line | ConvertFrom-Json)) + } catch { + Write-Debug "[Get-CIJobSummary] Skipping malformed JSONL line in $JsonlPath" + } + } + return $events +} + +# ── Validate log dir ──────────────────────────────────────────────────────── +if (-not (Test-Path $LogDir -PathType Container)) { + Write-Warning "Log directory not found: $LogDir" + exit 1 +} + +# ── Single-job detail mode ────────────────────────────────────────────────── +if ($JobId -ne '') { + $jsonlPath = Join-Path $LogDir (Join-Path $JobId 'invoke-ci.jsonl') + if (-not (Test-Path $jsonlPath)) { + Write-Warning "No JSONL log found for job '$JobId' at: $jsonlPath" + exit 1 + } + $events = Read-JobEvents -JsonlPath $jsonlPath + if ($events.Count -eq 0) { + Write-Warning "JSONL file is empty or unreadable: $jsonlPath" + exit 1 + } + Write-Host "`nJob: $JobId" + Write-Host ('=' * 60) + Write-Host ('{0,-35} {1,-10} {2}' -f 'Phase', 'Status', 'Timestamp') + Write-Host ('{0,-35} {1,-10} {2}' -f ('-'*35), ('-'*10), ('-'*24)) + foreach ($ev in $events) { + $ts = if ($ev.ts) { $ev.ts } else { '' } + Write-Host ('{0,-35} {1,-10} {2}' -f $ev.phase, $ev.status, $ts) + } + # Print error if present + $failEvent = $events | Where-Object { $_.status -eq 'failure' } | Select-Object -Last 1 + if ($failEvent -and $failEvent.data -and $failEvent.data.error) { + Write-Host "`nError: $($failEvent.data.error)" + } + exit 0 +} + +# ── Scan all job JSONL files ───────────────────────────────────────────────── +$jsonlFiles = Get-ChildItem -Path $LogDir -Recurse -Filter 'invoke-ci.jsonl' -ErrorAction SilentlyContinue | + Sort-Object { $_.LastWriteTime } -Descending + +if ($jsonlFiles.Count -eq 0) { + Write-Host "No invoke-ci.jsonl files found under $LogDir" + exit 0 +} + +$rows = [System.Collections.Generic.List[PSCustomObject]]::new() + +foreach ($file in $jsonlFiles) { + $events = Read-JobEvents -JsonlPath $file.FullName + if ($events.Count -eq 0) { continue } + + $jobEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'start' } | Select-Object -First 1 + $successEvent= $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'success' } | Select-Object -First 1 + $failEvent = $events | Where-Object { $_.phase -eq 'job' -and $_.status -eq 'failure' } | Select-Object -First 1 + + $jId = if ($events[0].jobId) { $events[0].jobId } else { (Split-Path (Split-Path $file.FullName -Parent) -Leaf) } + $started = if ($jobEvent -and $jobEvent.ts) { $jobEvent.ts } else { '' } + + if ($successEvent) { + $status = 'success' + $elSec = if ($successEvent.data -and $null -ne $successEvent.data.elapsedSec) { [int]$successEvent.data.elapsedSec } else { -1 } + $errMsg = '' + } elseif ($failEvent) { + $status = 'FAILED' + $elSec = if ($failEvent.data -and $null -ne $failEvent.data.elapsedSec) { [int]$failEvent.data.elapsedSec } else { -1 } + $errMsg = if ($failEvent.data -and $failEvent.data.error) { "$($failEvent.data.error)" } else { '' } + # Truncate error for table display + if ($errMsg.Length -gt 60) { $errMsg = $errMsg.Substring(0, 57) + '...' } + } else { + $status = 'in-progress' + $elSec = -1 + $errMsg = '' + } + + $rows.Add([PSCustomObject]@{ + JobId = $jId + Status = $status + Elapsed = Format-SecToHMS -Seconds $elSec + Started = $started + Error = $errMsg + }) +} + +# ── Apply filters + limit ──────────────────────────────────────────────────── +if ($Failed) { + $rows = @($rows | Where-Object { $_.Status -eq 'FAILED' }) +} + +$rows = @($rows | Select-Object -First $Last) + +if ($rows.Count -eq 0) { + Write-Host "No matching jobs found." + exit 0 +} + +# ── Print table ────────────────────────────────────────────────────────────── +Write-Host "`nCI Job Summary (last $($rows.Count) jobs):" +Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f 'JobId', 'Status', 'Elapsed', 'Started', 'Error') +Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f ('-'*40), ('-'*12), ('-'*10), ('-'*26), ('-'*30)) +foreach ($r in $rows) { + Write-Host ('{0,-40} {1,-12} {2,-10} {3,-26} {4}' -f $r.JobId, $r.Status, $r.Elapsed, $r.Started, $r.Error) +} +Write-Host '' diff --git a/scripts/Invoke-CIJob.ps1 b/scripts/Invoke-CIJob.ps1 index a522020..aa62674 100644 --- a/scripts/Invoke-CIJob.ps1 +++ b/scripts/Invoke-CIJob.ps1 @@ -1,691 +1,37 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Main CI orchestrator: creates an ephemeral VM, runs a build, collects - artifacts, and destroys the VM — in a guaranteed try/finally pattern. - -.DESCRIPTION - This script is called by act_runner as the primary build step. - It coordinates all sub-scripts: - 1. New-BuildVM.ps1 — linked clone from template - 2. vmrun start — start the clone - 3. Wait-VMReady.ps1 — poll until WinRM is ready - 4. Invoke-RemoteBuild.ps1 — execute build inside VM - 5. Get-BuildArtifacts.ps1 — copy artifacts to host - 6. Remove-BuildVM.ps1 — destroy VM (always runs via finally) - - Exit codes: - 0 = success - 1 = build or orchestration failure (VM destroyed, artifacts may be partial) - -.PARAMETER JobId - Unique job identifier (use ${{ github.run_id }} from Gitea Actions). - -.PARAMETER RepoUrl - Git URL of the repository to build (reachable from the build VM). - -.PARAMETER Branch - Branch to build. - -.PARAMETER Commit - Optional specific commit SHA to check out. - -.PARAMETER Configuration - Build configuration. Default: Release - -.PARAMETER TemplatePath - Full path to the template VM .vmx file. - Can also be set via env var GITEA_CI_TEMPLATE_PATH. - -.PARAMETER SnapshotName - Template snapshot to clone from. Default: BaseClean - -.PARAMETER CloneBaseDir - Base directory for clone VMs. - Can also be set via env var GITEA_CI_CLONE_BASE_DIR. - Default: F:\CI\BuildVMs - -.PARAMETER ArtifactBaseDir - Host directory where artifacts are stored per job. - Default: F:\CI\Artifacts - -.PARAMETER VMIPAddress - IP address to assign/expect for the build VM. - Must be unique per concurrent build. - Can be passed explicitly or derived from a registry/file-based IP allocator. - -.PARAMETER GuestCredentialTarget - Target name in Windows Credential Manager for guest credentials. - Default: BuildVMGuest - -.PARAMETER VmrunPath - Path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.EXAMPLE - # Called from Gitea Actions workflow: - .\scripts\Invoke-CIJob.ps1 ` - -JobId "${{ github.run_id }}" ` - -RepoUrl "http://gitea.local/org/myrepo.git" ` - -Branch "${{ github.ref_name }}" ` - -Commit "${{ github.sha }}" ` - -VMIPAddress "192.168.11.101" -#> -[CmdletBinding()] -param( - [Parameter(Mandatory)] - [ValidatePattern('^[A-Za-z0-9._-]+$')] - [string] $JobId, - - [Parameter(Mandatory)] - [string] $RepoUrl, - - [Parameter(Mandatory)] - [string] $Branch, - - [string] $Commit = '', - - [string] $Configuration = 'Release', - - # Pass -Submodules to clone with --recurse-submodules - [switch] $Submodules, - - # Custom build command to run inside the VM (empty = dotnet build) - [string] $BuildCommand = '', - - # Artifact source pattern relative to GuestWorkDir (used with BuildCommand) - [string] $GuestArtifactSource = 'dist', - - # Clone repo IN the VM instead of on host (requires Git in template §6.6) - # Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone. - # Prerequisite: Git for Windows installed in template. - [switch] $UseGitClone, - - # Use VMware shared folders for NuGet/pip cache (§SharedCache). - # Requires Set-TemplateSharedFolders.ps1 to have been run on the template. - [switch] $UseSharedCache, - - # Skip artifact packaging (Invoke-RemoteBuild) and collection (Phase 6). - # Use for jobs that intentionally produce no output (lint, test-only, etc.). - # When absent, the build must produce GuestArtifactSource or the job fails. - [switch] $SkipArtifact, - - # Credential Manager target for Gitea PAT used by -UseGitClone (private repos). - # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine - # Ignored when -UseGitClone is not set. - [string] $GiteaCredentialTarget = 'GiteaPAT', - - [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH, - - [string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }), - - [string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }), - - [string] $ArtifactBaseDir = 'F:\CI\Artifacts', - - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] - [string] $VMIPAddress = '', # if empty, auto-detected via vmrun getGuestIPAddress - - [string] $GuestCredentialTarget = 'BuildVMGuest', - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - # Directory for job execution logs. Each job gets its own sub-directory: - # $LogDir\$JobId\invoke-ci.log - [string] $LogDir = 'F:\CI\Logs', - - # Days to retain job logs. Directories older than this are purged at job end. - # Set to 0 to disable retention/purge. - [int] $LogRetentionDays = 30, - - # Guest OS type — determines transport (WinRM for Windows, SSH for Linux). - # When 'Auto', detected from the VMX guestOS field after clone. - [ValidateSet('Windows', 'Linux', 'Auto')] - [string] $GuestOS = 'Auto', - - [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - [string] $SshUser = 'ci_build', - - # Persistent known_hosts file used for CI-job SSH calls. - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). - # Ephemeral CI VMs on a local NAT network do not need strict host-key verification. - # Pass a file path only if you need accept-new mode for a specific deployment. - [string] $SshKnownHostsFile = '', - - # Extra environment variables injected into the guest VM before the build command (§6.5). - # Keys must be valid environment variable names. Values must be plain strings. - # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' } - [hashtable] $ExtraGuestEnv = @{}, - - # Optional webhook URL (Discord/Gitea). When set, a background job fires a - # [WARNING] once the job has been running for 90 minutes. - [string] $WebhookUrl = '', - - # Optional per-job VMX CPU and RAM override (default 0 = use template value). - # Applied to the linked clone after New-BuildVM.ps1, before vmrun start. - # The template VMX is never modified. - # Example: -GuestCPU 4 -GuestMemoryMB 8192 - [ValidateRange(0, 32)] - [int] $GuestCPU = 0, - - [ValidateRange(0, 131072)] - [int] $GuestMemoryMB = 0 -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' - -# ── Resolve script directory for dot-sourcing sibling scripts ───────────────── -$scriptDir = $PSScriptRoot - -# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ────────── -Import-Module (Join-Path $scriptDir '_Common.psm1') -Force - -# ── Validate required config ────────────────────────────────────────────────── -if ([string]::IsNullOrWhiteSpace($TemplatePath)) { - Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH." - exit 1 -} -if (-not (Test-Path $TemplatePath -PathType Leaf)) { - Write-Error "Template VMX not found: $TemplatePath" - exit 1 -} -# Sanity check: warn if template is outside the expected CI tree -if ($TemplatePath -notlike 'F:\CI\Templates\*') { - Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional." -} -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - Write-Error "vmrun.exe not found: $VmrunPath" - exit 1 -} - -# ── Validate ExtraGuestEnv keys ────────────────────────────────────────────── -foreach ($key in $ExtraGuestEnv.Keys) { - if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') { - Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$." - exit 1 - } -} - -# ── Load guest credentials from Windows Credential Manager ─────────────────── -# Requires the CredentialManager module: Install-Module CredentialManager -# In a lab setup you can fall back to prompting, but never hardcode credentials. -$credential = $null -try { - $credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop - Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)." -} -catch { - Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)." - Write-Warning " Using interactive prompt as fallback. Store credentials with:" - Write-Warning " cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password" - $credential = Get-Credential -Message "Enter guest VM credentials for job $JobId" -} - -# ── Setup paths ─────────────────────────────────────────────────────────────── -$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId -$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId" -$cloneVmxPath = $null -$leaseFile = $null # §2.1 IP lease file path — released in finally -$jsonLog = $null # §4.1 JSONL structured log path -$startTime = Get-Date - -# ── Phase-duration tracking (Task 5.5) ─────────────────────────────────────── -$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new() -$phaseStart = $startTime - -# ── Start transcript + JSONL log ────────────────────────────────────────────── -$transcriptStarted = $false -try { - $jobLogDir = Join-Path $LogDir $JobId - if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null } - $transcriptPath = Join-Path $jobLogDir 'invoke-ci.log' - $jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl' - # -Append allows starting a new transcript even when VS Code (or another host) - # has already opened a transcript on this PowerShell session. - Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop - $transcriptStarted = $true -} -catch { - Write-Warning "[Invoke-CIJob] Could not start transcript: $_" -} - -# ── §4.1 Structured event emitter ──────────────────────────────────────────── -# Appends a single JSON line to invoke-ci.jsonl per phase transition. -# Silent no-op if $jsonLog is null (transcript setup failed). -function Write-JobEvent { - param( - [Parameter(Mandatory)] [string] $Phase, - [Parameter(Mandatory)] [string] $Status, # start | success | failure | info - [hashtable] $Data = @{} - ) - if (-not $script:jsonLog) { return } - $logEntry = [ordered]@{ - ts = (Get-Date -Format 'o') - jobId = $script:JobId - phase = $Phase - status = $Status - data = $Data - } - try { - $logEntry | ConvertTo-Json -Compress -Depth 5 | - Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop - } catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" } # never let logging fail the build -} - -Write-Host "============================================================" -Write-Host "[Invoke-CIJob] Job : $JobId" -Write-Host "[Invoke-CIJob] Repository : $RepoUrl" -Write-Host "[Invoke-CIJob] Branch : $Branch" -if ($Commit) { Write-Host "[Invoke-CIJob] Commit : $Commit" } -Write-Host "[Invoke-CIJob] Build VM IP : $VMIPAddress" -Write-Host "[Invoke-CIJob] Template : $TemplatePath" -Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir" -Write-Host "[Invoke-CIJob] Started : $startTime" -Write-Host "============================================================" - -Write-JobEvent -Phase 'job' -Status 'start' -Data @{ - repo = $RepoUrl - branch = $Branch - commit = $Commit - template = $TemplatePath - snapshot = $SnapshotName -} - -$exitCode = 0 -$durationWarnJob = $null - -try { - # ── 90-minute duration warning (fires once via background job) ─────────────── - if ($WebhookUrl -ne '') { - $warnMsg = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck." - $warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress) - $durationWarnJob = Start-Job -ScriptBlock { - param($url, $body) - Start-Sleep -Seconds 5400 # 90 minutes - try { - Invoke-RestMethod -Uri $url -Method Post -Body $body ` - -ContentType 'application/json' -TimeoutSec 10 - } catch { } - } -ArgumentList $WebhookUrl, $warnBody - } - - # ── Phase 1: Clone repo (HOST or GUEST) ─────────────────────────────── - if ($UseGitClone) { - # §3.3: Clone in guest VM instead of host. Skip host clone phase. - # PAT will be injected at runtime (Phase 5) from Credential Manager. - Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..." - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' } - } - else { - # Default: Clone on host, zip, transfer to guest via WinRM. - Write-Host "`n[Phase 1/6] Cloning repository on host..." - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start' - if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force } - $gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch) - if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') } - $gitArgs += @($RepoUrl, $hostCloneDir) - Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')" - $ErrorActionPreference = 'Continue' - # Stream git stdout/stderr live (otherwise heavy clones with submodules - # look frozen for a long time). Tee for diagnostic in case of failure. - $gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" } - $gitExit = $LASTEXITCODE - $ErrorActionPreference = 'Stop' - if ($gitExit -ne 0) { - throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")" - } - if ($Commit -ne '') { - $ErrorActionPreference = 'Continue' - $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" } - $gitExit = $LASTEXITCODE - if ($gitExit -ne 0) { - # Commit not in shallow history (e.g. manual run on older SHA). - # Fetch full history and retry. - Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..." - & git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null - $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" } - $gitExit = $LASTEXITCODE - } - $ErrorActionPreference = 'Stop' - if ($gitExit -ne 0) { - throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")" - } - } - Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir" - Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir } - } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ──── - # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race - # and produce two VMs with the same DHCP lease. A file-based mutex ensures - # only one job is in the clone→start→IP phase at a time. After the IP is - # acquired and written to a lease file the lock is released, so build phases - # (the slow part) run fully in parallel. Lease is released in finally. - $stateDir = 'F:\CI\State' - $leasesDir = Join-Path $stateDir 'ip-leases' - $lockPath = Join-Path $stateDir 'vm-start.lock' - if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null } - - # ── Pre-clone disk space gate ───────────────────────────────────────── - $cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':') - $cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue - if ($null -ne $cloneVol) { - $freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1) - if ($freeGB -lt 20) { - throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone." - } - Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)." - } - - Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..." - Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start' - $lockStream = $null - $lockDeadline = (Get-Date).AddMinutes(10) - while ($true) { - try { - $lockStream = [System.IO.File]::Open( - $lockPath, - [System.IO.FileMode]::OpenOrCreate, - [System.IO.FileAccess]::ReadWrite, - [System.IO.FileShare]::None) - break - } catch [System.IO.IOException] { - if ((Get-Date) -ge $lockDeadline) { - throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck." - } - Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..." - Start-Sleep -Seconds 5 - } - } - Write-Host "[Invoke-CIJob] VM-start lock acquired." - - try { - # ── Phase 2: Clone VM ───────────────────────────────────────────── - Write-Host "[Phase 2/6] Cloning VM..." - $cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" ` - -TemplatePath $TemplatePath ` - -SnapshotName $SnapshotName ` - -CloneBaseDir $CloneBaseDir ` - -JobId $JobId ` - -VmrunPath $VmrunPath - - # ── Optional VMX CPU/RAM override (post-clone, pre-start) ───────────── - if ($GuestCPU -gt 0 -or $GuestMemoryMB -gt 0) { - Write-Host "[Invoke-CIJob] Applying VMX override: CPU=$GuestCPU, RAM=${GuestMemoryMB}MB" - $vmxLines = Get-Content -LiteralPath $cloneVmxPath - if ($GuestCPU -gt 0) { - if ($vmxLines -imatch '^numvcpus\s*=') { - $vmxLines = $vmxLines -replace '(?i)^numvcpus\s*=.*', "numvcpus = `"$GuestCPU`"" - } else { - $vmxLines = $vmxLines + "numvcpus = `"$GuestCPU`"" - } - } - if ($GuestMemoryMB -gt 0) { - if ($vmxLines -imatch '^memsize\s*=') { - $vmxLines = $vmxLines -replace '(?i)^memsize\s*=.*', "memsize = `"$GuestMemoryMB`"" - } else { - $vmxLines = $vmxLines + "memsize = `"$GuestMemoryMB`"" - } - } - Set-Content -LiteralPath $cloneVmxPath -Value $vmxLines -Encoding UTF8 - Write-Host "[Invoke-CIJob] VMX updated: $cloneVmxPath" - } - - # ── Phase 3: Start VM ───────────────────────────────────────────── - Write-Host "`n[Phase 3/6] Starting VM..." - $startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation start ` - -Arguments @($cloneVmxPath, 'nogui') ` - -TimeoutSeconds 180 - if ($startResult.TimedOut) { - throw "vmrun start timed out after 180s for: $cloneVmxPath" - } - if ($startResult.ExitCode -ne 0) { - throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)" - } - Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)." - - # ── Phase 3b: Resolve VM IP ─────────────────────────────────────── - if ([string]::IsNullOrWhiteSpace($VMIPAddress)) { - $detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath ` - -VmxPath $cloneVmxPath -TimeoutSeconds 120 - if ($detectedIP -eq '') { - throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)." - } - $VMIPAddress = $detectedIP - Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress" - } - - # ── Lease IP ────────────────────────────────────────────────────── - # If two VMs somehow got the same IP (DHCP edge case), fail fast here - # rather than letting both jobs write to the same WinRM target. - $leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease" - if (Test-Path $leaseFile) { - $holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim() - throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity." - } - [System.IO.File]::WriteAllText($leaseFile, $JobId) - Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId." - Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath } - - } finally { - # Release lock — next waiting job can now enter clone→start→IP phase - if ($lockStream) { - $lockStream.Close() - $lockStream.Dispose() - $lockStream = $null - } - Remove-Item $lockPath -Force -ErrorAction SilentlyContinue - Write-Host "[Invoke-CIJob] VM-start lock released." - } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - # ── Resolve GuestOS from VMX if Auto ──────────────────────────────────── - if ($GuestOS -eq 'Auto' -and $cloneVmxPath) { - $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue - if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') { - $detectedOS = $Matches[1] - $GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' } - Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)" - } else { - $GuestOS = 'Windows' - Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows" - } - } - # ── Phase 4: Wait for readiness ─────────────────────────────────────── - Write-Host "`n[Phase 4/6] Waiting for VM readiness..." - Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress } - $waitParams = @{ - VMPath = $cloneVmxPath - IPAddress = $VMIPAddress - VmrunPath = $VmrunPath - } - if ($GuestOS -eq 'Linux') { - $waitParams['Transport'] = 'SSH' - $waitParams['SshKeyPath'] = $SshKeyPath - $waitParams['SshUser'] = $SshUser - } - & "$scriptDir\Wait-VMReady.ps1" @waitParams - Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success' - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 5: Remote build (copy source + build inside VM) ───────────── - Write-Host "`n[Phase 5/6] Invoking remote build..." - Write-JobEvent -Phase 'phase5.build' -Status 'start' - $remoteBuildParams = @{ - IPAddress = $VMIPAddress - Credential = $credential - Configuration = $Configuration - } - if ($GuestOS -eq 'Linux') { - $remoteBuildParams['GuestOS'] = 'Linux' - $remoteBuildParams['SshKeyPath'] = $SshKeyPath - $remoteBuildParams['SshUser'] = $SshUser - $remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile - $remoteBuildParams.Remove('Credential') - if ($UseGitClone) { - # In-guest git clone (requires Gitea reachable + PAT/key configured in guest) - $remoteBuildParams['CloneUrl'] = $RepoUrl - $remoteBuildParams['CloneBranch'] = $Branch - $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget - if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } - if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } - } else { - # Default: host-cloned source shipped to guest via tar+scp - $remoteBuildParams['HostSourceDir'] = $hostCloneDir - } - } else { - if ($UseGitClone) { - # §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir - $remoteBuildParams['CloneUrl'] = $RepoUrl - $remoteBuildParams['CloneBranch'] = $Branch - $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget - if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit } - if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true } - } - else { - # Default: Use pre-cloned source from host - $remoteBuildParams['HostSourceDir'] = $hostCloneDir - } - } - if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand } - if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource } - if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv } - if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true } - if ($SkipArtifact) { $remoteBuildParams['SkipArtifact'] = $true } - & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams - Write-JobEvent -Phase 'phase5.build' -Status 'success' - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - $phaseStart = Get-Date - - # ── Phase 6: Collect artifacts ──────────────────────────────────────── - if ($SkipArtifact) { - Write-Host "`n[Phase 6/6] Artifact collection skipped (-SkipArtifact)." - Write-JobEvent -Phase 'phase6.artifacts' -Status 'skipped' - } else { - Write-Host "`n[Phase 6/6] Collecting artifacts..." - Write-JobEvent -Phase 'phase6.artifacts' -Status 'start' - if ($GuestOS -eq 'Linux') { - & "$scriptDir\Get-BuildArtifacts.ps1" ` - -IPAddress $VMIPAddress ` - -GuestOS 'Linux' ` - -SshKeyPath $SshKeyPath ` - -SshUser $SshUser ` - -SshKnownHostsFile $SshKnownHostsFile ` - -GuestArtifactPath '/opt/ci/output' ` - -HostArtifactDir $jobArtifactDir ` - -JobId $JobId ` - -Commit $Commit ` - -IncludeLogs - } else { - & "$scriptDir\Get-BuildArtifacts.ps1" ` - -IPAddress $VMIPAddress ` - -Credential $credential ` - -GuestArtifactPath 'C:\CI\output\artifacts.zip' ` - -HostArtifactDir $jobArtifactDir ` - -JobId $JobId ` - -Commit $Commit ` - -IncludeLogs - } - Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir } - } - $elapsed = (Get-Date) - $startTime - Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds } - $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds }) - Write-Host "`n============================================================" - Write-Host "[Invoke-CIJob] Phase durations:" - foreach ($pt in $phaseTimings) { - Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec) - } - Write-Host (" {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds) - Write-Host "============================================================" - Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))" - Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir" - Write-Host "============================================================" -} -catch { - $exitCode = 1 - $elapsed = (Get-Date) - $startTime - Write-JobEvent -Phase 'job' -Status 'failure' -Data @{ - elapsedSec = [int]$elapsed.TotalSeconds - error = "$_" - } - - # ── Best-effort guest diagnostics (skip if VM was never started) ───────── - # Only collect when a VM IP is known (Phase 3b completed). Never throws. - if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) { - try { - $diagDir = Join-Path $jobArtifactDir 'diagnostics' - $diagParams = @{ - IPAddress = $VMIPAddress - GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS } - DestinationDir = $diagDir - } - if ($diagParams.GuestOS -eq 'Windows' -and $credential) { - $diagParams['Credential'] = $credential - } - if ($diagParams.GuestOS -eq 'Linux') { - $diagParams['SshKeyPath'] = $SshKeyPath - $diagParams['SshUser'] = $SshUser - } - Get-GuestDiagnostics @diagParams - } catch { - Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_" - } - } - - Write-Host "`n============================================================" - Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))" - Write-Host "Error: $_" - Write-Host "============================================================" -} -finally { - # ── Release IP lease (§2.1) ─────────────────────────────────────────── - if ($leaseFile -and (Test-Path $leaseFile)) { - Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] IP lease released: $VMIPAddress" - } - - # ── Always destroy the VM ───────────────────────────────────────────── - if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) { - Write-Host "`n[Cleanup] Destroying ephemeral VM..." - & "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath - } - else { - Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)." - } - # ── Clean up host-side git clone (unless in-VM clone was used) ──────── - # When -UseGitClone, $hostCloneDir is never created - if (-not $UseGitClone -and (Test-Path $hostCloneDir)) { - Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..." - Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue - } - - # ── Cancel 90-minute warning job (build finished before it fired) ────────── - if ($null -ne $durationWarnJob) { - Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue - $durationWarnJob = $null - } - - # ── Stop transcript ─────────────────────────────────────────────────── - if ($transcriptStarted) { - try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" } - } - - # ── Log retention: purge directories older than $LogRetentionDays days ─ - if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) { - $cutoff = (Get-Date).AddDays(-$LogRetentionDays) - Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.LastWriteTime -lt $cutoff } | - ForEach-Object { - Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] Purged old log dir: $($_.FullName)" - } - } -} - -exit $exitCode +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A4-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator job @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Invoke-RemoteBuild.ps1 b/scripts/Invoke-RemoteBuild.ps1 index fd31f50..ed230dc 100644 --- a/scripts/Invoke-RemoteBuild.ps1 +++ b/scripts/Invoke-RemoteBuild.ps1 @@ -1,543 +1,91 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Executes a build inside a guest VM (via WinRM). Supports two clone modes: - (1) Host-side clone: copy pre-cloned source via WinRM zip, or - (2) Guest-side clone: clone repo directly in VM via git (§3.3, requires Git) +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Called by Invoke-CIJob.ps1 to orchestrate the build inside the VM. +# Shim: delegates to the Python ci_orchestrator CLI (`build run`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. +# +# Typed parameters (`[PSCredential]`, `[hashtable]`, `[switch]`) cannot cross +# a process boundary as-is; this shim translates them. Credentials are pulled +# inside Python via keyring (`--credential-target`), so the bound `-Credential` +# is intentionally discarded — callers should set `-CredentialTarget` instead +# (or rely on the default `BuildVMGuest` resolved by the Python CLI). - **Mode 1: Host-side (default)** — Caller provides pre-cloned source: - 1. Compresses the source tree on host, copies archive into VM via WinRM - 2. Runs dotnet restore + dotnet build inside VM - 3. Packages output as C:\CI\output\artifacts.zip - - **Mode 2: Guest-side (§3.3, -UseGitClone in Invoke-CIJob)** — Repo cloned in VM: - 1. Clone repo directly into VM via git (eliminates host-zip-transfer overhead) - 2. Runs dotnet restore + dotnet build inside VM - 3. Packages output same as Mode 1 - - Supply either -HostSourceDir (Mode 1) or -CloneUrl (Mode 2), not both. - - In Mode 2, PAT for private repos is read from Credential Manager inside - this script's WinRM session (§1.5: no argv, no log, cleanup at end). - -.PARAMETER IPAddress - IP of the build VM (e.g. 192.168.11.101). - -.PARAMETER Credential - PSCredential for the build user inside the VM. - Retrieve from Windows Credential Manager in production: - $cred = Get-StoredCredential -Target 'BuildVMGuest' - -.PARAMETER HostSourceDir - [Mode 1] Full path on HOST to already-cloned repository (host-side clone mode). - Mutually exclusive with -CloneUrl. When omitted, -CloneUrl must be provided. - Example: C:\Users\user\AppData\Local\Temp\ci-src-run-42 - -.PARAMETER CloneUrl - [Mode 2, §3.3] Git URL to clone directly in VM (guest-side clone mode). - Mutually exclusive with -HostSourceDir. When provided, repo cloned via git inside VM. - PAT (if needed) read from Credential Manager at runtime. - Example: https://gitea.emulab.it/Simone/myrepo.git - -.PARAMETER CloneBranch - [Mode 2] Branch to clone. Ignored in Mode 1. Default: main - -.PARAMETER CloneCommit - [Mode 2] Specific commit SHA to checkout after clone. Ignored in Mode 1. - -.PARAMETER CloneSubmodules - [Mode 2] Pass --recurse-submodules to git clone. Ignored in Mode 1. - -.PARAMETER Configuration - MSBuild/dotnet build configuration. Default: Release - -.PARAMETER GuestWorkDir - Working directory inside the VM where source will be placed. - Default: C:\CI\build - -.PARAMETER GuestArtifactZip - Full path inside the VM where the artifact archive will be written. - Default: C:\CI\output\artifacts.zip - -.EXAMPLE - $cred = Get-Credential - .\Invoke-RemoteBuild.ps1 ` - -IPAddress "192.168.11.101" ` - -Credential $cred ` - -HostSourceDir "C:\Temp\ci-src-run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] [string] $IPAddress, [Parameter()] [System.Management.Automation.PSCredential] $Credential, - # [Mode 1] Host-side clone — pre-cloned source directory on HOST - [ValidateScript({ - if ($_ -and -not (Test-Path $_ -PathType Container)) { - throw "HostSourceDir path does not exist: $_" - } - return $true - })] [string] $HostSourceDir = '', - - # [Mode 2, §3.3] Guest-side clone — Git URL for in-VM clone [string] $CloneUrl = '', - - # [Mode 2] Branch to clone (default: main). Ignored if HostSourceDir provided. [string] $CloneBranch = 'main', - - # [Mode 2] Specific commit SHA to checkout. Ignored if HostSourceDir provided. [string] $CloneCommit = '', - - # [Mode 2] Clone with --recurse-submodules. Ignored if HostSourceDir provided. [switch] $CloneSubmodules, - - # [Mode 2] Credential Manager target for Gitea PAT (private repos). - # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine - # Leave empty for public repos (no auth needed). [string] $GiteaCredentialTarget = 'GiteaPAT', - [string] $Configuration = 'Release', - - # Custom build command to run inside the VM (working dir = GuestWorkDir). - # When empty, defaults to: dotnet restore + dotnet build --configuration $Configuration - # Example: 'python build_plugin.py --final' [string] $BuildCommand = '', - - # Glob pattern (relative to GuestWorkDir) of files to include in the artifact zip. - # Used only when BuildCommand is set. Default: collect everything under 'dist'. - # Example: 'dist\**' or 'Contrib\nsInnoUnp\Build\**' [string] $GuestArtifactSource = 'dist', - [string] $GuestWorkDir = 'C:\CI\build', - [string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip', - - # Redirect dotnet NuGet package store and pip download cache to VMware shared - # folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip). - # Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1 - # and VMware Tools HGFS driver present in the guest. [switch] $UseSharedCache, - - # Skip artifact packaging entirely. Use for jobs that produce no output. - # When absent, missing GuestArtifactSource after a successful build is an error. [switch] $SkipArtifact, - # Guest OS type — Linux uses SSH instead of WinRM. [ValidateSet('Windows', 'Linux')] [string] $GuestOS = 'Windows', - # SSH private key path (used when GuestOS=Linux). [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when GuestOS=Linux). [string] $SshUser = 'ci_build', - - # Persistent known_hosts file for SSH calls (CI jobs). - # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL). [string] $SshKnownHostsFile = '', - - # Work directory inside the Linux guest. [string] $GuestLinuxWorkDir = '/opt/ci/build', - - # Output directory inside the Linux guest. [string] $GuestLinuxOutputDir = '/opt/ci/output', - # Extra environment variables injected into the guest before the build command (§6.5). - # Keys must be valid env var names. Values are plain strings (never logged). - [hashtable] $ExtraGuestEnv = @{} + [hashtable] $ExtraGuestEnv = @{}, + + # Optional override for the Credential Manager target (overrides any + # fallback resolved by Python). Discards $Credential. + [string] $CredentialTarget = '' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--ip-address', $IPAddress, + '--guest-os', $GuestOS.ToLower(), + '--ssh-user', $SshUser, + '--ssh-key-path', $SshKeyPath, + '--clone-branch', $CloneBranch, + '--gitea-credential-target', $GiteaCredentialTarget, + '--configuration', $Configuration, + '--guest-work-dir', $GuestWorkDir, + '--guest-artifact-zip', $GuestArtifactZip, + '--guest-linux-work-dir', $GuestLinuxWorkDir, + '--guest-linux-output-dir', $GuestLinuxOutputDir, + '--guest-artifact-source', $GuestArtifactSource +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +if ($HostSourceDir) { $pyArgs += @('--host-source-dir', $HostSourceDir) } +if ($CloneUrl) { $pyArgs += @('--clone-url', $CloneUrl) } +if ($CloneCommit) { $pyArgs += @('--clone-commit', $CloneCommit) } +if ($BuildCommand) { $pyArgs += @('--build-command', $BuildCommand) } +if ($SshKnownHostsFile) { $pyArgs += @('--ssh-known-hosts-file', $SshKnownHostsFile) } +if ($CredentialTarget) { $pyArgs += @('--credential-target', $CredentialTarget) } -# ── Validate mutually exclusive modes ───────────────────────────────────── -$hostCloneMode = -not [string]::IsNullOrWhiteSpace($HostSourceDir) -$guestCloneMode = -not [string]::IsNullOrWhiteSpace($CloneUrl) -if ($hostCloneMode -and $guestCloneMode) { - throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone), not both." -} -if (-not $hostCloneMode -and -not $guestCloneMode) { - throw "Provide either -HostSourceDir (host-side clone) or -CloneUrl (guest-side clone)." -} -if ($guestCloneMode -and -not $CloneBranch) { - throw "CloneBranch is required when using -CloneUrl mode." -} +if ($CloneSubmodules.IsPresent) { $pyArgs += '--clone-submodules' } +if ($UseSharedCache.IsPresent) { $pyArgs += '--use-shared-cache' } +if ($SkipArtifact.IsPresent) { $pyArgs += '--skip-artifact' } -if ($GuestOS -eq 'Windows' -and $null -eq $Credential) { - throw "Credential is required when GuestOS is Windows." -} - -if ($GuestOS -eq 'Linux') { - Import-Module (Join-Path $PSScriptRoot '_Transport.psm1') -Force - - Write-Host "[Invoke-RemoteBuild] Linux build mode — SSH transport" - - # Step 1: Clean work dir (parent permissions handled by template setup) - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "rm -rf '$GuestLinuxWorkDir' && mkdir -p '$GuestLinuxWorkDir'" - - if ($hostCloneMode) { - # ── Mode 1 (Linux): tar host source -> scp -> untar in guest ── - Write-Host "[Invoke-RemoteBuild] Packaging host source ($HostSourceDir) ..." - $tarName = "ci-src-$([Guid]::NewGuid().ToString('N').Substring(0,8)).tar.gz" - $hostTar = Join-Path $env:TEMP $tarName - $guestTar = "/tmp/$tarName" - if (Test-Path $hostTar) { Remove-Item $hostTar -Force } - # bsdtar on Windows 10/11 supports -C and gz natively. - & tar -czf $hostTar -C $HostSourceDir . - if ($LASTEXITCODE -ne 0) { - throw "[Invoke-RemoteBuild] tar failed packaging $HostSourceDir (exit $LASTEXITCODE)" - } - try { - Write-Host "[Invoke-RemoteBuild] Copying source archive to guest:$guestTar ..." - Copy-SshItem -Source $hostTar -Destination $guestTar ` - -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Direction ToGuest - - Write-Host "[Invoke-RemoteBuild] Extracting source in guest ($GuestLinuxWorkDir) ..." - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "tar -xzf '$guestTar' -C '$GuestLinuxWorkDir' && rm -f '$guestTar'" - } - finally { - if (Test-Path $hostTar) { Remove-Item $hostTar -Force -ErrorAction SilentlyContinue } - } - } - else { - # ── Mode 2 (Linux): in-guest git clone ── - $cloneCmd = "git clone --depth 1 --branch '$CloneBranch'" - if ($CloneSubmodules) { $cloneCmd += ' --recurse-submodules' } - - $cloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" # non-authed URL for logging - Write-Host "[Invoke-RemoteBuild] Cloning $CloneUrl into guest $GuestLinuxWorkDir ..." - - # PAT injection via git http.extraHeader — PAT never appears in argv or /proc/environ - $authHeader = '' - if ($GiteaCredentialTarget -ne '') { - try { - Import-Module CredentialManager -ErrorAction Stop - $credObj = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - if ($credObj) { - $b64 = [Convert]::ToBase64String( - [System.Text.Encoding]::UTF8.GetBytes( - "$($credObj.UserName):$($credObj.GetNetworkCredential().Password)")) - $authHeader = "Authorization: Basic $b64" - } - } catch { - Write-Warning "[Invoke-RemoteBuild] Could not load PAT from Credential Manager ($GiteaCredentialTarget) — cloning without auth (public repo?)." - } - } - - if ($authHeader -ne '') { - $authCloneCmd = "git -c credential.helper= -c 'http.extraHeader=$authHeader' clone --depth 1 --branch '$CloneBranch'" - if ($CloneSubmodules) { $authCloneCmd += ' --recurse-submodules' } - $authCloneCmd += " '$CloneUrl' '$GuestLinuxWorkDir'" - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $authCloneCmd - } else { - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $cloneCmd - } - - # Checkout specific commit if requested - if ($CloneCommit -ne '') { - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "git -C '$GuestLinuxWorkDir' checkout '$CloneCommit'" - } - } - - # Step 3: Run build command (always cd into workdir first) - # Prepend extra env exports (§6.5) — scoped to this SSH command only - $envPrefix = '' - foreach ($envKey in $ExtraGuestEnv.Keys) { - $envVal = $ExtraGuestEnv[$envKey] -replace "'", "'\''" # escape ' for POSIX single-quoted string - $envPrefix += "export $envKey='$envVal'; " - } - if ($BuildCommand -ne '') { - $buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && $BuildCommand" - } else { - $buildCmd = "${envPrefix}cd '$GuestLinuxWorkDir' && make" - } - $displayCmd = if ($BuildCommand -ne '') { $BuildCommand } else { 'make' } - Write-Host "[Invoke-RemoteBuild] Running build (env vars redacted): cd '$GuestLinuxWorkDir' && $displayCmd" - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile -Command $buildCmd - - # Step 4: Stage artifact tree under $GuestLinuxOutputDir for SCP collection - $outDir = $GuestLinuxOutputDir - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "rm -rf '$outDir' && mkdir -p '$outDir'" - - $artSrc = if ($GuestArtifactSource -ne '') { $GuestArtifactSource } else { 'dist' } - Invoke-SshCommand -IP $IPAddress -User $SshUser -KeyPath $SshKeyPath ` - -KnownHostsFile $SshKnownHostsFile ` - -Command "if [ -d '$GuestLinuxWorkDir/$artSrc' ]; then cp -r '$GuestLinuxWorkDir/$artSrc/.' '$outDir/'; elif [ -e '$GuestLinuxWorkDir/$artSrc' ]; then cp '$GuestLinuxWorkDir/$artSrc' '$outDir/'; else echo '[Invoke-RemoteBuild] WARNING: artifact source not found: $artSrc' >&2; fi" - - Write-Host "[Invoke-RemoteBuild] Linux build complete." - return -} - -function Compress-BuildArtifact { - param( - [Parameter(Mandatory)] - [string] $Path, - [Parameter(Mandatory)] - [string] $DestinationPath - ) - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Compress-BuildArtifact] Using 7-Zip (multi-threaded)..." - & $7zip a -mmt=on -mx1 $DestinationPath $Path 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { - throw "7-Zip compression failed (exit $LASTEXITCODE)" - } - } - else { - Write-Host "[Compress-BuildArtifact] 7-Zip not found, using Compress-Archive (fallback)..." - Compress-Archive -Path $Path -DestinationPath $DestinationPath -CompressionLevel Fastest -Force +if ($ExtraGuestEnv -and $ExtraGuestEnv.Count -gt 0) { + foreach ($key in $ExtraGuestEnv.Keys) { + $pyArgs += @('--extra-env', "$key=$($ExtraGuestEnv[$key])") } } -$sessionOptions = New-CISessionOption - -Write-Host "[Invoke-RemoteBuild] Connecting to VM at $IPAddress..." - -$session = New-PSSession ` - -ComputerName $IPAddress ` - -Credential $Credential ` - -SessionOption $sessionOptions ` - -UseSSL ` - -Port 5986 ` - -Authentication Basic ` - -ErrorAction Stop - -try { - # ── Ensure working directories exist on guest ──────────────────────── - Invoke-Command -Session $session -ScriptBlock { - param($workDir, $outputDir) - # Ensure artifact output parent exists - $outParent = Split-Path $outputDir -Parent - if (-not (Test-Path $outParent)) { - New-Item -ItemType Directory -Path $outParent -Force | Out-Null - } - # Clean and recreate build dir (ephemeral VM, but guard against partial prior run) - if (Test-Path $workDir) { Remove-Item $workDir -Recurse -Force } - New-Item -ItemType Directory -Path $workDir -Force | Out-Null - } -ArgumentList $GuestWorkDir, $GuestArtifactZip - - # ── Source preparation: Host clone (compress+transfer) or Guest clone (git) ── - if ($hostCloneMode) { - # Mode 1: Copy pre-cloned source from host to guest via WinRM zip - # Single-file transfer over WinRM is far faster than recursive Copy-Item. - $hostZip = [System.IO.Path]::GetTempFileName() -replace '\.tmp$', '.zip' - try { - Write-Host "[Invoke-RemoteBuild] Compressing source on host..." - Compress-BuildArtifact -Path "$HostSourceDir\*" -DestinationPath $hostZip - $zipSize = [math]::Round((Get-Item $hostZip).Length / 1MB, 1) - Write-Host "[Invoke-RemoteBuild] Transferring archive to guest ($zipSize MB)..." - $guestZip = 'C:\CI\src-transfer.zip' - Copy-Item -Path $hostZip -Destination $guestZip -ToSession $session -Force - Write-Host "[Invoke-RemoteBuild] Expanding archive in guest..." - Invoke-Command -Session $session -ScriptBlock { - param($zip, $dest) - Expand-Archive -Path $zip -DestinationPath $dest -Force - Remove-Item $zip -Force - } -ArgumentList $guestZip, $GuestWorkDir - Write-Host "[Invoke-RemoteBuild] Source transfer complete." - } - finally { - Remove-Item $hostZip -Force -ErrorAction SilentlyContinue - } - } - else { - # Mode 2 (§3.3): Clone repo directly in guest via git - # PAT (if needed) read from Credential Manager inside session, never logged - Write-Host "[Invoke-RemoteBuild] Cloning repository in guest (§3.3)..." - - # Read PAT from host Credential Manager (§1.5: never in argv or logs) - $gitPatUser = $null - $gitPatPass = $null - if (-not [string]::IsNullOrWhiteSpace($GiteaCredentialTarget)) { - try { - $stored = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - $gitPatUser = $stored.UserName - $gitPatPass = $stored.GetNetworkCredential().Password - Write-Host "[Invoke-RemoteBuild] PAT loaded from Credential Manager ($GiteaCredentialTarget)." - } catch { - Write-Warning "[Invoke-RemoteBuild] Could not read PAT from Credential Manager '$GiteaCredentialTarget' — proceeding without auth (public repo only)." - } - } - - # Clone in guest via WinRM - Invoke-Command -Session $session -ScriptBlock { - param($cloneUrl, $branch, $commit, $workDir, $patUser, $patPass, $submodules) - - Set-Location (Split-Path $workDir -Parent) - $gitArgs = @('clone', '--depth', '1', '--branch', $branch) - if ($submodules) { $gitArgs += '--recurse-submodules' } - $gitArgs += @($cloneUrl, (Split-Path $workDir -Leaf)) - - # Disable interactive credential prompts — WinRM has no TTY - $env:GIT_TERMINAL_PROMPT = '0' - - # Build git config args: - # - credential.helper= (empty) clears GCM and any system credential helper - # - http.extraHeader injects Basic auth without credential helpers, temp files, - # or modifying the clone URL (§1.5: token never in URL, argv display, or logs) - $gitConfigArgs = @('-c', 'credential.helper=') - if ($patUser -and $patPass) { - $b64 = [Convert]::ToBase64String( - [Text.Encoding]::UTF8.GetBytes("${patUser}:${patPass}")) - $gitConfigArgs += @('-c', "http.extraHeader=Authorization: Basic $b64") - } - - $gitArgs = $gitConfigArgs + $gitArgs - - Write-Host "[Guest] Cloning $cloneUrl (branch: $branch)..." - & git @gitArgs 2>&1 | ForEach-Object { Write-Host "[Guest Clone] $_" } - if ($LASTEXITCODE -ne 0) { - throw "git clone failed in guest (exit $LASTEXITCODE)" - } - - # Checkout specific commit if provided - if ($commit) { - Write-Host "[Guest] Checking out commit: $commit" - Set-Location $workDir - & git checkout $commit 2>&1 | ForEach-Object { Write-Host "[Guest Checkout] $_" } - if ($LASTEXITCODE -ne 0) { - throw "git checkout $commit failed in guest (exit $LASTEXITCODE)" - } - } - } -ArgumentList $CloneUrl, $CloneBranch, $CloneCommit, $GuestWorkDir, $gitPatUser, $gitPatPass, $CloneSubmodules.IsPresent - - Write-Host "[Invoke-RemoteBuild] Guest clone complete." - } - - # ── Build ────────────────────────────────────────────────────────── - if ($BuildCommand -ne '') { - # ── Custom build command (e.g. python, msbuild, cmake…) ────────── - Write-Host "[Invoke-RemoteBuild] Running build command: $BuildCommand" - $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $cmd, $artifactZip, $artifactSrc, $useCache, $extraEnv, $skipArt) - # Unbuffered Python output — lines appear in real-time instead of being held in buffer - $env:PYTHONUNBUFFERED = '1' - if ($useCache) { - $env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache' - } - # Inject extra env vars (§6.5 secret injection) - foreach ($pair in $extraEnv.GetEnumerator()) { - [System.Environment]::SetEnvironmentVariable($pair.Key, $pair.Value, 'Process') - } - Set-Location $workDir - # Stream each output line via Write-Host — WinRM forwards Write-Host in real-time - & cmd /c "$cmd 2>&1" | ForEach-Object { Write-Host $_ } - $exit = $LASTEXITCODE - - if ($exit -eq 0 -and -not $skipArt) { - $archiveDir = Split-Path $artifactZip -Parent - if (-not (Test-Path $archiveDir)) { - New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null - } - $srcPath = Join-Path $workDir $artifactSrc - if (-not (Test-Path $srcPath)) { - throw "Artifact source directory not found: $srcPath. Build succeeded (exit 0) but produced no output. Check GuestArtifactSource or set skip-artifact: 'true' in the workflow." - } - else { - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Build] Compressing artifacts with 7-Zip..." - & $7zip a -mmt=on -mx1 $artifactZip $srcPath 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" } - } - else { - Compress-Archive -Path $srcPath -DestinationPath $artifactZip -Force - } - } - } - - return $exit - } -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource, $UseSharedCache.IsPresent, $ExtraGuestEnv, $SkipArtifact.IsPresent - - if ($buildExit -ne 0) { - throw "Build command failed (exit $buildExit)" - } - } - else { - # ── Default: dotnet restore + dotnet build ──────────────────────── - Write-Host "[Invoke-RemoteBuild] Running dotnet restore..." - $restoreExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $useCache) - if ($useCache) { - $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' - } - Set-Location $workDir - dotnet restore 2>&1 | ForEach-Object { Write-Host $_ } - return $LASTEXITCODE - } -ArgumentList $GuestWorkDir, $UseSharedCache.IsPresent - - if ($restoreExit -ne 0) { - throw "dotnet restore failed (exit $restoreExit)" - } - - Write-Host "[Invoke-RemoteBuild] Running dotnet build (configuration: $Configuration)..." - $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $config, $artifactZip, $useCache) - if ($useCache) { - $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' - } - Set-Location $workDir - $outputDir = Join-Path $workDir 'bin\CIOutput' - dotnet build --configuration $config --output $outputDir 2>&1 | ForEach-Object { Write-Host $_ } - $exit = $LASTEXITCODE - - if ($exit -eq 0) { - $archiveDir = Split-Path $artifactZip -Parent - if (-not (Test-Path $archiveDir)) { - New-Item -ItemType Directory -Path $archiveDir -Force | Out-Null - } - if (-not (Test-Path $outputDir)) { - Write-Host "[Build] Output dir '$outputDir' not found — skipping packaging." - } - else { - $7zip = 'C:\Program Files\7-Zip\7z.exe' - if (Test-Path $7zip) { - Write-Host "[Build] Compressing output with 7-Zip..." - & $7zip a -mmt=on -mx1 $artifactZip "$outputDir\*" 2>&1 | Where-Object { $_ -notmatch '^7-Zip' } | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { throw "7-Zip failed" } - } - else { - Compress-Archive -Path "$outputDir\*" -DestinationPath $artifactZip -Force - } - } - } - - return $exit - } -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip, $UseSharedCache.IsPresent - - if ($buildExit -ne 0) { - throw "dotnet build failed (exit $buildExit)" - } - } - - if ($SkipArtifact) { - Write-Host '[Invoke-RemoteBuild] Build succeeded. Artifact packaging skipped (-SkipArtifact).' - } else { - Write-Host "[Invoke-RemoteBuild] Build succeeded. Artifact at $GuestArtifactZip" - } -} -finally { - Remove-PSSession $session -ErrorAction SilentlyContinue +if ($Credential) { + Write-Host '[Invoke-RemoteBuild] -Credential ignored; Python CLI uses keyring (set -CredentialTarget to override target name).' } + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator build run @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Measure-CIBenchmark.ps1 b/scripts/Measure-CIBenchmark.ps1 index cdb53e4..66bc05b 100644 --- a/scripts/Measure-CIBenchmark.ps1 +++ b/scripts/Measure-CIBenchmark.ps1 @@ -117,7 +117,7 @@ for ($i = 1; $i -le $Iterations; $i++) { Write-Host "[bench] Cloning..." $sw = [System.Diagnostics.Stopwatch]::StartNew() - $r = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` + $null = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) ` -ThrowOnError $t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2) diff --git a/scripts/New-BuildVM.ps1 b/scripts/New-BuildVM.ps1 index c1964b5..5470f77 100644 --- a/scripts/New-BuildVM.ps1 +++ b/scripts/New-BuildVM.ps1 @@ -1,43 +1,10 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Creates a linked clone of the template VM for an ephemeral CI build. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' -.DESCRIPTION - Uses vmrun.exe to create a linked clone from a frozen template snapshot. - Returns the full path to the new clone's .vmx file on success. - The template VM and its snapshot are never modified. +# Shim: delegates to the Python ci_orchestrator CLI (`vm new`). +# Original PS implementation moved to git history; see plans/A3-closeout.md. -.PARAMETER TemplatePath - Full path to the template VM's .vmx file. - Example: F:\CI\Templates\WinBuild\WinBuild.vmx - -.PARAMETER SnapshotName - Name of the snapshot on the template VM to clone from. - Default: BaseClean - -.PARAMETER CloneBaseDir - Directory under which the new clone folder will be created. - The clone folder is named after the job ID and timestamp. - Example: F:\CI\BuildVMs - -.PARAMETER JobId - Unique identifier for the CI job (e.g. run ID from Gitea Actions). - Used to name the clone: Clone_{JobId}_{yyyyMMdd_HHmmss} - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.OUTPUTS - [string] Full path to the new clone's .vmx file. - -.EXAMPLE - $vmxPath = .\New-BuildVM.ps1 ` - -TemplatePath "F:\CI\Templates\WinBuild\WinBuild.vmx" ` - -CloneBaseDir "F:\CI\BuildVMs" ` - -JobId "run-42" -#> [CmdletBinding()] param( [Parameter(Mandatory)] @@ -55,49 +22,15 @@ param( [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ) -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' +$pyArgs = @( + '--template-path', $TemplatePath, + '--snapshot-name', $SnapshotName, + '--clone-base-dir', $CloneBaseDir, + '--job-id', $JobId, + '--vmrun-path', $VmrunPath +) -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force - -Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null - -# --- Build clone path --- -$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' -$cloneName = "Clone_${JobId}_${timestamp}" -$cloneDir = Join-Path $CloneBaseDir $cloneName -$cloneVmx = Join-Path $cloneDir "$cloneName.vmx" - -# Ensure clone base dir exists -if (-not (Test-Path $CloneBaseDir)) { - New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null -} - -Write-Host "[New-BuildVM] Creating linked clone..." -Write-Host " Template : $TemplatePath" -Write-Host " Snapshot : $SnapshotName" -Write-Host " Clone VMX : $cloneVmx" - -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$cloneResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` - -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) - -$sw.Stop() - -if ($cloneResult.ExitCode -ne 0) { - # Clean up partial clone directory if it was created - if (Test-Path $cloneDir) { - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue - } - throw "vmrun clone failed (exit $($cloneResult.ExitCode)).`nOutput: $($cloneResult.Output)" -} - -if (-not (Test-Path $cloneVmx -PathType Leaf)) { - throw "vmrun reported success but clone VMX not found at: $cloneVmx" -} - -Write-Host "[New-BuildVM] Clone created in $($sw.Elapsed.TotalSeconds.ToString('F1'))s : $cloneVmx" - -# Return the VMX path as output -return $cloneVmx +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm new @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Remove-BuildVM.ps1 b/scripts/Remove-BuildVM.ps1 index 2d85fbf..0107b30 100644 --- a/scripts/Remove-BuildVM.ps1 +++ b/scripts/Remove-BuildVM.ps1 @@ -1,130 +1,40 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Stops and permanently deletes an ephemeral build VM. - -.DESCRIPTION - Performs a graceful shutdown (soft stop) with a timeout fallback to - a hard stop, then uses vmrun deleteVM to remove all VM files. - Finally, removes the clone directory from disk. - - This script is designed to be called from a try/finally block so - it always runs, even on build failure. - -.PARAMETER VMPath - Full path to the clone VM's .vmx file. - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER GracefulTimeoutSeconds - Seconds to wait for graceful shutdown before forcing power-off. - Default: 15 - -.EXAMPLE - .\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" -#> -[CmdletBinding(SupportsShouldProcess)] -param( - [Parameter(Mandatory)] - [string] $VMPath, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - [ValidateRange(1, 60)] - [int] $GracefulTimeoutSeconds = 15 -) - Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step +$ErrorActionPreference = 'Stop' -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# +# Callers (e.g. Measure-CIBenchmark.ps1) may pass PowerShell common +# parameters like -ErrorAction SilentlyContinue. Those must NOT be +# forwarded as --error-action etc. (the Python CLI rejects them). +# Value-taking common params consume the following token too. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } -$cloneDir = Split-Path $VMPath -Parent - -Write-Host "[Remove-BuildVM] Destroying VM: $VMPath" - -# ── Step 1: Check if VM exists at all ──────────────────────────────────────── -if (-not (Test-Path $VMPath -PathType Leaf)) { - Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath" - # Still attempt to remove the directory in case of partial clone - if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) { - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } - return -} - -if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return } - -# ── Step 2: Graceful stop ───────────────────────────────────────────────────── -Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..." -$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds -if ($stopSoft.TimedOut) { - Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s." -} elseif ($stopSoft.ExitCode -eq 0) { - Write-Host "[Remove-BuildVM] VM stopped gracefully." -} - -# ── Step 3: Force stop if still running ────────────────────────────────────── -if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) { - Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..." - $stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60 - if ($stopHard.TimedOut) { - Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s." + else { + $pyArgs += $a } } -# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly -# after stop. Waiting for the entry to disappear ensures deleteVM won't race the -# process exit. -$vmxNorm = $VMPath.Trim().ToLower() -$releaseDeadline = (Get-Date).AddSeconds(20) -while ((Get-Date) -lt $releaseDeadline) { - $listedVMs = & $VmrunPath -T ws list 2>&1 | - Where-Object { $_ -notmatch '^Total running' } | - ForEach-Object { "$_".Trim().ToLower() } - if ($listedVMs -notcontains $vmxNorm) { break } - Start-Sleep -Seconds 2 -} - -# ── Step 4: Delete VM via vmrun — retry up to 3× with backoff ──────────────── -Write-Host "[Remove-BuildVM] Deleting VM..." -$deleteOutput = '' -$deleteExit = -1 -foreach ($delay in @(0, 3, 6)) { - if ($delay -gt 0) { - Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..." - Start-Sleep -Seconds $delay - } - $delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90 - $deleteOutput = $delResult.Output - $deleteExit = $delResult.ExitCode - if ($delResult.TimedOut) { - Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s." - $deleteExit = -1 - } - if ($deleteExit -eq 0) { break } -} - -if ($deleteExit -ne 0) { - Write-Warning "[Remove-BuildVM] vmrun deleteVM failed after retries (exit $deleteExit): $deleteOutput" - Write-Warning "[Remove-BuildVM] Will attempt manual directory removal." -} - -# ── Step 5: Remove clone directory (catches any leftover files) ─────────────── -if (Test-Path $cloneDir) { - Write-Host "[Remove-BuildVM] Removing clone directory: $cloneDir" - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue - if (Test-Path $cloneDir) { - Write-Warning "[Remove-BuildVM] Clone directory could not be fully removed: $cloneDir" - Write-Warning " Manual cleanup required." - } else { - Write-Host "[Remove-BuildVM] Clone directory removed." - } -} - -Write-Host "[Remove-BuildVM] VM destruction complete." +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm remove @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Set-CIGuestCredential.ps1 b/scripts/Set-CIGuestCredential.ps1 new file mode 100644 index 0000000..abf562c --- /dev/null +++ b/scripts/Set-CIGuestCredential.ps1 @@ -0,0 +1,217 @@ +#Requires -Version 5.1 +#Requires -RunAsAdministrator +<# +.SYNOPSIS + Stores a CI guest-VM credential into the LocalSystem keyring vault. + +.DESCRIPTION + act_runner runs as the LocalSystem service account. Windows Credential + Manager / keyring vaults are per-user, so a credential added with + `cmdkey` or the Credential Manager UI under an interactive user is + invisible to the runner. The orchestrator then fails with: + + Credential '' not found in keyring. + + This script writes the credential into the SYSTEM vault using the + production venv's `keyring` itself (the exact backend + `ci_orchestrator` reads with `keyring.get_credential(target, None)`), + via a transient SYSTEM scheduled task. Using keyring for both write + and read avoids cmdkey/keyring format and persistence mismatches. + + Known-issue references: + plans/A1-closeout.md (KeyError: 'BuildVMGuest') + plans/implementation-plan-A-B.md (keyring under SYSTEM) + + The plaintext password unavoidably transits a temporary .py file. + The file is created with an Administrators/SYSTEM-only ACL, overwritten + with random bytes, then deleted. The password is never placed in a + scheduled-task argument or the command line. + +.PARAMETER Target + Credential target name. Must match the runner env + GITEA_CI_GUEST_CRED_TARGET. Default: BuildVMGuest + +.PARAMETER UserName + Guest login username in the form the transport expects (e.g. + 'WINBUILD\ci_build', '.\ci_build', or 'ci_build'). + +.PARAMETER Password + Guest password as a SecureString. If omitted, prompted securely. + +.PARAMETER VenvPython + Path to the production venv interpreter act_runner uses. Default: + CI_VENV_PYTHON env var, else F:\CI\python\venv\Scripts\python.exe + +.PARAMETER SkipVerify + Skip the post-write read-back verification (also run as SYSTEM). + +.EXAMPLE + .\Set-CIGuestCredential.ps1 -UserName 'WINBUILD\ci_build' + +.EXAMPLE + # Non-interactive (e.g. from another provisioning script) + $pw = Read-Host 'pw' -AsSecureString + .\Set-CIGuestCredential.ps1 -Target BuildVMGuest -UserName '.\ci_build' -Password $pw + +.EXAMPLE + .\Set-CIGuestCredential.ps1 -UserName '.\ci_build' -WhatIf +#> +[CmdletBinding(SupportsShouldProcess)] +param( + [ValidateNotNullOrEmpty()] + [string] $Target = 'BuildVMGuest', + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $UserName, + + [System.Security.SecureString] $Password, + + [string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON } + else { 'F:\CI\python\venv\Scripts\python.exe' }), + + [switch] $SkipVerify +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $VenvPython)) { + throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON." +} + +if (-not $Password) { + $Password = Read-Host "Password for '$UserName'" -AsSecureString +} + +# --- Run a script block as SYSTEM via a transient scheduled task ---------- +function Invoke-AsSystem { + param( + [Parameter(Mandatory)] [string] $TaskName, + [Parameter(Mandatory)] [string] $Execute, + [Parameter(Mandatory)] [string] $Argument, + [int] $TimeoutSec = 30 + ) + $action = New-ScheduledTaskAction -Execute $Execute -Argument $Argument + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' ` + -LogonType ServiceAccount -RunLevel Highest + Register-ScheduledTask -TaskName $TaskName -Action $action ` + -Principal $principal -Force | Out-Null + try { + Start-ScheduledTask -TaskName $TaskName + $deadline = (Get-Date).AddSeconds($TimeoutSec) + do { + Start-Sleep -Milliseconds 500 + $info = Get-ScheduledTaskInfo -TaskName $TaskName + $state = (Get-ScheduledTask -TaskName $TaskName).State + } while ($state -eq 'Running' -and (Get-Date) -lt $deadline) + return $info.LastTaskResult + } + finally { + Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false ` + -ErrorAction SilentlyContinue + } +} + +# --- Harden + shred helper for the transient secret file ------------------ +function Remove-Securely { + # Best-effort cleanup helper; ShouldProcess prompting is undesirable + # for an internal shred-and-delete of our own temp file. + [Diagnostics.CodeAnalysis.SuppressMessageAttribute( + 'PSUseShouldProcessForStateChangingFunctions', '')] + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + if (-not (Test-Path -LiteralPath $Path)) { return } + try { + $len = (Get-Item -LiteralPath $Path).Length + if ($len -gt 0) { + $rnd = New-Object byte[] $len + [Security.Cryptography.RandomNumberGenerator]::Create().GetBytes($rnd) + [IO.File]::WriteAllBytes($Path, $rnd) + } + } catch { + Write-Verbose "secure overwrite failed (continuing to delete): $_" + } + Remove-Item -LiteralPath $Path -Force -ErrorAction SilentlyContinue +} + +$tmpDir = $env:TEMP +$setPy = Join-Path $tmpDir ("setcred_{0}.py" -f ([guid]::NewGuid().ToString('N'))) +$verPy = Join-Path $tmpDir ("vercred_{0}.py" -f ([guid]::NewGuid().ToString('N'))) +$verOut = Join-Path $tmpDir ("vercred_{0}.txt" -f ([guid]::NewGuid().ToString('N'))) + +if (-not $PSCmdlet.ShouldProcess("$Target ($UserName)", + "store credential in SYSTEM keyring vault")) { + return +} + +try { + # Decrypt SecureString only at the moment of writing the temp script. + $bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($Password) + $plain = [Runtime.InteropServices.Marshal]::PtrToStringAuto($bstr) + [Runtime.InteropServices.Marshal]::ZeroFreeBSTR($bstr) + + # Python literals: pass via triple-quoted raw-ish strings. Reject quotes + # that would break out of the literal rather than risk silent corruption. + if ($UserName -match "'''" -or $plain -match "'''") { + throw "UserName/Password contains a triple-quote sequence; unsupported." + } + $setBody = @" +import keyring +keyring.set_password(r'''$Target''', r'''$UserName''', r'''$plain''') +print('OK') +"@ + Set-Content -LiteralPath $setPy -Value $setBody -Encoding UTF8 + Remove-Variable plain -ErrorAction SilentlyContinue + + # Lock the secret file down to SYSTEM + Administrators before it runs. + $acl = New-Object Security.AccessControl.FileSecurity + $acl.SetAccessRuleProtection($true, $false) + foreach ($id in 'SYSTEM', 'Administrators') { + $acl.AddAccessRule((New-Object Security.AccessControl.FileSystemAccessRule( + $id, 'FullControl', 'Allow'))) + } + Set-Acl -LiteralPath $setPy -AclObject $acl + + $rc = Invoke-AsSystem -TaskName 'CI-SetGuestCred' ` + -Execute $VenvPython -Argument "`"$setPy`"" + if ($rc -ne 0) { + throw "keyring.set_password task exited with code $rc." + } + Write-Host "Stored '$Target' (user '$UserName') in the SYSTEM vault." ` + -ForegroundColor Green +} +finally { + Remove-Securely -Path $setPy +} + +if ($SkipVerify) { return } + +try { + $verBody = @" +import keyring +c = keyring.get_credential(r'''$Target''', None) +open(r'''$verOut''', 'w', encoding='utf-8').write( + 'NONE' if c is None else 'OK:' + c.username) +"@ + Set-Content -LiteralPath $verPy -Value $verBody -Encoding UTF8 + Invoke-AsSystem -TaskName 'CI-VerifyGuestCred' ` + -Execute $VenvPython -Argument "`"$verPy`"" | Out-Null + + $result = if (Test-Path -LiteralPath $verOut) { + (Get-Content -LiteralPath $verOut -Raw).Trim() + } else { 'NONE' } + + if ($result -like 'OK:*') { + Write-Host "Verified: SYSTEM keyring returns user '$($result.Substring(3))'." ` + -ForegroundColor Green + } + else { + throw "Verification failed: SYSTEM keyring.get_credential('$Target', None) " + + "returned None. Credential did not persist." + } +} +finally { + Remove-Item -LiteralPath $verPy -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $verOut -Force -ErrorAction SilentlyContinue +} diff --git a/scripts/Test-CIGuestWinRM.ps1 b/scripts/Test-CIGuestWinRM.ps1 new file mode 100644 index 0000000..677e888 --- /dev/null +++ b/scripts/Test-CIGuestWinRM.ps1 @@ -0,0 +1,313 @@ +#Requires -Version 5.1 +#Requires -RunAsAdministrator +<# +.SYNOPSIS + Diagnoses WinRM reachability to a CI build VM from the runner's context. + +.DESCRIPTION + The orchestrator's readiness probe (WinRmTransport.is_ready) swallows + connection and auth errors and returns False, so a failing guest just + hangs "waiting for windows transport readiness" until timeout with no + cause in the log. + + This script reproduces the exact probe the runner performs, but + surfaces the real error: + + 1. TCP test to the WinRM HTTPS port (network / listener / firewall). + 2. A real pypsrp `execute_ps('$true')` against the guest, run as + SYSTEM (the act_runner account) so it reads the same SYSTEM + keyring vault credential the job uses. The full Python traceback + is captured and classified. + + Classifications: + CONNECT — port closed / refused / unreachable (guest WinRM + listener or firewall; check template WinRM 5986 setup). + SSL — TLS handshake/cert problem. + AUTH — credential rejected (wrong password, or username form; + try 'HOSTNAME\ci_build' instead of bare 'ci_build'). + KEYRING — credential missing from the SYSTEM vault (run + Set-CIGuestCredential.ps1). + OK — WinRM works; the hang is elsewhere. + +.PARAMETER IpAddress + Guest IP shown in the job log ("[job] guest IP: ..."). + +.PARAMETER Target + Credential target in the SYSTEM keyring. Default: BuildVMGuest + +.PARAMETER Port + WinRM HTTPS port. Default: 5986 + +.PARAMETER VenvPython + Production venv interpreter. Default: CI_VENV_PYTHON env, else + F:\CI\python\venv\Scripts\python.exe + +.PARAMETER TimeoutSec + Per-probe connection timeout (also the task wait budget). Default: 30 + +.EXAMPLE + .\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 + +.EXAMPLE + .\Test-CIGuestWinRM.ps1 -IpAddress 192.168.79.237 -Target BuildVMGuest -Port 5986 +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $IpAddress, + + [ValidateNotNullOrEmpty()] + [string] $Target = 'BuildVMGuest', + + [ValidateRange(1, 65535)] + [int] $Port = 5986, + + # Optional username override (password still read from the SYSTEM + # keyring). Use to test forms like 'HOSTNAME\ci_build' without + # re-storing the credential. Empty = use the keyring username. + [string] $UserName = '', + + # Auth packages to try, in order. Local/workgroup accounts usually + # need 'ntlm' (Negotiate->Kerberos is meaningless without a domain). + [ValidateNotNullOrEmpty()] + [string[]] $AuthOrder = @('ntlm', 'negotiate'), + + [string] $VenvPython = $(if ($env:CI_VENV_PYTHON) { $env:CI_VENV_PYTHON } + else { 'F:\CI\python\venv\Scripts\python.exe' }), + + [ValidateRange(5, 300)] + [int] $TimeoutSec = 30 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $VenvPython)) { + throw "venv interpreter not found: '$VenvPython'. Set -VenvPython or CI_VENV_PYTHON." +} + +# ── 1. TCP reachability ─────────────────────────────────────────────────── +Write-Host "[1/2] TCP $IpAddress`:$Port ..." -ForegroundColor Cyan +$tcp = Test-NetConnection -ComputerName $IpAddress -Port $Port ` + -WarningAction SilentlyContinue +if (-not $tcp.TcpTestSucceeded) { + Write-Host " RESULT: CONNECT — port $Port closed/unreachable." ` + -ForegroundColor Red + Write-Host " WinRM HTTPS listener or guest firewall. Check the" ` + -ForegroundColor Yellow + Write-Host " Windows template WinRM 5986 setup (WINDOWS-TEMPLATE-SETUP.md)." ` + -ForegroundColor Yellow + return +} +Write-Host " port open." -ForegroundColor Green + +# ── 2a. Discover guest hostname from the WinRM TLS certificate CN ────────── +# WinRM's self-signed cert CN is normally the guest computer name. Used to +# build the COMPUTERNAME\user form NTLM needs for a local/workgroup account. +$guestCn = '' +try { + $tcpc = New-Object Net.Sockets.TcpClient + $tcpc.Connect($IpAddress, $Port) + $ssl = New-Object Net.Security.SslStream( + $tcpc.GetStream(), $false, + ([Net.Security.RemoteCertificateValidationCallback] { $true })) + $ssl.AuthenticateAsClient($IpAddress) + $cn = $ssl.RemoteCertificate.Subject # e.g. "CN=WINBUILD" + if ($cn -match 'CN=([^,]+)') { $guestCn = $Matches[1].Trim() } + $ssl.Dispose(); $tcpc.Close() +} catch { + Write-Verbose "cert CN probe failed (hostname auto-detect skipped): $_" +} +if ($guestCn) { + Write-Host " guest cert CN (hostname): $guestCn" -ForegroundColor Cyan +} + +# ── 2b. Build the username candidate matrix ─────────────────────────────── +$authList = ($AuthOrder | ForEach-Object { "'" + $_ + "'" }) -join ', ' +Write-Host "[2/2] pypsrp execute_ps as SYSTEM — auth order: $($AuthOrder -join ', ')" ` + -ForegroundColor Cyan +if ($UserName) { + Write-Host " username override: $UserName" -ForegroundColor Cyan +} + +$probePy = Join-Path $env:TEMP ("winrmprobe_{0}.py" -f ([guid]::NewGuid().ToString('N'))) +$probeOut = Join-Path $env:TEMP ("winrmprobe_{0}.txt" -f ([guid]::NewGuid().ToString('N'))) + +$body = @" +import keyring, traceback +out = r'''$probeOut''' +auths = [$authList] +override = r'''$UserName''' +cn = r'''$guestCn''' +c = keyring.get_credential(r'''$Target''', None) +if c is None: + open(out, 'w', encoding='utf-8').write('KEYRING') +else: + base = override if override else c.username + leaf = base.split('\\')[-1] # strip any DOMAIN\ prefix + users = [base] + if cn: + users.append(cn + '\\' + leaf) # COMPUTERNAME\user + users.append('.\\' + leaf) # .\user + if cn: + users.append(leaf + '@' + cn) # user@COMPUTERNAME + # de-dup, keep order + seen, cand = set(), [] + for u in users: + if u not in seen: + seen.add(u); cand.append(u) + from pypsrp.client import Client + lines, success = [], None + lines.append('cred user=%r pwlen=%d' % (c.username, len(c.password or ''))) + for u in cand: + for a in auths: + try: + cl = Client(r'''$IpAddress''', username=u, password=c.password, + port=$Port, ssl=True, cert_validation=False, + auth=a, connection_timeout=$TimeoutSec) + so, se, rc = cl.execute_ps('`$true') + lines.append('user=%r auth=%s OK rc=%s' % (u, a, rc)) + success = (u, a) + break + except Exception as e: + msg = (str(e).splitlines() or [''])[0] + lines.append('user=%r auth=%s FAIL %s: %s' + % (u, a, type(e).__name__, msg)) + if success: + break + if success: + hdr = 'OK user=%r auth=%s' % success + tb = '' + else: + hdr = 'EXC (all combos failed)' + tb = '\n' + traceback.format_exc() + open(out, 'w', encoding='utf-8').write( + hdr + '\n' + '\n'.join(lines) + tb) +"@ +Set-Content -LiteralPath $probePy -Value $body -Encoding UTF8 + +try { + $action = New-ScheduledTaskAction -Execute $VenvPython -Argument "`"$probePy`"" + $principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' ` + -LogonType ServiceAccount -RunLevel Highest + Register-ScheduledTask -TaskName 'CI-WinRmProbe' -Action $action ` + -Principal $principal -Force | Out-Null + try { + Start-ScheduledTask -TaskName 'CI-WinRmProbe' + $deadline = (Get-Date).AddSeconds($TimeoutSec + 10) + do { + Start-Sleep -Milliseconds 500 + $state = (Get-ScheduledTask -TaskName 'CI-WinRmProbe').State + } while ($state -eq 'Running' -and (Get-Date) -lt $deadline) + } + finally { + Unregister-ScheduledTask -TaskName 'CI-WinRmProbe' -Confirm:$false ` + -ErrorAction SilentlyContinue + } + + $raw = if (Test-Path -LiteralPath $probeOut) { + (Get-Content -LiteralPath $probeOut -Raw) + } else { '' } + + if (-not $raw) { + Write-Host " RESULT: probe produced no output (task may have failed)." ` + -ForegroundColor Red + return + } + + if ($raw -eq 'KEYRING') { + Write-Host " RESULT: KEYRING — '$Target' not in the SYSTEM vault." ` + -ForegroundColor Red + Write-Host " Run: .\Set-CIGuestCredential.ps1 -UserName ''" ` + -ForegroundColor Yellow + return + } + if ($raw -like 'OK *') { + $okUser = if ($raw -match "OK user='([^']+)'") { $Matches[1] } else { '?' } + $okAuth = if ($raw -match 'auth=(\w+)') { $Matches[1] } else { '?' } + Write-Host " RESULT: OK — WinRM works." -ForegroundColor Green + Write-Host " Winning combo: user='$okUser' auth='$okAuth'" ` + -ForegroundColor Green + Write-Host $raw + Write-Host "" + $okLeaf = $okUser.Split('\')[-1].Split('@')[0] + $needsQualified = ($okUser -ne $okLeaf) + if ($needsQualified) { + Write-Host " -> FIX (credential): the stored bare username is rejected;" ` + -ForegroundColor Yellow + Write-Host " re-store in the working form:" -ForegroundColor Yellow + Write-Host " .\scripts\Set-CIGuestCredential.ps1 -UserName '$okUser'" ` + -ForegroundColor Yellow + } + if ($okAuth -ne 'negotiate') { + Write-Host " -> FIX (code): WinRmTransport (src/ci_orchestrator/transport/winrm.py)" ` + -ForegroundColor Yellow + Write-Host " builds pypsrp Client() without auth= (default 'negotiate')." ` + -ForegroundColor Yellow + Write-Host " Pass auth='$okAuth' for local-account guests." ` + -ForegroundColor Yellow + } + if (-not $needsQualified -and $okAuth -eq 'negotiate') { + Write-Host " Transport auth/connect is fine; the hang is elsewhere." ` + -ForegroundColor Yellow + } + return + } + + # EXC ... : classify the traceback. + Write-Host " RESULT: probe raised an exception:" -ForegroundColor Red + Write-Host $raw + $allFailed = $raw -like 'EXC (all combos failed)*' + $cls = switch -Regex ($raw) { + 'CertificateError|SSLError|SSL:|certificate' { 'SSL'; break } + '2146893043|UNKNOWN_CREDENTIALS|401|403|AuthenticationError|the user name or password|AccessDenied|access is denied' { 'AUTH'; break } + 'ConnectionError|timed out|TimeoutError|refused|No route|Failed to establish|getaddrinfo|Max retries' { 'CONNECT'; break } + default { 'UNKNOWN' } + } + Write-Host "" + switch ($cls) { + 'AUTH' { + if ($allFailed) { + Write-Host " -> AUTH: EVERY user-form x auth combo was rejected with" ` + -ForegroundColor Yellow + Write-Host " SEC_E_UNKNOWN_CREDENTIALS. Username form is NOT the cause." ` + -ForegroundColor Yellow + Write-Host " Most likely: the password stored in the SYSTEM keyring is" ` + -ForegroundColor Yellow + Write-Host " wrong / does not match the guest 'ci_build' account." ` + -ForegroundColor Yellow + Write-Host " 1. Confirm the real ci_build password in the template." ` + -ForegroundColor Yellow + Write-Host " 2. Re-run: .\scripts\Set-CIGuestCredential.ps1 -UserName 'ci_build'" ` + -ForegroundColor Yellow + Write-Host " If the password is definitely correct, suspect the" ` + -ForegroundColor Yellow + Write-Host " pyspnego SSPI provider under the SYSTEM account." ` + -ForegroundColor Yellow + } + else { + Write-Host " -> AUTH: credential rejected for the tested form(s)." ` + -ForegroundColor Yellow + Write-Host " Try a hostname-qualified form via Set-CIGuestCredential.ps1." ` + -ForegroundColor Yellow + } + } + 'CONNECT' { + Write-Host " -> CONNECT: WinRM service/listener not answering on the guest." ` + -ForegroundColor Yellow + } + 'SSL' { + Write-Host " -> SSL: TLS/cert problem (probe uses cert_validation=False; investigate listener cert)." ` + -ForegroundColor Yellow + } + default { + Write-Host " -> UNKNOWN: inspect the traceback above." ` + -ForegroundColor Yellow + } + } +} +finally { + Remove-Item -LiteralPath $probePy -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $probeOut -Force -ErrorAction SilentlyContinue +} diff --git a/scripts/Test-E2E-Section3.3.ps1 b/scripts/Test-E2E-Section3.3.ps1 index 85849ac..a41266c 100644 --- a/scripts/Test-E2E-Section3.3.ps1 +++ b/scripts/Test-E2E-Section3.3.ps1 @@ -89,7 +89,7 @@ function Format-Elapsed { } } -function Run-Test { +function Invoke-Test { param( [string] $JobId, [string] $Mode, @@ -164,11 +164,11 @@ $baselineResult = $null $guestCloneResult = $null if (-not $SkipBaseline) { - $baselineResult = Run-Test -JobId $BaselineJobId -Mode "BASELINE (host-side zip)" -CommonParams $commonParams + $baselineResult = Invoke-Test -JobId $BaselineJobId -Mode "BASELINE (host-side zip)" -CommonParams $commonParams } if (-not $SkipGuestClone) { - $guestCloneResult = Run-Test -JobId $GuestCloneJobId -Mode "GUEST CLONE (-UseGitClone)" -UseGitClone -CommonParams $commonParams + $guestCloneResult = Invoke-Test -JobId $GuestCloneJobId -Mode "GUEST CLONE (-UseGitClone)" -UseGitClone -CommonParams $commonParams } # ──────────────────────────────────────────────────────────────────────────── diff --git a/scripts/Wait-VMReady.ps1 b/scripts/Wait-VMReady.ps1 index 69b3633..605eff4 100644 --- a/scripts/Wait-VMReady.ps1 +++ b/scripts/Wait-VMReady.ps1 @@ -1,250 +1,37 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Waits until a build VM is fully ready to accept WinRM connections. - -.DESCRIPTION - Performs a three-phase readiness check in a polling loop: - 1. vmrun getState returns "running" - 2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH) - 3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux) - - Throws if the VM does not become ready within the timeout period. - - Transport modes: - WinRM (default) — existing Windows VM path: ping + WinRM port 5986. - SSH — Linux VM path: port 22 check + ssh echo test. - -.PARAMETER VMPath - Full path to the clone VM's .vmx file. - -.PARAMETER IPAddress - IP address of the VM on the build network (e.g. 192.168.11.101). - -.PARAMETER TimeoutSeconds - Maximum seconds to wait for the VM to become ready. - Default: 300 (5 minutes) - -.PARAMETER PollIntervalSeconds - Seconds between readiness checks. - Default: 5 - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER Transport - Transport protocol for readiness checks. - WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs). - SSH: checks TCP 22 + ssh echo (Linux VMs). - -.PARAMETER Credential - Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs - a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming - that WinRM authentication (not just port) is working. Useful to detect the - window where the WinRM listener is up but the LocalSystem session is not yet - accepting logins. When omitted the TCP check alone is used (existing behaviour). - -.PARAMETER SshKeyPath - Path to the SSH private key for key-based auth when Transport=SSH. - Default: F:\CI\keys\ci_linux - -.PARAMETER SshUser - SSH username when Transport=SSH. - Default: ci_build - -.EXAMPLE - .\Wait-VMReady.ps1 ` - -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" ` - -IPAddress "192.168.11.101" -#> -[CmdletBinding()] -param( - [Parameter(Mandatory)] - [string] $VMPath, - - [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] - [string] $IPAddress, - - [ValidateRange(3, 600)] - [int] $TimeoutSeconds = 300, - - [ValidateRange(1, 30)] - [int] $PollIntervalSeconds = 5, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - # Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP - # but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM) - # are still checked. - [switch] $SkipPing, - - # Transport protocol for phase 2/3 checks. - # WinRM (default) = Windows VM; SSH = Linux VM. - [ValidateSet('WinRM', 'SSH')] - [string] $Transport = 'WinRM', - - # SSH private key path (used when Transport=SSH). - [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when Transport=SSH). - [string] $SshUser = 'ci_build', - - # Optional credential for WinRM authentication probe (Transport=WinRM). - # When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds. - [PSCredential] $Credential = $null -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.) - -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - throw "vmrun.exe not found at: $VmrunPath" -} - -$deadline = (Get-Date).AddSeconds($TimeoutSeconds) -$attempt = 0 -$phase = 'vmrun-state' # tracks current phase for informative output -$lastPhase = '' - -Write-Host "[Wait-VMReady] Waiting for VM ready (timeout: ${TimeoutSeconds}s, IP: $IPAddress)..." - -while ((Get-Date) -lt $deadline) { - $attempt++ - - # ── Phase 1: VM must be running ───────────────────────────────────── - # vmrun has no getState. We previously used `getGuestIPAddress`, but that - # requires VMware Tools to be up and responding inside the guest. In - # headless boots Tools can take 30-60s to come online, during which the - # call exits non-zero even though the VM is fully powered on. That caused - # Phase 1 to appear "stuck" until a GUI was opened manually. - # `vmrun list` reports actual power state without needing Tools. - $vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue) - if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath } - $listOut = & $VmrunPath list 2>&1 - $isRunning = $false - foreach ($line in $listOut) { - if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) { - $isRunning = $true - break - } - } - - if (-not $isRunning) { - if ($phase -ne 'vmrun-state') { - $phase = 'vmrun-state' - } - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 1/3: waiting for VM to enter running state..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - - # ── Phase 2: Network check ────────────────────────────────────────── - if ($Transport -eq 'SSH') { - $port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue - if (-not $port22Open) { - $phase = 'ssh-port' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - elseif (-not $SkipPing) { - $pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue - - if (-not $pingOk) { - $phase = 'ping' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for network (ping $IPAddress)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - - # ── Phase 3: Transport-specific login check ──────────────────────── - if ($Transport -eq 'SSH') { - $sshArgs = @( - '-i', $SshKeyPath, - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=NUL', - '-o', 'ConnectTimeout=5', - '-o', 'BatchMode=yes', - "$SshUser@$IPAddress", - 'echo ready' - ) - # ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts") - # to stderr. Under $ErrorActionPreference='Stop' (set at script top), - # those stderr lines captured via 2>&1 are promoted to terminating errors - # and abort the script. Wrap the call to demote stderr to non-terminating. - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $sshOut = & ssh @sshArgs 2>&1 - $sshExit = $LASTEXITCODE - $ErrorActionPreference = $savedEap - if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') { - Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." - return - } - else { - $phase = 'ssh-echo' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - else { - # ── WinRM port 5986 accepting TCP connections ───────────────────── - # Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for - # self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready. - $winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue - if ($winrmUp) { - # Optional auth probe — ensures WinRM login is functional, not just TCP open - if ($null -ne $Credential) { - try { - $so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck - $null = Invoke-Command -ComputerName $IPAddress -Credential $Credential ` - -Authentication Basic -UseSSL -Port 5986 -SessionOption $so ` - -ScriptBlock { 'ready' } -ErrorAction Stop - Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress." - } catch { - $phase = 'winrm-auth' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." - return - } - else { - $phase = 'winrm' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } -} - -throw "[Wait-VMReady] Timed out after ${TimeoutSeconds}s waiting for VM at $IPAddress to become ready. Last phase: $phase" +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator wait-ready @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Watch-DiskSpace.ps1 b/scripts/Watch-DiskSpace.ps1 index 977cd89..aa4c7f9 100644 --- a/scripts/Watch-DiskSpace.ps1 +++ b/scripts/Watch-DiskSpace.ps1 @@ -1,96 +1,37 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook). - -.DESCRIPTION - Intended to run as a scheduled task every 15 minutes (registered by - Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB, - writes a Warning entry to the Windows Application Event Log under source - 'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler - marks the task as failed (visible in Task Scheduler history). - - If WebhookUrl is provided, also POSTs a JSON payload to that URL - (compatible with Discord and Gitea webhooks — content field only). - - A full drive silently causes vmrun clone to fail with cryptic errors - (not enough disk space for linked clone delta files). Alert early. - -.PARAMETER MinFreeGB - Alert threshold in GB. Default: 50 - -.PARAMETER DriveLetter - Single letter (no colon) of the drive to monitor. Default: F - -.PARAMETER WebhookUrl - Optional Discord/Gitea webhook URL. If empty, only Event Log is written. - -.EXAMPLE - # Manual check - .\Watch-DiskSpace.ps1 -MinFreeGB 50 - - # With Discord webhook - .\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." -#> -[CmdletBinding()] -param( - [ValidateRange(1, 2000)] - [int] $MinFreeGB = 50, - - [ValidatePattern('^[A-Za-z]$')] - [string] $DriveLetter = 'F', - - [string] $WebhookUrl = '' -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' - -$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue -if (-not $drive) { - Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found." - exit 2 -} - -$freeGB = [math]::Round($drive.Free / 1GB, 1) -$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1) -$pctFree = [math]::Round($freeGB / $totalGB * 100, 0) - -if ($freeGB -ge $MinFreeGB) { - Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)" - exit 0 -} - -# ── Alert ───────────────────────────────────────────────────────────────────── -$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " + - "the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " + - "Run Invoke-RetentionPolicy.ps1 or free disk space manually." - -Write-Warning "[DiskSpace] $msg" - -# Windows Event Log -$logSource = 'CI-DiskSpaceAlert' -try { - if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { - New-EventLog -LogName Application -Source $logSource -ErrorAction Stop - } - Write-EventLog -LogName Application -Source $logSource ` - -EventId 1001 -EntryType Warning -Message $msg - Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)." -} catch { - Write-Warning "[DiskSpace] Could not write Event Log: $_" -} - -# Optional webhook (Discord / Gitea) -if ($WebhookUrl) { - $body = @{ content = "[WARNING] CI Disk Alert -- $msg" } | ConvertTo-Json -Compress - try { - $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` - -Body $body -ContentType 'application/json' -TimeoutSec 10 - Write-Host "[DiskSpace] Webhook notified: $WebhookUrl" - } catch { - Write-Warning "[DiskSpace] Webhook POST failed: $_" - } -} - -exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer) +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator monitor disk @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Watch-RunnerHealth.ps1 b/scripts/Watch-RunnerHealth.ps1 index 119754a..b2515e9 100644 --- a/scripts/Watch-RunnerHealth.ps1 +++ b/scripts/Watch-RunnerHealth.ps1 @@ -1,214 +1,37 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Monitors the act_runner service and auto-restarts it if stopped. - -.DESCRIPTION - Intended to run as a scheduled task every 15 minutes (registered by - Register-CIScheduledTasks.ps1 as CI-RunnerHealth). - - Behaviour: - - If act_runner is Running: exits 0, no action. - - If not Running: writes a Warning to the Windows Application Event Log - (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service. - - Restart attempts are rate-limited to MaxRestarts per hour using a JSON - state file in StateDir. Once the limit is hit, further runs write an - alert (EventId 1004) and exit 1 without restarting — Task Scheduler - history shows the failure so the operator is alerted. - - On successful restart: writes EventId 1003 (Information). - - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and - Gitea webhooks on every restart or limit-exceeded event. - -.PARAMETER MaxRestarts - Maximum number of auto-restart attempts allowed per rolling 60-minute - window before giving up and requiring manual intervention. Default: 3 - -.PARAMETER ServiceName - Windows service name to monitor. Default: act_runner - -.PARAMETER StateDir - Directory used for the cooldown state file (runner-restart-log.json). - Default: F:\CI\State - -.PARAMETER WebhookUrl - Optional Discord/Gitea webhook URL. If empty, only Event Log is written. - -.EXAMPLE - # Manual health check - .\Watch-RunnerHealth.ps1 - - # With Discord webhook - .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." -#> -[CmdletBinding()] -param( - [ValidateRange(1, 10)] - [int] $MaxRestarts = 3, - - [string] $ServiceName = 'act_runner', - - [string] $StateDir = 'F:\CI\State', - - [string] $WebhookUrl = '', - - # Optional Gitea API runner-online verification. - # When $GiteaUrl is non-empty and the act_runner service is Running, this script - # queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online. - # Requires an admin Personal Access Token stored in Windows Credential Manager - # under $GiteaCredentialTarget (same module: CredentialManager). - # Silently skipped if GiteaUrl is empty or the credential is unavailable. - [string] $GiteaUrl = '', - - [string] $GiteaCredentialTarget = '' -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' - -$logSource = 'CI-RunnerHealth' - -# ── Helper: write to Event Log ──────────────────────────────────────────────── -function Write-CIEvent { - param([int] $EventId, [string] $EntryType, [string] $Message) - try { - if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { - New-EventLog -LogName Application -Source $logSource -ErrorAction Stop - } - Write-EventLog -LogName Application -Source $logSource ` - -EventId $EventId -EntryType $EntryType -Message $Message - } catch { - Write-Warning "[RunnerHealth] Could not write Event Log: $_" - } -} - -# ── Helper: POST webhook ────────────────────────────────────────────────────── -function Send-Webhook { - param([string] $Content) - if (-not $WebhookUrl) { return } - $body = @{ content = $Content } | ConvertTo-Json -Compress - try { - $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` - -Body $body -ContentType 'application/json' -TimeoutSec 10 - Write-Host "[RunnerHealth] Webhook notified." - } catch { - Write-Warning "[RunnerHealth] Webhook POST failed: $_" - } -} - -# ── Maintenance flag check ──────────────────────────────────────────────────── -# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic. -# Create the flag before planned maintenance; delete it when done. -$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag' -if (Test-Path $maintenanceFlag) { - Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic." - exit 0 -} - -# ── Step 1: Check service ───────────────────────────────────────────────────── -$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue -if (-not $svc) { - Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?" - exit 2 -} - -if ($svc.Status -eq 'Running') { - Write-Host "[RunnerHealth] $ServiceName is Running — OK" - - # ── Optional Gitea runner-online API check ──────────────────────────────── - if ($GiteaUrl -ne '') { - $pat = $null - if ($GiteaCredentialTarget -ne '') { - try { - $cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - $pat = $cred.GetNetworkCredential().Password - } catch { - Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_" - } - } - if ($pat) { - try { - $apiUrl = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners" - $runners = Invoke-RestMethod -Uri $apiUrl ` - -Headers @{ Authorization = "token $pat" } ` - -Method Get -TimeoutSec 10 -ErrorAction Stop - $online = @($runners | Where-Object { $_.online -eq $true }) - if ($online.Count -gt 0) { - Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK" - } else { - $msg = "Gitea API ($apiUrl) reports 0 online runners. " + - "$ServiceName service is Running — possible registration issue." - Write-Warning "[RunnerHealth] $msg" - Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg - Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg" - } - } catch { - Write-Warning "[RunnerHealth] Gitea API check failed: $_" - } - } - } - - exit 0 -} - -# ── Step 2: Service not running — load cooldown state ───────────────────────── -if (-not (Test-Path $StateDir)) { - New-Item -ItemType Directory -Path $StateDir -Force | Out-Null -} - -$cooldownFile = Join-Path $StateDir 'runner-restart-log.json' -$restartLog = @() - -if (Test-Path $cooldownFile) { - try { - $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue - if ($raw) { $restartLog = @($raw | ConvertFrom-Json) } - } catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" } -} - -# Prune timestamps older than 1 hour -$cutoff = (Get-Date).AddHours(-1) -$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff }) -$recentCount = $restartLog.Count - -$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts." -Write-Warning "[RunnerHealth] $stateMsg" -Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg - -# ── Step 3: Rate-limit check ────────────────────────────────────────────────── -if ($recentCount -ge $MaxRestarts) { - $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " + - "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\." - Write-Warning "[RunnerHealth] $limitMsg" - Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg - Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg" - exit 1 -} - -# ── Step 4: Attempt restart ─────────────────────────────────────────────────── -Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..." -try { - Restart-Service -Name $ServiceName -Force -ErrorAction Stop - Start-Sleep -Seconds 5 - $newStatus = (Get-Service -Name $ServiceName).Status - $restartMsg = "$ServiceName restarted — new status: $newStatus." - Write-Host "[RunnerHealth] $restartMsg" - - # Persist this restart in the cooldown log - $restartLog += (Get-Date -Format 'o') - $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue - - $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' } - Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg - - $prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' } - Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg" - - exit $(if ($newStatus -eq 'Running') { 0 } else { 1 }) -} -catch { - $errMsg = "$ServiceName restart failed: $_" - Write-Warning "[RunnerHealth] $errMsg" - Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg - Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg" - exit 1 -} +#Requires -Version 5.1 +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +# PowerShell common params (-ErrorAction etc.) must not be forwarded +# to the Python CLI. Value-taking ones also consume the next token. +$commonWithValue = @{ + erroraction = $true; warningaction = $true; informationaction = $true + progressaction = $true; errorvariable = $true; warningvariable = $true + informationvariable = $true; outvariable = $true; outbuffer = $true + pipelinevariable = $true +} +$commonSwitch = @{ verbose = $true; debug = $true; whatif = $true; confirm = $true } + +$pyArgs = @() +for ($i = 0; $i -lt $args.Count; $i++) { + $a = $args[$i] + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $lname = $name.ToLower() + if ($commonSwitch.ContainsKey($lname)) { continue } + if ($commonWithValue.ContainsKey($lname)) { $i++; continue } + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator monitor runner @pyArgs +exit $LASTEXITCODE diff --git a/src/ci_orchestrator/__init__.py b/src/ci_orchestrator/__init__.py new file mode 100644 index 0000000..84646d9 --- /dev/null +++ b/src/ci_orchestrator/__init__.py @@ -0,0 +1,9 @@ +"""Local CI/CD orchestrator package. + +Phase A: Python rewrite of the PowerShell-based orchestrator. +See plans/implementation-plan-A-B.md for the full scope. +""" + +__version__ = "0.1.0" + +__all__ = ["__version__"] diff --git a/src/ci_orchestrator/__main__.py b/src/ci_orchestrator/__main__.py new file mode 100644 index 0000000..d9ce664 --- /dev/null +++ b/src/ci_orchestrator/__main__.py @@ -0,0 +1,48 @@ +"""CLI entry point. + +Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor`` +(disk/runner) and ``report`` (job). Phase A3 adds ``vm new``, ``build``, +``artifacts``. Subsequent phases add ``job``. +""" + +from __future__ import annotations + +import time as time + +import click + +from ci_orchestrator import __version__ +from ci_orchestrator.commands.artifacts import artifacts +from ci_orchestrator.commands.build import build +from ci_orchestrator.commands.job import job +from ci_orchestrator.commands.monitor import monitor +from ci_orchestrator.commands.report import report +from ci_orchestrator.commands.vm import vm + +# Re-export so legacy A1 tests that patched ``cli_module.`` keep working. +from ci_orchestrator.commands.wait import ( # noqa: F401 (intentional re-exports) + KeyringCredentialStore, + SshTransport, + WinRmTransport, + load_backend, + wait_ready, +) + + +@click.group() +@click.version_option(__version__, prog_name="ci-orchestrator") +def cli() -> None: + """Local CI/CD orchestrator.""" + + +cli.add_command(wait_ready) +cli.add_command(vm) +cli.add_command(build) +cli.add_command(artifacts) +cli.add_command(monitor) +cli.add_command(report) +cli.add_command(job) + + +if __name__ == "__main__": # pragma: no cover + cli() diff --git a/src/ci_orchestrator/backends/__init__.py b/src/ci_orchestrator/backends/__init__.py new file mode 100644 index 0000000..761a213 --- /dev/null +++ b/src/ci_orchestrator/backends/__init__.py @@ -0,0 +1,30 @@ +"""VM backend implementations. + +The :class:`~ci_orchestrator.backends.protocol.VmBackend` Protocol defines +the neutral interface every backend must satisfy. Phase A ships only the +:class:`~ci_orchestrator.backends.workstation.WorkstationVmrunBackend`; +Phase C will add an ESXi backend behind the same interface. +""" + +from __future__ import annotations + +from ci_orchestrator.backends.protocol import VmBackend, VmHandle, VmState +from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + +__all__ = ["VmBackend", "VmHandle", "VmState", "WorkstationVmrunBackend", "load_backend"] + + +def load_backend(config: object) -> VmBackend: + """Factory: pick a backend implementation from ``config.backend.type``. + + Phase C hook: when the ESXi backend lands, this function will dispatch + on ``config.backend.type``. For now only ``workstation`` is supported. + """ + backend_type = getattr(getattr(config, "backend", None), "type", "workstation") + if backend_type != "workstation": + raise NotImplementedError( + f"Backend type '{backend_type}' is not implemented. " + "Only 'workstation' is available in Phase A." + ) + vmrun_path = getattr(config, "vmrun_path", None) + return WorkstationVmrunBackend(vmrun_path=vmrun_path) diff --git a/src/ci_orchestrator/backends/errors.py b/src/ci_orchestrator/backends/errors.py new file mode 100644 index 0000000..7b6a161 --- /dev/null +++ b/src/ci_orchestrator/backends/errors.py @@ -0,0 +1,24 @@ +"""Backend error hierarchy.""" + +from __future__ import annotations + + +class BackendError(RuntimeError): + """Base error for VM backend failures.""" + + +class BackendNotAvailable(BackendError): + """Backend tooling (e.g. ``vmrun``) is missing or unusable.""" + + +class BackendOperationFailed(BackendError): + """A backend operation (clone, start, ...) returned a non-zero status.""" + + def __init__(self, operation: str, returncode: int, output: str) -> None: + super().__init__( + f"Backend operation '{operation}' failed " + f"(exit={returncode}): {output.strip() or ''}" + ) + self.operation = operation + self.returncode = returncode + self.output = output diff --git a/src/ci_orchestrator/backends/protocol.py b/src/ci_orchestrator/backends/protocol.py new file mode 100644 index 0000000..19af3b9 --- /dev/null +++ b/src/ci_orchestrator/backends/protocol.py @@ -0,0 +1,81 @@ +"""Neutral VM backend Protocol. + +Phase C hook: keep names hypervisor-agnostic. ``clone_linked``, ``start``, +``stop``, ``delete``, ``get_ip``, ``list_snapshots`` map cleanly onto both +``vmrun`` and pyVmomi (vSphere). Identifiers are opaque strings (a VMX path +on Workstation, a managed object reference on ESXi). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import Protocol + + +class VmState(StrEnum): + """Coarse VM power state, normalised across backends.""" + + POWERED_OFF = "powered_off" + POWERED_ON = "powered_on" + SUSPENDED = "suspended" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class VmHandle: + """Opaque reference to a VM managed by a backend. + + ``identifier`` is backend-specific (a VMX path for Workstation, a + managed object reference for vSphere). Callers MUST treat it as opaque. + """ + + identifier: str + name: str | None = None + + +class VmBackend(Protocol): + """Hypervisor-agnostic VM operations. + + All methods raise :class:`~ci_orchestrator.backends.errors.BackendError` + on failure. Implementations are free to add backend-specific kwargs but + MUST honour the documented signature for the listed parameters. + """ + + def clone_linked( + self, + template: str, + snapshot: str, + name: str, + destination: str | None = None, + ) -> VmHandle: + """Create a linked clone of ``template`` at snapshot ``snapshot``.""" + ... + + def start(self, handle: VmHandle, headless: bool = True) -> None: + """Power on the VM.""" + ... + + def stop(self, handle: VmHandle, hard: bool = False) -> None: + """Power off the VM. ``hard=True`` requests a forced power-off.""" + ... + + def delete(self, handle: VmHandle) -> None: + """Remove the VM from inventory and delete its files.""" + ... + + def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None: + """Return the guest's primary IP, or ``None`` if not yet known.""" + ... + + def list_snapshots(self, handle: VmHandle) -> list[str]: + """Return snapshot names defined on ``handle``.""" + ... + + def is_running(self, handle: VmHandle) -> bool: + """Return ``True`` if the VM appears in the running-VM list. + + Implementations MUST NOT rely on guest tools (e.g. ``vmrun + getGuestIPAddress``) for this check — see ``AGENTS.md`` error #10. + """ + ... diff --git a/src/ci_orchestrator/backends/workstation.py b/src/ci_orchestrator/backends/workstation.py new file mode 100644 index 0000000..726a8ef --- /dev/null +++ b/src/ci_orchestrator/backends/workstation.py @@ -0,0 +1,162 @@ +"""VMware Workstation backend (``vmrun -T ws`` wrapper). + +Mirrors the behaviour of ``scripts/_Common.psm1`` :func:`Invoke-Vmrun` plus +the higher-level helpers used by ``New-BuildVM.ps1`` and +``Wait-VMReady.ps1``. Only ``subprocess`` is used — no PowerShell. + +Cross-platform: ``vmrun`` lives at different paths on Windows vs Linux; +when ``vmrun_path`` is not provided, :func:`shutil.which` is consulted and +finally a Windows default is tried. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +from ci_orchestrator.backends.errors import BackendNotAvailable, BackendOperationFailed +from ci_orchestrator.backends.protocol import VmHandle + +_DEFAULT_WIN_VMRUN = r"C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe" + + +def _resolve_vmrun(explicit: str | None) -> str: + """Pick a usable ``vmrun`` binary path.""" + if explicit: + if not Path(explicit).is_file(): + raise BackendNotAvailable(f"vmrun not found at: {explicit}") + return explicit + found = shutil.which("vmrun") + if found: + return found + if Path(_DEFAULT_WIN_VMRUN).is_file(): + return _DEFAULT_WIN_VMRUN + raise BackendNotAvailable( + "vmrun is not available. Install VMware Workstation Pro or set vmrun_path." + ) + + +class WorkstationVmrunBackend: + """``vmrun -T ws`` backend for VMware Workstation Pro.""" + + def __init__(self, vmrun_path: str | None = None) -> None: + self._vmrun_path = _resolve_vmrun(vmrun_path) + + # ------------------------------------------------------------------ utils + @property + def vmrun_path(self) -> str: + return self._vmrun_path + + def _run( + self, + operation: str, + *args: str, + check: bool = True, + timeout: float | None = None, + ) -> subprocess.CompletedProcess[str]: + """Run ``vmrun -T ws `` and capture output.""" + cmd = [self._vmrun_path, "-T", "ws", operation, *args] + result = subprocess.run( + cmd, + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + ) + if check and result.returncode != 0: + raise BackendOperationFailed( + operation=operation, + returncode=result.returncode, + output=(result.stdout or "") + (result.stderr or ""), + ) + return result + + # ------------------------------------------------------------- VmBackend + def clone_linked( + self, + template: str, + snapshot: str, + name: str, + destination: str | None = None, + ) -> VmHandle: + if destination is None: + destination = str(Path(template).parent.parent / name / f"{name}.vmx") + # vmrun clone linked -snapshot= -cloneName= + self._run( + "clone", + template, + destination, + "linked", + f"-snapshot={snapshot}", + f"-cloneName={name}", + ) + return VmHandle(identifier=destination, name=name) + + def start(self, handle: VmHandle, headless: bool = True) -> None: + mode = "nogui" if headless else "gui" + self._run("start", handle.identifier, mode) + + def stop(self, handle: VmHandle, hard: bool = False) -> None: + mode = "hard" if hard else "soft" + # stop may legitimately fail if the VM is already off; treat that as ok + result = self._run("stop", handle.identifier, mode, check=False) + if result.returncode != 0: + stderr = (result.stderr or "").lower() + if "not powered on" in stderr or "is not running" in stderr: + return + raise BackendOperationFailed( + "stop", + result.returncode, + (result.stdout or "") + (result.stderr or ""), + ) + + def delete(self, handle: VmHandle) -> None: + self._run("deleteVM", handle.identifier) + + def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None: + # ``getGuestIPAddress`` requires VMware Tools; tolerate failure. + args: list[str] = [handle.identifier] + if timeout > 0: + args.extend(["-wait"]) + try: + result = self._run("getGuestIPAddress", *args, check=False, timeout=timeout or None) + except subprocess.TimeoutExpired: + return None + if result.returncode != 0: + return None + ip = (result.stdout or "").strip() + if not ip or ip.lower().startswith("error:"): + return None + return ip + + def list_snapshots(self, handle: VmHandle) -> list[str]: + result = self._run("listSnapshots", handle.identifier) + lines = [line.strip() for line in (result.stdout or "").splitlines() if line.strip()] + # First line is "Total snapshots: N" + if lines and lines[0].lower().startswith("total snapshots"): + lines = lines[1:] + return lines + + def is_running(self, handle: VmHandle) -> bool: + """``vmrun list`` lookup — does NOT call ``getGuestIPAddress``. + + See ``AGENTS.md`` error #10: ``getGuestIPAddress`` blocks for 30-60s + without VMware Tools. ``vmrun list`` returns immediately. + """ + result = self._run("list", check=False) + if result.returncode != 0: + return False + target = Path(handle.identifier).resolve() + for line in (result.stdout or "").splitlines(): + line = line.strip() + if not line or line.lower().startswith("total running"): + continue + try: + if Path(line).resolve() == target: + return True + except OSError: + continue + return False diff --git a/src/ci_orchestrator/commands/__init__.py b/src/ci_orchestrator/commands/__init__.py new file mode 100644 index 0000000..574b144 --- /dev/null +++ b/src/ci_orchestrator/commands/__init__.py @@ -0,0 +1,10 @@ +"""Sub-command modules for the ``ci_orchestrator`` CLI. + +Each module exposes one or more :class:`click.Command` instances that the +top-level :mod:`ci_orchestrator.__main__` registers on the root group. +Phase A2 ports the leaf PowerShell scripts (Wait-VMReady, Remove-BuildVM, +Cleanup-OrphanedBuildVMs, Watch-DiskSpace, Watch-RunnerHealth, +Get-CIJobSummary). +""" + +from __future__ import annotations diff --git a/src/ci_orchestrator/commands/artifacts.py b/src/ci_orchestrator/commands/artifacts.py new file mode 100644 index 0000000..e972a1b --- /dev/null +++ b/src/ci_orchestrator/commands/artifacts.py @@ -0,0 +1,294 @@ +"""``artifacts`` sub-commands. + +Phase A3 ports the artifact-collection script: + +* ``artifacts collect`` — replaces ``scripts/Get-BuildArtifacts.ps1`` + +Uses the abstract :meth:`~ci_orchestrator.transport.winrm.WinRmTransport.fetch` +and the SSH transport (via a tar+fetch helper) so no platform-specific +``Copy-Item`` / ``scp.exe`` invocations leak into command code. +""" + +from __future__ import annotations + +import hashlib +import json +import shlex +import tarfile +import tempfile +import zipfile +from datetime import UTC, datetime +from pathlib import Path + +import click + +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportCommandError, TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +# ---------------------------------------------------------------- group + + +@click.group("artifacts") +def artifacts() -> None: + """Artifact transfer commands.""" + + +# ---------------------------------------------------------------- helpers + + +def _sh_quote(value: str) -> str: + return shlex.quote(value) + + +def _ps_quote(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def _sha256_hex(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(64 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def _write_manifest(host_dir: Path, job_id: str, commit: str) -> int: + """Write ``manifest.json`` summarising every collected file. Returns count.""" + files = [ + p + for p in host_dir.rglob("*") + if p.is_file() and p.name != "manifest.json" + ] + entries = [] + for f in files: + rel = f.relative_to(host_dir).as_posix() + entries.append( + { + "name": rel, + "size": f.stat().st_size, + "sha256": _sha256_hex(f), + } + ) + payload = { + "ts": datetime.now(tz=UTC).isoformat(), + "jobId": job_id, + "commit": commit, + "files": entries, + } + out = host_dir / "manifest.json" + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return len(entries) + + +# ---------------------------------------------------------------- Linux flow + + +def _linux_collect( + *, + ip_address: str, + ssh_user: str, + key_path: str | None, + known_hosts: str | None, + guest_path: str, + host_dir: Path, +) -> None: + """Tar the guest directory, fetch the single archive, extract locally.""" + click.echo(f"[artifacts collect] tarring guest:{guest_path}") + archive_remote = f"/tmp/ci-artifacts-{ip_address.replace('.', '-')}.tar.gz" + with SshTransport( + ip_address, + username=ssh_user, + key_path=key_path, + known_hosts=known_hosts, + ) as t: + # Build a tarball whose contents are *inside* guest_path. Strip the + # parent component so files land directly under host_dir. + t.run( + f"tar -czf {_sh_quote(archive_remote)} " + f"-C {_sh_quote(guest_path)} ." + ) + try: + with tempfile.TemporaryDirectory() as td: + local_tar = Path(td) / "artifacts.tar.gz" + t.fetch(archive_remote, str(local_tar)) + with tarfile.open(local_tar, "r:gz") as tf: + # ``data`` filter (Python 3.12+) protects against path + # traversal in malicious tarballs. + tf.extractall(host_dir, filter="data") + finally: + t.run(f"rm -f {_sh_quote(archive_remote)}", check=False) + + +# ---------------------------------------------------------------- Windows flow + + +def _windows_collect( + *, + ip_address: str, + credential_target: str, + guest_path: str, + host_dir: Path, + include_logs: bool, +) -> None: + cred = KeyringCredentialStore().get(credential_target) + remote_zip = "C:\\CI\\ci-artifacts-transfer.zip" + with WinRmTransport(ip_address, cred.username, cred.password) as t: + # guest_path is the collect directory (raw build outputs). + # Verify it exists and is non-empty. + probe = t.run( + f"if ((Test-Path {_ps_quote(guest_path)}) -and " + f"(Get-ChildItem -Force -- {_ps_quote(guest_path)})) " + f"{{ 'OK' }} else {{ 'MISSING' }}", + check=False, + ) + if "MISSING" in probe.stdout: + raise click.ClickException( + f"artifact not found or empty in guest: {guest_path}" + ) + + # Zip the directory contents as a single transport file, fetch, + # extract into host_dir. Mirrors the Linux tar flow so both OSes + # deposit raw files; the final single archive is produced by + # actions/upload-artifact (no zip-in-zip). + t.run( + f"if (Test-Path {_ps_quote(remote_zip)}) " + f"{{ Remove-Item {_ps_quote(remote_zip)} -Force }}; " + f"Compress-Archive -Path (Join-Path {_ps_quote(guest_path)} '*') " + f"-DestinationPath {_ps_quote(remote_zip)} -Force" + ) + click.echo(f"[artifacts collect] fetching artifacts from {guest_path}") + try: + with tempfile.TemporaryDirectory() as td: + local_zip = Path(td) / "artifacts.zip" + t.fetch(remote_zip, str(local_zip)) + with zipfile.ZipFile(local_zip) as zf: + zf.extractall(host_dir) + finally: + t.run( + f"Remove-Item {_ps_quote(remote_zip)} -Force " + f"-ErrorAction SilentlyContinue", + check=False, + ) + + if include_logs: + # List log files in the build dir and fetch each one. + list_ps = ( + "Get-ChildItem -Path 'C:\\CI\\build' -Filter '*.log' -Recurse " + "-ErrorAction SilentlyContinue | " + "ForEach-Object { $_.FullName }" + ) + listing = t.run(list_ps, check=False) + for line in (listing.stdout or "").splitlines(): + line = line.strip() + if not line: + continue + dest = host_dir / Path(line).name + click.echo(f"[artifacts collect] fetching log: {Path(line).name}") + try: + t.fetch(line, str(dest)) + except TransportError as exc: + click.echo( + f"[artifacts collect] log fetch failed ({line}): {exc}", + err=True, + ) + + if not any(host_dir.iterdir()): + raise click.ClickException( + f"no artifacts after collect from guest: {guest_path}" + ) + + +# ---------------------------------------------------------------- CLI + + +@artifacts.command("collect") +@click.option("--ip-address", "ip_address", required=True) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, +) +@click.option( + "--guest-artifact-path", + "guest_artifact_path", + required=True, + help="Guest collect directory (Windows and Linux).", +) +@click.option( + "--host-artifact-dir", + "host_artifact_dir", + required=True, + help="Local directory to deposit artifacts (created if missing).", +) +@click.option("--include-logs", "include_logs", is_flag=True, default=False) +@click.option("--credential-target", "credential_target", default=None) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option("--ssh-key-path", "ssh_key_path", default=None) +@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None) +@click.option("--job-id", "job_id", default="") +@click.option("--commit", "commit", default="") +def artifacts_collect( + ip_address: str, + guest_os: str, + guest_artifact_path: str, + host_artifact_dir: str, + include_logs: bool, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts_file: str | None, + job_id: str, + commit: str, +) -> None: + """Copy build artifacts from a guest VM to a local directory. + + Mirrors ``scripts/Get-BuildArtifacts.ps1``. Writes ``manifest.json`` + when ``--job-id`` or ``--commit`` is provided. + """ + guest_os = guest_os.lower() + host_dir = Path(host_artifact_dir) + host_dir.mkdir(parents=True, exist_ok=True) + config = load_config() + + try: + if guest_os == "linux": + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + _linux_collect( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + guest_path=guest_artifact_path, + host_dir=host_dir, + ) + else: + target = credential_target or config.guest_cred_target + _windows_collect( + ip_address=ip_address, + credential_target=target, + guest_path=guest_artifact_path, + host_dir=host_dir, + include_logs=include_logs, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + + transferred = [p for p in host_dir.rglob("*") if p.is_file()] + if not transferred: + raise click.ClickException( + f"no files were transferred into {host_dir}" + ) + click.echo(f"[artifacts collect] {len(transferred)} file(s) collected.") + + if job_id or commit: + count = _write_manifest(host_dir, job_id, commit) + click.echo(f"[artifacts collect] manifest.json written ({count} file(s)).") + + +__all__ = ["artifacts", "artifacts_collect"] diff --git a/src/ci_orchestrator/commands/build.py b/src/ci_orchestrator/commands/build.py new file mode 100644 index 0000000..a917515 --- /dev/null +++ b/src/ci_orchestrator/commands/build.py @@ -0,0 +1,595 @@ +"""``build`` sub-commands. + +Phase A3 ports the remote build pipeline: + +* ``build run`` — replaces ``scripts/Invoke-RemoteBuild.ps1`` + +The original PS script supports two source-provisioning modes (host-side +clone uploaded via WinRM/SCP, and in-guest ``git clone``) plus a custom +build command and optional artifact packaging. This Python port preserves +the mechanics while keeping the guest workdir and artifact paths fully +parameterised — see Phase C hooks below. + +Phase C hook: the guest workdir / artifact paths are passed as opaque +strings; this module makes no assumption about Windows vs POSIX path +syntax beyond a sensible default per ``--guest-os``. ``build run`` does +not import ``WorkstationVmrunBackend`` directly; it relies on the +transport layer to talk to whatever guest the orchestrator points it at. +""" + +from __future__ import annotations + +import shlex +import sys +import tarfile +import tempfile +import zipfile +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportCommandError, TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + pass + + +# ---------------------------------------------------------------- group + + +@click.group("build") +def build() -> None: + """Remote build commands.""" + + +# ---------------------------------------------------------------- helpers + + +def _parse_extra_env(items: Iterable[str]) -> dict[str, str]: + """Parse ``KEY=VALUE`` pairs into a dict. + + Empty entries are ignored. A pair without ``=`` raises ``BadParameter``. + """ + out: dict[str, str] = {} + for raw in items: + if not raw: + continue + if "=" not in raw: + raise click.BadParameter( + f"--extra-env value must be KEY=VALUE, got: {raw!r}" + ) + key, value = raw.split("=", 1) + if not key: + raise click.BadParameter(f"--extra-env key cannot be empty: {raw!r}") + out[key] = value + return out + + +def _zip_dir(source_dir: Path, archive_path: Path) -> None: + """Create a deterministic ZIP archive containing the contents of ``source_dir``.""" + with zipfile.ZipFile( + archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=1 + ) as zf: + for item in source_dir.rglob("*"): + if item.is_file(): + zf.write(item, arcname=item.relative_to(source_dir)) + + +def _tar_dir(source_dir: Path, archive_path: Path) -> None: + """Create a gzip tar archive containing the contents of ``source_dir``.""" + with tarfile.open(archive_path, "w:gz") as tf: + for item in source_dir.iterdir(): + tf.add(item, arcname=item.name) + + +def _ps_quote(value: str) -> str: + """Quote a value for embedding in a PowerShell single-quoted string.""" + return "'" + value.replace("'", "''") + "'" + + +def _sh_quote(value: str) -> str: + """POSIX shell single-quote escaping.""" + return shlex.quote(value) + + +def _report_build_output(stdout: str, stderr: str) -> str: + """Emit guest build stdout/stderr to the CI log, flushed. + + Returns the last few lines for the failure message so the cause is + visible even if stream buffering hides the body on a non-zero exit. + """ + click.echo("[build run] ----- build output -----") + if stdout: + sys.stdout.write(stdout if stdout.endswith("\n") else stdout + "\n") + sys.stdout.flush() + if stderr: + sys.stderr.write(stderr if stderr.endswith("\n") else stderr + "\n") + sys.stderr.flush() + click.echo("[build run] ----- end build output -----") + return _tail_lines(stdout, stderr) + + +def _tail_lines(stdout: str, stderr: str) -> str: + """Last few output lines for a failure message (no printing).""" + combined = (stderr or "").strip() or (stdout or "").strip() + lines = combined.splitlines() + return " | ".join(lines[-5:]) if lines else "(no build output)" + + +# ---------------------------------------------------------------- Linux flow + + +def _linux_build( + *, + ip_address: str, + ssh_user: str, + key_path: str | None, + known_hosts: str | None, + workdir: str, + output_dir: str, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + build_command: str, + artifact_source: str, + extra_env: dict[str, str], + skip_artifact: bool, +) -> None: + click.echo("[build run] Linux build mode (SSH transport)") + + with SshTransport( + ip_address, + username=ssh_user, + key_path=key_path, + known_hosts=known_hosts, + ) as t: + t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}") + + if host_source_dir: + click.echo(f"[build run] packaging host source: {host_source_dir}") + with tempfile.TemporaryDirectory() as td: + tar_path = Path(td) / "ci-src.tar.gz" + _tar_dir(Path(host_source_dir), tar_path) + guest_tar = f"/tmp/{tar_path.name}" + t.copy(str(tar_path), guest_tar) + click.echo(f"[build run] extracting source in guest: {workdir}") + t.run( + f"tar -xzf {_sh_quote(guest_tar)} -C {_sh_quote(workdir)} " + f"&& rm -f {_sh_quote(guest_tar)}" + ) + elif clone_url: + click.echo(f"[build run] cloning {clone_url} into guest:{workdir}") + cmd = ( + f"GIT_TERMINAL_PROMPT=0 " + f"git clone --depth 1 --branch {_sh_quote(clone_branch)} " + f"{'--recurse-submodules ' if clone_submodules else ''}" + f"{_sh_quote(clone_url)} {_sh_quote(workdir)}" + ) + t.run(cmd) + if clone_commit: + # The shallow --depth 1 --branch clone only has the branch + # tip. If the branch advanced between workflow trigger and + # runner pickup, the pinned commit is absent ("reference is + # not a tree"). Fetch exactly that commit, then check out. + t.run( + f"git -C {_sh_quote(workdir)} fetch --depth 1 origin " + f"{_sh_quote(clone_commit)} && " + f"git -C {_sh_quote(workdir)} checkout " + f"{_sh_quote(clone_commit)}" + ) + else: + raise click.ClickException( + "Linux build requires either --host-source-dir or --clone-url." + ) + + env_prefix = "".join( + f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items() + ) + cmd = build_command or "make" + # PYTHONUNBUFFERED so the guest python flushes promptly and the + # streamed output is timely rather than emitted in one block. + full_cmd = ( + f"{env_prefix}export PYTHONUNBUFFERED=1; " + f"cd {_sh_quote(workdir)} && {cmd}" + ) + click.echo(f"[build run] running build (env vars redacted): cd {workdir} && {cmd}") + click.echo("[build run] ----- build output (live) -----") + result = t.run_streaming(full_cmd, check=False) + click.echo("[build run] ----- end build output -----") + if result.returncode != 0: + raise click.ClickException( + "build command failed " + f"(exit {result.returncode}): {_tail_lines(result.stdout, result.stderr)}" + ) + + if skip_artifact: + click.echo("[build run] artifact packaging skipped (--skip-artifact).") + return + + # Resolve the artifact source: absolute path as-is, else relative + # to the build workdir. (artifact_source may be absolute, e.g. + # /opt/ci/output; the old `workdir + '/' + src` produced + # /opt/ci/build//opt/ci/output and silently found nothing.) + src = artifact_source or "dist" + srcpath = src.rstrip("/") if src.startswith("/") else workdir.rstrip("/") + "/" + src + out = output_dir.rstrip("/") + + if srcpath == out or srcpath.startswith(out + "/"): + # The build wrote directly into the collect dir; do NOT wipe it + # (that deleted the build output) and nothing to copy. Fail + # loudly if it is missing or empty rather than collecting an + # empty artifact. + click.echo( + f"[build run] artifact source is the output dir ({out}); collecting in place" + ) + t.run( + f"if [ ! -d {_sh_quote(out)} ] || " + f'[ -z "$(ls -A {_sh_quote(out)} 2>/dev/null)" ]; then ' + f"echo '[build run] artifact source not found or empty: {out}' >&2; " + f"exit 1; fi" + ) + else: + t.run(f"rm -rf {_sh_quote(out)} && mkdir -p {_sh_quote(out)}") + cp_cmd = ( + f"if [ -d {_sh_quote(srcpath)} ]; then " + f"cp -r {_sh_quote(srcpath + '/.')} {_sh_quote(out + '/')}; " + f"elif [ -e {_sh_quote(srcpath)} ]; then " + f"cp {_sh_quote(srcpath)} {_sh_quote(out + '/')}; " + f"else echo '[build run] artifact source not found: {srcpath}' >&2; " + f"exit 1; fi" + ) + t.run(cp_cmd) + click.echo(f"[build run] artifact staged at {out}") + + +# ---------------------------------------------------------------- Windows flow + + +def _windows_build( + *, + ip_address: str, + credential_target: str, + workdir: str, + output_dir: str, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + build_command: str, + artifact_source: str, + configuration: str, + extra_env: dict[str, str], + skip_artifact: bool, + use_shared_cache: bool, +) -> None: + click.echo("[build run] Windows build mode (WinRM transport)") + cred = KeyringCredentialStore().get(credential_target) + + with WinRmTransport(ip_address, cred.username, cred.password) as t: + # Prepare workdir + artifact output directory. + prep_ps = ( + f"$work = {_ps_quote(workdir)}; " + f"$out = {_ps_quote(output_dir)}; " + "if (-not (Test-Path $out)) { New-Item -ItemType Directory -Path $out -Force | Out-Null }; " + "if (Test-Path $work) { Remove-Item $work -Recurse -Force }; " + "New-Item -ItemType Directory -Path $work -Force | Out-Null" + ) + t.run(prep_ps) + + if host_source_dir: + click.echo(f"[build run] zipping host source: {host_source_dir}") + with tempfile.TemporaryDirectory() as td: + zip_path = Path(td) / "ci-src.zip" + _zip_dir(Path(host_source_dir), zip_path) + size_mb = round(zip_path.stat().st_size / (1024 * 1024), 1) + click.echo(f"[build run] uploading source archive ({size_mb} MB)") + guest_zip = "C:\\CI\\src-transfer.zip" + t.copy(str(zip_path), guest_zip) + t.run( + f"Expand-Archive -Path {_ps_quote(guest_zip)} " + f"-DestinationPath {_ps_quote(workdir)} -Force; " + f"Remove-Item {_ps_quote(guest_zip)} -Force" + ) + elif clone_url: + click.echo(f"[build run] cloning {clone_url} into guest:{workdir}") + sub_flag = " --recurse-submodules" if clone_submodules else "" + t.run( + f"$env:GIT_TERMINAL_PROMPT = '0'; " + f"Set-Location (Split-Path {_ps_quote(workdir)} -Parent); " + f"git clone --depth 1 --branch {_ps_quote(clone_branch)}{sub_flag} " + f"{_ps_quote(clone_url)} (Split-Path {_ps_quote(workdir)} -Leaf)" + ) + if clone_commit: + # Shallow clone has only the branch tip; fetch the pinned + # commit explicitly so checkout works even if the branch + # advanced between trigger and runner pickup. + t.run( + f"git -C {_ps_quote(workdir)} fetch --depth 1 origin " + f"{_ps_quote(clone_commit)}; " + f"git -C {_ps_quote(workdir)} checkout {_ps_quote(clone_commit)}" + ) + else: + raise click.ClickException( + "Windows build requires either --host-source-dir or --clone-url." + ) + + # Inject env vars + run the build. + env_setup = "; ".join( + f"$env:{k} = {_ps_quote(v)}" for k, v in extra_env.items() + ) + if env_setup: + env_setup += "; " + if use_shared_cache: + env_setup += ( + "$env:NUGET_PACKAGES = '\\\\vmware-host\\Shared Folders\\nuget-cache'; " + "$env:PIP_CACHE_DIR = '\\\\vmware-host\\Shared Folders\\pip-cache'; " + ) + if build_command: + run_cmd = build_command + click.echo(f"[build run] custom build command: {run_cmd}") + else: + click.echo( + f"[build run] default build: dotnet restore + build (configuration={configuration})" + ) + run_cmd = ( + "dotnet restore; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; " + f"dotnet build --configuration {_ps_quote(configuration)} " + "--output (Join-Path (Get-Location) 'bin\\CIOutput')" + ) + + # Run the build command as PowerShell on the Windows guest. It is + # authored in PS by the workflow (New-Item, Set-Content, ...) and + # may also invoke native exes (python, dotnet). Wrapping it in + # `cmd /c` broke PS-native commands ("'New-Item' is not + # recognized"). LASTEXITCODE is seeded to 0 so a PS-only command + # (which never sets it) exits cleanly; native tools still set it. + full_ps = ( + f"{env_setup}Set-Location {_ps_quote(workdir)}; " + f"$ErrorActionPreference='Continue'; " + f"$global:LASTEXITCODE = 0; " + f"{run_cmd}; " + "if ($null -eq $LASTEXITCODE) { exit 0 } else { exit $LASTEXITCODE }" + ) + result = t.run(full_ps, check=False) + tail = _report_build_output(result.stdout, result.stderr) + if result.returncode != 0: + raise click.ClickException( + f"build command failed (exit {result.returncode}): {tail}" + ) + + if skip_artifact: + click.echo("[build run] artifact packaging skipped (--skip-artifact).") + return + + # Stage the artifact source into the collect dir as raw files + # (no intermediate zip — `actions/upload-artifact` produces the + # single final archive; mirrors the Linux flow, avoids zip-in-zip). + # artifact_source may be absolute (e.g. C:\CI\output) or relative + # to the build workdir; PowerShell Join-Path does not treat a + # rooted segment as absolute, so resolve explicitly. When the + # source IS the collect dir (build wrote there directly), collect + # in place — do NOT wipe it (that deleted the build output). + src_rel = (artifact_source or "dist") if build_command else "bin\\CIOutput" + + pkg_ps = ( + f"$work = {_ps_quote(workdir)}; " + f"$rel = {_ps_quote(src_rel)}; " + "$src = if ([System.IO.Path]::IsPathRooted($rel)) " + "{ $rel } else { Join-Path $work $rel }; " + f"$out = {_ps_quote(output_dir)}; " + "$sf = [System.IO.Path]::GetFullPath($src).TrimEnd('\\'); " + "$of = [System.IO.Path]::GetFullPath($out).TrimEnd('\\'); " + "$inPlace = ($sf -ieq $of) -or " + "$sf.StartsWith($of + '\\', " + "[System.StringComparison]::OrdinalIgnoreCase); " + "if ($inPlace) { " + " if (-not (Test-Path $out) -or " + " -not (Get-ChildItem -Force -- $out)) { " + " throw \"[build run] artifact source not found or empty: $out\" } " + "} else { " + " if (-not (Test-Path $src)) { " + " throw \"[build run] artifact source not found: $src\" }; " + " if (Test-Path $out) { Remove-Item $out -Recurse -Force }; " + " New-Item -ItemType Directory -Path $out -Force | Out-Null; " + " if (Test-Path $src -PathType Leaf) { " + " Copy-Item $src -Destination $out -Force " + " } else { " + " Copy-Item (Join-Path $src '*') -Destination $out -Recurse -Force } " + "}" + ) + t.run(pkg_ps) + click.echo(f"[build run] artifact staged at {output_dir}") + + +# ---------------------------------------------------------------- CLI + + +@build.command("run") +@click.option("--ip-address", "ip_address", required=True, help="Guest IP address.") +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, +) +@click.option( + "--credential-target", + "credential_target", + default=None, + help="Keyring target for the guest user (Windows). Defaults to config.guest_cred_target.", +) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option( + "--ssh-key-path", + "ssh_key_path", + default=None, + help="SSH private key path (Linux). Defaults to config.ssh_key_path.", +) +@click.option( + "--ssh-known-hosts-file", + "ssh_known_hosts_file", + default=None, + help="Persistent known_hosts file (Linux). Default: permissive (None).", +) +@click.option("--host-source-dir", "host_source_dir", default=None) +@click.option("--clone-url", "clone_url", default=None) +@click.option("--clone-branch", "clone_branch", default="main", show_default=True) +@click.option("--clone-commit", "clone_commit", default="") +@click.option("--clone-submodules", "clone_submodules", is_flag=True, default=False) +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="GiteaPAT", + show_default=True, + help="(Reserved) Keyring target for Gitea PAT — currently unused by the Python port.", +) +@click.option("--build-command", "build_command", default="") +@click.option( + "--guest-work-dir", + "guest_work_dir", + default=None, + help="Workdir inside the guest (defaults: Win=C:\\CI\\build, Linux=/opt/ci/build).", +) +@click.option( + "--guest-output-dir", + "guest_output_dir", + default=None, + help="Collect dir inside the Windows guest (default: C:\\CI\\output). " + "Raw build outputs are staged here; upload-artifact zips them.", +) +@click.option( + "--guest-linux-work-dir", + "guest_linux_work_dir", + default=None, + help="Alias for --guest-work-dir, kept for shim compatibility (Linux).", +) +@click.option( + "--guest-linux-output-dir", + "guest_linux_output_dir", + default="/opt/ci/output", + show_default=True, +) +@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) +@click.option("--configuration", default="Release", show_default=True) +@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) +@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option( + "--extra-env", + "extra_env", + multiple=True, + help="Repeatable KEY=VALUE pairs injected into the guest environment.", +) +def build_run( + ip_address: str, + guest_os: str, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts_file: str | None, + host_source_dir: str | None, + clone_url: str | None, + clone_branch: str, + clone_commit: str, + clone_submodules: bool, + gitea_credential_target: str, + build_command: str, + guest_work_dir: str | None, + guest_output_dir: str | None, + guest_linux_work_dir: str | None, + guest_linux_output_dir: str, + guest_artifact_source: str, + configuration: str, + skip_artifact: bool, + use_shared_cache: bool, + extra_env: tuple[str, ...], +) -> None: + """Run a build inside a guest VM, optionally packaging the result. + + Mirrors ``scripts/Invoke-RemoteBuild.ps1``. Source is provisioned via + one of: + + * ``--host-source-dir`` — local directory zipped/tarred and uploaded; + * ``--clone-url`` — ``git clone`` invoked inside the guest. + + Phase C hook: ``--guest-work-dir`` and ``--guest-output-dir`` are + parameterised; defaults are applied here, never inside the transport + layer. + """ + del gitea_credential_target # not yet honoured by Python port + guest_os = guest_os.lower() + + if host_source_dir and clone_url: + raise click.BadParameter( + "Provide either --host-source-dir or --clone-url, not both." + ) + if not host_source_dir and not clone_url: + raise click.BadParameter( + "Provide either --host-source-dir or --clone-url." + ) + if host_source_dir and not Path(host_source_dir).is_dir(): + raise click.BadParameter(f"--host-source-dir does not exist: {host_source_dir}") + + extra = _parse_extra_env(extra_env) + config = load_config() + + if guest_os == "linux": + workdir = guest_linux_work_dir or guest_work_dir or "/opt/ci/build" + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + try: + _linux_build( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + workdir=workdir, + output_dir=guest_linux_output_dir, + host_source_dir=host_source_dir, + clone_url=clone_url, + clone_branch=clone_branch, + clone_commit=clone_commit, + clone_submodules=clone_submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + extra_env=extra, + skip_artifact=skip_artifact, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + else: + workdir = guest_work_dir or "C:\\CI\\build" + output_dir = guest_output_dir or "C:\\CI\\output" + target = credential_target or config.guest_cred_target + try: + _windows_build( + ip_address=ip_address, + credential_target=target, + workdir=workdir, + output_dir=output_dir, + host_source_dir=host_source_dir, + clone_url=clone_url, + clone_branch=clone_branch, + clone_commit=clone_commit, + clone_submodules=clone_submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + configuration=configuration, + extra_env=extra, + skip_artifact=skip_artifact, + use_shared_cache=use_shared_cache, + ) + except (TransportError, TransportCommandError) as exc: + raise click.ClickException(str(exc)) from exc + + +__all__ = ["build", "build_run"] diff --git a/src/ci_orchestrator/commands/job.py b/src/ci_orchestrator/commands/job.py new file mode 100644 index 0000000..afef004 --- /dev/null +++ b/src/ci_orchestrator/commands/job.py @@ -0,0 +1,638 @@ +"""``job`` sub-command — full CI orchestration. + +Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a +template snapshot, wait for the guest to become reachable, run the build +inside the guest, optionally collect artifacts, and **always** destroy the +clone (even on failure or ``KeyboardInterrupt``). + +Composition over re-spawn: this module imports the in-process helpers +from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather +than re-invoking ``python -m ci_orchestrator`` for each phase. + +Phase C hook: the backend is selected via :func:`load_backend` so that an +ESXi backend can be plugged in by changing ``[backend].type`` in +``config.toml``. ``WorkstationVmrunBackend`` is never imported directly. +""" + +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import sys +import tempfile +import time +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.commands.artifacts import ( + _linux_collect, + _windows_collect, + _write_manifest, +) +from ci_orchestrator.commands.build import _linux_build, _windows_build +from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir +from ci_orchestrator.config import Config, load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse") + + +# ────────────────────────────────────────────────────────── helpers + + +def _now() -> float: + return time.monotonic() + + +def _read_guest_os_from_vmx(vmx_path: Path) -> str: + """Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line.""" + try: + text = vmx_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "windows" + m = re.search(r'guestOS\s*=\s*"([^"]+)"', text) + if not m: + return "windows" + detected = m.group(1).lower() + if any(h in detected for h in _GUEST_OS_LINUX_HINTS): + return "linux" + return "windows" + + +def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None: + """Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0.""" + if cpu <= 0 and memory_mb <= 0: + return + lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines() + out: list[str] = [] + seen_cpu = False + seen_mem = False + for line in lines: + low = line.strip().lower() + if cpu > 0 and low.startswith("numvcpus"): + out.append(f'numvcpus = "{cpu}"') + seen_cpu = True + elif memory_mb > 0 and low.startswith("memsize"): + out.append(f'memsize = "{memory_mb}"') + seen_mem = True + else: + out.append(line) + if cpu > 0 and not seen_cpu: + out.append(f'numvcpus = "{cpu}"') + if memory_mb > 0 and not seen_mem: + out.append(f'memsize = "{memory_mb}"') + vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8") + + +def _parse_extra_env_json(raw: str) -> dict[str, str]: + """Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict.""" + raw = (raw or "").strip() + if not raw or raw == "{}": + return {} + try: + loaded = json.loads(raw) + except json.JSONDecodeError as exc: + raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc + if not isinstance(loaded, dict): + raise click.BadParameter("--extra-env-json must be a JSON object.") + out: dict[str, str] = {} + for key, value in loaded.items(): + if not isinstance(key, str) or not _VALID_ENV_KEY.match(key): + raise click.BadParameter( + f"--extra-env-json key {key!r} is not a valid env var name." + ) + out[key] = "" if value is None else str(value) + return out + + +def _wait_running( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> bool: + while _now() < deadline: + try: + if backend.is_running(handle): + return True + except BackendError as exc: + click.echo(f"[job] backend probe error: {exc}", err=True) + time.sleep(poll) + return False + + +def _wait_for_ip( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> str | None: + while (now := _now()) < deadline: + call_timeout = min(poll, deadline - now) + try: + ip = backend.get_ip(handle, timeout=call_timeout) + except BackendError: + ip = None + if ip: + return ip + return None + + +def _probe_transport( + *, + guest_os: str, + ip_address: str, + deadline: float, + poll: float, + credential_target: str, + ssh_user: str, + ssh_key_path: str | None, + ssh_known_hosts: str | None, +) -> bool: + """Loop until the guest answers WinRM (Windows) or SSH (Linux).""" + if guest_os == "windows": + cred = KeyringCredentialStore().get(credential_target) + while _now() < deadline: + try: + with WinRmTransport(ip_address, cred.username, cred.password) as t: + if t.is_ready(): + return True + except TransportError as exc: + click.echo(f"[job] WinRM probe failed: {exc}", err=True) + time.sleep(poll) + return False + while _now() < deadline: + try: + with SshTransport( + ip_address, + username=ssh_user, + key_path=ssh_key_path, + known_hosts=ssh_known_hosts, + ) as t: + if t.is_ready(): + return True + except TransportError as exc: + click.echo(f"[job] SSH probe failed: {exc}", err=True) + time.sleep(poll) + return False + + +def _make_clone_name(job_id: str) -> str: + timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + # uuid4 suffix avoids collisions when several clones share a JobId. + suffix = uuid.uuid4().hex[:6] + return f"Clone_{job_id}_{timestamp}_{suffix}" + + +def _redact_url_creds(text: str) -> str: + """Mask ``scheme://user:pass@host`` credentials in arbitrary text.""" + return re.sub(r"(\w+://)[^/\s:@]+:[^/\s@]+@", r"\1***@", text) + + +def _host_clone( + repo_url: str, branch: str, commit: str, submodules: bool +) -> tuple[Path, Path]: + """Clone the repo on the host into a temp dir (host-side transport). + + Returns ``(tmp_root, src_dir)``. ``src_dir`` is passed as + ``host_source_dir`` to the build (zipped and transferred to the + guest). The caller must remove ``tmp_root`` when done. git stderr is + surfaced with embedded URL credentials redacted. + """ + tmp_root = Path(tempfile.mkdtemp(prefix="ci-host-clone-")) + src_dir = tmp_root / "src" + clone = ["git", "clone", "--depth", "1", "--branch", branch] + if submodules: + clone.append("--recurse-submodules") + clone += [repo_url, str(src_dir)] + try: + subprocess.run(clone, check=True, capture_output=True, text=True) + if commit: + subprocess.run( + ["git", "-C", str(src_dir), "fetch", "--depth", "1", + "origin", commit], + check=True, capture_output=True, text=True, + ) + subprocess.run( + ["git", "-C", str(src_dir), "checkout", commit], + check=True, capture_output=True, text=True, + ) + except subprocess.CalledProcessError as exc: + shutil.rmtree(tmp_root, ignore_errors=True) + detail = _redact_url_creds((exc.stderr or "").strip()) + tail = " | ".join(detail.splitlines()[-3:]) if detail else "(no stderr)" + raise click.ClickException( + f"host-side git clone failed (exit {exc.returncode}): {tail}" + ) from None + return tmp_root, src_dir + + +def _destroy_clone( + backend: VmBackend | None, + handle: VmHandle | None, + clone_dir: Path | None, +) -> None: + """Final cleanup. Tolerates every error — must never raise.""" + if backend is not None and handle is not None: + try: + _stop_with_fallback(backend, handle) + except Exception as exc: # pragma: no cover - belt and braces + click.echo(f"[job] stop ignored: {exc}", err=True) + try: + backend.delete(handle) + except BackendError as exc: + click.echo( + f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.", + err=True, + ) + except Exception as exc: # pragma: no cover + click.echo(f"[job] delete ignored: {exc}", err=True) + if clone_dir is not None and clone_dir.exists(): + if _try_remove_dir(clone_dir): + click.echo(f"[job] clone directory removed: {clone_dir}") + else: + click.echo( + f"[job] could not fully remove: {clone_dir}", err=True + ) + + +# ────────────────────────────────────────────────────────── command + + +@click.command("job") +@click.option("--job-id", "job_id", required=True) +@click.option("--repo-url", "repo_url", required=True) +@click.option("--branch", required=True) +@click.option("--commit", default="") +@click.option("--configuration", default="Release", show_default=True) +@click.option("--build-command", "build_command", default="") +@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True) +@click.option( + "--submodules/--no-submodules", + "submodules", + default=True, + show_default=True, + help="Recurse submodules when cloning (default: on).", +) +@click.option( + "--use-git-clone/--host-clone", + "use_git_clone", + default=True, + show_default=True, + help="In-guest git clone (default) vs host-side clone + zip transfer.", +) +@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False) +@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False) +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="GiteaPAT", + show_default=True, +) +@click.option( + "--template-path", + "template_path", + default=None, + help="Template VMX path (or backend-specific identifier).", +) +@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True) +@click.option("--clone-base-dir", "clone_base_dir", default=None) +@click.option("--artifact-base-dir", "artifact_base_dir", default=None) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux", "auto"], case_sensitive=False), + default="auto", + show_default=True, +) +@click.option( + "--guest-credential-target", + "guest_credential_target", + default=None, + help="Override config.guest_cred_target (Windows).", +) +@click.option("--ssh-key-path", "ssh_key_path", default=None) +@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True) +@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None) +@click.option( + "--extra-env-json", + "extra_env_json", + default="", + help="JSON object of env vars to inject into the guest before the build.", +) +@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0) +@click.option( + "--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0 +) +@click.option( + "--ready-timeout", + "ready_timeout", + type=click.FloatRange(10.0, 3600.0), + default=600.0, + show_default=True, + help="Total seconds to wait for VM running + IP + transport ready.", +) +@click.option( + "--poll-interval", + "poll_interval", + type=click.FloatRange(1.0, 60.0), + default=5.0, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None) +def job( + job_id: str, + repo_url: str, + branch: str, + commit: str, + configuration: str, + build_command: str, + guest_artifact_source: str, + submodules: bool, + use_git_clone: bool, + use_shared_cache: bool, + skip_artifact: bool, + gitea_credential_target: str, + template_path: str | None, + snapshot_name: str, + clone_base_dir: str | None, + artifact_base_dir: str | None, + guest_os: str, + guest_credential_target: str | None, + ssh_key_path: str | None, + ssh_user: str, + ssh_known_hosts_file: str | None, + extra_env_json: str, + guest_cpu: int, + guest_memory_mb: int, + ready_timeout: float, + poll_interval: float, + vmrun_path: str | None, +) -> None: + """Run the full CI pipeline against an ephemeral build VM. + + Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is + guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``. + """ + # Transport: in-guest git clone (default) or host-side clone + zip + # transfer (``--host-clone``). Both honour ``--submodules``. + + # Inject Gitea credentials into HTTP(S) clone URL so git doesn't prompt. + authed_repo_url = repo_url + if repo_url and repo_url.startswith(("http://", "https://")): + from urllib.parse import urlparse, urlunparse + + try: + gitea_cred = KeyringCredentialStore().get(gitea_credential_target) + parsed = urlparse(repo_url) + host = parsed.hostname or "" + port_part = f":{parsed.port}" if parsed.port else "" + netloc = f"{gitea_cred.username}:{gitea_cred.password}@{host}{port_part}" + authed_repo_url = urlunparse(parsed._replace(netloc=netloc)) + except Exception: + pass # no credential stored — try unauthenticated (public repo) + + config: Config = load_config() + if not template_path: + raise click.ClickException( + "--template-path is required." + ) + template_p = Path(template_path) + if not template_p.is_file(): + raise click.ClickException(f"Template VMX not found: {template_path}") + + base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms + base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts + base_clone_dir.mkdir(parents=True, exist_ok=True) + job_artifact_dir = base_artifact_dir / job_id + job_artifact_dir.mkdir(parents=True, exist_ok=True) + + extra_env = _parse_extra_env_json(extra_env_json) + + # Build backend from config; allow CLI override for vmrun path. + if vmrun_path is not None: + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path) + else: + try: + backend = load_backend(config) + except BackendNotAvailable as exc: + raise click.ClickException(f"backend unavailable: {exc}") from exc + + clone_name = _make_clone_name(job_id) + clone_dir = base_clone_dir / clone_name + clone_vmx = clone_dir / f"{clone_name}.vmx" + + handle: VmHandle | None = None + started = datetime.now(tz=UTC) + phase_timings: list[tuple[str, float]] = [] + phase_start = _now() + + click.echo("=" * 60) + click.echo(f"[job] Job : {job_id}") + click.echo(f"[job] Repository : {repo_url}") + click.echo(f"[job] Branch : {branch}") + if commit: + click.echo(f"[job] Commit : {commit}") + click.echo(f"[job] Template : {template_path}") + click.echo(f"[job] Snapshot : {snapshot_name}") + click.echo(f"[job] Artifacts : {job_artifact_dir}") + click.echo(f"[job] Started : {started.isoformat()}") + click.echo("=" * 60) + + try: + # ── Phase 1/5: clone VM ───────────────────────────────────────── + click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}") + try: + handle = backend.clone_linked( + template=str(template_p), + snapshot=snapshot_name, + name=clone_name, + destination=str(clone_vmx), + ) + except BackendError as exc: + raise click.ClickException(f"clone failed: {exc}") from exc + if not clone_vmx.is_file(): + raise click.ClickException( + f"backend reported success but clone VMX is missing: {clone_vmx}" + ) + + if guest_cpu > 0 or guest_memory_mb > 0: + click.echo( + f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB" + ) + _apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb) + + phase_timings.append(("clone", _now() - phase_start)) + phase_start = _now() + + # ── Phase 2/5: start VM ───────────────────────────────────────── + click.echo("\n[job] Phase 2/5 — starting VM (headless)") + try: + backend.start(handle, headless=True) + except BackendError as exc: + raise click.ClickException(f"vmrun start failed: {exc}") from exc + + # Resolve guest OS now that the clone exists. + if guest_os.lower() == "auto": + guest_os = _read_guest_os_from_vmx(clone_vmx) + click.echo(f"[job] auto-detected guest OS: {guest_os}") + else: + guest_os = guest_os.lower() + + deadline = _now() + ready_timeout + if not _wait_running(backend, handle, deadline, poll_interval): + raise click.ClickException("timeout waiting for VM to be running.") + ip_address = _wait_for_ip(backend, handle, deadline, poll_interval) + if not ip_address: + raise click.ClickException("timeout waiting for guest IP.") + click.echo(f"[job] guest IP: {ip_address}") + phase_timings.append(("start", _now() - phase_start)) + phase_start = _now() + + # ── Phase 3/5: wait for transport readiness ──────────────────── + click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness") + target = guest_credential_target or config.guest_cred_target + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + + if not _probe_transport( + guest_os=guest_os, + ip_address=ip_address, + deadline=deadline, + poll=poll_interval, + credential_target=target, + ssh_user=ssh_user, + ssh_key_path=key_path, + ssh_known_hosts=ssh_known_hosts_file, + ): + raise click.ClickException( + "timeout waiting for guest transport to be ready." + ) + phase_timings.append(("wait-ready", _now() - phase_start)) + phase_start = _now() + + # ── Phase 4/5: build inside guest ────────────────────────────── + click.echo("\n[job] Phase 4/5 — invoking remote build") + host_clone_root: Path | None = None + if use_git_clone: + build_clone_url: str | None = authed_repo_url + build_host_src: str | None = None + click.echo("[job] transport: in-guest git clone") + else: + click.echo("[job] transport: host-side clone + zip transfer") + host_clone_root, src_dir = _host_clone( + authed_repo_url, branch, commit, submodules + ) + build_clone_url = None + build_host_src = str(src_dir) + try: + if guest_os == "linux": + _linux_build( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + workdir="/opt/ci/build", + output_dir="/opt/ci/output", + host_source_dir=build_host_src, + clone_url=build_clone_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + extra_env=extra_env, + skip_artifact=skip_artifact, + ) + else: + _windows_build( + ip_address=ip_address, + credential_target=target, + workdir="C:\\CI\\build", + output_dir="C:\\CI\\output", + host_source_dir=build_host_src, + clone_url=build_clone_url, + clone_branch=branch, + clone_commit=commit, + clone_submodules=submodules, + build_command=build_command, + artifact_source=guest_artifact_source, + configuration=configuration, + extra_env=extra_env, + skip_artifact=skip_artifact, + use_shared_cache=use_shared_cache, + ) + finally: + if host_clone_root is not None: + shutil.rmtree(host_clone_root, ignore_errors=True) + phase_timings.append(("build", _now() - phase_start)) + phase_start = _now() + + # ── Phase 5/5: collect artifacts ─────────────────────────────── + if skip_artifact: + click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)") + else: + click.echo("\n[job] Phase 5/5 — collecting artifacts") + if guest_os == "linux": + _linux_collect( + ip_address=ip_address, + ssh_user=ssh_user, + key_path=key_path, + known_hosts=ssh_known_hosts_file, + guest_path="/opt/ci/output", + host_dir=job_artifact_dir, + ) + else: + _windows_collect( + ip_address=ip_address, + credential_target=target, + guest_path="C:\\CI\\output", + host_dir=job_artifact_dir, + include_logs=True, + ) + try: + count = _write_manifest(job_artifact_dir, job_id, commit) + click.echo(f"[job] manifest.json written ({count} file(s))") + except OSError as exc: + click.echo(f"[job] manifest write failed: {exc}", err=True) + phase_timings.append(("artifacts", _now() - phase_start)) + + elapsed = (datetime.now(tz=UTC) - started).total_seconds() + click.echo("\n" + "=" * 60) + click.echo("[job] phase durations:") + for name, secs in phase_timings: + click.echo(f" {name:<14} {int(secs):>5}s") + click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s") + click.echo("=" * 60) + click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s") + click.echo(f"[job] Artifacts: {job_artifact_dir}") + click.echo("=" * 60) + except KeyboardInterrupt: + click.echo("\n[job] interrupted by user; cleaning up...", err=True) + _destroy_clone(backend, handle, clone_dir) + sys.exit(130) + except click.ClickException: + _destroy_clone(backend, handle, clone_dir) + raise + except Exception as exc: + click.echo(f"\n[job] FAILURE: {exc}", err=True) + _destroy_clone(backend, handle, clone_dir) + raise click.ClickException(str(exc)) from exc + else: + _destroy_clone(backend, handle, clone_dir) + + +__all__ = ["job"] diff --git a/src/ci_orchestrator/commands/monitor.py b/src/ci_orchestrator/commands/monitor.py new file mode 100644 index 0000000..8b7f1b8 --- /dev/null +++ b/src/ci_orchestrator/commands/monitor.py @@ -0,0 +1,459 @@ +"""``monitor`` sub-commands. + +Phase A2 ports the maintenance/monitoring leaf scripts: + +* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1`` +* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1`` + +Both commands are designed to run from a scheduler (Windows Task Scheduler +in Phase A, systemd timers in Phase B) and emit warnings via the platform +event log when available, plus optional Discord/Gitea webhooks via stdlib +``urllib`` (no extra dependency). +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import click + +logger = logging.getLogger(__name__) + + +# ──────────────────────────────────────────────────────────────────── helpers + + +def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool: + """POST a Discord/Gitea-compatible JSON payload. Returns True on success.""" + if not url: + return False + payload = json.dumps({"content": content}).encode("utf-8") + req = urllib.request.Request( + url, + data=payload, + method="POST", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return bool(200 <= resp.status < 300) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + logger.warning("Webhook POST failed: %s", exc) + return False + + +def _write_windows_event( + source: str, event_id: int, entry_type: str, message: str +) -> bool: + """Write a Windows Application Event Log entry via ``eventcreate.exe``. + + Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts. + Returns True on success. + """ + if os.name != "nt": + return False + eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe") + if eventcreate is None: + return False + type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"} + cmd = [ + eventcreate, + "/T", type_map.get(entry_type, "WARNING"), + "/ID", str(event_id), + "/L", "APPLICATION", + "/SO", source, + "/D", message, + ] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, timeout=10.0 + ) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("eventcreate failed: %s", exc) + return False + if result.returncode != 0: + logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr) + return False + return True + + +# ─────────────────────────────────────────────────────────────────────── group + + +@click.group("monitor") +def monitor() -> None: + """Host monitoring commands (disk, runner).""" + + +# ──────────────────────────────────────────────────────────────────────── disk + + +def _resolve_drive_path(drive: str) -> str: + """Map a single-letter Windows drive (``F``) to a usable path. + + On non-Windows the ``drive`` value is treated as a path itself. + """ + if os.name == "nt" and len(drive) == 1 and drive.isalpha(): + return f"{drive.upper()}:\\" + return drive + + +@monitor.command("disk") +@click.option( + "--min-free-gb", + "min_free_gb", + type=click.IntRange(1, 2000), + default=50, + show_default=True, +) +@click.option( + "--drive-letter", + "drive_letter", + default="F" if os.name == "nt" else "/", + show_default=True, + help="Drive letter (Windows) or mount path (Linux).", +) +@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.") +@click.option( + "--json", + "as_json", + is_flag=True, + default=False, + help="Emit a single-line JSON object instead of human-readable output.", +) +def monitor_disk( + min_free_gb: int, + drive_letter: str, + webhook_url: str, + as_json: bool, +) -> None: + """Check free space on a CI host drive and alert if below threshold. + + Exit codes mirror the original PowerShell script: + + * 0 = drive OK + * 1 = below threshold (alert raised) + * 2 = drive not found + """ + target = _resolve_drive_path(drive_letter) + try: + usage = shutil.disk_usage(target) + except (FileNotFoundError, OSError) as exc: + click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True) + sys.exit(2) + + gb = 1024.0 ** 3 + free_gb = round(usage.free / gb, 1) + total_gb = round(usage.total / gb, 1) + pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0 + + payload = { + "drive": drive_letter, + "free_gb": free_gb, + "total_gb": total_gb, + "percent_free": pct_free, + "threshold_gb": min_free_gb, + "status": "ok" if free_gb >= min_free_gb else "alert", + } + + if free_gb >= min_free_gb: + if as_json: + click.echo(json.dumps(payload)) + else: + click.echo( + f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB " + f"({pct_free}%) - OK (threshold: {min_free_gb} GB)" + ) + return + + msg = ( + f"CI host drive {drive_letter}: free space {free_gb} GB " + f"({pct_free}%) is below the {min_free_gb} GB threshold. " + "vmrun linked clones may fail silently." + ) + if as_json: + payload["message"] = msg + click.echo(json.dumps(payload)) + else: + click.echo(f"[disk] WARNING: {msg}", err=True) + + _write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg) + if webhook_url: + _post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}") + sys.exit(1) + + +# ────────────────────────────────────────────────────────────────────── runner + + +def _service_status(service_name: str) -> str | None: + """Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed.""" + if os.name == "nt": + sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" + try: + r = subprocess.run( + [sc, "query", service_name], + capture_output=True, + text=True, + check=False, + timeout=10.0, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if r.returncode != 0: + return None + out = r.stdout.upper() + if "RUNNING" in out: + return "Running" + if "STOPPED" in out: + return "Stopped" + return "Unknown" + # Linux / systemd + systemctl = shutil.which("systemctl") + if systemctl is None: + return None + try: + r = subprocess.run( + [systemctl, "is-active", service_name], + capture_output=True, + text=True, + check=False, + timeout=10.0, + ) + except (subprocess.TimeoutExpired, OSError): + return None + state = r.stdout.strip().lower() + if state == "active": + return "Running" + if state in {"inactive", "failed", "deactivating"}: + return "Stopped" + return None # unknown / not installed + + +def _restart_service(service_name: str) -> bool: + """Attempt to (re)start ``service_name``. Returns True if now running.""" + if os.name == "nt": + sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" + # `sc start` returns 1056 if already running; ignore. + try: + subprocess.run( + [sc, "start", service_name], + capture_output=True, + text=True, + check=False, + timeout=30.0, + ) + except (subprocess.TimeoutExpired, OSError): + return False + else: + systemctl = shutil.which("systemctl") + if systemctl is None: + return False + try: + subprocess.run( + [systemctl, "restart", service_name], + capture_output=True, + text=True, + check=False, + timeout=30.0, + ) + except (subprocess.TimeoutExpired, OSError): + return False + time.sleep(5.0) + return _service_status(service_name) == "Running" + + +def _load_restart_log(path: Path) -> list[str]: + if not path.is_file(): + return [] + try: + raw = path.read_text(encoding="utf-8") + except OSError: + return [] + if not raw.strip(): + return [] + try: + data = json.loads(raw) + except json.JSONDecodeError: + return [] + if isinstance(data, list): + return [str(x) for x in data] + return [] + + +def _save_restart_log(path: Path, entries: list[str]) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(entries), encoding="utf-8") + except OSError as exc: + logger.warning("Could not persist restart log: %s", exc) + + +def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None: + """Return number of online runners reported by Gitea, or ``None`` on failure.""" + api = api_url.rstrip("/") + "/api/v1/admin/runners" + req = urllib.request.Request( + api, headers={"Authorization": f"token {token}"} + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: + logger.warning("Gitea API call failed: %s", exc) + return None + if isinstance(data, list): + return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True) + return None + + +@monitor.command("runner") +@click.option( + "--service-name", + "service_name", + default="act_runner", + show_default=True, +) +@click.option( + "--max-restarts", + type=click.IntRange(1, 10), + default=3, + show_default=True, +) +@click.option( + "--state-dir", + "state_dir", + default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state", + show_default=True, +) +@click.option("--webhook-url", "webhook_url", default="") +@click.option("--gitea-url", "gitea_url", default="") +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="", + help="Credential target name for the Gitea admin PAT.", +) +def monitor_runner( + service_name: str, + max_restarts: int, + state_dir: str, + webhook_url: str, + gitea_url: str, + gitea_credential_target: str, +) -> None: + """Monitor act_runner; auto-restart up to ``--max-restarts`` per hour. + + Exit codes: + 0 = healthy (or restart succeeded) + 1 = service down + restart failed/limit reached + 2 = service not installed + """ + state_dir_path = Path(state_dir) + maintenance_flag = state_dir_path / "runner-maintenance.flag" + if maintenance_flag.is_file(): + click.echo( + f"[runner] maintenance mode active ({maintenance_flag}) - skipping." + ) + return + + status = _service_status(service_name) + if status is None: + click.echo( + f"[runner] service {service_name!r} not found - is act_runner installed?", + err=True, + ) + sys.exit(2) + + if status == "Running": + click.echo(f"[runner] {service_name} is Running - OK") + if gitea_url and gitea_credential_target: + try: + from ci_orchestrator.credentials import KeyringCredentialStore + + cred = KeyringCredentialStore().get(gitea_credential_target) + online = _gitea_runners_online(gitea_url, cred.password) + except KeyError as exc: + click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True) + online = None + if online is None: + click.echo("[runner] Gitea API check skipped/failed.") + elif online == 0: + msg = ( + f"Gitea API ({gitea_url}) reports 0 online runners. " + f"{service_name} is Running - possible registration issue." + ) + click.echo(f"[runner] WARNING: {msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1005, "Warning", msg) + if webhook_url: + _post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}") + else: + click.echo(f"[runner] Gitea API: {online} runner(s) online - OK") + return + + # Not running → check rate limit and try to restart. + cooldown_file = state_dir_path / "runner-restart-log.json" + log = _load_restart_log(cooldown_file) + cutoff = datetime.now(tz=UTC) - timedelta(hours=1) + pruned: list[str] = [] + for entry in log: + try: + ts = datetime.fromisoformat(entry) + except ValueError: + continue + if ts.tzinfo is None: + ts = ts.replace(tzinfo=UTC) + if ts > cutoff: + pruned.append(entry) + log = pruned + + state_msg = ( + f"{service_name} service is '{status}'. " + f"Auto-restarts in last hour: {len(log)} / {max_restarts}." + ) + click.echo(f"[runner] WARNING: {state_msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg) + + if len(log) >= max_restarts: + limit_msg = ( + f"{service_name} auto-restart limit ({max_restarts}/h) reached - " + "manual intervention required." + ) + click.echo(f"[runner] {limit_msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg) + if webhook_url: + _post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}") + sys.exit(1) + + click.echo( + f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..." + ) + ok = _restart_service(service_name) + log.append(datetime.now(tz=UTC).isoformat()) + _save_restart_log(cooldown_file, log) + new_status = _service_status(service_name) or "Unknown" + restart_msg = f"{service_name} restarted - new status: {new_status}." + entry_type = "Information" if ok else "Warning" + click.echo(f"[runner] {restart_msg}") + _write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg) + prefix = "[WARNING]" if ok else "[ALERT]" + if webhook_url: + _post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}") + sys.exit(0 if ok else 1) + + +def _ignored(_name: str, _value: Any) -> None: + """Marker so type checkers see we deliberately drop a value.""" + + +__all__ = ["monitor"] diff --git a/src/ci_orchestrator/commands/report.py b/src/ci_orchestrator/commands/report.py new file mode 100644 index 0000000..e6de6be --- /dev/null +++ b/src/ci_orchestrator/commands/report.py @@ -0,0 +1,258 @@ +"""``report`` sub-commands. + +Phase A2 ports ``scripts/Get-CIJobSummary.ps1`` to ``report job``. Reads +JSONL log files written by the (still-PowerShell) ``Invoke-CIJob.ps1`` +under ``F:\\CI\\Logs\\\\invoke-ci.jsonl`` and prints either a +summary table of recent jobs or a per-phase breakdown for one job. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path + +import click + + +@dataclass(frozen=True) +class _Event: + phase: str + status: str + ts: str + elapsed_sec: int | None + error: str | None + raw: dict[str, object] + + +def _read_events(jsonl_path: Path) -> list[_Event]: + out: list[_Event] = [] + try: + text = jsonl_path.read_text(encoding="utf-8") + except OSError: + return out + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + data = obj.get("data") if isinstance(obj.get("data"), dict) else {} + elapsed: int | None = None + if isinstance(data, dict): + raw_elapsed = data.get("elapsedSec") + if isinstance(raw_elapsed, int | float): + elapsed = int(raw_elapsed) + err: str | None = None + if isinstance(data, dict) and isinstance(data.get("error"), str): + err = str(data["error"]) + out.append( + _Event( + phase=str(obj.get("phase", "")), + status=str(obj.get("status", "")), + ts=str(obj.get("ts", "")), + elapsed_sec=elapsed, + error=err, + raw=obj, + ) + ) + return out + + +def _format_hms(sec: int) -> str: + if sec < 0: + return "?" + h, rem = divmod(sec, 3600) + m, s = divmod(rem, 60) + if h > 0: + return f"{h}h{m:02d}m{s:02d}s" + return f"{m}m{s:02d}s" + + +@dataclass(frozen=True) +class _Row: + job_id: str + status: str + elapsed: str + started: str + error: str + + +def _summarise(events: list[_Event], default_id: str) -> _Row: + job_start = next( + (e for e in events if e.phase == "job" and e.status == "start"), None + ) + job_success = next( + (e for e in events if e.phase == "job" and e.status == "success"), None + ) + job_failure = next( + (e for e in events if e.phase == "job" and e.status == "failure"), None + ) + raw_id = events[0].raw.get("jobId") if events else None + job_id = str(raw_id) if isinstance(raw_id, str) and raw_id else default_id + started = job_start.ts if job_start else "" + if job_success: + return _Row( + job_id=job_id, + status="success", + elapsed=_format_hms(job_success.elapsed_sec or -1), + started=started, + error="", + ) + if job_failure: + err = job_failure.error or "" + if len(err) > 60: + err = err[:57] + "..." + return _Row( + job_id=job_id, + status="FAILED", + elapsed=_format_hms(job_failure.elapsed_sec or -1), + started=started, + error=err, + ) + return _Row( + job_id=job_id, + status="in-progress", + elapsed="?", + started=started, + error="", + ) + + +@click.group("report") +def report() -> None: + """Reporting commands (job summaries).""" + + +@report.command("job") +@click.option( + "--log-dir", + "log_dir", + default=r"F:\CI\Logs", + show_default=True, + help="Base directory containing per-job log subdirectories.", +) +@click.option( + "--last", + type=click.IntRange(1, 1000), + default=20, + show_default=True, + help="Show only the N most recent jobs.", +) +@click.option( + "--job-id", + "job_id", + default="", + help="Show a phase-by-phase breakdown for a single job.", +) +@click.option( + "--failed", + is_flag=True, + default=False, + help="Filter to failed jobs only.", +) +@click.option( + "--json", + "as_json", + is_flag=True, + default=False, + help="Emit JSON instead of a human-readable table.", +) +def report_job( + log_dir: str, + last: int, + job_id: str, + failed: bool, + as_json: bool, +) -> None: + """Display CI job summaries parsed from invoke-ci.jsonl files.""" + base = Path(log_dir) + if not base.is_dir(): + click.echo(f"Log directory not found: {log_dir}", err=True) + sys.exit(1) + + if job_id: + jsonl = base / job_id / "invoke-ci.jsonl" + if not jsonl.is_file(): + click.echo(f"No JSONL log found for job '{job_id}' at: {jsonl}", err=True) + sys.exit(1) + events = _read_events(jsonl) + if not events: + click.echo(f"JSONL file is empty or unreadable: {jsonl}", err=True) + sys.exit(1) + if as_json: + click.echo(json.dumps([e.raw for e in events], default=str)) + return + click.echo(f"\nJob: {job_id}") + click.echo("=" * 60) + click.echo(f"{'Phase':<35} {'Status':<10} {'Timestamp'}") + click.echo(f"{'-' * 35:<35} {'-' * 10:<10} {'-' * 24}") + for ev in events: + click.echo(f"{ev.phase:<35} {ev.status:<10} {ev.ts}") + fail = next( + (e for e in reversed(events) if e.status == "failure"), + None, + ) + if fail and fail.error: + click.echo(f"\nError: {fail.error}") + return + + files = sorted( + base.rglob("invoke-ci.jsonl"), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + if not files: + click.echo(f"No invoke-ci.jsonl files found under {log_dir}") + return + + rows: list[_Row] = [] + for f in files: + events = _read_events(f) + if not events: + continue + default_id = f.parent.name + rows.append(_summarise(events, default_id)) + + if failed: + rows = [r for r in rows if r.status == "FAILED"] + rows = rows[:last] + if not rows: + click.echo("No matching jobs found.") + return + + if as_json: + click.echo( + json.dumps( + [ + { + "jobId": r.job_id, + "status": r.status, + "elapsed": r.elapsed, + "started": r.started, + "error": r.error, + } + for r in rows + ] + ) + ) + return + + click.echo(f"\nCI Job Summary (last {len(rows)} jobs):") + header = f"{'JobId':<40} {'Status':<12} {'Elapsed':<10} {'Started':<26} Error" + click.echo(header) + click.echo( + f"{'-' * 40:<40} {'-' * 12:<12} {'-' * 10:<10} {'-' * 26:<26} {'-' * 30}" + ) + for r in rows: + click.echo( + f"{r.job_id:<40} {r.status:<12} {r.elapsed:<10} {r.started:<26} {r.error}" + ) + + +__all__ = ["report"] diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py new file mode 100644 index 0000000..97e3011 --- /dev/null +++ b/src/ci_orchestrator/commands/vm.py @@ -0,0 +1,469 @@ +"""``vm`` sub-commands. + +Phase A2 ports the leaf VM management scripts: + +* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1`` +* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1`` + +Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend` +dependency so an ESXi backend can later supply a folder-scoped VM list +instead of scanning the local filesystem. +""" + +from __future__ import annotations + +import shutil +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.config import load_config + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +# ────────────────────────────────────────────────────────────────────── helpers + + +def _make_backend(vmrun_path: str | None) -> VmBackend: + """Build a backend honouring an optional vmrun override.""" + config = load_config() + if vmrun_path is not None: + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + return WorkstationVmrunBackend(vmrun_path=vmrun_path) + return load_backend(config) + + +def _try_remove_dir(path: Path) -> bool: + """Best-effort recursive directory removal. Returns True if path is gone.""" + if not path.exists(): + return True + shutil.rmtree(path, ignore_errors=True) + return not path.exists() + + +def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None: + """Soft-stop, then hard-stop on failure. Both errors are tolerated.""" + try: + backend.stop(handle, hard=False) + except BackendError as exc: + click.echo(f"[vm remove] soft stop failed: {exc}", err=True) + try: + if backend.is_running(handle): + click.echo("[vm remove] still running, forcing hard stop...") + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm remove] hard stop failed: {exc}", err=True) + + +# ─────────────────────────────────────────────────────────────────────── group + + +@click.group("vm") +def vm() -> None: + """VM lifecycle commands.""" + + +# ─────────────────────────────────────────────────────────────────────── remove + + +@vm.command("remove") +@click.option( + "--vmx", + "--vm-path", + "vmx", + required=True, + help="Path to the clone VMX file to destroy.", +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--graceful-timeout", + "--graceful-timeout-seconds", + "graceful_timeout", + type=click.IntRange(1, 60), + default=15, + show_default=True, + help="Seconds to wait for soft stop before forcing.", +) +@click.option( + "--force", + is_flag=True, + default=False, + help="Skip soft stop and go straight to hard stop.", +) +def vm_remove( + vmx: str, + vmrun_path: str | None, + graceful_timeout: int, + force: bool, +) -> None: + """Stop and permanently delete an ephemeral build VM.""" + vmx_path = Path(vmx) + clone_dir = vmx_path.parent + click.echo(f"[vm remove] destroying VM: {vmx}") + + if not vmx_path.is_file(): + click.echo( + f"[vm remove] VMX file not found - nothing to remove: {vmx}", + err=True, + ) + # Still try to clean up a partial clone directory. + if clone_dir.exists() and clone_dir != vmx_path: + _try_remove_dir(clone_dir) + return + + try: + backend = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + click.echo( + f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.", + err=True, + ) + _try_remove_dir(clone_dir) + return + + handle = VmHandle(identifier=vmx) + + if force: + try: + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm remove] hard stop failed: {exc}", err=True) + else: + click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...") + _stop_with_fallback(backend, handle) + + # Brief settle: VMware may keep file locks for a moment after stop. + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + if not backend.is_running(handle): + break + except BackendError: + break + time.sleep(2.0) + + click.echo("[vm remove] deleting VM...") + deleted = False + last_err: str = "" + for delay in (0, 3, 6): + if delay > 0: + click.echo(f"[vm remove] retrying deleteVM in {delay}s...") + time.sleep(delay) + try: + backend.delete(handle) + deleted = True + break + except BackendError as exc: + last_err = str(exc) + + if not deleted: + click.echo( + f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True + ) + click.echo("[vm remove] falling back to manual directory removal.") + + if clone_dir.exists(): + click.echo(f"[vm remove] removing clone directory: {clone_dir}") + if _try_remove_dir(clone_dir): + click.echo("[vm remove] clone directory removed.") + else: + click.echo( + f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.", + err=True, + ) + + click.echo("[vm remove] VM destruction complete.") + + +# ────────────────────────────────────────────────────────────────────── cleanup + + +def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]: + """Return orphan clone directories older than ``max_age_hours``.""" + if not clone_base.is_dir(): + return [] + cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) + out: list[Path] = [] + for entry in clone_base.iterdir(): + if not entry.is_dir(): + continue + try: + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) + except OSError: + continue + if mtime < cutoff: + out.append(entry) + return out + + +def _find_vmx(clone_dir: Path) -> Path | None: + for entry in clone_dir.iterdir(): + if entry.is_file() and entry.suffix.lower() == ".vmx": + return entry + return None + + +def cleanup_orphans( + clone_base: Path, + max_age_hours: int, + backend: VmBackend | None, + *, + dry_run: bool = False, + lock_file: Path | None = None, + lock_max_age_minutes: int = 30, +) -> int: + """Remove orphaned clone directories. Returns count removed. + + Phase C hook: ``backend`` is used to stop+delete each orphan. If the + backend is unavailable, directory removal is still attempted. + """ + orphans = _list_orphans(clone_base, max_age_hours) + if not orphans: + click.echo( + f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)." + ) + else: + click.echo( + f"[vm cleanup] found {len(orphans)} orphaned director" + f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h." + ) + + removed = 0 + now = datetime.now(tz=UTC) + for entry in orphans: + age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600) + if dry_run: + click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)") + continue + click.echo(f"[vm cleanup] processing: {entry}") + vmx = _find_vmx(entry) + if vmx is not None and backend is not None: + handle = VmHandle(identifier=str(vmx)) + try: + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True) + time.sleep(2.0) + try: + backend.delete(handle) + except BackendError as exc: + click.echo( + f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.", + err=True, + ) + elif vmx is None: + click.echo( + f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True + ) + + if _try_remove_dir(entry): + click.echo(f"[vm cleanup] removed: {entry}") + removed += 1 + else: + click.echo( + f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.", + err=True, + ) + + # Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1). + if lock_file is not None and lock_file.is_file(): + cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes) + mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC) + if mtime < cutoff and not dry_run: + age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60) + try: + lock_file.unlink() + click.echo( + f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}" + ) + except OSError as exc: + click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True) + + click.echo("[vm cleanup] done.") + return removed + + +@vm.command("cleanup") +@click.option( + "--clone-base-dir", + "--clone-base", + "clone_base_dir", + default=None, + help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).", +) +@click.option( + "--max-age-hours", + type=click.IntRange(0, 168), + default=4, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--lock-file", + default=None, + help="Path to a stale vm-start lock file to remove if older than 30 minutes.", +) +@click.option( + "--what-if", + "--whatif", + "what_if", + is_flag=True, + default=False, + help="List orphans without destroying them.", +) +def vm_cleanup( + clone_base_dir: str | None, + max_age_hours: int, + vmrun_path: str | None, + lock_file: str | None, + what_if: bool, +) -> None: + """Destroy ephemeral build VMs left behind after a crash or timeout.""" + config = load_config() + base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms + if not base_path.exists(): + click.echo( + f"[vm cleanup] clone base dir not found: {base_path} - nothing to do." + ) + return + + try: + backend: VmBackend | None = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + click.echo( + f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.", + err=True, + ) + backend = None + + cleanup_orphans( + clone_base=base_path, + max_age_hours=max_age_hours, + backend=backend, + dry_run=what_if, + lock_file=Path(lock_file) if lock_file else None, + ) + + +# ─────────────────────────────────────────────────────────────────────── new + + +@vm.command("new") +@click.option( + "--template", + "--template-path", + "template", + required=True, + help="Identifier of the template VM (Workstation: VMX path; ESXi: managed-object ref).", +) +@click.option( + "--snapshot", + "--snapshot-name", + "snapshot", + default="BaseClean", + show_default=True, + help="Snapshot name on the template to clone from.", +) +@click.option( + "--clone-base-dir", + "clone_base_dir", + required=True, + help="Directory under which the new clone folder will be created.", +) +@click.option( + "--job-id", + "job_id", + required=True, + help="Unique job identifier (used to name the clone folder).", +) +@click.option( + "--vmrun-path", + "vmrun_path", + default=None, + help="Override vmrun binary path.", +) +@click.option( + "--guest-os", + type=click.Choice(["windows", "linux"], case_sensitive=False), + default="windows", + show_default=True, + help="Informational; reserved for backend hints in Phase C.", +) +def vm_new( + template: str, + snapshot: str, + clone_base_dir: str, + job_id: str, + vmrun_path: str | None, + guest_os: str, +) -> None: + """Create a linked clone of the template VM for an ephemeral CI build. + + Mirrors ``scripts/New-BuildVM.ps1``. Prints the clone identifier + (Workstation: VMX path) on stdout on success. + + Phase C hook: ``--template`` is treated as an opaque string; the backend + is responsible for interpretation. Workstation builds the destination + path under ``--clone-base-dir`` from ``--job-id`` + timestamp. + """ + del guest_os # currently informational only + # Verify the template identifier looks plausible: for Workstation it is + # a VMX path. For other backends in Phase C this validation will move + # into the backend itself. + template_path = Path(template) + if not template_path.is_file(): + raise click.ClickException(f"Template VMX not found: {template}") + + base = Path(clone_base_dir) + base.mkdir(parents=True, exist_ok=True) + + timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + clone_name = f"Clone_{job_id}_{timestamp}" + clone_dir = base / clone_name + clone_vmx = clone_dir / f"{clone_name}.vmx" + + click.echo("[vm new] creating linked clone...") + click.echo(f" template : {template}") + click.echo(f" snapshot : {snapshot}") + click.echo(f" clone vmx: {clone_vmx}") + + try: + backend = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + raise click.ClickException(f"vmrun unavailable: {exc}") from exc + + start = time.monotonic() + try: + handle = backend.clone_linked( + template=str(template_path), + snapshot=snapshot, + name=clone_name, + destination=str(clone_vmx), + ) + except BackendError as exc: + # Clean up partial clone directory if vmrun left one behind. + if clone_dir.exists(): + _try_remove_dir(clone_dir) + raise click.ClickException(f"clone failed: {exc}") from exc + + if not clone_vmx.is_file(): + raise click.ClickException( + f"backend reported success but clone VMX is missing: {clone_vmx}" + ) + + elapsed = time.monotonic() - start + click.echo(f"[vm new] clone created in {elapsed:.1f}s") + # Final stdout line: the identifier itself, so PS callers can capture it. + click.echo(handle.identifier) + + +__all__ = ["cleanup_orphans", "vm"] diff --git a/src/ci_orchestrator/commands/wait.py b/src/ci_orchestrator/commands/wait.py new file mode 100644 index 0000000..4e41520 --- /dev/null +++ b/src/ci_orchestrator/commands/wait.py @@ -0,0 +1,203 @@ +"""``wait-ready`` sub-command. + +Replaces ``scripts/Wait-VMReady.ps1``. Polls a guest VM until WinRM +(Windows) or SSH (Linux) is responsive. Designed to be invoked from the +PowerShell shim ``scripts/Wait-VMReady.ps1`` which translates PS-style +PascalCase parameters (``-VMPath``, ``-IPAddress``, ``-Transport``, +``-SshKeyPath``, ``-SshUser``) into click kebab-case options. +""" + +from __future__ import annotations + +import sys +import time +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +# Exit codes (kept in sync with the PowerShell original where possible): +# 0 ready, 2 timeout-running, 3 timeout-ip, 4 timeout-transport +_EXIT_TIMEOUT_RUNNING = 2 +_EXIT_TIMEOUT_IP = 3 +_EXIT_TIMEOUT_TRANSPORT = 4 + + +def _normalise_guest_os(value: str) -> str: + """Map Transport/GuestOS aliases to ``windows``/``linux``.""" + v = value.strip().lower() + if v in {"winrm", "windows", "win"}: + return "windows" + if v in {"ssh", "linux", "lnx"}: + return "linux" + raise click.BadParameter(f"Unknown guest-os/transport value: {value!r}") + + +def _wait_running( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> bool: + while time.monotonic() < deadline: + try: + if backend.is_running(handle): + return True + except BackendError as exc: + click.echo(f"[wait-ready] backend probe error: {exc}", err=True) + time.sleep(poll) + return False + + +def _wait_for_ip( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> str | None: + while (now := time.monotonic()) < deadline: + call_timeout = min(poll, deadline - now) + try: + ip = backend.get_ip(handle, timeout=call_timeout) + except BackendError: + ip = None + if ip: + return ip + return None + + +@click.command("wait-ready") +@click.option( + "--vmx", + "--vm-path", + "vmx", + required=True, + help="Path to the guest VMX file.", +) +@click.option( + "--ip-address", + "ip_address", + default=None, + help="Guest IP address. If omitted the backend is polled for it.", +) +@click.option( + "--guest-os", + "--transport", + "guest_os", + default="windows", + show_default=True, + callback=lambda _ctx, _p, v: _normalise_guest_os(v), + help="windows/WinRM or linux/SSH (case-insensitive).", +) +@click.option("--timeout", "--timeout-seconds", "timeout", type=float, default=300.0, show_default=True) +@click.option( + "--poll-interval", + "--poll-interval-seconds", + "poll_interval", + type=float, + default=5.0, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--credential-target", + default=None, + help="Override credential target name (defaults to config.guest_cred_target).", +) +@click.option("--ssh-user", default="ci_build", show_default=True) +@click.option( + "--ssh-key-path", + default=None, + help="SSH private key (defaults to config.ssh_key_path).", +) +@click.option( + "--skip-ping", + is_flag=True, + default=False, + help="Accepted for PS compatibility; ignored (Python uses transport probe directly).", +) +def wait_ready( + vmx: str, + ip_address: str | None, + guest_os: str, + timeout: float, + poll_interval: float, + vmrun_path: str | None, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + skip_ping: bool, +) -> None: + """Poll a guest until WinRM (Windows) or SSH (Linux) is responsive.""" + del skip_ping # accepted for PS compatibility, no-op + config = load_config() + if vmrun_path is not None: + # Re-load backend with override; cheap, no real connections. + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path) + else: + backend = load_backend(config) + handle = VmHandle(identifier=vmx) + deadline = time.monotonic() + timeout + + click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}") + if not _wait_running(backend, handle, deadline, poll_interval): + click.echo("[wait-ready] timeout waiting for VM to be running.", err=True) + sys.exit(_EXIT_TIMEOUT_RUNNING) + + if ip_address is None: + click.echo("[wait-ready] waiting for guest IP (VMware Tools must be running)...") + ip = _wait_for_ip(backend, handle, deadline, poll_interval) + if not ip: + click.echo("[wait-ready] timeout waiting for guest IP.", err=True) + sys.exit(_EXIT_TIMEOUT_IP) + click.echo(f"[wait-ready] guest IP: {ip}") + else: + ip = ip_address + click.echo(f"[wait-ready] using provided IP: {ip}") + + if guest_os == "windows": + store = KeyringCredentialStore() + target = credential_target or config.guest_cred_target + cred = store.get(target) + while time.monotonic() < deadline: + try: + with WinRmTransport(ip, cred.username, cred.password) as t: + if t.is_ready(): + click.echo("[wait-ready] WinRM ready.") + return + except TransportError as exc: + click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True) + time.sleep(poll_interval) + else: + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + while time.monotonic() < deadline: + try: + with SshTransport(ip, username=ssh_user, key_path=key_path) as t: + if t.is_ready(): + click.echo("[wait-ready] SSH ready.") + return + except TransportError as exc: + click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True) + time.sleep(poll_interval) + + click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True) + sys.exit(_EXIT_TIMEOUT_TRANSPORT) + + +__all__ = [ + "KeyringCredentialStore", + "SshTransport", + "WinRmTransport", + "load_backend", + "wait_ready", +] diff --git a/src/ci_orchestrator/config.py b/src/ci_orchestrator/config.py new file mode 100644 index 0000000..df3744b --- /dev/null +++ b/src/ci_orchestrator/config.py @@ -0,0 +1,163 @@ +"""Configuration loading for the CI orchestrator. + +Resolution order (later wins): + 1. OS-aware defaults (Windows: F:\\CI\\..., Linux: /var/lib/ci/...). + 2. Optional ``config.toml`` file (path via ``CI_CONFIG`` env var or + ``$CI_ROOT/config.toml`` if present). + 3. Environment variables (``CI_ROOT``, ``CI_TEMPLATES``, ``CI_BUILD_VMS``, + ``CI_ARTIFACTS``, ``CI_KEYS``). + +Phase C hook: a ``[backend]`` section in ``config.toml`` selects the VM +backend implementation; the field is read but currently only the +``workstation`` value is supported. +""" + +from __future__ import annotations + +import os +import tomllib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +_ENV_KEYS = ("CI_ROOT", "CI_TEMPLATES", "CI_BUILD_VMS", "CI_ARTIFACTS", "CI_KEYS") + + +@dataclass(frozen=True) +class Paths: + """Filesystem paths used by the orchestrator.""" + + root: Path + templates: Path + build_vms: Path + artifacts: Path + keys: Path + + +@dataclass(frozen=True) +class BackendConfig: + """Backend selector. ``type`` is currently always 'workstation'.""" + + type: str = "workstation" + options: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Config: + """Top-level configuration object.""" + + paths: Paths + backend: BackendConfig + vmrun_path: str | None = None + ssh_key_path: Path | None = None + guest_cred_target: str = "BuildVMGuest" + + +def _platform_defaults() -> Paths: + root = Path(r"F:\CI") if os.name == "nt" else Path("/var/lib/ci") + return Paths( + root=root, + templates=root / "Templates" if os.name == "nt" else root / "templates", + build_vms=root / "BuildVMs" if os.name == "nt" else root / "build-vms", + artifacts=root / "Artifacts" if os.name == "nt" else root / "artifacts", + keys=root / "keys" if os.name == "nt" else Path("/etc/ci/keys"), + ) + + +def _load_toml(path: Path) -> dict[str, Any]: + with path.open("rb") as fh: + return tomllib.load(fh) + + +def _merge_paths(base: Paths, overrides: dict[str, Any]) -> Paths: + def pick(key: str, current: Path) -> Path: + val = overrides.get(key) + return Path(val) if val else current + + return Paths( + root=pick("root", base.root), + templates=pick("templates", base.templates), + build_vms=pick("build_vms", base.build_vms), + artifacts=pick("artifacts", base.artifacts), + keys=pick("keys", base.keys), + ) + + +def _apply_env(paths: Paths, env: dict[str, str]) -> Paths: + mapping = { + "CI_ROOT": "root", + "CI_TEMPLATES": "templates", + "CI_BUILD_VMS": "build_vms", + "CI_ARTIFACTS": "artifacts", + "CI_KEYS": "keys", + } + overrides: dict[str, Any] = {} + for env_key, attr in mapping.items(): + val = env.get(env_key) + if val: + overrides[attr] = val + return _merge_paths(paths, overrides) + + +def load_config( + config_path: Path | None = None, + env: dict[str, str] | None = None, +) -> Config: + """Build a :class:`Config` from defaults, optional TOML file, and env vars. + + Parameters + ---------- + config_path: + Explicit path to a TOML file. If ``None``, looks at ``$CI_CONFIG`` then + ``$CI_ROOT/config.toml``. + env: + Environment dict (defaults to :data:`os.environ`). Injected for tests. + """ + if env is None: + env = dict(os.environ) + + paths = _platform_defaults() + + # Resolve config file + toml_path: Path | None = None + if config_path is not None: + toml_path = config_path + elif env.get("CI_CONFIG"): + toml_path = Path(env["CI_CONFIG"]) + else: + candidate = paths.root / "config.toml" + if candidate.is_file(): + toml_path = candidate + + backend = BackendConfig() + vmrun_path: str | None = env.get("CI_VMRUN_PATH") + ssh_key_path = Path(env["CI_SSH_KEY_PATH"]) if env.get("CI_SSH_KEY_PATH") else None + guest_cred_target = env.get("CI_GUEST_CRED_TARGET", "BuildVMGuest") + + if toml_path and toml_path.is_file(): + data = _load_toml(toml_path) + paths = _merge_paths(paths, data.get("paths", {})) + backend_section = data.get("backend", {}) + if backend_section: + backend = BackendConfig( + type=str(backend_section.get("type", "workstation")), + options={k: v for k, v in backend_section.items() if k != "type"}, + ) + if vmrun_path is None: + vmrun_path = data.get("vmrun_path") + if ssh_key_path is None and data.get("ssh_key_path"): + ssh_key_path = Path(data["ssh_key_path"]) + guest_cred_target = data.get("guest_cred_target", guest_cred_target) + + paths = _apply_env(paths, env) + + return Config( + paths=paths, + backend=backend, + vmrun_path=vmrun_path, + ssh_key_path=ssh_key_path, + guest_cred_target=guest_cred_target, + ) + + +__all__ = ["BackendConfig", "Config", "Paths", "load_config"] diff --git a/src/ci_orchestrator/credentials.py b/src/ci_orchestrator/credentials.py new file mode 100644 index 0000000..d192e59 --- /dev/null +++ b/src/ci_orchestrator/credentials.py @@ -0,0 +1,59 @@ +"""Credential store abstraction. + +Wraps the ``keyring`` library so callers do not depend on it directly. +Phase B note: the same interface is used on both Windows (Credential +Manager) and Linux (Secret Service / file-based vault); see +``plans/implementation-plan-A-B.md`` step B3. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol + + +@dataclass(frozen=True) +class Credential: + """Username + secret pair retrieved from a credential store.""" + + username: str + password: str + + +class CredentialStore(Protocol): + """Read-only credential lookup.""" + + def get(self, target: str) -> Credential: + """Return the credential associated with ``target``. + + Raises :class:`KeyError` if the target is unknown. + """ + ... + + +class KeyringCredentialStore: + """Default :class:`CredentialStore` backed by the ``keyring`` library.""" + + def __init__(self, service: str = "ci") -> None: + self._service = service + + def get(self, target: str) -> Credential: + try: + import keyring + except ImportError as exc: # pragma: no cover - dep declared + raise RuntimeError( + "keyring is not installed; run `pip install keyring`." + ) from exc + + cred = keyring.get_credential(target, None) + if cred is None: + # Fallback: some backends only support get_password and need the + # username to be the same as the target. + password = keyring.get_password(self._service, target) + if password is None: + raise KeyError(f"Credential '{target}' not found in keyring.") + return Credential(username=target, password=password) + return Credential(username=cred.username, password=cred.password) + + +__all__ = ["Credential", "CredentialStore", "KeyringCredentialStore"] diff --git a/src/ci_orchestrator/transport/__init__.py b/src/ci_orchestrator/transport/__init__.py new file mode 100644 index 0000000..ba9b3eb --- /dev/null +++ b/src/ci_orchestrator/transport/__init__.py @@ -0,0 +1,9 @@ +"""Guest transport layer (WinRM for Windows, SSH for Linux).""" + +from __future__ import annotations + +from ci_orchestrator.transport.errors import TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +__all__ = ["SshTransport", "TransportError", "WinRmTransport"] diff --git a/src/ci_orchestrator/transport/errors.py b/src/ci_orchestrator/transport/errors.py new file mode 100644 index 0000000..1a5eb06 --- /dev/null +++ b/src/ci_orchestrator/transport/errors.py @@ -0,0 +1,23 @@ +"""Transport error hierarchy.""" + +from __future__ import annotations + + +class TransportError(RuntimeError): + """Base class for transport-layer (WinRM/SSH) failures.""" + + +class TransportConnectError(TransportError): + """Raised when the underlying connection cannot be established.""" + + +class TransportCommandError(TransportError): + """Raised when a remote command exits non-zero.""" + + def __init__(self, returncode: int, stdout: str, stderr: str) -> None: + super().__init__( + f"Remote command failed (exit={returncode}): {(stderr or stdout).strip()[:500]}" + ) + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr diff --git a/src/ci_orchestrator/transport/ssh.py b/src/ci_orchestrator/transport/ssh.py new file mode 100644 index 0000000..d5d3f69 --- /dev/null +++ b/src/ci_orchestrator/transport/ssh.py @@ -0,0 +1,210 @@ +"""SSH transport via ``paramiko``. + +Replaces the ``ssh.exe`` / ``scp.exe`` calls used in +``scripts/_Transport.psm1``. By default the host-key policy is +``AutoAddPolicy`` with an empty ``known_hosts`` file (i.e. permissive), +which mirrors the ``StrictHostKeyChecking=no UserKnownHostsFile=NUL`` +default used for ephemeral CI build VMs. + +See ``AGENTS.md`` error #12: in PowerShell with ``$ErrorActionPreference = +'Stop'`` the native ``ssh-keygen -R`` call could raise terminating errors +on stderr output. With paramiko there is no equivalent issue. +""" + +from __future__ import annotations + +import contextlib +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError + +if TYPE_CHECKING: # pragma: no cover + import paramiko + + +@dataclass(frozen=True) +class SshResult: + """Result of a remote SSH command.""" + + stdout: str + stderr: str + returncode: int + + @property + def ok(self) -> bool: + return self.returncode == 0 + + +class SshTransport: + """Thin wrapper around :class:`paramiko.SSHClient`.""" + + def __init__( + self, + host: str, + *, + username: str = "ci_build", + key_path: str | Path | None = None, + port: int = 22, + connect_timeout: float = 10.0, + known_hosts: str | Path | None = None, + ) -> None: + self._host = host + self._username = username + self._key_path = Path(key_path) if key_path else None + self._port = port + self._connect_timeout = connect_timeout + self._known_hosts = Path(known_hosts) if known_hosts else None + self._client: paramiko.SSHClient | None = None + + # ----------------------------------------------------------- connection + def _connect(self) -> paramiko.SSHClient: + if self._client is not None: + return self._client + try: + import paramiko + except ImportError as exc: # pragma: no cover - dep declared + raise TransportConnectError( + "paramiko is not installed; run `pip install paramiko`." + ) from exc + + client = paramiko.SSHClient() + if self._known_hosts is not None and self._known_hosts.is_file(): + client.load_host_keys(str(self._known_hosts)) + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + else: + # Permissive mode for ephemeral CI VMs (see _Transport.psm1). + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + try: + client.connect( + hostname=self._host, + port=self._port, + username=self._username, + key_filename=str(self._key_path) if self._key_path else None, + timeout=self._connect_timeout, + allow_agent=False, + look_for_keys=False, + ) + except Exception as exc: + raise TransportConnectError( + f"SSH connection to {self._username}@{self._host}:{self._port} failed: {exc}" + ) from exc + self._client = client + return client + + def close(self) -> None: + if self._client is not None: + with contextlib.suppress(Exception): + self._client.close() + self._client = None + + def __enter__(self) -> SshTransport: + self._connect() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + # --------------------------------------------------------------- ops + def run(self, command: str, *, check: bool = True, timeout: float | None = None) -> SshResult: + """Run ``command`` in the guest shell.""" + client = self._connect() + try: + _stdin, stdout, stderr = client.exec_command(command, timeout=timeout) + out = stdout.read().decode("utf-8", errors="replace") + err = stderr.read().decode("utf-8", errors="replace") + rc = stdout.channel.recv_exit_status() + except Exception as exc: + raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc + result = SshResult(stdout=out, stderr=err, returncode=rc) + if check and not result.ok: + raise TransportCommandError(result.returncode, result.stdout, result.stderr) + return result + + def run_streaming( + self, command: str, *, check: bool = True, timeout: float | None = None + ) -> SshResult: + """Run ``command`` streaming stdout/stderr live to this process. + + Unlike :meth:`run` (which blocks on a full channel read), output + is forwarded chunk-by-chunk as it arrives so long builds show + progress instead of one block at the end. Captured output is + still returned in :class:`SshResult`. + """ + client = self._connect() + try: + chan = client.get_transport().open_session() # type: ignore[union-attr] + if timeout: + chan.settimeout(timeout) + chan.exec_command(command) + except Exception as exc: + raise TransportConnectError(f"SSH exec_command failed: {exc}") from exc + + out_parts: list[str] = [] + err_parts: list[str] = [] + try: + while True: + progressed = False + while chan.recv_ready(): + d = chan.recv(65536).decode("utf-8", errors="replace") + out_parts.append(d) + sys.stdout.write(d) + sys.stdout.flush() + progressed = True + while chan.recv_stderr_ready(): + d = chan.recv_stderr(65536).decode("utf-8", errors="replace") + err_parts.append(d) + sys.stderr.write(d) + sys.stderr.flush() + progressed = True + if ( + chan.exit_status_ready() + and not chan.recv_ready() + and not chan.recv_stderr_ready() + ): + break + if not progressed: + time.sleep(0.05) + rc = chan.recv_exit_status() + except Exception as exc: + raise TransportConnectError(f"SSH stream failed: {exc}") from exc + finally: + with contextlib.suppress(Exception): + chan.close() + + result = SshResult( + stdout="".join(out_parts), stderr="".join(err_parts), returncode=rc + ) + if check and not result.ok: + raise TransportCommandError(result.returncode, result.stdout, result.stderr) + return result + + def copy(self, local_path: str | Path, remote_path: str) -> None: + """Upload via SFTP.""" + client = self._connect() + try: + with client.open_sftp() as sftp: + sftp.put(str(local_path), remote_path) + except Exception as exc: + raise TransportConnectError(f"SFTP put failed: {exc}") from exc + + def fetch(self, remote_path: str, local_path: str | Path) -> None: + """Download via SFTP.""" + client = self._connect() + try: + with client.open_sftp() as sftp: + sftp.get(remote_path, str(local_path)) + except Exception as exc: + raise TransportConnectError(f"SFTP get failed: {exc}") from exc + + def is_ready(self) -> bool: + """Lightweight readiness probe.""" + try: + result = self.run("true", check=False, timeout=5.0) + except (TransportConnectError, TransportCommandError): + return False + return result.ok diff --git a/src/ci_orchestrator/transport/winrm.py b/src/ci_orchestrator/transport/winrm.py new file mode 100644 index 0000000..09e361a --- /dev/null +++ b/src/ci_orchestrator/transport/winrm.py @@ -0,0 +1,205 @@ +"""WinRM transport via ``pypsrp``. + +Replaces the ``New-PSSession`` / ``Invoke-Command`` calls used in the +PowerShell scripts. CI build VMs use a self-signed TLS certificate on +WinRM port 5986, so SSL validation is disabled by default — appropriate +for an isolated lab network (see ``scripts/_Common.psm1``). +""" + +from __future__ import annotations + +import contextlib +import time +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from ci_orchestrator.transport.errors import TransportCommandError, TransportConnectError + +# WSMan faults that are transient — the guest momentarily cannot launch +# the WSMan host process (wsmprovhost), typically right after a +# memory-heavy build. Retrying after a short settle usually succeeds. +_WSMAN_TRANSIENT_MARKERS = ( + "1018", # could not launch a host process to process the given request + "could not launch a host process", + "the wsman provider host server", + "1726", # RPC server unavailable (service still coming back) + "connection was aborted", +) + +if TYPE_CHECKING: # pragma: no cover + from pypsrp.client import Client + + +@dataclass(frozen=True) +class WinRmResult: + """Result of a remote PowerShell invocation.""" + + stdout: str + stderr: str + returncode: int + + @property + def ok(self) -> bool: + return self.returncode == 0 + + +class WinRmTransport: + """Thin wrapper around :class:`pypsrp.client.Client`. + + Parameters + ---------- + host: + IP or DNS name of the guest. + username, password: + Credentials. Pass via :class:`~ci_orchestrator.credentials.Credential` + in caller code; do not embed plaintext. + port: + WinRM HTTPS port (default 5986). + cert_validation: + When ``False`` (default), self-signed certs are accepted — required + for the ephemeral build VMs in this lab. + """ + + def __init__( + self, + host: str, + username: str, + password: str, + *, + port: int = 5986, + ssl: bool = True, + cert_validation: bool = False, + connection_timeout: int = 30, + auth: str = "ntlm", + ) -> None: + self._host = host + self._username = username + self._password = password + self._port = port + self._ssl = ssl + self._cert_validation = cert_validation + self._connection_timeout = connection_timeout + # Build VMs use local (workgroup) accounts. pypsrp defaults to + # 'negotiate', which tries Kerberos with no domain and fails under + # SSPI with SEC_E_UNKNOWN_CREDENTIALS. NTLM is required for local + # accounts. The guest username must also be host-qualified + # (WINBUILD-2025\ci_build), stored that way in the credential vault. + self._auth = auth + self._client: Client | None = None + + # ----------------------------------------------------------- connection + def _connect(self) -> Client: + if self._client is not None: + return self._client + try: + from pypsrp.client import Client # local import: optional dep at runtime + except ImportError as exc: # pragma: no cover - dep declared in pyproject + raise TransportConnectError( + "pypsrp is not installed; run `pip install pypsrp`." + ) from exc + try: + self._client = Client( + self._host, + username=self._username, + password=self._password, + port=self._port, + ssl=self._ssl, + auth=self._auth, + cert_validation=self._cert_validation, + connection_timeout=self._connection_timeout, + ) + except Exception as exc: + raise TransportConnectError( + f"WinRM connection to {self._host}:{self._port} failed: {exc}" + ) from exc + return self._client + + def close(self) -> None: + if self._client is not None: + with contextlib.suppress(Exception): + self._client.wsman.close() + self._client = None + + def __enter__(self) -> WinRmTransport: + self._connect() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + # --------------------------------------------------------------- ops + def run(self, script: str, *, check: bool = True) -> WinRmResult: + """Run a PowerShell script block in the guest. + + Retries transient WSMan host-process-launch failures (e.g. fault + 1018 right after a memory-heavy build) with a short backoff. + """ + attempts = 4 + delay = 5.0 + last_exc: Exception | None = None + for attempt in range(attempts): + client = self._connect() + try: + stdout, stderr, returncode = client.execute_ps(script) + break + except Exception as exc: + last_exc = exc + msg = str(exc).lower() + transient = any(m in msg for m in _WSMAN_TRANSIENT_MARKERS) + if transient and attempt < attempts - 1: + # Drop the cached client so the next try opens a fresh + # WSMan shell once the guest can spawn the host again. + self.close() + time.sleep(delay) + delay *= 2 + continue + raise TransportConnectError( + f"WinRM execute_ps failed: {exc}" + ) from exc + else: # pragma: no cover - loop always breaks or raises + raise TransportConnectError( + f"WinRM execute_ps failed: {last_exc}" + ) from last_exc + # pypsrp returns stderr as a PSDataStreams object (newer versions) or a + # list of error records (older versions); normalise to a plain string. + stderr_any: Any = stderr + if isinstance(stderr_any, str): + stderr_str = stderr_any + elif hasattr(stderr_any, "error"): + # PSDataStreams — extract .error records. + error_records = getattr(stderr_any, "error", None) or [] + stderr_str = "\n".join(str(e) for e in error_records) + else: + try: + stderr_str = "\n".join(str(s) for s in stderr_any) + except TypeError: + stderr_str = str(stderr_any) + result = WinRmResult(stdout=stdout, stderr=stderr_str, returncode=int(returncode)) + if check and not result.ok: + raise TransportCommandError(result.returncode, result.stdout, result.stderr) + return result + + def copy(self, local_path: str | Path, remote_path: str) -> None: + """Upload a file to the guest via PSRP.""" + client = self._connect() + try: + client.copy(str(local_path), remote_path) + except Exception as exc: + raise TransportConnectError(f"WinRM copy failed: {exc}") from exc + + def fetch(self, remote_path: str, local_path: str | Path) -> None: + """Download a file from the guest via PSRP.""" + client = self._connect() + try: + client.fetch(remote_path, str(local_path)) + except Exception as exc: + raise TransportConnectError(f"WinRM fetch failed: {exc}") from exc + + def is_ready(self) -> bool: + """Lightweight readiness probe: run ``$true`` and check exit code.""" + try: + result = self.run("$true | Out-Null", check=False) + except (TransportConnectError, TransportCommandError): + return False + return result.ok diff --git a/template/Deploy-LinuxBuild2404.ps1 b/template/Deploy-LinuxBuild2404.ps1 index cdfcb10..f06a81a 100644 --- a/template/Deploy-LinuxBuild2404.ps1 +++ b/template/Deploy-LinuxBuild2404.ps1 @@ -242,11 +242,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) { } function New-IsoFromFolder { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $SourceFolder, [Parameter(Mandatory)] [string] $IsoPath, [Parameter(Mandatory)] [string] $VolumeLabel ) + if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return } if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force } $fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage $result = $null @@ -296,11 +298,13 @@ function Invoke-Vmrun { } function Set-VmxKey { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $VmxPath, [Parameter(Mandatory)] [string] $Key, [Parameter(Mandatory)] [string] $Value ) + if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return } $lines = Get-Content -LiteralPath $VmxPath $pattern = "^\s*$([regex]::Escape($Key))\s*=" $found = $false diff --git a/template/Deploy-WinBuild2022.ps1 b/template/Deploy-WinBuild2022.ps1 index 7596903..9d8f0cf 100644 --- a/template/Deploy-WinBuild2022.ps1 +++ b/template/Deploy-WinBuild2022.ps1 @@ -270,11 +270,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) { } function New-IsoFromFolder { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $SourceFolder, [Parameter(Mandatory)] [string] $IsoPath, [Parameter(Mandatory)] [string] $VolumeLabel ) + if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return } if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force } $fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage $result = $null @@ -305,10 +307,12 @@ function New-WinIsoNoPrompt { Result: UEFI boot proceeds straight into Windows Setup, no "Press any key to boot from CD" prompt. #> + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $SourceIsoPath, [Parameter(Mandatory)] [string] $OutputIsoPath ) + if (-not $PSCmdlet.ShouldProcess($OutputIsoPath, 'Rebuild Windows ISO without boot prompt')) { return } if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force } $mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly @@ -329,7 +333,7 @@ function New-WinIsoNoPrompt { $fsi.VolumeName = $label $fsi.ChooseImageDefaultsForMediaType(13) # 5+ GB ISO needs explicit large-volume defaults if available. - try { $fsi.UDFRevision = 0x102 } catch {} + try { $fsi.UDFRevision = 0x102 } catch { Write-Verbose "UDF revision not supported on this IMAPI version." } Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow $fsi.Root.AddTree($letter, $false) @@ -383,11 +387,13 @@ function Invoke-Vmrun { } function Set-VmxKey { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $VmxPath, [Parameter(Mandatory)] [string] $Key, [Parameter(Mandatory)] [string] $Value ) + if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return } $lines = Get-Content -LiteralPath $VmxPath $pattern = "^\s*$([regex]::Escape($Key))\s*=" $found = $false @@ -400,8 +406,10 @@ function Set-VmxKey { } function Remove-VmxLines { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param([Parameter(Mandatory)] [string] $VmxPath, [Parameter(Mandatory)] [string] $KeyPrefix) + if (-not $PSCmdlet.ShouldProcess($VmxPath, "Remove VMX lines starting with '$KeyPrefix'")) { return } $pat = "^\s*$([regex]::Escape($KeyPrefix))" $lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat } Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII @@ -428,8 +436,10 @@ function Invoke-VmrunBounded { } function Stop-Vm { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param([Parameter(Mandatory)] [string] $VmxPath, [int] $SoftTimeoutMinutes = 5) + if (-not $PSCmdlet.ShouldProcess($VmxPath, 'Stop VM')) { return } # Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s. # Guest OS continues shutdown independently once the signal is received. Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30 @@ -1114,4 +1124,4 @@ Write-Host " .\Validate-DeployState.ps1 -VMIPAddress $guestIP" Write-Host " .\Prepare-WinBuild2022.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]" Write-Host "" Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green - + diff --git a/template/Deploy-WinBuild2025.ps1 b/template/Deploy-WinBuild2025.ps1 index de5cd16..0cb2f8b 100644 --- a/template/Deploy-WinBuild2025.ps1 +++ b/template/Deploy-WinBuild2025.ps1 @@ -270,11 +270,13 @@ if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) { } function New-IsoFromFolder { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $SourceFolder, [Parameter(Mandatory)] [string] $IsoPath, [Parameter(Mandatory)] [string] $VolumeLabel ) + if (-not $PSCmdlet.ShouldProcess($IsoPath, 'Create ISO from folder')) { return } if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force } $fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage $result = $null @@ -305,10 +307,12 @@ function New-WinIsoNoPrompt { Result: UEFI boot proceeds straight into Windows Setup, no "Press any key to boot from CD" prompt. #> + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $SourceIsoPath, [Parameter(Mandatory)] [string] $OutputIsoPath ) + if (-not $PSCmdlet.ShouldProcess($OutputIsoPath, 'Rebuild Windows ISO without boot prompt')) { return } if (Test-Path $OutputIsoPath) { Remove-Item $OutputIsoPath -Force } $mount = Mount-DiskImage -ImagePath $SourceIsoPath -PassThru -StorageType ISO -Access ReadOnly @@ -329,7 +333,7 @@ function New-WinIsoNoPrompt { $fsi.VolumeName = $label $fsi.ChooseImageDefaultsForMediaType(13) # 5+ GB ISO needs explicit large-volume defaults if available. - try { $fsi.UDFRevision = 0x102 } catch {} + try { $fsi.UDFRevision = 0x102 } catch { Write-Verbose "UDF revision not supported on this IMAPI version." } Write-Host " Adding ISO tree (this takes a few minutes for 5+ GB)..." -ForegroundColor Yellow $fsi.Root.AddTree($letter, $false) @@ -383,11 +387,13 @@ function Invoke-Vmrun { } function Set-VmxKey { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param( [Parameter(Mandatory)] [string] $VmxPath, [Parameter(Mandatory)] [string] $Key, [Parameter(Mandatory)] [string] $Value ) + if (-not $PSCmdlet.ShouldProcess($VmxPath, "Set VMX key '$Key'")) { return } $lines = Get-Content -LiteralPath $VmxPath $pattern = "^\s*$([regex]::Escape($Key))\s*=" $found = $false @@ -400,8 +406,10 @@ function Set-VmxKey { } function Remove-VmxLines { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param([Parameter(Mandatory)] [string] $VmxPath, [Parameter(Mandatory)] [string] $KeyPrefix) + if (-not $PSCmdlet.ShouldProcess($VmxPath, "Remove VMX lines starting with '$KeyPrefix'")) { return } $pat = "^\s*$([regex]::Escape($KeyPrefix))" $lines = Get-Content -LiteralPath $VmxPath | Where-Object { $_ -notmatch $pat } Set-Content -LiteralPath $VmxPath -Value $lines -Encoding ASCII @@ -428,8 +436,10 @@ function Invoke-VmrunBounded { } function Stop-Vm { + [CmdletBinding(SupportsShouldProcess, ConfirmImpact='Low')] param([Parameter(Mandatory)] [string] $VmxPath, [int] $SoftTimeoutMinutes = 5) + if (-not $PSCmdlet.ShouldProcess($VmxPath, 'Stop VM')) { return } # Send soft-stop (ACPI); vmrun may block until guest finishes — kill vmrun after 30 s. # Guest OS continues shutdown independently once the signal is received. Invoke-VmrunBounded -Arguments @('-T','ws','stop',$VmxPath,'soft') -TimeoutSeconds 30 @@ -1114,4 +1124,4 @@ Write-Host " .\Validate-DeployState.ps1 -VMIPAddress $guestIP" Write-Host " .\Prepare-WinBuild2025.ps1 -VMIPAddress $guestIP [-SkipWindowsUpdate]" Write-Host "" Write-Host " VM is powered on with vmxnet3 + DHCP. Snapshot '$SnapshotName' taken." -ForegroundColor Green - + diff --git a/template/Install-CIToolchain-WinBuild2022.ps1 b/template/Install-CIToolchain-WinBuild2022.ps1 index 427f557..e6d3f03 100644 --- a/template/Install-CIToolchain-WinBuild2022.ps1 +++ b/template/Install-CIToolchain-WinBuild2022.ps1 @@ -123,6 +123,8 @@ Set-ExecutionPolicy Bypass -Scope Process -Force .\Install-CIToolchain-WinBuild2022.ps1 -BuildPassword 'YourPassword' #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for local user creation. Intentional by design.')] [CmdletBinding()] param( # Local username for the CI build account inside the VM @@ -865,7 +867,6 @@ else { $vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe' $vsInstallerPath = 'C:\CI\vs_BuildTools.exe' $vsResultPath = 'C:\CI\vs_result.txt' - $vsLogPath = 'C:\CI\vs_log.txt' Write-Host "Downloading VS Build Tools installer..." Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy diff --git a/template/Install-CIToolchain-WinBuild2025.ps1 b/template/Install-CIToolchain-WinBuild2025.ps1 index 3fe7661..d6e368f 100644 --- a/template/Install-CIToolchain-WinBuild2025.ps1 +++ b/template/Install-CIToolchain-WinBuild2025.ps1 @@ -123,6 +123,8 @@ Set-ExecutionPolicy Bypass -Scope Process -Force .\Install-CIToolchain-WinBuild2025.ps1 -BuildPassword 'YourPassword' #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for local user creation. Intentional by design.')] [CmdletBinding()] param( # Local username for the CI build account inside the VM @@ -865,7 +867,6 @@ else { $vsInstallerUrl = 'https://aka.ms/vs/stable/vs_buildtools.exe' $vsInstallerPath = 'C:\CI\vs_BuildTools.exe' $vsResultPath = 'C:\CI\vs_result.txt' - $vsLogPath = 'C:\CI\vs_log.txt' Write-Host "Downloading VS Build Tools installer..." Remove-Item $vsInstallerPath -ErrorAction SilentlyContinue # remove any stale/corrupt copy diff --git a/template/Prepare-WinBuild2022.ps1 b/template/Prepare-WinBuild2022.ps1 index 1e924ff..2acfb97 100644 --- a/template/Prepare-WinBuild2022.ps1 +++ b/template/Prepare-WinBuild2022.ps1 @@ -113,6 +113,8 @@ -VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 ` -BuildPassword 'StrongCIPass!2026' -StoreCredential #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for WinRM PSCredential. Intentional by design.')] [CmdletBinding()] param( # Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot. @@ -436,7 +438,7 @@ try { -Authentication Basic -ErrorAction Stop Remove-PSSession $probe -ErrorAction SilentlyContinue return $true - } catch { } + } catch { Write-Verbose "WinRM probe attempt failed." } } return $false } @@ -479,7 +481,7 @@ try { if ($session) { try { Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue - } catch { } + } catch { Write-Verbose "Reboot command sent; session may have already dropped." } Remove-PSSession $session -ErrorAction SilentlyContinue } @@ -595,7 +597,7 @@ try { try { Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue } - catch { } + catch { Write-Verbose "Reboot command sent; session may have already dropped." } Remove-PSSession $session -ErrorAction SilentlyContinue $session = $null @@ -721,5 +723,5 @@ finally { } Write-Host "[Prepare] Host TrustedHosts restored." } - - + + diff --git a/template/Prepare-WinBuild2025.ps1 b/template/Prepare-WinBuild2025.ps1 index 14c98fa..d0d12bf 100644 --- a/template/Prepare-WinBuild2025.ps1 +++ b/template/Prepare-WinBuild2025.ps1 @@ -113,6 +113,8 @@ -VMXPath 'F:\CI\Templates\WinBuild\WinBuild.vmx' -VMIPAddress 192.168.79.130 ` -BuildPassword 'StrongCIPass!2026' -StoreCredential #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template setup script: plain-text password from CI credential store converted to SecureString for WinRM PSCredential. Intentional by design.')] [CmdletBinding()] param( # Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot. @@ -436,7 +438,7 @@ try { -Authentication Basic -ErrorAction Stop Remove-PSSession $probe -ErrorAction SilentlyContinue return $true - } catch { } + } catch { Write-Verbose "WinRM probe attempt failed." } } return $false } @@ -479,7 +481,7 @@ try { if ($session) { try { Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue - } catch { } + } catch { Write-Verbose "Reboot command sent; session may have already dropped." } Remove-PSSession $session -ErrorAction SilentlyContinue } @@ -595,7 +597,7 @@ try { try { Invoke-Command -Session $session -ScriptBlock { Restart-Computer -Force } -ErrorAction SilentlyContinue } - catch { } + catch { Write-Verbose "Reboot command sent; session may have already dropped." } Remove-PSSession $session -ErrorAction SilentlyContinue $session = $null @@ -721,5 +723,5 @@ finally { } Write-Host "[Prepare] Host TrustedHosts restored." } - - + + diff --git a/template/Validate-DeployState.ps1 b/template/Validate-DeployState.ps1 index e4c4359..94654c0 100644 --- a/template/Validate-DeployState.ps1 +++ b/template/Validate-DeployState.ps1 @@ -41,6 +41,8 @@ # Linux .\Validate-DeployState.ps1 -VMIPAddress 192.168.79.200 -GuestOS Linux #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template validation script: plain-text password converted to SecureString for WinRM PSCredential. Intentional by design.')] [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VMIPAddress, diff --git a/template/Validate-SetupState.ps1 b/template/Validate-SetupState.ps1 index 7bd9776..cd6da63 100644 --- a/template/Validate-SetupState.ps1 +++ b/template/Validate-SetupState.ps1 @@ -37,6 +37,8 @@ .EXAMPLE .\Validate-SetupState.ps1 -VMIPAddress 192.168.79.129 -Remediate #> +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingConvertToSecureStringWithPlainText', '', + Justification = 'Template setup script: plain-text password required to bootstrap WinRM credential chain before Credential Manager is available.')] [CmdletBinding()] param( [Parameter(Mandatory)] [string] $VMIPAddress, @@ -273,8 +275,6 @@ try { Write-Host '' $failed = 0 - $deployCnt = 0 - $setupCnt = 0 $inSetup = $false foreach ($r in $checks) { diff --git a/tests/New-BuildVM.Tests.ps1 b/tests/New-BuildVM.Tests.ps1 deleted file mode 100644 index 19ba063..0000000 --- a/tests/New-BuildVM.Tests.ps1 +++ /dev/null @@ -1,106 +0,0 @@ -#Requires -Version 5.1 -<# -.SYNOPSIS - Pester v5 tests for scripts/New-BuildVM.ps1 -#> - -BeforeAll { - $script:ScriptPath = Join-Path $PSScriptRoot '..\scripts\New-BuildVM.ps1' - $script:CloneBaseDir = Join-Path $env:TEMP "CI-Tests-CloneBase-$PID" - New-Item -ItemType Directory -Path $script:CloneBaseDir -Force | Out-Null - - # Fake vmrun.ps1 — exits with $env:FAKE_VMRUN_EXIT (default 0) - # On success it also creates the destination VMX so the post-clone Test-Path passes. - # Using .ps1 instead of .cmd avoids Windows cmd redirect limitations inside if blocks. - $script:FakeVmrun = Join-Path $env:TEMP "fake-vmrun-$PID.ps1" - Set-Content $script:FakeVmrun -Value @' -# Args: -T ws clone