diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md new file mode 100644 index 0000000..544b1a9 --- /dev/null +++ b/docs/CHANGELOG.md @@ -0,0 +1,151 @@ +# CHANGELOG — Local CI/CD System + +Completed sprint work, in reverse-chronological order. +Open backlog items remain in [TODO.md](../TODO.md). + +--- + +## 2026-05-13 — Sprint 16: Medium/Low hardening (final-master-plan) + +- **Watch-DiskSpace.ps1**: Event Log source renamed `CI-DiskAlert` → `CI-DiskSpaceAlert` + (aligned with task name registered by `Register-CIScheduledTasks.ps1`). +- **Watch-RunnerHealth.ps1**: maintenance flag support — if `F:\CI\State\runner-maintenance.flag` + exists, restart logic is skipped entirely. +- **Invoke-CIJob.ps1**: + - New `WebhookUrl` parameter (default `''`). + - `TemplatePath` sanity check: `Write-Warning` if not under `F:\CI\Templates\`. + - 90-minute duration warning: `Start-Job` fires `[WARNING]` webhook after 5400 s; cancelled + in `finally` if the build completes first. +- **BEST-PRACTICES.md** section 11: VMware HGFS shared folder write semantics, cache-poisoning + risk model, safe-use rules for `UseSharedCache`, and cache invalidation procedure. +- **Validate-HostState.ps1** (new): checks SSH key ACLs, Credential Manager target + accessibility, CI directory structure, and vmrun.exe availability. +- **Validate-SetupState.ps1** docblock: `HTTP/Basic` → `HTTPS/Basic, port 5986`. +- **TEST-PLAN-v1.3-to-HEAD.md** §3.6: benchmark JSONL field names corrected to match actual + script output (`cloneSec`, `startSec`, `ipSec`, `winrmSec`, `destroySec`, `totalBootSec`). +- **CI-FLOW.md** failure table: "collects partial artifacts" replaced with accurate description + of `Get-GuestDiagnostics` post-failure collection. +- **final-master-plan.md**: all Medium and most Low items marked done. + +--- + +## 2026-05-12 — Sprint 15b: Hardcoded path fix + +- **action.yml** + **runner/config.yaml**: replaced hardcoded `N:\Code\...` script path with + `GITEA_CI_SCRIPT_ROOT` env var injected by the runner. act_runner restarted with labels + `windows-build:host dotnet:host msbuild:host linux-build:host`. + +--- + +## 2026-05-11 — Sprint 15: Rename Setup-* → Install-CIToolchain-* + +Renamed three guest-side provisioning scripts and updated all references across 14 files: + +| Old name | New name | +| ---------------------------------------- | ---------------------------------------------- | +| `template/Setup-WinBuild2025.ps1` | `template/Install-CIToolchain-WinBuild2025.ps1` | +| `template/Setup-WinBuild2022.ps1` | `template/Install-CIToolchain-WinBuild2022.ps1` | +| `template/Setup-LinuxBuild2404.sh` | `template/Install-CIToolchain-Linux2404.sh` | + +References updated in: Prepare-WinBuild2025/2022.ps1, Prepare-LinuxBuild2404.ps1, +Deploy-WinBuild2025/2022.ps1, Deploy-LinuxBuild2404.ps1, README.md, TODO.md, +docs/WINDOWS-TEMPLATE-SETUP.md, docs/LINUX-TEMPLATE-SETUP.md, +docs/TEST-PLAN-v1.3-to-HEAD.md, gitea/actions/local-ci-build/action.yml. + +--- + +## 2026-05-11 — Sprint 14: §6.4 Build matrix + §6.5 Secret injection + +**§6.4** — `build-nsInnoUnp.yml` rewritten with matrix strategy `target: [windows, linux]`. +Separate artifact dirs; `fail-fast: false`; composite action used for both targets. + +**§6.5** — `ExtraGuestEnv [hashtable]` added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`: +- Windows: `[System.Environment]::SetEnvironmentVariable` in session ScriptBlock. +- Linux: `export KEY='val'; ` prefix before build command. +- `action.yml`: `extra-guest-env-json` input (JSON object), never logged. + +--- + +## 2026-05-11 — Sprint 13: §6.2 Composite Action + +**action.yml** (`gitea/actions/local-ci-build/action.yml`) fully implemented. +Inputs: `build-command`, `artifact-source`, `submodules`, `guest-os`, `use-git-clone`, +`configuration`, `template-path`, `snapshot-name`, `artifact-name`, `artifact-retention-days`, +`extra-guest-env-json`, `use-shared-cache`, `job-id-suffix`, `repo-url`, `webhook-url`. +Outputs: `artifact-path`, `artifact-name`. +Artifact upload included; no orchestration logic required in calling repos. + +--- + +## 2026-05-10 — Sprint 12: §6.1 Linux Build VM + +End-to-end Linux build pipeline implemented: +- `Deploy-LinuxBuild2404.ps1`: Ubuntu 24.04 cloud image deployment via VMware vmrun. +- `Prepare-LinuxBuild2404.ps1`: SSH key injection, CI dirs, IP reporting service. +- `Install-CIToolchain-Linux2404.sh`: GCC, G++, Clang, CMake, Ninja, Python 3, Git, 7-Zip. +- `scripts/_Transport.psm1`: `Invoke-SshCommand`, `Copy-SshItem`, `Test-SshReady` with + `StrictHostKeyChecking` and `KnownHostsFile` parametrisation. +- E2E verified: `nsis7z.dll` 2272 KB on parity with Windows build. + +--- + +## 2026-05-10 — Sprint 11: §6.6 Tier-2 Toolchain (WinBuild2025) + +Tier-1 (Step 11) and Tier-2 (Step 12) added to `Install-CIToolchain-WinBuild2025.ps1`: + +| Tool | Version | +| ---------------- | ---------------- | +| Git for Windows | 2.54.0.windows.1 | +| 7-Zip | 26.01 | +| PowerShell 7 | 7.6.1 | +| NSIS | 3.12 | +| gh CLI | 2.92.0 | +| CMake | 3.31.6 | +| Node.js LTS | 22.14.0 | +| WiX Toolset | 4.0.5 | +| Sysinternals | latest | +| vcpkg | HEAD | + +All tools hash-pinned (SHA256), path-extended machine-wide, validated via `where.exe` in +`Assert-Step` final gate. + +--- + +## 2026-05-10 — Sprint 10: §7 Refactor Deploy ↔ Setup + +Separated OS hardening (Deploy responsibility) from CI toolchain setup (Install-CIToolchain +responsibility). Setup Steps 1/3/4b/5b/5c converted to assertion-only (`Assert-Step`). + +New scripts: `Validate-DeployState.ps1`, `Validate-SetupState.ps1`. +WU lifecycle strategy B: Deploy sets GPO `NoAutoUpdate`; Install-CIToolchain runs WU then +disables wuauserv/UsoSvc permanently. + +--- + +## 2026-05-10 — Sprint 9: §3.3 In-VM Git Clone + +`-UseGitClone` switch added to `Invoke-CIJob.ps1` and `Invoke-RemoteBuild.ps1`. +- Skips host git clone + zip + WinRM transfer overhead. +- PAT injected at runtime, never persisted in snapshot. +- E2E result: 34 s saving, 25.7% reduction vs baseline on nsis-plugin-nsinnounp. + +--- + +## 2026-05-10 — Sprints 1-8: Core infrastructure + +| Sprint | Area | Key deliverable | +| ------ | ----------------------------- | ------------------------------------------------------ | +| 1 | §1.1 WinRM HTTP → HTTPS | Self-signed cert, port 5986, AllowUnencrypted=false | +| 2 | §1.4 IP validation | Per-octet regex extracted to `_Common.psm1` | +| 3 | §1.6 Threat model | `BEST-PRACTICES.md` §2.1 Threat Model documented | +| 4 | §2.1 IP race / lease | File-based IP allocator in `Invoke-CIJob.ps1` | +| 5 | §2.2-2.7 Reliability | Orphan cleanup, retention, WhatIf, snapshot versioning, backup, health watch | +| 6 | §3.1 NuGet/pip shared cache | `Set-TemplateSharedFolders.ps1`, `UseSharedCache` switch | +| 7 | §3.2 7-Zip compression | 7-Zip replaces Compress-Archive in `Invoke-RemoteBuild.ps1` | +| 8 | §3.6 Benchmark baseline | `Measure-CIBenchmark.ps1` — 5-phase JSONL output | +| - | §4.1 Structured logging | `Write-JobEvent` + `invoke-ci.jsonl` in `Invoke-CIJob.ps1` | +| - | §4.3 Disk alert | `Watch-DiskSpace.ps1` + scheduled task | +| - | §4.4 Runbook | `docs/RUNBOOK.md` — 5 incident categories | +| - | §5.1 Pester tests | `tests/` — 4 test files, fake vmrun harness | +| - | §5.2 `_Common.psm1` | `New-CISessionOption`, `Resolve-VmrunPath`, `Invoke-Vmrun` | +| - | §5.4 PSScriptAnalyzer | `PSScriptAnalyzerSettings.psd1` | diff --git a/docs/CI-FLOW.md b/docs/CI-FLOW.md index 0648dfd..328656c 100644 --- a/docs/CI-FLOW.md +++ b/docs/CI-FLOW.md @@ -270,7 +270,7 @@ automatically via `vmrun getGuestIPAddress` — no static assignment required. | -------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | | VM clone fails | `Invoke-CIJob.ps1` throws; no VM to destroy; job fails immediately | | VM never reaches ready state (timeout) | `Wait-VMReady.ps1` throws; `finally` destroys the (running) VM | -| Build fails inside VM | `Invoke-RemoteBuild.ps1` throws; `finally` collects partial artifacts (if any), destroys VM | +| Build fails inside VM | `Invoke-RemoteBuild.ps1` throws; `finally` runs `Get-GuestDiagnostics` (best-effort, to `F:\CI\Artifacts\\diagnostics\`), destroys VM | | Artifact collection fails | Job marked failed; VM is still destroyed in `finally` | | Host machine loses power mid-job | `act_runner` resumes on reboot, marks in-flight job as failed; orphan VMs must be cleaned up manually (see TODO.md) | diff --git a/docs/TEST-PLAN-v1.3-to-HEAD.md b/docs/TEST-PLAN-v1.3-to-HEAD.md index 515b104..153fb4a 100644 --- a/docs/TEST-PLAN-v1.3-to-HEAD.md +++ b/docs/TEST-PLAN-v1.3-to-HEAD.md @@ -471,22 +471,23 @@ Test-Path 'F:\CI\Logs\benchmark.jsonl' # Expected: True, file has 5 lines (one per iteration) # 3. Parse JSONL and examine timing breakdown +# Fields: iteration, template, snapshot, cloneSec, startSec, ipSec, winrmSec, destroySec, totalBootSec, vmIP, error $benchmarks = Get-Content 'F:\CI\Logs\benchmark.jsonl' | ConvertFrom-Json -$benchmarks | Select iteration, phase, duration_ms | Format-Table -# Expected: Phases (clone, start, ip, winrm, destroy) with millisecond durations +$benchmarks | Select-Object iteration, cloneSec, startSec, ipSec, winrmSec, destroySec, totalBootSec | Format-Table +# Expected: one row per iteration; all Sec fields populated; totalBootSec = cloneSec+startSec+ipSec+winrmSec -# 4. Calculate statistics -$times = $benchmarks | Where phase -eq 'clone' | Select -ExpandProperty duration_ms -$avg = ($times | Measure-Object -Average).Average -$min = ($times | Measure-Object -Minimum).Minimum -$max = ($times | Measure-Object -Maximum).Maximum -Write-Host "Clone phase: avg=${avg}ms, min=${min}ms, max=${max}ms" +# 4. Calculate clone statistics (values in seconds) +$cloneTimes = $benchmarks | Select-Object -ExpandProperty cloneSec +$avg = ($cloneTimes | Measure-Object -Average).Average +$min = ($cloneTimes | Measure-Object -Minimum).Minimum +$max = ($cloneTimes | Measure-Object -Maximum).Maximum +Write-Host "Clone phase: avg=${avg}s, min=${min}s, max=${max}s" ``` **Pass Criteria**: - [x] Script completes N iterations without error - [x] JSONL output created with correct structure -- [x] Five phases present (clone, start, ip, winrm, destroy) +- [x] Five phase fields present (cloneSec, startSec, ipSec, winrmSec, destroySec) - [x] Durations reasonable (clone 1-5s, start 2-5s, IP <5s, WinRM 1-3s, destroy <2s) - [x] Variance visible across iterations (natural fluctuation) diff --git a/plans/final-master-plan.md b/plans/final-master-plan.md index a6c05f0..7d54029 100644 --- a/plans/final-master-plan.md +++ b/plans/final-master-plan.md @@ -343,13 +343,13 @@ Every item is concrete, references a file, and ends with an effort estimate. - [x] done Delete deprecated runner installer — runner/Install-Runner.ps1 — file already absent from repo — confirmed 2026-05-13. - [x] done Align Setup-Host password documentation — [Setup-Host.ps1](../Setup-Host.ps1) help block has `Default: CIBuild!ChangeMe2026` with explicit "CHANGE THIS before production use" note — sufficient for homelab — 2026-05-13. - [x] done Remove emoji from webhook payloads — Watch-DiskSpace uses `[WARNING]`, Watch-RunnerHealth uses `[ALERT]`/`[WARNING]` — no emoji found — confirmed 2026-05-13. -- [ ] Fix Validate-* docblocks to reflect HTTPS/5986 — [template/Validate-DeployState.ps1](../template/Validate-DeployState.ps1), [template/Validate-SetupState.ps1](../template/Validate-SetupState.ps1) — update HTTP/Basic to HTTPS/Basic — 10min. -- [ ] Align benchmark JSON schema with test plan — [scripts/Measure-CIBenchmark.ps1](../scripts/Measure-CIBenchmark.ps1) or [docs/TEST-PLAN-v1.3-to-HEAD.md](../docs/TEST-PLAN-v1.3-to-HEAD.md) — pick one and update the other — 30min. +- [x] done Fix Validate-* docblocks to reflect HTTPS/5986 — Validate-SetupState.ps1 L7: `HTTP/Basic` → `HTTPS/Basic, port 5986` — done 2026-05-13. (Validate-DeployState already correct.) +- [x] done Align benchmark JSON schema with test plan — [docs/TEST-PLAN-v1.3-to-HEAD.md](../docs/TEST-PLAN-v1.3-to-HEAD.md) § 3.6 updated to match actual script output fields (`cloneSec`, `startSec`, `ipSec`, `winrmSec`, `destroySec`, `totalBootSec`) — done 2026-05-13. - [x] done Add `TemplatePath` sanity check — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — `Write-Warning` if not under `F:\CI\Templates\` — done 2026-05-13. - [x] done Add a maintenance flag for Watch-RunnerHealth — [scripts/Watch-RunnerHealth.ps1](../scripts/Watch-RunnerHealth.ps1) — skip restart if `F:\CI\State\runner-maintenance.flag` exists — done 2026-05-13. -- [ ] Add ACL validation script — new `scripts/Validate-HostState.ps1` — assert ACLs on `F:\CI\keys\ci_linux` and Credential Manager target accessible — 30min. -- [ ] Move done sections of [TODO.md](../TODO.md) into a new `docs/CHANGELOG.md` — 1h. -- [ ] Fix [docs/CI-FLOW.md](../docs/CI-FLOW.md) failure-scenario table — change "partial artifacts collected" to "artifacts collected only after successful build packaging" or implement H-level diagnostics; if H-level diagnostics added, leave docs as-is — 15min. +- [x] done Add ACL validation script — [scripts/Validate-HostState.ps1](../scripts/Validate-HostState.ps1) — SSH key ACLs, Credential Manager target, CI dir structure, vmrun.exe check — done 2026-05-13. +- [x] done Move done sections of [TODO.md](../TODO.md) into a new [docs/CHANGELOG.md](../docs/CHANGELOG.md) — reverse-chronological log of all completed sprints — done 2026-05-13. +- [x] done Fix [docs/CI-FLOW.md](../docs/CI-FLOW.md) failure-scenario table — "Build fails inside VM" row updated: `Get-GuestDiagnostics` to `diagnostics\` folder — done 2026-05-13. ### Nice to have [ ] diff --git a/scripts/Validate-HostState.ps1 b/scripts/Validate-HostState.ps1 new file mode 100644 index 0000000..fa734c4 --- /dev/null +++ b/scripts/Validate-HostState.ps1 @@ -0,0 +1,190 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Validates host-side security prerequisites for the CI system. + +.DESCRIPTION + Checks that: + 1. The SSH private key for Linux guest access exists and has restrictive ACLs + (only the current user and SYSTEM/Administrators may read it). + 2. The Windows Credential Manager target used by CI guest sessions is accessible. + 3. The CI key directories exist with expected permissions. + + Run after `Setup-Host.ps1` and any time credentials or keys are rotated. + Exits with code 0 on all-pass, non-zero if any check fails. + +.PARAMETER SshKeyPath + Path to the SSH private key used for Linux guest access. + Default: F:\CI\keys\ci_linux + +.PARAMETER GuestCredTarget + Windows Credential Manager target name for Windows guest credentials. + Default: BuildVMGuest + +.PARAMETER CIRootDir + Root CI directory to verify existence. + Default: F:\CI + +.EXAMPLE + .\Validate-HostState.ps1 + +.EXAMPLE + .\Validate-HostState.ps1 -SshKeyPath 'F:\CI\keys\ci_linux' -Verbose +#> +[CmdletBinding()] +param( + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + [string] $GuestCredTarget = 'BuildVMGuest', + [string] $CIRootDir = 'F:\CI' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +$script:passCount = 0 +$script:failCount = 0 + +function Write-Check { + param([string]$Label, [scriptblock]$Test) + try { + $result = & $Test + if ($result) { + Write-Host " [OK] $Label" + $script:passCount++ + } else { + Write-Host " [FAIL] $Label" + $script:failCount++ + } + } catch { + Write-Host " [FAIL] $Label — $_" + $script:failCount++ + } +} + +# ── Check 1: SSH key file exists ────────────────────────────────────────────── + +Write-Host "`n=== SSH Key ($SshKeyPath) ===" + +Write-Check "SSH private key file exists" { + Test-Path $SshKeyPath -PathType Leaf +} + +Write-Check "SSH public key file exists" { + Test-Path "${SshKeyPath}.pub" -PathType Leaf +} + +# Check ACLs: only SYSTEM, Administrators, and the current user should have access +Write-Check "SSH key ACL — no unexpected identities" { + if (-not (Test-Path $SshKeyPath -PathType Leaf)) { return $false } + + $acl = Get-Acl -Path $SshKeyPath + $currentUser = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name + + # Allowed identities (case-insensitive prefix matching) + $allowedPatterns = @( + 'NT AUTHORITY\SYSTEM', + 'BUILTIN\Administrators', + $currentUser + ) + + $unexpected = @() + foreach ($ace in $acl.Access) { + $identity = $ace.IdentityReference.Value + $matched = $false + foreach ($pattern in $allowedPatterns) { + if ($identity -ieq $pattern) { $matched = $true; break } + } + if (-not $matched) { + $unexpected += $identity + } + } + + if ($unexpected.Count -gt 0) { + Write-Verbose "[ACL] Unexpected identities on SSH key: $($unexpected -join ', ')" + return $false + } + return $true +} + +Write-Check "SSH key not world-readable (Everyone/Users absent)" { + if (-not (Test-Path $SshKeyPath -PathType Leaf)) { return $false } + + $acl = Get-Acl -Path $SshKeyPath + $broad = $acl.Access | Where-Object { + $id = $_.IdentityReference.Value + $id -ilike '*Everyone*' -or $id -ilike 'BUILTIN\Users' + } + return ($null -eq $broad -or @($broad).Count -eq 0) +} + +# ── Check 2: Credential Manager target ──────────────────────────────────────── + +Write-Host "`n=== Credential Manager (target: $GuestCredTarget) ===" + +Write-Check "CredentialManager module available" { + $null -ne (Get-Module -Name CredentialManager -ListAvailable -ErrorAction SilentlyContinue) +} + +Write-Check "Credential target '$GuestCredTarget' accessible" { + # Use cmdkey.exe as a fallback — avoids requiring CredentialManager module + $cmdkeyOut = & cmdkey.exe /list:$GuestCredTarget 2>&1 | Out-String + if ($cmdkeyOut -imatch [regex]::Escape($GuestCredTarget)) { + return $true + } + + # Try via CredentialManager module if available + if ($null -ne (Get-Module -Name CredentialManager -ListAvailable -ErrorAction SilentlyContinue)) { + Import-Module CredentialManager -ErrorAction SilentlyContinue + $cred = Get-StoredCredential -Target $GuestCredTarget -ErrorAction SilentlyContinue + return ($null -ne $cred) + } + + return $false +} + +# ── Check 3: CI directory structure ─────────────────────────────────────────── + +Write-Host "`n=== CI Directory Structure ($CIRootDir) ===" + +$requiredDirs = @( + 'BuildVMs', + 'Artifacts', + 'Logs', + 'State', + 'keys', + 'Cache\NuGet', + 'Cache\pip', + 'Templates' +) + +foreach ($rel in $requiredDirs) { + $full = Join-Path $CIRootDir $rel + Write-Check "Directory exists: $rel" { Test-Path $full -PathType Container } +} + +# ── Check 4: vmrun.exe reachable ────────────────────────────────────────────── + +Write-Host "`n=== VMware ===" + +Write-Check "vmrun.exe found in PATH or default location" { + $vmrunDefault = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' + $inPath = $null -ne (Get-Command 'vmrun.exe' -ErrorAction SilentlyContinue) + $atDefault = Test-Path $vmrunDefault -PathType Leaf + $inPath -or $atDefault +} + +# ── Summary ─────────────────────────────────────────────────────────────────── + +Write-Host "`n=== Summary ===" +Write-Host " Passed : $($script:passCount)" +Write-Host " Failed : $($script:failCount)" + +if ($script:failCount -gt 0) { + Write-Host "`n[Validate-HostState] $($script:failCount) check(s) failed. Review output above." -ForegroundColor Red + exit 1 +} else { + Write-Host "`n[Validate-HostState] All checks passed." -ForegroundColor Green + exit 0 +} diff --git a/template/Validate-SetupState.ps1 b/template/Validate-SetupState.ps1 index c8bdd41..7bd9776 100644 --- a/template/Validate-SetupState.ps1 +++ b/template/Validate-SetupState.ps1 @@ -4,7 +4,7 @@ Validates full post-Setup state of a template VM (Deploy + Setup). .DESCRIPTION - Connects to the VM via WinRM (HTTP/Basic) and runs all checks for the + Connects to the VM via WinRM (HTTPS/Basic, port 5986) and runs all checks for the final BaseClean snapshot state: Deploy OS hardening + Setup CI toolchain. Run this after Prepare-WinBuild2025.ps1 completes (exit 0) to confirm the VM is ready for snapshot and production use.