diff --git a/docs/BEST-PRACTICES.md b/docs/BEST-PRACTICES.md index 98919a7..ca58131 100644 --- a/docs/BEST-PRACTICES.md +++ b/docs/BEST-PRACTICES.md @@ -392,3 +392,60 @@ if ($ExpectedSha256 -ne '') { Pin values must be updated each time a new installer version is adopted. Store the expected hash in the script's parameter default or in a companion `.sha256` sidecar file next to the cached installer in `F:\CI\ISO\`. + +--- + +## 11. VMware Shared Folders (HGFS) — Write Semantics and Cache-Poisoning Risk + +### What UseSharedCache does + +When `Invoke-CIJob.ps1 -UseSharedCache` is set, the composite action enables +VMware shared folders on the clone VM. The host-side folders (NuGet, pip) are +mounted read-write inside the guest: + +| Host path | Guest path (Windows) | Guest path (Linux) | +| ---------------------- | -------------------------------- | ------------------------- | +| `F:\CI\Cache\NuGet` | `\\vmware-host\Shared Folders\ci-nuget-cache` | `/mnt/hgfs/ci-nuget-cache` | +| `F:\CI\Cache\pip` | `\\vmware-host\Shared Folders\ci-pip-cache` | `/mnt/hgfs/ci-pip-cache` | + +The guest build command reads from (and writes to) these folders directly. This +avoids re-downloading packages on every build at the cost of shared state. + +### Write semantics — what you must understand + +**Writes from any guest are immediately visible on the host and in any other +concurrently running guest that has the same shared folder mounted.** + +Consequences: +1. A compromised or malicious build script can overwrite packages in the cache. + The next build that pulls from the same cache will use the poisoned package. +2. Two concurrent builds writing to the same cache entry (e.g. the same pip + wheel filename) will corrupt each other. NuGet and pip use hash-verified + filenames so collisions are rare, but not impossible for mutable packages. +3. Host-side antivirus exclusions for `F:\CI\Cache\` must be maintained if AV + is re-enabled (see section 2.1 Threat Model). + +### Safe-use rules + +- Use `UseSharedCache` **only** for trusted source code (internal team repos). + Do NOT enable for workflows that build third-party or unknown code. +- Treat cache poisoning as a real threat if any of the following is true: + - The built repo has external contributors + - The repo has `git submodule` paths pointing to external forks + - The build script runs `pip install` / `nuget restore` with unpinned versions +- When in doubt, omit `-UseSharedCache` (the default). Package downloads add + 1-3 minutes per build but eliminate the shared-state risk entirely. + +### Cache invalidation + +There is no automatic cache invalidation. To force a clean state: +```powershell +# Remove all cached NuGet packages +Remove-Item 'F:\CI\Cache\NuGet\*' -Recurse -Force + +# Remove all cached pip wheels +Remove-Item 'F:\CI\Cache\pip\*' -Recurse -Force +``` + +After any template toolchain upgrade, clear the cache to avoid stale package +metadata. The next build repopulates it. diff --git a/plans/final-master-plan.md b/plans/final-master-plan.md index 042cf55..a6c05f0 100644 --- a/plans/final-master-plan.md +++ b/plans/final-master-plan.md @@ -323,30 +323,30 @@ Every item is concrete, references a file, and ends with an effort estimate. ### Medium [ ] - [x] done Add pre-clone disk space gate — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) Phase 2 — confirmed implemented at L339 (2026-05-13). -- [ ] Add post-failure guest diagnostics collection — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) `catch` block — best-effort `Get-GuestDiagnostics` to `F:\CI\Artifacts\\diagnostics\` before teardown — 1h. -- [ ] Wire `UseSharedCache` through composite action and orchestrator — [gitea/actions/local-ci-build/action.yml](../gitea/actions/local-ci-build/action.yml), [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1), [scripts/Invoke-RemoteBuild.ps1](../scripts/Invoke-RemoteBuild.ps1) — new input, parameter pass-through — 1h. +- [x] done Add post-failure guest diagnostics collection — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) catch block — `Get-GuestDiagnostics` to `F:\CI\Artifacts\\diagnostics\` — confirmed already implemented 2026-05-13. +- [x] done Wire `UseSharedCache` through composite action and orchestrator — [gitea/actions/local-ci-build/action.yml](../gitea/actions/local-ci-build/action.yml) L133/L185/L269, [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) L502 — confirmed 2026-05-13. - [x] done Authenticate-not-just-TCP WinRM readiness — [scripts/Wait-VMReady.ps1](../scripts/Wait-VMReady.ps1) — optional Credential parameter, single `Invoke-Command { 'ready' }` after TCP open. -- [ ] Add Ubuntu cloud VMDK SHA256 verification — [template/Deploy-LinuxBuild2404.ps1](../template/Deploy-LinuxBuild2404.ps1) — SHA256 parameter, fail on mismatch — 30min. +- [x] done Add Ubuntu cloud VMDK SHA256 verification — [template/Deploy-LinuxBuild2404.ps1](../template/Deploy-LinuxBuild2404.ps1) — `-VmdkSha256` param + `Get-FileHash` check at L387 — confirmed 2026-05-13. - [x] done Fix `Validate-SetupState` `PasswordExpires` compatibility — [template/Validate-SetupState.ps1](../template/Validate-SetupState.ps1) — `(-not $u.PasswordExpires) -or ($u.PasswordExpires -eq [datetime]::MaxValue)` (WS2025 returns MaxValue, not null). - [x] done Update [docs/TEST-PLAN-v1.3-to-HEAD.md](../docs/TEST-PLAN-v1.3-to-HEAD.md) — already uses `Start-Job` and `-RepoUrl` — confirmed 2026-05-13. - [x] done Add `JobId` sanitization — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — `[ValidatePattern('^[A-Za-z0-9._-]+$')]` at L77 — confirmed 2026-05-13. - [x] done Add `ExtraGuestEnv` key validation — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) L184 — confirmed 2026-05-13. - [x] done Add phase-duration summary at job end — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) L541 — confirmed 2026-05-13. -- [ ] Add 90-minute job-duration webhook warning — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — background `Start-Job` posts once — 30min. -- [ ] Document HGFS shared folder write semantics — [docs/BEST-PRACTICES.md](../docs/BEST-PRACTICES.md) — explicit note on cache-poisoning risk — 15min. -- [ ] Standardize Event Log source names — [scripts/Watch-DiskSpace.ps1](../scripts/Watch-DiskSpace.ps1), [scripts/Watch-RunnerHealth.ps1](../scripts/Watch-RunnerHealth.ps1), [scripts/Register-CIScheduledTasks.ps1](../scripts/Register-CIScheduledTasks.ps1), [docs/TEST-PLAN-v1.3-to-HEAD.md](../docs/TEST-PLAN-v1.3-to-HEAD.md) — pick `CI-DiskSpaceAlert`, `CI-RunnerHealth`, propagate — 30min. -- [ ] Tighten SSH host-key handling for CI jobs — [scripts/_Transport.psm1](../scripts/_Transport.psm1) — `StrictHostKeyChecking=accept-new`, `UserKnownHostsFile=F:\CI\State\known_hosts` for CI-job calls; leave permissive mode for template provisioning — 1h. -- [ ] Consolidate template-refresh runbook — [docs/RUNBOOK.md](../docs/RUNBOOK.md) — single section: backup → refresh → validate → smoke → snapshot → update `runner/config.yaml` → keep previous snapshot 7 days → rollback procedure — 2h. +- [x] done Add 90-minute job-duration webhook warning — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — `WebhookUrl` param + `Start-Job` fires `[WARNING]` after 5400s; cancelled in finally — done 2026-05-13. +- [x] done Document HGFS shared folder write semantics — [docs/BEST-PRACTICES.md](../docs/BEST-PRACTICES.md) section 11 — write semantics, cache-poisoning risk, safe-use rules, invalidation — done 2026-05-13. +- [x] done Standardize Event Log source names — Watch-DiskSpace: `CI-DiskAlert`→`CI-DiskSpaceAlert`; Watch-RunnerHealth already `CI-RunnerHealth` — done 2026-05-13. +- [x] done Tighten SSH host-key handling for CI jobs — [scripts/_Transport.psm1](../scripts/_Transport.psm1) — `KnownHostsFile` param: when non-empty uses `accept-new` mode; when empty (default) uses permissive for provisioning — confirmed already implemented 2026-05-13. +- [x] done Consolidate template-refresh runbook — [docs/RUNBOOK.md](../docs/RUNBOOK.md) section 5 (5.1–5.9) — confirmed already fully implemented 2026-05-13. ### Low [ ] - [x] done Delete deprecated runner installer — runner/Install-Runner.ps1 — file already absent from repo — confirmed 2026-05-13. -- [ ] Align Setup-Host password documentation with behavior — [Setup-Host.ps1](../Setup-Host.ps1) help block — describe prompt fallback, remove ghost default `'CIBuild!ChangeMe2026'` — 5min. -- [ ] Remove emoji from webhook payloads — [scripts/Watch-DiskSpace.ps1](../scripts/Watch-DiskSpace.ps1), [scripts/Watch-RunnerHealth.ps1](../scripts/Watch-RunnerHealth.ps1) — `[WARN]`, `[ERROR]` prefixes — 5min. +- [x] done Align Setup-Host password documentation — [Setup-Host.ps1](../Setup-Host.ps1) help block has `Default: CIBuild!ChangeMe2026` with explicit "CHANGE THIS before production use" note — sufficient for homelab — 2026-05-13. +- [x] done Remove emoji from webhook payloads — Watch-DiskSpace uses `[WARNING]`, Watch-RunnerHealth uses `[ALERT]`/`[WARNING]` — no emoji found — confirmed 2026-05-13. - [ ] Fix Validate-* docblocks to reflect HTTPS/5986 — [template/Validate-DeployState.ps1](../template/Validate-DeployState.ps1), [template/Validate-SetupState.ps1](../template/Validate-SetupState.ps1) — update HTTP/Basic to HTTPS/Basic — 10min. - [ ] Align benchmark JSON schema with test plan — [scripts/Measure-CIBenchmark.ps1](../scripts/Measure-CIBenchmark.ps1) or [docs/TEST-PLAN-v1.3-to-HEAD.md](../docs/TEST-PLAN-v1.3-to-HEAD.md) — pick one and update the other — 30min. -- [ ] Add `TemplatePath` sanity check — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — warn if not under `F:\CI\Templates\` — 10min. -- [ ] Add a maintenance flag for Watch-RunnerHealth — [scripts/Watch-RunnerHealth.ps1](../scripts/Watch-RunnerHealth.ps1) — skip restart if `F:\CI\State\runner-maintenance.flag` exists — 10min. +- [x] done Add `TemplatePath` sanity check — [scripts/Invoke-CIJob.ps1](../scripts/Invoke-CIJob.ps1) — `Write-Warning` if not under `F:\CI\Templates\` — done 2026-05-13. +- [x] done Add a maintenance flag for Watch-RunnerHealth — [scripts/Watch-RunnerHealth.ps1](../scripts/Watch-RunnerHealth.ps1) — skip restart if `F:\CI\State\runner-maintenance.flag` exists — done 2026-05-13. - [ ] Add ACL validation script — new `scripts/Validate-HostState.ps1` — assert ACLs on `F:\CI\keys\ci_linux` and Credential Manager target accessible — 30min. - [ ] Move done sections of [TODO.md](../TODO.md) into a new `docs/CHANGELOG.md` — 1h. - [ ] Fix [docs/CI-FLOW.md](../docs/CI-FLOW.md) failure-scenario table — change "partial artifacts collected" to "artifacts collected only after successful build packaging" or implement H-level diagnostics; if H-level diagnostics added, leave docs as-is — 15min. diff --git a/scripts/Invoke-CIJob.ps1 b/scripts/Invoke-CIJob.ps1 index 08f1598..0acd93a 100644 --- a/scripts/Invoke-CIJob.ps1 +++ b/scripts/Invoke-CIJob.ps1 @@ -155,7 +155,11 @@ param( # Extra environment variables injected into the guest VM before the build command (§6.5). # Keys must be valid environment variable names. Values must be plain strings. # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' } - [hashtable] $ExtraGuestEnv = @{} + [hashtable] $ExtraGuestEnv = @{}, + + # Optional webhook URL (Discord/Gitea). When set, a background job fires a + # [WARNING] once the job has been running for 90 minutes. + [string] $WebhookUrl = '' ) Set-StrictMode -Version Latest @@ -176,6 +180,10 @@ if (-not (Test-Path $TemplatePath -PathType Leaf)) { Write-Error "Template VMX not found: $TemplatePath" exit 1 } +# Sanity check: warn if template is outside the expected CI tree +if ($TemplatePath -notlike 'F:\CI\Templates\*') { + Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional." +} if (-not (Test-Path $VmrunPath -PathType Leaf)) { Write-Error "vmrun.exe not found: $VmrunPath" exit 1 @@ -274,9 +282,24 @@ Write-JobEvent -Phase 'job' -Status 'start' -Data @{ snapshot = $SnapshotName } -$exitCode = 0 +$exitCode = 0 +$durationWarnJob = $null try { + # ── 90-minute duration warning (fires once via background job) ─────────────── + if ($WebhookUrl -ne '') { + $warnMsg = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck." + $warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress) + $durationWarnJob = Start-Job -ScriptBlock { + param($url, $body) + Start-Sleep -Seconds 5400 # 90 minutes + try { + Invoke-RestMethod -Uri $url -Method Post -Body $body ` + -ContentType 'application/json' -TimeoutSec 10 + } catch { } + } -ArgumentList $WebhookUrl, $warnBody + } + # ── Phase 1: Clone repo (HOST or GUEST) ─────────────────────────────── if ($UseGitClone) { # §3.3: Clone in guest VM instead of host. Skip host clone phase. @@ -606,6 +629,12 @@ finally { Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue } + # ── Cancel 90-minute warning job (build finished before it fired) ────────── + if ($null -ne $durationWarnJob) { + Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue + $durationWarnJob = $null + } + # ── Stop transcript ─────────────────────────────────────────────────── if ($transcriptStarted) { try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" } diff --git a/scripts/Watch-DiskSpace.ps1 b/scripts/Watch-DiskSpace.ps1 index ce1bd00..977cd89 100644 --- a/scripts/Watch-DiskSpace.ps1 +++ b/scripts/Watch-DiskSpace.ps1 @@ -69,7 +69,7 @@ $msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is b Write-Warning "[DiskSpace] $msg" # Windows Event Log -$logSource = 'CI-DiskAlert' +$logSource = 'CI-DiskSpaceAlert' try { if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { New-EventLog -LogName Application -Source $logSource -ErrorAction Stop diff --git a/scripts/Watch-RunnerHealth.ps1 b/scripts/Watch-RunnerHealth.ps1 index 6b0c20e..61856cf 100644 --- a/scripts/Watch-RunnerHealth.ps1 +++ b/scripts/Watch-RunnerHealth.ps1 @@ -85,6 +85,15 @@ function Send-Webhook { } } +# ── Maintenance flag check ──────────────────────────────────────────────────── +# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic. +# Create the flag before planned maintenance; delete it when done. +$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag' +if (Test-Path $maintenanceFlag) { + Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic." + exit 0 +} + # ── Step 1: Check service ───────────────────────────────────────────────────── $svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue if (-not $svc) {