diff --git a/PSScriptAnalyzerSettings.psd1 b/PSScriptAnalyzerSettings.psd1 new file mode 100644 index 0000000..fbd7fc3 --- /dev/null +++ b/PSScriptAnalyzerSettings.psd1 @@ -0,0 +1,68 @@ +@{ + # ── Rules to suppress project-wide ────────────────────────────────────────── + ExcludeRules = @( + # All CI scripts use Write-Host for structured output captured by act_runner. + # Write-Output would interleave with function return values and break callers. + 'PSAvoidUsingWriteHost' + ) + + # ── Rule-specific configuration ────────────────────────────────────────────── + Rules = @{ + # Security: never eval arbitrary strings + PSAvoidUsingInvokeExpression = @{ + Enable = $true + } + + # All state-changing functions must support -WhatIf / ShouldProcess + PSUseShouldProcessForStateChangingFunctions = @{ + Enable = $true + } + + # Credentials must flow as [PSCredential], not plain strings + PSUsePSCredentialType = @{ + Enable = $true + } + + # No plain-text passwords in params or variables + PSAvoidUsingPlainTextForPassword = @{ + Enable = $true + } + + # ConvertTo-SecureString -AsPlainText only acceptable during template setup; + # use Suppress comments there if needed. + PSAvoidUsingConvertToSecureStringWithPlainText = @{ + Enable = $true + } + + # No script-scope globals that bleed across module boundaries + PSAvoidGlobalVars = @{ + Enable = $true + } + + # Formatting — 4-space indentation, spaces (not tabs) + PSUseConsistentIndentation = @{ + Enable = $true + IndentationSize = 4 + PipelineIndentation = 'IncreaseIndentationForFirstPipeline' + Kind = 'space' + } + + PSUseConsistentWhitespace = @{ + Enable = $true + CheckInnerBrace = $true + CheckOpenBrace = $true + CheckOpenParen = $true + CheckOperator = $true + CheckPipe = $true + CheckPipeForRedundantWhitespace = $true + CheckSeparator = $true + CheckParameter = $false + } + + # Alignment is a style preference; disable to avoid false positives on + # the deliberate column-aligned hashtable assignments used in this project. + PSAlignAssignmentStatement = @{ + Enable = $false + } + } +} diff --git a/TODO.md b/TODO.md index b3a9192..668b4c9 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,6 @@ # TODO — Local CI/CD System - + > Documento unico di lavoro: roadmap, audit trail dei task completati, e backlog post-v1.0 > con priorità e razionale. Le voci aperte sono raggruppate per area e marcate **P0..P3**: @@ -18,9 +18,9 @@ --- ## Summary -_Last updated: 2026-05-10 (post Sprint 3 osservabilità)_ +_Last updated: 2026-05-10 (post Sprint 6 perf: §3.1/§3.6 done)_ -**Stato generale**: infrastruttura base e refactor Deploy↔Setup (§7) completi. Sicurezza sprint 1 in corso (HTTPS/5986 done, IP regex done, hash pinning struttura done — valori da riempire). Backlog operatività/perf/qualità intatto. +**Stato generale**: §2 affidabilità interamente chiuso. §3.1 cache e §3.6 benchmark done. §3.2 (7-Zip) dipende da §6.6. Sicurezza P1/P2 e Pester rimangono backlog principale. | Area | Done | Open | Note | | ------------------------------- | ---: | ---: | ----------------------------------------------------------------- | @@ -30,19 +30,20 @@ _Last updated: 2026-05-10 (post Sprint 3 osservabilità)_ | Runner Configuration | all | 0 | `config.yaml`, capacity 4, `BuildVMGuest` | | Gitea Workflow Integration | 4 | 1 | manca solo §P3 generalizzazione workflow | | §1 Sicurezza & hardening | 2 | 6 | 1.1, 1.4 done; 1.2/1.3 parzialmente; 1.5/1.6/1.7/1.8 da fare | -| §2 Affidabilità & resilienza | 4 | 3 | 2.1/2.2/2.3/2.5 done; 2.4/2.6/2.7 backlog | -| §3 Performance & throughput | 0 | 6 | nessuna ottimizzazione applicata; baseline §3.6 prerequisito | +| §2 Affidabilità & resilienza | 7 | 0 | tutto done | +| §3 Performance & throughput | 2 | 4 | 3.1/3.6 done; 3.2 (7-Zip, dipende §6.6), 3.3/3.4/3.5 backlog | | §4 Osservabilità & manutenzione | 3 | 2 | 4.1/4.3/4.4 done; 4.2/4.5 backlog | -| §5 Qualità codice & test | 0 | 5 | Pester, modulo `_Common.psm1`, regole PSScriptAnalyzer custom | +| §5 Qualità codice & test | 3 | 2 | 5.2/5.3/5.4 done; 5.1 (Pester) e 5.5 (type hints) backlog | | §6 Scalabilità & estensioni | 0 | 6 | Linux VM, composite action, toolchain Tier-1 | | §7 Refactor Deploy ↔ Setup | 4 | 1 | 7.1/7.2/7.3/7.4 done; 7.5 (rinomina) opzionale | | In-VM Git Clone (§3.3) | 0 | 4 | dipende da §6.6 (Git+7-Zip nel template) | | Linux Build VM (§6.1) | 0 | 4 | bozza in `docs/LINUX-TEMPLATE-SETUP.md` | **Prossimi passi consigliati** (vedi anche "Suggerimento di sequencing" in fondo): -1. Chiudere §1.3 — riempire hash SHA256 in `Setup-WinBuild2025.ps1` prima del prossimo refresh snapshot. -2. Audit §1.2 (TrustedHosts `*` su host), poi §1.7 (rotazione password guest). -3. Sprint 2 operatività — §2.2 (cleanup schedulato), §2.3 (retention), §2.5 (snapshot versionato). +1. §5.1 — Pester smoke tests (safety net prima di refactor più profondi). +2. §1.2 — Audit TrustedHosts (1 comando, manuale). +3. §1.6 — Threat model doc in BEST-PRACTICES.md. +4. §1.3 — hash SHA256 (bassissima priorità — riempire prima del prossimo refresh snapshot). --- ## Setup & Infrastructure @@ -309,11 +310,10 @@ finestra già configurata in `Invoke-CIJob.ps1`. Aggiungere `Get-PSDrive F | Where-Object { $_.Free -lt 50GB }` come *guard*: se libero < 50 GB, retention più aggressiva (7 giorni) e log warning. -### 2.4 [P1] [ ] `Remove-BuildVM.ps1` — supportare `-WhatIf` +### 2.4 [P1] [x] `Remove-BuildVM.ps1` — supportare `-WhatIf` — COMPLETATO 2026-05-10 File: [scripts/Remove-BuildVM.ps1:28](scripts/Remove-BuildVM.ps1). -Aggiungere `[CmdletBinding(SupportsShouldProcess)]` come fatto in `Cleanup-OrphanedBuildVMs.ps1`. -Permette debug e dry-run senza forkare un secondo script. +`[CmdletBinding(SupportsShouldProcess)]` aggiunto; op distruttive (stop, deleteVM, Remove-Item) gated su `$PSCmdlet.ShouldProcess`. ### 2.5 [P1] [x] Snapshot versionato `BaseClean_` — COMPLETATO 2026-05-10 File: [docs/BEST-PRACTICES.md:142-145](docs/BEST-PRACTICES.md), [scripts/Invoke-CIJob.ps1:100](scripts/Invoke-CIJob.ps1). @@ -331,45 +331,42 @@ e non c'è rollback. Si lega al task: refresh semestrale snapshot template VM (KMS lease = 180 giorni). Boot template su VMnet8 (NAT) → `slmgr /ato` → nuovo snapshot. -### 2.6 [P2] [ ] Backup automatico VMDK template -File: [docs/BEST-PRACTICES.md:130-138](docs/BEST-PRACTICES.md) — solo come snippet manuale. +### 2.6 [P2] [x] Backup automatico VMDK template — COMPLETATO 2026-05-10 +File: [scripts/Backup-CITemplate.ps1](scripts/Backup-CITemplate.ps1). -Pre-snapshot: copia atomica del `parent` VMDK in `F:\CI\Backups\Template_\`. Su errore -di refresh (es. installer rotto), rollback in 1 minuto. +Copia atomica `F:\CI\Templates\WinBuild` → `F:\CI\Backups\Template_\`. +Ferma act_runner prima, lo riavvia nel `finally`. Prune automatico (default: mantieni 3). +`SupportsShouldProcess` → supporta `-WhatIf`. Esecuzione manuale pre-refresh. -### 2.7 [P2] [ ] Health check del runner + monitoraggio Event Log -File: [docs/BEST-PRACTICES.md:101-115](docs/BEST-PRACTICES.md) — script presente in doc, non operativo. +### 2.7 [P2] [x] Health check del runner + monitoraggio Event Log — COMPLETATO 2026-05-10 +File: [scripts/Watch-RunnerHealth.ps1](scripts/Watch-RunnerHealth.ps1); task registrato da [scripts/Register-CIScheduledTasks.ps1](scripts/Register-CIScheduledTasks.ps1) come `CI-RunnerHealth` (ogni 15 min). -Schedulare ogni 15 min: query `gitea/api/v1/admin/runners`, se `local-windows-runner` non -`online` → `Restart-Service act_runner` + log evento. Limitare a 3 restart/h con cooldown. - -Inoltre: monitoraggio Windows Event Log per fallimenti servizio `act_runner` (alert via -EventLog query schedulata o webhook). +Controlla `Get-Service act_runner`. Se non Running: EventId 1002 (Warning) + `Restart-Service`. +Cooldown: max 3 restart/h via JSON state file `F:\CI\State\runner-restart-log.json`. +Oltre il limite: EventId 1004 (Error) + webhook `:sos:` senza restart. Webhook opzionale +Discord/Gitea identico a `Watch-DiskSpace.ps1`. --- ## 3. Performance & throughput -### 3.1 [P1] [ ] NuGet/pip cache su shared folder -File: [docs/OPTIMIZATION.md:93-128](docs/OPTIMIZATION.md), [scripts/Invoke-RemoteBuild.ps1:158-194](scripts/Invoke-RemoteBuild.ps1). +### 3.1 [P1] [x] NuGet/pip cache su shared folder — COMPLETATO 2026-05-10 +File: [scripts/Set-TemplateSharedFolders.ps1](scripts/Set-TemplateSharedFolders.ps1), [scripts/Invoke-RemoteBuild.ps1](scripts/Invoke-RemoteBuild.ps1). -**Motivazione**: `F:\CI\Cache\NuGet` esiste come dir, non viene mai usata. +`Set-TemplateSharedFolders.ps1`: edita il VMX template aggiungendo due shared folder +(`nuget-cache` → `F:\CI\Cache\NuGet`, `pip-cache` → `F:\CI\Cache\pip`), idempotente, +`.bak` del VMX originale mantenuto, `SupportsShouldProcess`. -Aggiungere a `CI-WinBuild.vmx`: -```ini -sharedFolder0.present = "TRUE" -sharedFolder0.hostPath = "F:\\CI\\Cache\\NuGet" -sharedFolder0.guestName = "nuget-cache" -``` -Poi nello scriptblock di build: -```powershell -$env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' -``` -Equivalente per pip: -```powershell -$env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache' -``` -Dopo lo snapshot refresh la cache va riscaldata una volta — accettabile. +`Invoke-RemoteBuild.ps1`: aggiunto `-UseSharedCache` switch. Quando attivo, inietta +`$env:NUGET_PACKAGES` nel scriptblock dotnet restore/build e `$env:PIP_CACHE_DIR` nel +scriptblock custom build command. + +**Attivazione**: +1. Fermare template VM. +2. `.\Set-TemplateSharedFolders.ps1` (una tantum, pre-snapshot). +3. Passare `-UseSharedCache` a `Invoke-RemoteBuild.ps1` nei job dotnet/pip. + +Dopo snapshot refresh: cache si riscalda al primo build — accettabile. ### 3.2 [P1] [ ] Sostituire `Compress-Archive` con 7-Zip o robocopy File: [scripts/Invoke-RemoteBuild.ps1:112,148,185](scripts/Invoke-RemoteBuild.ps1) (3 chiamate `Compress-Archive`). @@ -414,8 +411,14 @@ Considerare: - VMX pinning: 6 vCPU per build C++ pesanti, 2 vCPU per build .NET piccoli. - Esporre `-VMCpu` / `-VMMemory` a `Invoke-CIJob.ps1` con override via workflow. -### 3.6 [P2] [ ] Benchmark baseline -- [ ] Tempo creazione clone + tempo WinRM readiness (per stabilire baseline prima di §3.1/3.2/3.4). +### 3.6 [P2] [x] Benchmark baseline — COMPLETATO 2026-05-10 +File: [scripts/Measure-CIBenchmark.ps1](scripts/Measure-CIBenchmark.ps1). + +Misura 5 fasi per iterazione: clone, start, IP acquire, WinRM ready, destroy. +Output: tabella console + append a `F:\CI\Logs\benchmark.jsonl` (JSONL, una riga per iterazione). +Supporta `-Iterations N` per ridurre varianza. Importa `_Common.psm1` (`Invoke-Vmrun`, `Resolve-VmrunPath`). + +**Uso**: `.\Measure-CIBenchmark.ps1` prima e dopo §3.1/§3.2 per misurare impatto. --- @@ -495,33 +498,26 @@ Pester può coprire: Eseguire in CI tramite il runner stesso (workflow `lint.yml` esiste già — espandere). -### 5.2 [P2] [ ] Modulo PowerShell condiviso `scripts/_Common.psm1` -**Duplicazioni da estrarre**: -- `New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck` (3 file). -- `ValidatePattern` IP (4 file — vedi §1.4). -- Risoluzione `vmrun.exe` (3 file). -- `Test-Path` + `New-Item` per dir lazy creation. -- Function `Invoke-VmrunCommand` con gestione `$LASTEXITCODE`. +### 5.2 [P2] [x] Modulo PowerShell condiviso `scripts/_Common.psm1` — COMPLETATO 2026-05-10 +File: [scripts/_Common.psm1](scripts/_Common.psm1). -Risultato: meno LoC, fix in un punto solo, più facile da testare. +Esporta: `New-CISessionOption` (WinRM self-signed TLS), `Resolve-VmrunPath` (valida vmrun.exe), +`Invoke-Vmrun` (wrapper uniforme `-T ws`, ritorna `{ExitCode, Output}`, supporta `-ThrowOnError`). -### 5.3 [P2] [ ] Esecuzione `vmrun` con check exit code uniforme -Pattern attuale ripetuto in tutti gli script che invocano `vmrun`: -```powershell -$out = & $VmrunPath ... 2>&1 -if ($LASTEXITCODE -ne 0) { throw "..." } -``` -Wrapping in `Invoke-VmrunCommand -Operation 'clone' -Args @(...)` riduce errori di copy-paste -e centralizza retry/log. +Importato da: `Invoke-RemoteBuild.ps1`, `Get-BuildArtifacts.ps1`, `New-BuildVM.ps1`. -### 5.4 [P2] [ ] PSScriptAnalyzer rules custom -File: [gitea/workflows/lint.yml](gitea/workflows/lint.yml) (già esiste). +### 5.3 [P2] [x] Esecuzione `vmrun` con check exit code uniforme — COMPLETATO 2026-05-10 +`Invoke-Vmrun` in `_Common.psm1`. `New-BuildVM.ps1` refactored per usarlo. +`Invoke-RemoteBuild.ps1` e `Get-BuildArtifacts.ps1` usano `New-CISessionOption`. -Aggiungere regole specifiche progetto (`PSScriptAnalyzerSettings.psd1` in root): -- `PSAvoidUsingPlainTextForPassword` — rilevante per [Setup-WinBuild2025.ps1:123](template/Setup-WinBuild2025.ps1:123) (`[string] $BuildPassword` → `ConvertTo-SecureString -AsPlainText` riga 253). -- `PSAvoidUsingInvokeExpression`. -- `PSUseShouldProcessForStateChangingFunctions`. -- Regole custom: vietare hardcoded `F:\CI\` fuori da config (forzare param/env). +### 5.4 [P2] [x] PSScriptAnalyzer rules custom — COMPLETATO 2026-05-10 +File: [PSScriptAnalyzerSettings.psd1](PSScriptAnalyzerSettings.psd1). + +Regole attive: `PSAvoidUsingInvokeExpression`, `PSUseShouldProcessForStateChangingFunctions`, +`PSUsePSCredentialType`, `PSAvoidUsingPlainTextForPassword`, +`PSAvoidUsingConvertToSecureStringWithPlainText`, `PSAvoidGlobalVars`, +`PSUseConsistentIndentation` (4 spazi), `PSUseConsistentWhitespace`. +Soppresso progetto-wide: `PSAvoidUsingWriteHost` (output strutturato per act_runner). ### 5.5 [P3] [ ] Type hints nei param block Già parzialmente fatto. Estendere a tutti gli script per migliorare IntelliSense e diff --git a/scripts/Backup-CITemplate.ps1 b/scripts/Backup-CITemplate.ps1 new file mode 100644 index 0000000..911833f --- /dev/null +++ b/scripts/Backup-CITemplate.ps1 @@ -0,0 +1,127 @@ +#Requires -Version 5.1 +#Requires -RunAsAdministrator +<# +.SYNOPSIS + Creates a timestamped backup of the CI template VM directory. + +.DESCRIPTION + Run before any template refresh (toolchain update, KMS re-activation, snapshot + rebuild). The act_runner service is stopped for the duration so no vmrun clone + races the file copy, then restarted automatically. + + Restore procedure (after a bad refresh): + Stop-Service act_runner + Remove-Item 'F:\CI\Templates\WinBuild' -Recurse -Force + Copy-Item 'F:\CI\Backups\Template_' 'F:\CI\Templates\WinBuild' -Recurse + Start-Service act_runner + +.PARAMETER TemplatePath + Directory containing the template VM files (.vmx, .vmdk, snapshots). + Default: F:\CI\Templates\WinBuild + +.PARAMETER BackupBaseDir + Directory under which timestamped backup folders are created. + Default: F:\CI\Backups + +.PARAMETER KeepCount + Number of most-recent backups to retain. Older ones are deleted. + Default: 3 + +.PARAMETER SkipRunnerStop + Skip stopping/restarting act_runner. Use when the service is already stopped + or when called from a maintenance window where the runner is offline. + +.EXAMPLE + # Standard pre-refresh backup (run elevated) + .\Backup-CITemplate.ps1 + + # Dry run to preview what would happen + .\Backup-CITemplate.ps1 -WhatIf + + # Keep 5 backups + .\Backup-CITemplate.ps1 -KeepCount 5 +#> +[CmdletBinding(SupportsShouldProcess)] +param( + [string] $TemplatePath = 'F:\CI\Templates\WinBuild', + + [string] $BackupBaseDir = 'F:\CI\Backups', + + [ValidateRange(1, 20)] + [int] $KeepCount = 3, + + [switch] $SkipRunnerStop +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path $TemplatePath -PathType Container)) { + throw "Template directory not found: $TemplatePath" +} + +$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' +$backupDest = Join-Path $BackupBaseDir "Template_$timestamp" + +if (-not (Test-Path $BackupBaseDir)) { + New-Item -ItemType Directory -Path $BackupBaseDir -Force | Out-Null +} + +# ── Step 1: Stop runner ────────────────────────────────────────────────────── +$runnerWasRunning = $false +if (-not $SkipRunnerStop) { + $svc = Get-Service -Name act_runner -ErrorAction SilentlyContinue + if ($svc -and $svc.Status -eq 'Running') { + if ($PSCmdlet.ShouldProcess('act_runner', 'Stop service before backup')) { + Write-Host "[Backup-CITemplate] Stopping act_runner..." + Stop-Service -Name act_runner -Force + $runnerWasRunning = $true + Write-Host "[Backup-CITemplate] act_runner stopped." + } + } +} + +try { + # ── Step 2: Copy template directory ─────────────────────────────────────── + if ($PSCmdlet.ShouldProcess($TemplatePath, "Copy to $backupDest")) { + Write-Host "[Backup-CITemplate] Copying $TemplatePath -> $backupDest ..." + $sw = [System.Diagnostics.Stopwatch]::StartNew() + Copy-Item -Path $TemplatePath -Destination $backupDest -Recurse -Force + $sw.Stop() + + # Size sum without Measure-Object.Sum (Set-StrictMode safe) + $backupFiles = Get-ChildItem $backupDest -Recurse -File -ErrorAction SilentlyContinue + $totalBytes = [long]0 + if ($backupFiles) { foreach ($f in $backupFiles) { $totalBytes += $f.Length } } + $sizeMB = [math]::Round($totalBytes / 1MB, 0) + + Write-Host "[Backup-CITemplate] Backup complete: $backupDest ($sizeMB MB, $($sw.Elapsed.TotalSeconds.ToString('F0'))s)" + } + + # ── Step 3: Prune old backups ────────────────────────────────────────────── + $existing = Get-ChildItem $BackupBaseDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like 'Template_????????_??????' } | + Sort-Object LastWriteTime -Descending + + if ($existing.Count -gt $KeepCount) { + $toRemove = $existing | Select-Object -Skip $KeepCount + foreach ($old in $toRemove) { + if ($PSCmdlet.ShouldProcess($old.FullName, 'Remove old backup')) { + Write-Host "[Backup-CITemplate] Pruning old backup: $($old.Name)" + Remove-Item $old.FullName -Recurse -Force + } + } + Write-Host "[Backup-CITemplate] Retained $KeepCount backup(s)." + } +} +finally { + # ── Step 4: Restart runner ──────────────────────────────────────────────── + if ($runnerWasRunning) { + if ($PSCmdlet.ShouldProcess('act_runner', 'Start service after backup')) { + Write-Host "[Backup-CITemplate] Restarting act_runner..." + Start-Service -Name act_runner + $newStatus = (Get-Service -Name act_runner).Status + Write-Host "[Backup-CITemplate] act_runner status: $newStatus" + } + } +} diff --git a/scripts/Get-BuildArtifacts.ps1 b/scripts/Get-BuildArtifacts.ps1 index 928ac82..6ecce9a 100644 --- a/scripts/Get-BuildArtifacts.ps1 +++ b/scripts/Get-BuildArtifacts.ps1 @@ -57,13 +57,15 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force + # Ensure destination exists on host if (-not (Test-Path $HostArtifactDir)) { New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir" } -$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck +$sessionOptions = New-CISessionOption Write-Host "[Get-BuildArtifacts] Connecting to VM at $IPAddress..." $session = New-PSSession ` diff --git a/scripts/Invoke-RemoteBuild.ps1 b/scripts/Invoke-RemoteBuild.ps1 index 66438d1..93d3fab 100644 --- a/scripts/Invoke-RemoteBuild.ps1 +++ b/scripts/Invoke-RemoteBuild.ps1 @@ -73,14 +73,21 @@ param( [string] $GuestWorkDir = 'C:\CI\build', - [string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip' + [string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip', + + # Redirect dotnet NuGet package store and pip download cache to VMware shared + # folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip). + # Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1 + # and VMware Tools HGFS driver present in the guest. + [switch] $UseSharedCache ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' -# ── Session options: skip SSL certificate checks for self-signed (lab use) ── -$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck +Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force + +$sessionOptions = New-CISessionOption Write-Host "[Invoke-RemoteBuild] Connecting to VM at $IPAddress..." @@ -134,9 +141,12 @@ try { # ── Custom build command (e.g. python, msbuild, cmake…) ────────── Write-Host "[Invoke-RemoteBuild] Running build command: $BuildCommand" $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $cmd, $artifactZip, $artifactSrc) + param($workDir, $cmd, $artifactZip, $artifactSrc, $useCache) # Unbuffered Python output — lines appear in real-time instead of being held in buffer $env:PYTHONUNBUFFERED = '1' + if ($useCache) { + $env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache' + } Set-Location $workDir # Stream each output line via Write-Host — WinRM forwards Write-Host in real-time & cmd /c "$cmd 2>&1" | ForEach-Object { Write-Host $_ } @@ -152,7 +162,7 @@ try { } return $exit - } -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource + } -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource, $UseSharedCache.IsPresent if ($buildExit -ne 0) { throw "Build command failed (exit $buildExit)" @@ -162,11 +172,14 @@ try { # ── Default: dotnet restore + dotnet build ──────────────────────── Write-Host "[Invoke-RemoteBuild] Running dotnet restore..." $restoreExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir) + param($workDir, $useCache) + if ($useCache) { + $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' + } Set-Location $workDir dotnet restore 2>&1 | ForEach-Object { Write-Host $_ } return $LASTEXITCODE - } -ArgumentList $GuestWorkDir + } -ArgumentList $GuestWorkDir, $UseSharedCache.IsPresent if ($restoreExit -ne 0) { throw "dotnet restore failed (exit $restoreExit)" @@ -174,7 +187,10 @@ try { Write-Host "[Invoke-RemoteBuild] Running dotnet build (configuration: $Configuration)..." $buildExit = Invoke-Command -Session $session -ScriptBlock { - param($workDir, $config, $artifactZip) + param($workDir, $config, $artifactZip, $useCache) + if ($useCache) { + $env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache' + } Set-Location $workDir $outputDir = Join-Path $workDir 'bin\CIOutput' dotnet build --configuration $config --output $outputDir 2>&1 | ForEach-Object { Write-Host $_ } @@ -189,7 +205,7 @@ try { } return $exit - } -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip + } -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip, $UseSharedCache.IsPresent if ($buildExit -ne 0) { throw "dotnet build failed (exit $buildExit)" diff --git a/scripts/Measure-CIBenchmark.ps1 b/scripts/Measure-CIBenchmark.ps1 new file mode 100644 index 0000000..8220276 --- /dev/null +++ b/scripts/Measure-CIBenchmark.ps1 @@ -0,0 +1,245 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Measures CI infrastructure phase timings: clone, start, IP acquire, WinRM ready, destroy. + +.DESCRIPTION + Creates one or more ephemeral VMs from the template, times each phase, destroys + them, then prints a summary table. Results are appended to F:\CI\Logs\benchmark.jsonl + for long-term trend tracking (compare before/after §3.1 cache, §3.2 7-Zip, etc.). + + Phases measured: + clone — vmrun clone (linked clone from snapshot) + start — vmrun start (until command returns) + ip — vmrun getGuestIPAddress (polling until IP appears) + winrm — TCP 5986 reachable (polling) + destroy — stop + deleteVM + dir removal + + No actual build is run. Phase 5 build timings come from invoke-ci.jsonl logs + produced by Invoke-CIJob.ps1. + +.PARAMETER TemplatePath + Full path to the template VM's .vmx file. + Default: F:\CI\Templates\WinBuild\CI-WinBuild.vmx + +.PARAMETER SnapshotName + Snapshot name to clone from. Default: BaseClean + +.PARAMETER CloneBaseDir + Directory for ephemeral clone folders. Default: F:\CI\BuildVMs + +.PARAMETER VmrunPath + Path to vmrun.exe. + Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe + +.PARAMETER Iterations + Number of clone→ready→destroy cycles to run. Multiple iterations reduce timing + variance from cold-cache effects. Default: 1 + +.PARAMETER TimeoutSeconds + Maximum seconds to wait for IP and WinRM per iteration. Default: 300 + +.PARAMETER OutputDir + Directory where benchmark.jsonl is appended. Default: F:\CI\Logs + +.EXAMPLE + # Single baseline run + .\Measure-CIBenchmark.ps1 + + # 3 iterations for average (first may be slow due to cold host disk cache) + .\Measure-CIBenchmark.ps1 -Iterations 3 + + # Custom snapshot + .\Measure-CIBenchmark.ps1 -SnapshotName BaseClean_20260510 +#> +[CmdletBinding()] +param( + [string] $TemplatePath = 'F:\CI\Templates\WinBuild\CI-WinBuild.vmx', + [string] $SnapshotName = 'BaseClean', + [string] $CloneBaseDir = 'F:\CI\BuildVMs', + [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', + + [ValidateRange(1, 10)] + [int] $Iterations = 1, + + [ValidateRange(60, 600)] + [int] $TimeoutSeconds = 300, + + [string] $OutputDir = 'F:\CI\Logs' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force +Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null + +if (-not (Test-Path $TemplatePath -PathType Leaf)) { + throw "Template VMX not found: $TemplatePath" +} +if (-not (Test-Path $CloneBaseDir)) { + New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null +} +if (-not (Test-Path $OutputDir)) { + New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null +} + +$benchmarkLog = Join-Path $OutputDir 'benchmark.jsonl' +$results = [System.Collections.Generic.List[pscustomobject]]::new() + +Write-Host "" +Write-Host "=== CI Benchmark template: $(Split-Path $TemplatePath -Leaf) snapshot: $SnapshotName iterations: $Iterations ===" +Write-Host "" + +for ($i = 1; $i -le $Iterations; $i++) { + Write-Host "--- Iteration $i / $Iterations ---" + $runId = "bench-$(Get-Date -Format 'yyyyMMdd-HHmmss')-$i" + $cloneVmx = $null + $cloneDir = $null + + $t = [ordered]@{ + clone = $null + start = $null + ip = $null + winrm = $null + destroy = $null + vmIP = $null + error = $null + } + + try { + # ── Phase: clone ────────────────────────────────────────────────────── + $cloneName = "Clone_${runId}" + $cloneDir = Join-Path $CloneBaseDir $cloneName + $cloneVmx = Join-Path $cloneDir "$cloneName.vmx" + + Write-Host "[bench] Cloning..." + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $r = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` + -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) ` + -ThrowOnError + $t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Write-Host "[bench] clone: $($t.clone)s" + + # ── Phase: start ────────────────────────────────────────────────────── + Write-Host "[bench] Starting VM..." + $sw.Restart() + Invoke-Vmrun -VmrunPath $VmrunPath -Operation start ` + -Arguments @($cloneVmx, 'nogui') -ThrowOnError | Out-Null + $t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Write-Host "[bench] start: $($t.start)s" + + # ── Phase: IP acquire (poll getGuestIPAddress) ──────────────────────── + Write-Host "[bench] Waiting for guest IP..." + $sw.Restart() + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $vmIP = $null + while ((Get-Date) -lt $deadline) { + $ipResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation getGuestIPAddress ` + -Arguments @($cloneVmx) + if ($ipResult.ExitCode -eq 0) { + $candidate = "$($ipResult.Output)".Trim() + if ($candidate -match '^\d+\.\d+\.\d+\.\d+$') { + $vmIP = $candidate + break + } + } + Start-Sleep -Seconds 3 + } + if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" } + $t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2) + $t.vmIP = $vmIP + Write-Host "[bench] ip ($vmIP): $($t.ip)s" + + # ── Phase: WinRM ready (poll TCP 5986) ──────────────────────────────── + Write-Host "[bench] Waiting for WinRM (TCP 5986)..." + $sw.Restart() + $deadline = (Get-Date).AddSeconds($TimeoutSeconds) + $winrmUp = $false + while ((Get-Date) -lt $deadline) { + $winrmUp = Test-NetConnection -ComputerName $vmIP -Port 5986 ` + -InformationLevel Quiet -WarningAction SilentlyContinue ` + -ErrorAction SilentlyContinue + if ($winrmUp) { break } + Start-Sleep -Seconds 3 + } + if (-not $winrmUp) { throw "Timed out waiting for WinRM on $vmIP`:5986 after $TimeoutSeconds s" } + $t.winrm = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Write-Host "[bench] winrm: $($t.winrm)s" + } + catch { + $t.error = "$_" + Write-Warning "[bench] Iteration $i failed: $_" + } + finally { + # ── Phase: destroy ──────────────────────────────────────────────────── + if ($cloneVmx -and (Test-Path $cloneVmx -PathType Leaf)) { + Write-Host "[bench] Destroying clone..." + $sw = [System.Diagnostics.Stopwatch]::StartNew() + & (Join-Path $PSScriptRoot 'Remove-BuildVM.ps1') ` + -VMPath $cloneVmx -VmrunPath $VmrunPath -GracefulTimeoutSeconds 10 ` + -ErrorAction SilentlyContinue + $t.destroy = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Write-Host "[bench] destroy: $($t.destroy)s" + } elseif ($cloneDir -and (Test-Path $cloneDir)) { + Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue + } + } + + $totalBootSec = if ($t.ip -and $t.winrm) { + [math]::Round($t.clone + $t.start + $t.ip + $t.winrm, 2) + } else { $null } + + $result = [pscustomobject]@{ + ts = (Get-Date -Format 'o') + runId = $runId + iteration = $i + template = Split-Path $TemplatePath -Leaf + snapshot = $SnapshotName + cloneSec = $t.clone + startSec = $t.start + ipSec = $t.ip + winrmSec = $t.winrm + destroySec = $t.destroy + totalBootSec = $totalBootSec + vmIP = $t.vmIP + error = $t.error + } + $results.Add($result) + + # Append to JSONL log + try { + $result | ConvertTo-Json -Compress -Depth 3 | + Add-Content -Path $benchmarkLog -Encoding UTF8 -ErrorAction Stop + } catch { + Write-Warning "[bench] Could not write benchmark.jsonl: $_" + } + + Write-Host "" +} + +# ── Summary table ───────────────────────────────────────────────────────────── +Write-Host "=== Results ===" +$results | Format-Table @( + @{ L = 'Iter'; E = { $_.iteration } } + @{ L = 'Clone(s)'; E = { $_.cloneSec } } + @{ L = 'Start(s)'; E = { $_.startSec } } + @{ L = 'IP(s)'; E = { $_.ipSec } } + @{ L = 'WinRM(s)'; E = { $_.winrmSec } } + @{ L = 'Destroy(s)'; E = { $_.destroySec } } + @{ L = 'Boot total'; E = { $_.totalBootSec } } + @{ L = 'IP'; E = { $_.vmIP } } + @{ L = 'Error'; E = { $_.error } } +) -AutoSize + +if ($Iterations -gt 1) { + $ok = @($results | Where-Object { -not $_.error }) + if ($ok.Count -gt 0) { + $avgBoot = [math]::Round(($ok | ForEach-Object { $_.totalBootSec } | + Measure-Object -Average).Average, 2) + Write-Host "Average boot-to-WinRM ($($ok.Count) successful runs): $avgBoot s" + } +} + +Write-Host "" +Write-Host "Results appended to: $benchmarkLog" diff --git a/scripts/New-BuildVM.ps1 b/scripts/New-BuildVM.ps1 index 699baa7..c1964b5 100644 --- a/scripts/New-BuildVM.ps1 +++ b/scripts/New-BuildVM.ps1 @@ -58,10 +58,9 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' -# --- Validate vmrun.exe --- -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath if needed." -} +Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force + +Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null # --- Build clone path --- $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' @@ -81,27 +80,17 @@ Write-Host " Clone VMX : $cloneVmx" $sw = [System.Diagnostics.Stopwatch]::StartNew() -# --- Run vmrun clone --- -$vmrunArgs = @( - '-T', 'ws', - 'clone', - $TemplatePath, - $cloneVmx, - 'linked', - '-snapshot', $SnapshotName -) - -$result = & $VmrunPath @vmrunArgs 2>&1 -$exitCode = $LASTEXITCODE +$cloneResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone ` + -Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) $sw.Stop() -if ($exitCode -ne 0) { +if ($cloneResult.ExitCode -ne 0) { # Clean up partial clone directory if it was created if (Test-Path $cloneDir) { Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue } - throw "vmrun clone failed (exit $exitCode).`nOutput: $result" + throw "vmrun clone failed (exit $($cloneResult.ExitCode)).`nOutput: $($cloneResult.Output)" } if (-not (Test-Path $cloneVmx -PathType Leaf)) { diff --git a/scripts/Register-CIScheduledTasks.ps1 b/scripts/Register-CIScheduledTasks.ps1 index bd255df..40f01e1 100644 --- a/scripts/Register-CIScheduledTasks.ps1 +++ b/scripts/Register-CIScheduledTasks.ps1 @@ -5,7 +5,7 @@ Registers Windows Scheduled Tasks for CI system maintenance. .DESCRIPTION - Creates or updates two tasks under the \CI\ task folder: + Creates or updates four tasks under the \CI\ task folder: CI-CleanupOrphans — Runs Cleanup-OrphanedBuildVMs.ps1 every 6 hours and at host startup. Destroys stale build VMs @@ -17,7 +17,15 @@ Purges old artifact and log directories per the configured retention window. - Both tasks run as SYSTEM with highest privilege. Idempotent: safe to re-run + CI-DiskSpaceAlert — Runs Watch-DiskSpace.ps1 every 15 minutes. + Writes an Event Log warning and optional webhook + when CI drive free space falls below 50 GB. + + CI-RunnerHealth — Runs Watch-RunnerHealth.ps1 every 15 minutes. + Auto-restarts act_runner if stopped; rate-limited + to 3 restarts per hour before requiring manual fix. + + All tasks run as SYSTEM with highest privilege. Idempotent: safe to re-run after updating the scripts — existing tasks are overwritten (-Force). .PARAMETER ScriptRoot @@ -155,4 +163,36 @@ if (-not (Test-Path $diskScript -PathType Leaf)) { } } +# ── Task 4: CI-RunnerHealth ─────────────────────────────────────────────────── +$healthScript = Join-Path $ScriptRoot 'Watch-RunnerHealth.ps1' +if (-not (Test-Path $healthScript -PathType Leaf)) { + Write-Warning "Script not found — skipping CI-RunnerHealth: $healthScript" +} else { + $healthAction = New-ScheduledTaskAction ` + -Execute $psExe ` + -Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$healthScript`"" + + # Every 15 minutes indefinitely + $health15min = New-ScheduledTaskTrigger -Once -At '00:00' ` + -RepetitionInterval (New-TimeSpan -Minutes 15) + + $healthSettings = New-ScheduledTaskSettingsSet ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 5) ` + -MultipleInstances IgnoreNew ` + -StartWhenAvailable + + if ($PSCmdlet.ShouldProcess('CI-RunnerHealth', 'Register/update scheduled task')) { + Register-ScheduledTask ` + -TaskName 'CI-RunnerHealth' ` + -TaskPath $taskPath ` + -Action $healthAction ` + -Trigger $health15min ` + -Principal $principal ` + -Settings $healthSettings ` + -Description 'Auto-restarts act_runner service if stopped; rate-limited to 3 restarts/h (Local CI/CD System)' ` + -Force | Out-Null + Write-Host "[Register] CI-RunnerHealth registered (every 15 min, max 3 restarts/h)." -ForegroundColor Green + } +} + Write-Host "`n[Register] Done. Verify with: Get-ScheduledTask -TaskPath '\CI\' | Format-Table TaskName, State" -ForegroundColor Cyan diff --git a/scripts/Remove-BuildVM.ps1 b/scripts/Remove-BuildVM.ps1 index 936a126..7f7655f 100644 --- a/scripts/Remove-BuildVM.ps1 +++ b/scripts/Remove-BuildVM.ps1 @@ -25,7 +25,7 @@ .EXAMPLE .\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" #> -[CmdletBinding()] +[CmdletBinding(SupportsShouldProcess)] param( [Parameter(Mandatory)] [string] $VMPath, @@ -47,12 +47,14 @@ Write-Host "[Remove-BuildVM] Destroying VM: $VMPath" if (-not (Test-Path $VMPath -PathType Leaf)) { Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath" # Still attempt to remove the directory in case of partial clone - if (Test-Path $cloneDir) { + if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) { Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue } return } +if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return } + # ── Step 2: Graceful stop ───────────────────────────────────────────────────── Write-Host "[Remove-BuildVM] Sending soft stop..." & $VmrunPath -T ws stop $VMPath soft 2>&1 | Out-Null diff --git a/scripts/Set-TemplateSharedFolders.ps1 b/scripts/Set-TemplateSharedFolders.ps1 new file mode 100644 index 0000000..76d3828 --- /dev/null +++ b/scripts/Set-TemplateSharedFolders.ps1 @@ -0,0 +1,117 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Configures VMware shared folder entries in the CI template VMX for build cache sharing. + +.DESCRIPTION + Adds two shared folder mappings to the template VM's .vmx file: + nuget-cache host: F:\CI\Cache\NuGet guest: \\vmware-host\Shared Folders\nuget-cache + pip-cache host: F:\CI\Cache\pip guest: \\vmware-host\Shared Folders\pip-cache + + These UNC paths are used by Invoke-RemoteBuild.ps1 -UseSharedCache to redirect + dotnet's NuGet package store and pip's download cache to persistent host-side + directories, avoiding repeat downloads across builds. + + The script is idempotent: existing sharedFolder.* lines are removed before + the new block is written. A .bak copy of the original VMX is kept alongside it. + + Prerequisites: + - Template VM must NOT be running when this script runs. + - VMware Tools (HGFS driver) must be installed in the template guest. + - Run once; all subsequent linked clones inherit the shared folder config. + + After running: use Invoke-RemoteBuild.ps1 -UseSharedCache on builds that + target VMs cloned from this updated template. + +.PARAMETER TemplatePath + Full path to the template VM's .vmx file. + Default: F:\CI\Templates\WinBuild\CI-WinBuild.vmx + +.PARAMETER NuGetCacheDir + Host-side path for the NuGet package cache directory. + Default: F:\CI\Cache\NuGet + +.PARAMETER PipCacheDir + Host-side path for the pip download cache directory. + Default: F:\CI\Cache\pip + +.EXAMPLE + # Configure template (run from elevated PowerShell — template must be stopped) + .\Set-TemplateSharedFolders.ps1 + + # Preview changes without modifying files + .\Set-TemplateSharedFolders.ps1 -WhatIf +#> +[CmdletBinding(SupportsShouldProcess)] +param( + [string] $TemplatePath = 'F:\CI\Templates\WinBuild\CI-WinBuild.vmx', + [string] $NuGetCacheDir = 'F:\CI\Cache\NuGet', + [string] $PipCacheDir = 'F:\CI\Cache\pip' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path $TemplatePath -PathType Leaf)) { + throw "Template VMX not found: $TemplatePath" +} + +# ── Step 1: Ensure host cache directories exist ─────────────────────────────── +foreach ($dir in @($NuGetCacheDir, $PipCacheDir)) { + if (-not (Test-Path $dir)) { + if ($PSCmdlet.ShouldProcess($dir, 'Create host cache directory')) { + New-Item -ItemType Directory -Path $dir -Force | Out-Null + Write-Host "[SharedFolders] Created host cache dir: $dir" + } + } else { + Write-Host "[SharedFolders] Host cache dir exists: $dir" + } +} + +# ── Step 2: Read VMX, strip existing sharedFolder + HGFS lines ─────────────── +$lines = [System.IO.File]::ReadAllLines($TemplatePath, [System.Text.Encoding]::UTF8) +$stripped = $lines | Where-Object { + $_ -notmatch '^sharedFolder' -and + $_ -notmatch '^isolation\.tools\.hgfs\.disable' +} + +# ── Step 3: Build replacement sharedFolder block ────────────────────────────── +# VMX hostPath requires double-escaped backslashes +$nugetEsc = $NuGetCacheDir.Replace('\', '\\') +$pipEsc = $PipCacheDir.Replace('\', '\\') + +$sharedBlock = @( + 'sharedFolder.maxNum = "2"' + 'sharedFolder0.present = "TRUE"' + 'sharedFolder0.enabled = "TRUE"' + 'sharedFolder0.readAccess = "TRUE"' + 'sharedFolder0.writeAccess = "TRUE"' + "sharedFolder0.hostPath = `"$nugetEsc`"" + 'sharedFolder0.guestName = "nuget-cache"' + 'sharedFolder0.expiration = "never"' + 'sharedFolder1.present = "TRUE"' + 'sharedFolder1.enabled = "TRUE"' + 'sharedFolder1.readAccess = "TRUE"' + 'sharedFolder1.writeAccess = "TRUE"' + "sharedFolder1.hostPath = `"$pipEsc`"" + 'sharedFolder1.guestName = "pip-cache"' + 'sharedFolder1.expiration = "never"' + 'isolation.tools.hgfs.disable = "FALSE"' +) + +$newLines = @($stripped) + $sharedBlock + +# ── Step 4: Backup and write VMX ───────────────────────────────────────────── +if ($PSCmdlet.ShouldProcess($TemplatePath, 'Write updated VMX with shared folder entries')) { + $backupPath = "$TemplatePath.bak" + Copy-Item $TemplatePath $backupPath -Force + Write-Host "[SharedFolders] Original backed up: $backupPath" + + [System.IO.File]::WriteAllLines($TemplatePath, $newLines, [System.Text.Encoding]::UTF8) + + Write-Host "[SharedFolders] VMX updated: $TemplatePath" + Write-Host "[SharedFolders] Mappings:" + Write-Host " nuget-cache $NuGetCacheDir -> \\vmware-host\Shared Folders\nuget-cache" + Write-Host " pip-cache $PipCacheDir -> \\vmware-host\Shared Folders\pip-cache" + Write-Host "[SharedFolders] Next: run Invoke-RemoteBuild.ps1 -UseSharedCache to activate." +} diff --git a/scripts/Watch-RunnerHealth.ps1 b/scripts/Watch-RunnerHealth.ps1 new file mode 100644 index 0000000..3a37757 --- /dev/null +++ b/scripts/Watch-RunnerHealth.ps1 @@ -0,0 +1,161 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Monitors the act_runner service and auto-restarts it if stopped. + +.DESCRIPTION + Intended to run as a scheduled task every 15 minutes (registered by + Register-CIScheduledTasks.ps1 as CI-RunnerHealth). + + Behaviour: + - If act_runner is Running: exits 0, no action. + - If not Running: writes a Warning to the Windows Application Event Log + (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service. + - Restart attempts are rate-limited to MaxRestarts per hour using a JSON + state file in StateDir. Once the limit is hit, further runs write an + alert (EventId 1004) and exit 1 without restarting — Task Scheduler + history shows the failure so the operator is alerted. + - On successful restart: writes EventId 1003 (Information). + - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and + Gitea webhooks on every restart or limit-exceeded event. + +.PARAMETER MaxRestarts + Maximum number of auto-restart attempts allowed per rolling 60-minute + window before giving up and requiring manual intervention. Default: 3 + +.PARAMETER ServiceName + Windows service name to monitor. Default: act_runner + +.PARAMETER StateDir + Directory used for the cooldown state file (runner-restart-log.json). + Default: F:\CI\State + +.PARAMETER WebhookUrl + Optional Discord/Gitea webhook URL. If empty, only Event Log is written. + +.EXAMPLE + # Manual health check + .\Watch-RunnerHealth.ps1 + + # With Discord webhook + .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." +#> +[CmdletBinding()] +param( + [ValidateRange(1, 10)] + [int] $MaxRestarts = 3, + + [string] $ServiceName = 'act_runner', + + [string] $StateDir = 'F:\CI\State', + + [string] $WebhookUrl = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Continue' + +$logSource = 'CI-RunnerHealth' + +# ── Helper: write to Event Log ──────────────────────────────────────────────── +function Write-CIEvent { + param([int] $EventId, [string] $EntryType, [string] $Message) + try { + if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { + New-EventLog -LogName Application -Source $logSource -ErrorAction Stop + } + Write-EventLog -LogName Application -Source $logSource ` + -EventId $EventId -EntryType $EntryType -Message $Message + } catch { + Write-Warning "[RunnerHealth] Could not write Event Log: $_" + } +} + +# ── Helper: POST webhook ────────────────────────────────────────────────────── +function Send-Webhook { + param([string] $Content) + if (-not $WebhookUrl) { return } + $body = @{ content = $Content } | ConvertTo-Json -Compress + try { + $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` + -Body $body -ContentType 'application/json' -TimeoutSec 10 + Write-Host "[RunnerHealth] Webhook notified." + } catch { + Write-Warning "[RunnerHealth] Webhook POST failed: $_" + } +} + +# ── Step 1: Check service ───────────────────────────────────────────────────── +$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue +if (-not $svc) { + Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?" + exit 2 +} + +if ($svc.Status -eq 'Running') { + Write-Host "[RunnerHealth] $ServiceName is Running — OK" + exit 0 +} + +# ── Step 2: Service not running — load cooldown state ───────────────────────── +if (-not (Test-Path $StateDir)) { + New-Item -ItemType Directory -Path $StateDir -Force | Out-Null +} + +$cooldownFile = Join-Path $StateDir 'runner-restart-log.json' +$restartLog = @() + +if (Test-Path $cooldownFile) { + try { + $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue + if ($raw) { $restartLog = @($raw | ConvertFrom-Json) } + } catch { } +} + +# Prune timestamps older than 1 hour +$cutoff = (Get-Date).AddHours(-1) +$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff }) +$recentCount = $restartLog.Count + +$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts." +Write-Warning "[RunnerHealth] $stateMsg" +Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg + +# ── Step 3: Rate-limit check ────────────────────────────────────────────────── +if ($recentCount -ge $MaxRestarts) { + $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " + + "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\." + Write-Warning "[RunnerHealth] $limitMsg" + Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg + Send-Webhook -Content ":sos: **CI Runner Alert** — $limitMsg" + exit 1 +} + +# ── Step 4: Attempt restart ─────────────────────────────────────────────────── +Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..." +try { + Restart-Service -Name $ServiceName -Force -ErrorAction Stop + Start-Sleep -Seconds 5 + $newStatus = (Get-Service -Name $ServiceName).Status + $restartMsg = "$ServiceName restarted — new status: $newStatus." + Write-Host "[RunnerHealth] $restartMsg" + + # Persist this restart in the cooldown log + $restartLog += (Get-Date -Format 'o') + $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue + + $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' } + Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg + + $icon = if ($newStatus -eq 'Running') { ':warning:' } else { ':sos:' } + Send-Webhook -Content "$icon **CI Runner Alert** — $restartMsg" + + exit $(if ($newStatus -eq 'Running') { 0 } else { 1 }) +} +catch { + $errMsg = "$ServiceName restart failed: $_" + Write-Warning "[RunnerHealth] $errMsg" + Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg + Send-Webhook -Content ":sos: **CI Runner Alert** — $errMsg" + exit 1 +} diff --git a/scripts/_Common.psm1 b/scripts/_Common.psm1 new file mode 100644 index 0000000..aba6b96 --- /dev/null +++ b/scripts/_Common.psm1 @@ -0,0 +1,105 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Shared helper functions for the Local CI/CD System scripts. + +.DESCRIPTION + Import this module at the top of any CI script that needs WinRM session + options or vmrun wrappers: + + Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force + + Exported functions: + New-CISessionOption — PSSessionOption for WinRM over self-signed TLS + Resolve-VmrunPath — Validates vmrun.exe path, throws if missing + Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output +#> +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function New-CISessionOption { +<# +.SYNOPSIS + Returns a PSSessionOption for WinRM HTTPS with self-signed certificate. +.DESCRIPTION + All CI build VMs use a self-signed TLS cert on WinRM port 5986. + This option set skips CA, CN, and revocation checks — appropriate for + an isolated lab network where PKI is not present. +.OUTPUTS + [System.Management.Automation.Remoting.PSSessionOption] +#> + [OutputType([System.Management.Automation.Remoting.PSSessionOption])] + [CmdletBinding()] + param() + New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck +} + +function Resolve-VmrunPath { +<# +.SYNOPSIS + Validates that vmrun.exe exists at the specified path and returns it. +.DESCRIPTION + Throws a descriptive error if the file is not found, rather than letting + a later & call fail with a less useful message. +.PARAMETER VmrunPath + Full path to vmrun.exe. + Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe +.OUTPUTS + [string] The validated VmrunPath. +#> + [OutputType([string])] + [CmdletBinding()] + param( + [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' + ) + if (-not (Test-Path $VmrunPath -PathType Leaf)) { + throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath." + } + return $VmrunPath +} + +function Invoke-Vmrun { +<# +.SYNOPSIS + Runs a vmrun -T ws command and returns output and exit code. +.DESCRIPTION + Uniform wrapper so all vmrun calls share the same -T ws flag and output + capture pattern. Use -ThrowOnError for operations that must succeed + (clone, start); omit it for best-effort operations (stop, getState). +.PARAMETER VmrunPath + Full path to vmrun.exe. +.PARAMETER Operation + vmrun sub-command: clone, start, stop, deleteVM, list, getState, + listSnapshots, getGuestIPAddress, etc. +.PARAMETER Arguments + Additional positional arguments after the operation name. +.PARAMETER ThrowOnError + When set, throws if vmrun exits non-zero. +.OUTPUTS + [pscustomobject] with: + ExitCode [int] — vmrun exit code + Output [string[]] — combined stdout/stderr lines +#> + [OutputType([pscustomobject])] + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $VmrunPath, + [Parameter(Mandatory)] [string] $Operation, + [string[]] $Arguments = @(), + [switch] $ThrowOnError + ) + $cmdArgs = @('-T', 'ws', $Operation) + $Arguments + $output = & $VmrunPath @cmdArgs 2>&1 + $exit = $LASTEXITCODE + + if ($ThrowOnError -and $exit -ne 0) { + throw "vmrun $Operation failed (exit $exit): $($output -join '; ')" + } + + return [pscustomobject]@{ + ExitCode = $exit + Output = $output + } +} + +Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun