feat(sprint4-6): quality, reliability, perf baseline

Sprint 4 — qualità codice:
- scripts/_Common.psm1: New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun
- PSScriptAnalyzerSettings.psd1: project-wide PSSA rules (security + formatting)
- Remove-BuildVM.ps1: SupportsShouldProcess (-WhatIf support)
- Invoke-RemoteBuild.ps1, Get-BuildArtifacts.ps1: use New-CISessionOption from module
- New-BuildVM.ps1: use Resolve-VmrunPath + Invoke-Vmrun from module

Sprint 5 — affidabilità:
- Backup-CITemplate.ps1: stop runner, timestamped VMDK backup, prune old, restart
- Watch-RunnerHealth.ps1: auto-restart act_runner, cooldown 3/h, EventLog + webhook
- Register-CIScheduledTasks.ps1: add CI-RunnerHealth task (every 15 min)

Sprint 6 — perf baseline:
- Set-TemplateSharedFolders.ps1: idempotent VMX editor for NuGet/pip shared folders
- Invoke-RemoteBuild.ps1: -UseSharedCache injects NUGET_PACKAGES + PIP_CACHE_DIR in guest
- Measure-CIBenchmark.ps1: times clone/start/IP/WinRM/destroy, appends to benchmark.jsonl

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-10 20:08:19 +02:00
parent 1962c977ba
commit 552eb628b7
12 changed files with 966 additions and 98 deletions
+127
View File
@@ -0,0 +1,127 @@
#Requires -Version 5.1
#Requires -RunAsAdministrator
<#
.SYNOPSIS
Creates a timestamped backup of the CI template VM directory.
.DESCRIPTION
Run before any template refresh (toolchain update, KMS re-activation, snapshot
rebuild). The act_runner service is stopped for the duration so no vmrun clone
races the file copy, then restarted automatically.
Restore procedure (after a bad refresh):
Stop-Service act_runner
Remove-Item 'F:\CI\Templates\WinBuild' -Recurse -Force
Copy-Item 'F:\CI\Backups\Template_<stamp>' 'F:\CI\Templates\WinBuild' -Recurse
Start-Service act_runner
.PARAMETER TemplatePath
Directory containing the template VM files (.vmx, .vmdk, snapshots).
Default: F:\CI\Templates\WinBuild
.PARAMETER BackupBaseDir
Directory under which timestamped backup folders are created.
Default: F:\CI\Backups
.PARAMETER KeepCount
Number of most-recent backups to retain. Older ones are deleted.
Default: 3
.PARAMETER SkipRunnerStop
Skip stopping/restarting act_runner. Use when the service is already stopped
or when called from a maintenance window where the runner is offline.
.EXAMPLE
# Standard pre-refresh backup (run elevated)
.\Backup-CITemplate.ps1
# Dry run to preview what would happen
.\Backup-CITemplate.ps1 -WhatIf
# Keep 5 backups
.\Backup-CITemplate.ps1 -KeepCount 5
#>
[CmdletBinding(SupportsShouldProcess)]
param(
[string] $TemplatePath = 'F:\CI\Templates\WinBuild',
[string] $BackupBaseDir = 'F:\CI\Backups',
[ValidateRange(1, 20)]
[int] $KeepCount = 3,
[switch] $SkipRunnerStop
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not (Test-Path $TemplatePath -PathType Container)) {
throw "Template directory not found: $TemplatePath"
}
$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss'
$backupDest = Join-Path $BackupBaseDir "Template_$timestamp"
if (-not (Test-Path $BackupBaseDir)) {
New-Item -ItemType Directory -Path $BackupBaseDir -Force | Out-Null
}
# ── Step 1: Stop runner ──────────────────────────────────────────────────────
$runnerWasRunning = $false
if (-not $SkipRunnerStop) {
$svc = Get-Service -Name act_runner -ErrorAction SilentlyContinue
if ($svc -and $svc.Status -eq 'Running') {
if ($PSCmdlet.ShouldProcess('act_runner', 'Stop service before backup')) {
Write-Host "[Backup-CITemplate] Stopping act_runner..."
Stop-Service -Name act_runner -Force
$runnerWasRunning = $true
Write-Host "[Backup-CITemplate] act_runner stopped."
}
}
}
try {
# ── Step 2: Copy template directory ───────────────────────────────────────
if ($PSCmdlet.ShouldProcess($TemplatePath, "Copy to $backupDest")) {
Write-Host "[Backup-CITemplate] Copying $TemplatePath -> $backupDest ..."
$sw = [System.Diagnostics.Stopwatch]::StartNew()
Copy-Item -Path $TemplatePath -Destination $backupDest -Recurse -Force
$sw.Stop()
# Size sum without Measure-Object.Sum (Set-StrictMode safe)
$backupFiles = Get-ChildItem $backupDest -Recurse -File -ErrorAction SilentlyContinue
$totalBytes = [long]0
if ($backupFiles) { foreach ($f in $backupFiles) { $totalBytes += $f.Length } }
$sizeMB = [math]::Round($totalBytes / 1MB, 0)
Write-Host "[Backup-CITemplate] Backup complete: $backupDest ($sizeMB MB, $($sw.Elapsed.TotalSeconds.ToString('F0'))s)"
}
# ── Step 3: Prune old backups ──────────────────────────────────────────────
$existing = Get-ChildItem $BackupBaseDir -Directory -ErrorAction SilentlyContinue |
Where-Object { $_.Name -like 'Template_????????_??????' } |
Sort-Object LastWriteTime -Descending
if ($existing.Count -gt $KeepCount) {
$toRemove = $existing | Select-Object -Skip $KeepCount
foreach ($old in $toRemove) {
if ($PSCmdlet.ShouldProcess($old.FullName, 'Remove old backup')) {
Write-Host "[Backup-CITemplate] Pruning old backup: $($old.Name)"
Remove-Item $old.FullName -Recurse -Force
}
}
Write-Host "[Backup-CITemplate] Retained $KeepCount backup(s)."
}
}
finally {
# ── Step 4: Restart runner ────────────────────────────────────────────────
if ($runnerWasRunning) {
if ($PSCmdlet.ShouldProcess('act_runner', 'Start service after backup')) {
Write-Host "[Backup-CITemplate] Restarting act_runner..."
Start-Service -Name act_runner
$newStatus = (Get-Service -Name act_runner).Status
Write-Host "[Backup-CITemplate] act_runner status: $newStatus"
}
}
}
+3 -1
View File
@@ -57,13 +57,15 @@ param(
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
# Ensure destination exists on host
if (-not (Test-Path $HostArtifactDir)) {
New-Item -ItemType Directory -Path $HostArtifactDir -Force | Out-Null
Write-Host "[Get-BuildArtifacts] Created host artifact directory: $HostArtifactDir"
}
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
$sessionOptions = New-CISessionOption
Write-Host "[Get-BuildArtifacts] Connecting to VM at $IPAddress..."
$session = New-PSSession `
+25 -9
View File
@@ -73,14 +73,21 @@ param(
[string] $GuestWorkDir = 'C:\CI\build',
[string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip'
[string] $GuestArtifactZip = 'C:\CI\output\artifacts.zip',
# Redirect dotnet NuGet package store and pip download cache to VMware shared
# folders on the host (F:\CI\Cache\NuGet and F:\CI\Cache\pip).
# Requires the template VMX to have been configured by Set-TemplateSharedFolders.ps1
# and VMware Tools HGFS driver present in the guest.
[switch] $UseSharedCache
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
# ── Session options: skip SSL certificate checks for self-signed (lab use) ──
$sessionOptions = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
$sessionOptions = New-CISessionOption
Write-Host "[Invoke-RemoteBuild] Connecting to VM at $IPAddress..."
@@ -134,9 +141,12 @@ try {
# ── Custom build command (e.g. python, msbuild, cmake…) ──────────
Write-Host "[Invoke-RemoteBuild] Running build command: $BuildCommand"
$buildExit = Invoke-Command -Session $session -ScriptBlock {
param($workDir, $cmd, $artifactZip, $artifactSrc)
param($workDir, $cmd, $artifactZip, $artifactSrc, $useCache)
# Unbuffered Python output — lines appear in real-time instead of being held in buffer
$env:PYTHONUNBUFFERED = '1'
if ($useCache) {
$env:PIP_CACHE_DIR = '\\vmware-host\Shared Folders\pip-cache'
}
Set-Location $workDir
# Stream each output line via Write-Host — WinRM forwards Write-Host in real-time
& cmd /c "$cmd 2>&1" | ForEach-Object { Write-Host $_ }
@@ -152,7 +162,7 @@ try {
}
return $exit
} -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource
} -ArgumentList $GuestWorkDir, $BuildCommand, $GuestArtifactZip, $GuestArtifactSource, $UseSharedCache.IsPresent
if ($buildExit -ne 0) {
throw "Build command failed (exit $buildExit)"
@@ -162,11 +172,14 @@ try {
# ── Default: dotnet restore + dotnet build ────────────────────────
Write-Host "[Invoke-RemoteBuild] Running dotnet restore..."
$restoreExit = Invoke-Command -Session $session -ScriptBlock {
param($workDir)
param($workDir, $useCache)
if ($useCache) {
$env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache'
}
Set-Location $workDir
dotnet restore 2>&1 | ForEach-Object { Write-Host $_ }
return $LASTEXITCODE
} -ArgumentList $GuestWorkDir
} -ArgumentList $GuestWorkDir, $UseSharedCache.IsPresent
if ($restoreExit -ne 0) {
throw "dotnet restore failed (exit $restoreExit)"
@@ -174,7 +187,10 @@ try {
Write-Host "[Invoke-RemoteBuild] Running dotnet build (configuration: $Configuration)..."
$buildExit = Invoke-Command -Session $session -ScriptBlock {
param($workDir, $config, $artifactZip)
param($workDir, $config, $artifactZip, $useCache)
if ($useCache) {
$env:NUGET_PACKAGES = '\\vmware-host\Shared Folders\nuget-cache'
}
Set-Location $workDir
$outputDir = Join-Path $workDir 'bin\CIOutput'
dotnet build --configuration $config --output $outputDir 2>&1 | ForEach-Object { Write-Host $_ }
@@ -189,7 +205,7 @@ try {
}
return $exit
} -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip
} -ArgumentList $GuestWorkDir, $Configuration, $GuestArtifactZip, $UseSharedCache.IsPresent
if ($buildExit -ne 0) {
throw "dotnet build failed (exit $buildExit)"
+245
View File
@@ -0,0 +1,245 @@
#Requires -Version 5.1
<#
.SYNOPSIS
Measures CI infrastructure phase timings: clone, start, IP acquire, WinRM ready, destroy.
.DESCRIPTION
Creates one or more ephemeral VMs from the template, times each phase, destroys
them, then prints a summary table. Results are appended to F:\CI\Logs\benchmark.jsonl
for long-term trend tracking (compare before/after §3.1 cache, §3.2 7-Zip, etc.).
Phases measured:
clone — vmrun clone (linked clone from snapshot)
start — vmrun start (until command returns)
ip — vmrun getGuestIPAddress (polling until IP appears)
winrm — TCP 5986 reachable (polling)
destroy — stop + deleteVM + dir removal
No actual build is run. Phase 5 build timings come from invoke-ci.jsonl logs
produced by Invoke-CIJob.ps1.
.PARAMETER TemplatePath
Full path to the template VM's .vmx file.
Default: F:\CI\Templates\WinBuild\CI-WinBuild.vmx
.PARAMETER SnapshotName
Snapshot name to clone from. Default: BaseClean
.PARAMETER CloneBaseDir
Directory for ephemeral clone folders. Default: F:\CI\BuildVMs
.PARAMETER VmrunPath
Path to vmrun.exe.
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
.PARAMETER Iterations
Number of clone→ready→destroy cycles to run. Multiple iterations reduce timing
variance from cold-cache effects. Default: 1
.PARAMETER TimeoutSeconds
Maximum seconds to wait for IP and WinRM per iteration. Default: 300
.PARAMETER OutputDir
Directory where benchmark.jsonl is appended. Default: F:\CI\Logs
.EXAMPLE
# Single baseline run
.\Measure-CIBenchmark.ps1
# 3 iterations for average (first may be slow due to cold host disk cache)
.\Measure-CIBenchmark.ps1 -Iterations 3
# Custom snapshot
.\Measure-CIBenchmark.ps1 -SnapshotName BaseClean_20260510
#>
[CmdletBinding()]
param(
[string] $TemplatePath = 'F:\CI\Templates\WinBuild\CI-WinBuild.vmx',
[string] $SnapshotName = 'BaseClean',
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
[ValidateRange(1, 10)]
[int] $Iterations = 1,
[ValidateRange(60, 600)]
[int] $TimeoutSeconds = 300,
[string] $OutputDir = 'F:\CI\Logs'
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
throw "Template VMX not found: $TemplatePath"
}
if (-not (Test-Path $CloneBaseDir)) {
New-Item -ItemType Directory -Path $CloneBaseDir -Force | Out-Null
}
if (-not (Test-Path $OutputDir)) {
New-Item -ItemType Directory -Path $OutputDir -Force | Out-Null
}
$benchmarkLog = Join-Path $OutputDir 'benchmark.jsonl'
$results = [System.Collections.Generic.List[pscustomobject]]::new()
Write-Host ""
Write-Host "=== CI Benchmark template: $(Split-Path $TemplatePath -Leaf) snapshot: $SnapshotName iterations: $Iterations ==="
Write-Host ""
for ($i = 1; $i -le $Iterations; $i++) {
Write-Host "--- Iteration $i / $Iterations ---"
$runId = "bench-$(Get-Date -Format 'yyyyMMdd-HHmmss')-$i"
$cloneVmx = $null
$cloneDir = $null
$t = [ordered]@{
clone = $null
start = $null
ip = $null
winrm = $null
destroy = $null
vmIP = $null
error = $null
}
try {
# ── Phase: clone ──────────────────────────────────────────────────────
$cloneName = "Clone_${runId}"
$cloneDir = Join-Path $CloneBaseDir $cloneName
$cloneVmx = Join-Path $cloneDir "$cloneName.vmx"
Write-Host "[bench] Cloning..."
$sw = [System.Diagnostics.Stopwatch]::StartNew()
$r = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName) `
-ThrowOnError
$t.clone = [math]::Round($sw.Elapsed.TotalSeconds, 2)
Write-Host "[bench] clone: $($t.clone)s"
# ── Phase: start ──────────────────────────────────────────────────────
Write-Host "[bench] Starting VM..."
$sw.Restart()
Invoke-Vmrun -VmrunPath $VmrunPath -Operation start `
-Arguments @($cloneVmx, 'nogui') -ThrowOnError | Out-Null
$t.start = [math]::Round($sw.Elapsed.TotalSeconds, 2)
Write-Host "[bench] start: $($t.start)s"
# ── Phase: IP acquire (poll getGuestIPAddress) ────────────────────────
Write-Host "[bench] Waiting for guest IP..."
$sw.Restart()
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
$vmIP = $null
while ((Get-Date) -lt $deadline) {
$ipResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation getGuestIPAddress `
-Arguments @($cloneVmx)
if ($ipResult.ExitCode -eq 0) {
$candidate = "$($ipResult.Output)".Trim()
if ($candidate -match '^\d+\.\d+\.\d+\.\d+$') {
$vmIP = $candidate
break
}
}
Start-Sleep -Seconds 3
}
if (-not $vmIP) { throw "Timed out waiting for guest IP after $TimeoutSeconds s" }
$t.ip = [math]::Round($sw.Elapsed.TotalSeconds, 2)
$t.vmIP = $vmIP
Write-Host "[bench] ip ($vmIP): $($t.ip)s"
# ── Phase: WinRM ready (poll TCP 5986) ────────────────────────────────
Write-Host "[bench] Waiting for WinRM (TCP 5986)..."
$sw.Restart()
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
$winrmUp = $false
while ((Get-Date) -lt $deadline) {
$winrmUp = Test-NetConnection -ComputerName $vmIP -Port 5986 `
-InformationLevel Quiet -WarningAction SilentlyContinue `
-ErrorAction SilentlyContinue
if ($winrmUp) { break }
Start-Sleep -Seconds 3
}
if (-not $winrmUp) { throw "Timed out waiting for WinRM on $vmIP`:5986 after $TimeoutSeconds s" }
$t.winrm = [math]::Round($sw.Elapsed.TotalSeconds, 2)
Write-Host "[bench] winrm: $($t.winrm)s"
}
catch {
$t.error = "$_"
Write-Warning "[bench] Iteration $i failed: $_"
}
finally {
# ── Phase: destroy ────────────────────────────────────────────────────
if ($cloneVmx -and (Test-Path $cloneVmx -PathType Leaf)) {
Write-Host "[bench] Destroying clone..."
$sw = [System.Diagnostics.Stopwatch]::StartNew()
& (Join-Path $PSScriptRoot 'Remove-BuildVM.ps1') `
-VMPath $cloneVmx -VmrunPath $VmrunPath -GracefulTimeoutSeconds 10 `
-ErrorAction SilentlyContinue
$t.destroy = [math]::Round($sw.Elapsed.TotalSeconds, 2)
Write-Host "[bench] destroy: $($t.destroy)s"
} elseif ($cloneDir -and (Test-Path $cloneDir)) {
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
}
}
$totalBootSec = if ($t.ip -and $t.winrm) {
[math]::Round($t.clone + $t.start + $t.ip + $t.winrm, 2)
} else { $null }
$result = [pscustomobject]@{
ts = (Get-Date -Format 'o')
runId = $runId
iteration = $i
template = Split-Path $TemplatePath -Leaf
snapshot = $SnapshotName
cloneSec = $t.clone
startSec = $t.start
ipSec = $t.ip
winrmSec = $t.winrm
destroySec = $t.destroy
totalBootSec = $totalBootSec
vmIP = $t.vmIP
error = $t.error
}
$results.Add($result)
# Append to JSONL log
try {
$result | ConvertTo-Json -Compress -Depth 3 |
Add-Content -Path $benchmarkLog -Encoding UTF8 -ErrorAction Stop
} catch {
Write-Warning "[bench] Could not write benchmark.jsonl: $_"
}
Write-Host ""
}
# ── Summary table ─────────────────────────────────────────────────────────────
Write-Host "=== Results ==="
$results | Format-Table @(
@{ L = 'Iter'; E = { $_.iteration } }
@{ L = 'Clone(s)'; E = { $_.cloneSec } }
@{ L = 'Start(s)'; E = { $_.startSec } }
@{ L = 'IP(s)'; E = { $_.ipSec } }
@{ L = 'WinRM(s)'; E = { $_.winrmSec } }
@{ L = 'Destroy(s)'; E = { $_.destroySec } }
@{ L = 'Boot total'; E = { $_.totalBootSec } }
@{ L = 'IP'; E = { $_.vmIP } }
@{ L = 'Error'; E = { $_.error } }
) -AutoSize
if ($Iterations -gt 1) {
$ok = @($results | Where-Object { -not $_.error })
if ($ok.Count -gt 0) {
$avgBoot = [math]::Round(($ok | ForEach-Object { $_.totalBootSec } |
Measure-Object -Average).Average, 2)
Write-Host "Average boot-to-WinRM ($($ok.Count) successful runs): $avgBoot s"
}
}
Write-Host ""
Write-Host "Results appended to: $benchmarkLog"
+7 -18
View File
@@ -58,10 +58,9 @@ param(
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
# --- Validate vmrun.exe ---
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath if needed."
}
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
Resolve-VmrunPath -VmrunPath $VmrunPath | Out-Null
# --- Build clone path ---
$timestamp = Get-Date -Format 'yyyyMMdd_HHmmss'
@@ -81,27 +80,17 @@ Write-Host " Clone VMX : $cloneVmx"
$sw = [System.Diagnostics.Stopwatch]::StartNew()
# --- Run vmrun clone ---
$vmrunArgs = @(
'-T', 'ws',
'clone',
$TemplatePath,
$cloneVmx,
'linked',
'-snapshot', $SnapshotName
)
$result = & $VmrunPath @vmrunArgs 2>&1
$exitCode = $LASTEXITCODE
$cloneResult = Invoke-Vmrun -VmrunPath $VmrunPath -Operation clone `
-Arguments @($TemplatePath, $cloneVmx, 'linked', '-snapshot', $SnapshotName)
$sw.Stop()
if ($exitCode -ne 0) {
if ($cloneResult.ExitCode -ne 0) {
# Clean up partial clone directory if it was created
if (Test-Path $cloneDir) {
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
}
throw "vmrun clone failed (exit $exitCode).`nOutput: $result"
throw "vmrun clone failed (exit $($cloneResult.ExitCode)).`nOutput: $($cloneResult.Output)"
}
if (-not (Test-Path $cloneVmx -PathType Leaf)) {
+42 -2
View File
@@ -5,7 +5,7 @@
Registers Windows Scheduled Tasks for CI system maintenance.
.DESCRIPTION
Creates or updates two tasks under the \CI\ task folder:
Creates or updates four tasks under the \CI\ task folder:
CI-CleanupOrphans — Runs Cleanup-OrphanedBuildVMs.ps1 every 6 hours
and at host startup. Destroys stale build VMs
@@ -17,7 +17,15 @@
Purges old artifact and log directories per the
configured retention window.
Both tasks run as SYSTEM with highest privilege. Idempotent: safe to re-run
CI-DiskSpaceAlert — Runs Watch-DiskSpace.ps1 every 15 minutes.
Writes an Event Log warning and optional webhook
when CI drive free space falls below 50 GB.
CI-RunnerHealth — Runs Watch-RunnerHealth.ps1 every 15 minutes.
Auto-restarts act_runner if stopped; rate-limited
to 3 restarts per hour before requiring manual fix.
All tasks run as SYSTEM with highest privilege. Idempotent: safe to re-run
after updating the scripts — existing tasks are overwritten (-Force).
.PARAMETER ScriptRoot
@@ -155,4 +163,36 @@ if (-not (Test-Path $diskScript -PathType Leaf)) {
}
}
# ── Task 4: CI-RunnerHealth ───────────────────────────────────────────────────
$healthScript = Join-Path $ScriptRoot 'Watch-RunnerHealth.ps1'
if (-not (Test-Path $healthScript -PathType Leaf)) {
Write-Warning "Script not found — skipping CI-RunnerHealth: $healthScript"
} else {
$healthAction = New-ScheduledTaskAction `
-Execute $psExe `
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$healthScript`""
# Every 15 minutes indefinitely
$health15min = New-ScheduledTaskTrigger -Once -At '00:00' `
-RepetitionInterval (New-TimeSpan -Minutes 15)
$healthSettings = New-ScheduledTaskSettingsSet `
-ExecutionTimeLimit (New-TimeSpan -Minutes 5) `
-MultipleInstances IgnoreNew `
-StartWhenAvailable
if ($PSCmdlet.ShouldProcess('CI-RunnerHealth', 'Register/update scheduled task')) {
Register-ScheduledTask `
-TaskName 'CI-RunnerHealth' `
-TaskPath $taskPath `
-Action $healthAction `
-Trigger $health15min `
-Principal $principal `
-Settings $healthSettings `
-Description 'Auto-restarts act_runner service if stopped; rate-limited to 3 restarts/h (Local CI/CD System)' `
-Force | Out-Null
Write-Host "[Register] CI-RunnerHealth registered (every 15 min, max 3 restarts/h)." -ForegroundColor Green
}
}
Write-Host "`n[Register] Done. Verify with: Get-ScheduledTask -TaskPath '\CI\' | Format-Table TaskName, State" -ForegroundColor Cyan
+4 -2
View File
@@ -25,7 +25,7 @@
.EXAMPLE
.\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx"
#>
[CmdletBinding()]
[CmdletBinding(SupportsShouldProcess)]
param(
[Parameter(Mandatory)]
[string] $VMPath,
@@ -47,12 +47,14 @@ Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
if (-not (Test-Path $VMPath -PathType Leaf)) {
Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath"
# Still attempt to remove the directory in case of partial clone
if (Test-Path $cloneDir) {
if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) {
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
}
return
}
if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
# ── Step 2: Graceful stop ─────────────────────────────────────────────────────
Write-Host "[Remove-BuildVM] Sending soft stop..."
& $VmrunPath -T ws stop $VMPath soft 2>&1 | Out-Null
+117
View File
@@ -0,0 +1,117 @@
#Requires -Version 5.1
<#
.SYNOPSIS
Configures VMware shared folder entries in the CI template VMX for build cache sharing.
.DESCRIPTION
Adds two shared folder mappings to the template VM's .vmx file:
nuget-cache host: F:\CI\Cache\NuGet guest: \\vmware-host\Shared Folders\nuget-cache
pip-cache host: F:\CI\Cache\pip guest: \\vmware-host\Shared Folders\pip-cache
These UNC paths are used by Invoke-RemoteBuild.ps1 -UseSharedCache to redirect
dotnet's NuGet package store and pip's download cache to persistent host-side
directories, avoiding repeat downloads across builds.
The script is idempotent: existing sharedFolder.* lines are removed before
the new block is written. A .bak copy of the original VMX is kept alongside it.
Prerequisites:
- Template VM must NOT be running when this script runs.
- VMware Tools (HGFS driver) must be installed in the template guest.
- Run once; all subsequent linked clones inherit the shared folder config.
After running: use Invoke-RemoteBuild.ps1 -UseSharedCache on builds that
target VMs cloned from this updated template.
.PARAMETER TemplatePath
Full path to the template VM's .vmx file.
Default: F:\CI\Templates\WinBuild\CI-WinBuild.vmx
.PARAMETER NuGetCacheDir
Host-side path for the NuGet package cache directory.
Default: F:\CI\Cache\NuGet
.PARAMETER PipCacheDir
Host-side path for the pip download cache directory.
Default: F:\CI\Cache\pip
.EXAMPLE
# Configure template (run from elevated PowerShell — template must be stopped)
.\Set-TemplateSharedFolders.ps1
# Preview changes without modifying files
.\Set-TemplateSharedFolders.ps1 -WhatIf
#>
[CmdletBinding(SupportsShouldProcess)]
param(
[string] $TemplatePath = 'F:\CI\Templates\WinBuild\CI-WinBuild.vmx',
[string] $NuGetCacheDir = 'F:\CI\Cache\NuGet',
[string] $PipCacheDir = 'F:\CI\Cache\pip'
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
throw "Template VMX not found: $TemplatePath"
}
# ── Step 1: Ensure host cache directories exist ───────────────────────────────
foreach ($dir in @($NuGetCacheDir, $PipCacheDir)) {
if (-not (Test-Path $dir)) {
if ($PSCmdlet.ShouldProcess($dir, 'Create host cache directory')) {
New-Item -ItemType Directory -Path $dir -Force | Out-Null
Write-Host "[SharedFolders] Created host cache dir: $dir"
}
} else {
Write-Host "[SharedFolders] Host cache dir exists: $dir"
}
}
# ── Step 2: Read VMX, strip existing sharedFolder + HGFS lines ───────────────
$lines = [System.IO.File]::ReadAllLines($TemplatePath, [System.Text.Encoding]::UTF8)
$stripped = $lines | Where-Object {
$_ -notmatch '^sharedFolder' -and
$_ -notmatch '^isolation\.tools\.hgfs\.disable'
}
# ── Step 3: Build replacement sharedFolder block ──────────────────────────────
# VMX hostPath requires double-escaped backslashes
$nugetEsc = $NuGetCacheDir.Replace('\', '\\')
$pipEsc = $PipCacheDir.Replace('\', '\\')
$sharedBlock = @(
'sharedFolder.maxNum = "2"'
'sharedFolder0.present = "TRUE"'
'sharedFolder0.enabled = "TRUE"'
'sharedFolder0.readAccess = "TRUE"'
'sharedFolder0.writeAccess = "TRUE"'
"sharedFolder0.hostPath = `"$nugetEsc`""
'sharedFolder0.guestName = "nuget-cache"'
'sharedFolder0.expiration = "never"'
'sharedFolder1.present = "TRUE"'
'sharedFolder1.enabled = "TRUE"'
'sharedFolder1.readAccess = "TRUE"'
'sharedFolder1.writeAccess = "TRUE"'
"sharedFolder1.hostPath = `"$pipEsc`""
'sharedFolder1.guestName = "pip-cache"'
'sharedFolder1.expiration = "never"'
'isolation.tools.hgfs.disable = "FALSE"'
)
$newLines = @($stripped) + $sharedBlock
# ── Step 4: Backup and write VMX ─────────────────────────────────────────────
if ($PSCmdlet.ShouldProcess($TemplatePath, 'Write updated VMX with shared folder entries')) {
$backupPath = "$TemplatePath.bak"
Copy-Item $TemplatePath $backupPath -Force
Write-Host "[SharedFolders] Original backed up: $backupPath"
[System.IO.File]::WriteAllLines($TemplatePath, $newLines, [System.Text.Encoding]::UTF8)
Write-Host "[SharedFolders] VMX updated: $TemplatePath"
Write-Host "[SharedFolders] Mappings:"
Write-Host " nuget-cache $NuGetCacheDir -> \\vmware-host\Shared Folders\nuget-cache"
Write-Host " pip-cache $PipCacheDir -> \\vmware-host\Shared Folders\pip-cache"
Write-Host "[SharedFolders] Next: run Invoke-RemoteBuild.ps1 -UseSharedCache to activate."
}
+161
View File
@@ -0,0 +1,161 @@
#Requires -Version 5.1
<#
.SYNOPSIS
Monitors the act_runner service and auto-restarts it if stopped.
.DESCRIPTION
Intended to run as a scheduled task every 15 minutes (registered by
Register-CIScheduledTasks.ps1 as CI-RunnerHealth).
Behaviour:
- If act_runner is Running: exits 0, no action.
- If not Running: writes a Warning to the Windows Application Event Log
(source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
- Restart attempts are rate-limited to MaxRestarts per hour using a JSON
state file in StateDir. Once the limit is hit, further runs write an
alert (EventId 1004) and exit 1 without restarting — Task Scheduler
history shows the failure so the operator is alerted.
- On successful restart: writes EventId 1003 (Information).
- Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
Gitea webhooks on every restart or limit-exceeded event.
.PARAMETER MaxRestarts
Maximum number of auto-restart attempts allowed per rolling 60-minute
window before giving up and requiring manual intervention. Default: 3
.PARAMETER ServiceName
Windows service name to monitor. Default: act_runner
.PARAMETER StateDir
Directory used for the cooldown state file (runner-restart-log.json).
Default: F:\CI\State
.PARAMETER WebhookUrl
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
.EXAMPLE
# Manual health check
.\Watch-RunnerHealth.ps1
# With Discord webhook
.\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
#>
[CmdletBinding()]
param(
[ValidateRange(1, 10)]
[int] $MaxRestarts = 3,
[string] $ServiceName = 'act_runner',
[string] $StateDir = 'F:\CI\State',
[string] $WebhookUrl = ''
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Continue'
$logSource = 'CI-RunnerHealth'
# ── Helper: write to Event Log ────────────────────────────────────────────────
function Write-CIEvent {
param([int] $EventId, [string] $EntryType, [string] $Message)
try {
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
}
Write-EventLog -LogName Application -Source $logSource `
-EventId $EventId -EntryType $EntryType -Message $Message
} catch {
Write-Warning "[RunnerHealth] Could not write Event Log: $_"
}
}
# ── Helper: POST webhook ──────────────────────────────────────────────────────
function Send-Webhook {
param([string] $Content)
if (-not $WebhookUrl) { return }
$body = @{ content = $Content } | ConvertTo-Json -Compress
try {
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
-Body $body -ContentType 'application/json' -TimeoutSec 10
Write-Host "[RunnerHealth] Webhook notified."
} catch {
Write-Warning "[RunnerHealth] Webhook POST failed: $_"
}
}
# ── Step 1: Check service ─────────────────────────────────────────────────────
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
if (-not $svc) {
Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
exit 2
}
if ($svc.Status -eq 'Running') {
Write-Host "[RunnerHealth] $ServiceName is Running — OK"
exit 0
}
# ── Step 2: Service not running — load cooldown state ─────────────────────────
if (-not (Test-Path $StateDir)) {
New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
}
$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
$restartLog = @()
if (Test-Path $cooldownFile) {
try {
$raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
} catch { }
}
# Prune timestamps older than 1 hour
$cutoff = (Get-Date).AddHours(-1)
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
$recentCount = $restartLog.Count
$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
Write-Warning "[RunnerHealth] $stateMsg"
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg
# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
if ($recentCount -ge $MaxRestarts) {
$limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
"Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
Write-Warning "[RunnerHealth] $limitMsg"
Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
Send-Webhook -Content ":sos: **CI Runner Alert** — $limitMsg"
exit 1
}
# ── Step 4: Attempt restart ───────────────────────────────────────────────────
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
try {
Restart-Service -Name $ServiceName -Force -ErrorAction Stop
Start-Sleep -Seconds 5
$newStatus = (Get-Service -Name $ServiceName).Status
$restartMsg = "$ServiceName restarted — new status: $newStatus."
Write-Host "[RunnerHealth] $restartMsg"
# Persist this restart in the cooldown log
$restartLog += (Get-Date -Format 'o')
$restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue
$evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg
$icon = if ($newStatus -eq 'Running') { ':warning:' } else { ':sos:' }
Send-Webhook -Content "$icon **CI Runner Alert** — $restartMsg"
exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
}
catch {
$errMsg = "$ServiceName restart failed: $_"
Write-Warning "[RunnerHealth] $errMsg"
Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
Send-Webhook -Content ":sos: **CI Runner Alert** — $errMsg"
exit 1
}
+105
View File
@@ -0,0 +1,105 @@
#Requires -Version 5.1
<#
.SYNOPSIS
Shared helper functions for the Local CI/CD System scripts.
.DESCRIPTION
Import this module at the top of any CI script that needs WinRM session
options or vmrun wrappers:
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
Exported functions:
New-CISessionOption — PSSessionOption for WinRM over self-signed TLS
Resolve-VmrunPath — Validates vmrun.exe path, throws if missing
Invoke-Vmrun — Uniform vmrun -T ws wrapper; returns ExitCode + Output
#>
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
function New-CISessionOption {
<#
.SYNOPSIS
Returns a PSSessionOption for WinRM HTTPS with self-signed certificate.
.DESCRIPTION
All CI build VMs use a self-signed TLS cert on WinRM port 5986.
This option set skips CA, CN, and revocation checks — appropriate for
an isolated lab network where PKI is not present.
.OUTPUTS
[System.Management.Automation.Remoting.PSSessionOption]
#>
[OutputType([System.Management.Automation.Remoting.PSSessionOption])]
[CmdletBinding()]
param()
New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
}
function Resolve-VmrunPath {
<#
.SYNOPSIS
Validates that vmrun.exe exists at the specified path and returns it.
.DESCRIPTION
Throws a descriptive error if the file is not found, rather than letting
a later & call fail with a less useful message.
.PARAMETER VmrunPath
Full path to vmrun.exe.
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
.OUTPUTS
[string] The validated VmrunPath.
#>
[OutputType([string])]
[CmdletBinding()]
param(
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
)
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
throw "vmrun.exe not found at: $VmrunPath`nInstall VMware Workstation Pro and update -VmrunPath."
}
return $VmrunPath
}
function Invoke-Vmrun {
<#
.SYNOPSIS
Runs a vmrun -T ws <Operation> command and returns output and exit code.
.DESCRIPTION
Uniform wrapper so all vmrun calls share the same -T ws flag and output
capture pattern. Use -ThrowOnError for operations that must succeed
(clone, start); omit it for best-effort operations (stop, getState).
.PARAMETER VmrunPath
Full path to vmrun.exe.
.PARAMETER Operation
vmrun sub-command: clone, start, stop, deleteVM, list, getState,
listSnapshots, getGuestIPAddress, etc.
.PARAMETER Arguments
Additional positional arguments after the operation name.
.PARAMETER ThrowOnError
When set, throws if vmrun exits non-zero.
.OUTPUTS
[pscustomobject] with:
ExitCode [int] — vmrun exit code
Output [string[]] — combined stdout/stderr lines
#>
[OutputType([pscustomobject])]
[CmdletBinding()]
param(
[Parameter(Mandatory)] [string] $VmrunPath,
[Parameter(Mandatory)] [string] $Operation,
[string[]] $Arguments = @(),
[switch] $ThrowOnError
)
$cmdArgs = @('-T', 'ws', $Operation) + $Arguments
$output = & $VmrunPath @cmdArgs 2>&1
$exit = $LASTEXITCODE
if ($ThrowOnError -and $exit -ne 0) {
throw "vmrun $Operation failed (exit $exit): $($output -join '; ')"
}
return [pscustomobject]@{
ExitCode = $exit
Output = $output
}
}
Export-ModuleMember -Function New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun