Simone
509d1fc284
H4: Invoke-VmrunBounded added to _Common.psm1 (Start-Process + Wait-Process + taskkill on
timeout, returns {ExitCode;Output;TimedOut;ElapsedSeconds}). Wired in Invoke-CIJob.ps1
for vmrun start (180s) and Remove-BuildVM.ps1 for stop/deleteVM (60s/90s).
H5: Cleanup-OrphanedBuildVMs.ps1 removes stale vm-start.lock (>30 min). ValidateRange
lowered to 0 to allow emergency -MaxAgeHours 0. Invoke-RetentionPolicy.ps1 same.
H8: _Transport.psm1 SSH hardening: StrictHostKeyChecking=accept-new, F:\CI\State\known_hosts.
H9: Validate-DeployState.ps1 Linux branch added: checks ci-report-ip.service enabled,
ci-report-ip.sh executable via SSH. New params: -GuestOS, -SshKeyPath, -SshUser.
H12: Get-GuestIPAddress extracted into _Common.psm1 (guestinfo.ci-ip primary, getGuestIPAddress
fallback). Invoke-CIJob.ps1 Phase 3b replaced with single call. Measure-CIBenchmark.ps1
updated to match.
Also: _Common.psm1 adds Write-CIRedactedCommand and ConvertTo-SafeShellSingleQuotedString.
Pester suite updated: 30/30 pass.
119 lines
4.5 KiB
PowerShell
119 lines
4.5 KiB
PowerShell
#Requires -Version 5.1
|
|
<#
|
|
.SYNOPSIS
|
|
Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss).
|
|
|
|
.DESCRIPTION
|
|
Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours.
|
|
For each stale directory, stops the VM (hard), deletes via vmrun deleteVM,
|
|
then removes the directory.
|
|
|
|
Run as a daily scheduled task or on host startup to prevent disk accumulation.
|
|
Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours.
|
|
|
|
.PARAMETER CloneBaseDir
|
|
Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs
|
|
|
|
.PARAMETER MaxAgeHours
|
|
Age threshold in hours. Directories with LastWriteTime older than this are
|
|
treated as orphaned. Must exceed the longest expected build duration.
|
|
Default: 4 (runner.timeout is 2h — 4h gives double margin)
|
|
|
|
.PARAMETER VmrunPath
|
|
Path to vmrun.exe.
|
|
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
|
|
.PARAMETER WhatIf
|
|
List orphaned VMs without destroying them.
|
|
|
|
.EXAMPLE
|
|
# Dry run
|
|
.\Cleanup-OrphanedBuildVMs.ps1 -WhatIf
|
|
|
|
# Live cleanup (run elevated)
|
|
.\Cleanup-OrphanedBuildVMs.ps1
|
|
#>
|
|
[CmdletBinding(SupportsShouldProcess)]
|
|
param(
|
|
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
|
|
|
[ValidateRange(0, 168)]
|
|
[int] $MaxAgeHours = 4,
|
|
|
|
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors
|
|
|
|
if (-not (Test-Path $CloneBaseDir)) {
|
|
Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do."
|
|
exit 0
|
|
}
|
|
|
|
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath"
|
|
Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only."
|
|
}
|
|
|
|
$cutoff = (Get-Date).AddHours(-$MaxAgeHours)
|
|
$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
|
Where-Object { $_.LastWriteTime -lt $cutoff })
|
|
|
|
if (-not $orphans) {
|
|
Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)."
|
|
exit 0
|
|
}
|
|
|
|
Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h."
|
|
|
|
foreach ($dir in $orphans) {
|
|
$vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue |
|
|
Select-Object -First 1
|
|
|
|
if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) {
|
|
Write-Host "[Cleanup] Processing: $($dir.FullName)"
|
|
|
|
if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) {
|
|
# Hard stop — don't wait for graceful shutdown on an orphan
|
|
& $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null
|
|
Start-Sleep -Seconds 2
|
|
$deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut"
|
|
Write-Warning " Falling back to directory removal."
|
|
}
|
|
}
|
|
elseif (-not $vmx) {
|
|
Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only."
|
|
}
|
|
|
|
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
|
if (Test-Path $dir.FullName) {
|
|
Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed."
|
|
}
|
|
else {
|
|
Write-Host "[Cleanup] Removed: $($dir.FullName)"
|
|
}
|
|
}
|
|
else {
|
|
Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)"
|
|
}
|
|
}
|
|
|
|
Write-Host "[Cleanup] Done."
|
|
|
|
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
|
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
|
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
|
# than 30 minutes.
|
|
$lockFile = 'F:\CI\State\vm-start.lock'
|
|
$lockCutoff = (Get-Date).AddMinutes(-30)
|
|
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
|
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
|
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
|
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
|
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
|
}
|
|
}
|