feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -1,118 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss).
|
||||
|
||||
.DESCRIPTION
|
||||
Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours.
|
||||
For each stale directory, stops the VM (hard), deletes via vmrun deleteVM,
|
||||
then removes the directory.
|
||||
|
||||
Run as a daily scheduled task or on host startup to prevent disk accumulation.
|
||||
Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours.
|
||||
|
||||
.PARAMETER CloneBaseDir
|
||||
Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs
|
||||
|
||||
.PARAMETER MaxAgeHours
|
||||
Age threshold in hours. Directories with LastWriteTime older than this are
|
||||
treated as orphaned. Must exceed the longest expected build duration.
|
||||
Default: 4 (runner.timeout is 2h — 4h gives double margin)
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER WhatIf
|
||||
List orphaned VMs without destroying them.
|
||||
|
||||
.EXAMPLE
|
||||
# Dry run
|
||||
.\Cleanup-OrphanedBuildVMs.ps1 -WhatIf
|
||||
|
||||
# Live cleanup (run elevated)
|
||||
.\Cleanup-OrphanedBuildVMs.ps1
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
||||
|
||||
[ValidateRange(0, 168)]
|
||||
[int] $MaxAgeHours = 4,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not (Test-Path $CloneBaseDir)) {
|
||||
Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do."
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath"
|
||||
Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only."
|
||||
}
|
||||
|
||||
$cutoff = (Get-Date).AddHours(-$MaxAgeHours)
|
||||
$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.LastWriteTime -lt $cutoff })
|
||||
|
||||
if (-not $orphans) {
|
||||
Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h."
|
||||
|
||||
foreach ($dir in $orphans) {
|
||||
$vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue |
|
||||
Select-Object -First 1
|
||||
|
||||
if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) {
|
||||
Write-Host "[Cleanup] Processing: $($dir.FullName)"
|
||||
|
||||
if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
# Hard stop — don't wait for graceful shutdown on an orphan
|
||||
& $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut"
|
||||
Write-Warning " Falling back to directory removal."
|
||||
}
|
||||
}
|
||||
elseif (-not $vmx) {
|
||||
Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only."
|
||||
}
|
||||
|
||||
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path $dir.FullName) {
|
||||
Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed."
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] Removed: $($dir.FullName)"
|
||||
}
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)"
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Done."
|
||||
|
||||
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
||||
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
||||
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
||||
# than 30 minutes.
|
||||
$lockFile = 'F:\CI\State\vm-start.lock'
|
||||
$lockCutoff = (Get-Date).AddMinutes(-30)
|
||||
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
||||
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
||||
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
||||
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
||||
}
|
||||
}
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator vm cleanup @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
Reference in New Issue
Block a user