feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -1,118 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss).
|
||||
|
||||
.DESCRIPTION
|
||||
Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours.
|
||||
For each stale directory, stops the VM (hard), deletes via vmrun deleteVM,
|
||||
then removes the directory.
|
||||
|
||||
Run as a daily scheduled task or on host startup to prevent disk accumulation.
|
||||
Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours.
|
||||
|
||||
.PARAMETER CloneBaseDir
|
||||
Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs
|
||||
|
||||
.PARAMETER MaxAgeHours
|
||||
Age threshold in hours. Directories with LastWriteTime older than this are
|
||||
treated as orphaned. Must exceed the longest expected build duration.
|
||||
Default: 4 (runner.timeout is 2h — 4h gives double margin)
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER WhatIf
|
||||
List orphaned VMs without destroying them.
|
||||
|
||||
.EXAMPLE
|
||||
# Dry run
|
||||
.\Cleanup-OrphanedBuildVMs.ps1 -WhatIf
|
||||
|
||||
# Live cleanup (run elevated)
|
||||
.\Cleanup-OrphanedBuildVMs.ps1
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
||||
|
||||
[ValidateRange(0, 168)]
|
||||
[int] $MaxAgeHours = 4,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not (Test-Path $CloneBaseDir)) {
|
||||
Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do."
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath"
|
||||
Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only."
|
||||
}
|
||||
|
||||
$cutoff = (Get-Date).AddHours(-$MaxAgeHours)
|
||||
$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.LastWriteTime -lt $cutoff })
|
||||
|
||||
if (-not $orphans) {
|
||||
Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h."
|
||||
|
||||
foreach ($dir in $orphans) {
|
||||
$vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue |
|
||||
Select-Object -First 1
|
||||
|
||||
if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) {
|
||||
Write-Host "[Cleanup] Processing: $($dir.FullName)"
|
||||
|
||||
if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
# Hard stop — don't wait for graceful shutdown on an orphan
|
||||
& $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut"
|
||||
Write-Warning " Falling back to directory removal."
|
||||
}
|
||||
}
|
||||
elseif (-not $vmx) {
|
||||
Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only."
|
||||
}
|
||||
|
||||
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path $dir.FullName) {
|
||||
Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed."
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] Removed: $($dir.FullName)"
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)"
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Done."
|
||||
|
||||
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
||||
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
||||
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
||||
# than 30 minutes.
|
||||
$lockFile = 'F:\CI\State\vm-start.lock'
|
||||
$lockCutoff = (Get-Date).AddMinutes(-30)
|
||||
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
||||
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
||||
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
||||
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
||||
}
|
||||
}
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator vm cleanup @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -1,7 +1,27 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Displays a summary table of CI job results parsed from JSONL log files.
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator report job @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
|
||||
.DESCRIPTION
|
||||
Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and
|
||||
|
||||
+16
-123
@@ -1,130 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Stops and permanently deletes an ephemeral build VM.
|
||||
|
||||
.DESCRIPTION
|
||||
Performs a graceful shutdown (soft stop) with a timeout fallback to
|
||||
a hard stop, then uses vmrun deleteVM to remove all VM files.
|
||||
Finally, removes the clone directory from disk.
|
||||
|
||||
This script is designed to be called from a try/finally block so
|
||||
it always runs, even on build failure.
|
||||
|
||||
.PARAMETER VMPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER GracefulTimeoutSeconds
|
||||
Seconds to wait for graceful shutdown before forcing power-off.
|
||||
Default: 15
|
||||
|
||||
.EXAMPLE
|
||||
.\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx"
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $VMPath,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
||||
|
||||
[ValidateRange(1, 60)]
|
||||
[int] $GracefulTimeoutSeconds = 15
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
||||
|
||||
$cloneDir = Split-Path $VMPath -Parent
|
||||
|
||||
Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
|
||||
|
||||
# ── Step 1: Check if VM exists at all ────────────────────────────────────────
|
||||
if (-not (Test-Path $VMPath -PathType Leaf)) {
|
||||
Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath"
|
||||
# Still attempt to remove the directory in case of partial clone
|
||||
if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) {
|
||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
|
||||
|
||||
# ── Step 2: Graceful stop ─────────────────────────────────────────────────────
|
||||
Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..."
|
||||
$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds
|
||||
if ($stopSoft.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s."
|
||||
} elseif ($stopSoft.ExitCode -eq 0) {
|
||||
Write-Host "[Remove-BuildVM] VM stopped gracefully."
|
||||
}
|
||||
|
||||
# ── Step 3: Force stop if still running ──────────────────────────────────────
|
||||
if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) {
|
||||
Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..."
|
||||
$stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60
|
||||
if ($stopHard.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s."
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly
|
||||
# after stop. Waiting for the entry to disappear ensures deleteVM won't race the
|
||||
# process exit.
|
||||
$vmxNorm = $VMPath.Trim().ToLower()
|
||||
$releaseDeadline = (Get-Date).AddSeconds(20)
|
||||
while ((Get-Date) -lt $releaseDeadline) {
|
||||
$listedVMs = & $VmrunPath -T ws list 2>&1 |
|
||||
Where-Object { $_ -notmatch '^Total running' } |
|
||||
ForEach-Object { "$_".Trim().ToLower() }
|
||||
if ($listedVMs -notcontains $vmxNorm) { break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
# ── Step 4: Delete VM via vmrun — retry up to 3× with backoff ────────────────
|
||||
Write-Host "[Remove-BuildVM] Deleting VM..."
|
||||
$deleteOutput = ''
|
||||
$deleteExit = -1
|
||||
foreach ($delay in @(0, 3, 6)) {
|
||||
if ($delay -gt 0) {
|
||||
Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
}
|
||||
$delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90
|
||||
$deleteOutput = $delResult.Output
|
||||
$deleteExit = $delResult.ExitCode
|
||||
if ($delResult.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s."
|
||||
$deleteExit = -1
|
||||
}
|
||||
if ($deleteExit -eq 0) { break }
|
||||
}
|
||||
|
||||
if ($deleteExit -ne 0) {
|
||||
Write-Warning "[Remove-BuildVM] vmrun deleteVM failed after retries (exit $deleteExit): $deleteOutput"
|
||||
Write-Warning "[Remove-BuildVM] Will attempt manual directory removal."
|
||||
}
|
||||
|
||||
# ── Step 5: Remove clone directory (catches any leftover files) ───────────────
|
||||
if (Test-Path $cloneDir) {
|
||||
Write-Host "[Remove-BuildVM] Removing clone directory: $cloneDir"
|
||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path $cloneDir) {
|
||||
Write-Warning "[Remove-BuildVM] Clone directory could not be fully removed: $cloneDir"
|
||||
Write-Warning " Manual cleanup required."
|
||||
} else {
|
||||
Write-Host "[Remove-BuildVM] Clone directory removed."
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[Remove-BuildVM] VM destruction complete."
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator vm remove @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+14
-241
@@ -1,250 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Waits until a build VM is fully ready to accept WinRM connections.
|
||||
|
||||
.DESCRIPTION
|
||||
Performs a three-phase readiness check in a polling loop:
|
||||
1. vmrun getState returns "running"
|
||||
2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH)
|
||||
3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux)
|
||||
|
||||
Throws if the VM does not become ready within the timeout period.
|
||||
|
||||
Transport modes:
|
||||
WinRM (default) — existing Windows VM path: ping + WinRM port 5986.
|
||||
SSH — Linux VM path: port 22 check + ssh echo test.
|
||||
|
||||
.PARAMETER VMPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
|
||||
.PARAMETER IPAddress
|
||||
IP address of the VM on the build network (e.g. 192.168.11.101).
|
||||
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to wait for the VM to become ready.
|
||||
Default: 300 (5 minutes)
|
||||
|
||||
.PARAMETER PollIntervalSeconds
|
||||
Seconds between readiness checks.
|
||||
Default: 5
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER Transport
|
||||
Transport protocol for readiness checks.
|
||||
WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs).
|
||||
SSH: checks TCP 22 + ssh echo (Linux VMs).
|
||||
|
||||
.PARAMETER Credential
|
||||
Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs
|
||||
a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming
|
||||
that WinRM authentication (not just port) is working. Useful to detect the
|
||||
window where the WinRM listener is up but the LocalSystem session is not yet
|
||||
accepting logins. When omitted the TCP check alone is used (existing behaviour).
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Path to the SSH private key for key-based auth when Transport=SSH.
|
||||
Default: F:\CI\keys\ci_linux
|
||||
|
||||
.PARAMETER SshUser
|
||||
SSH username when Transport=SSH.
|
||||
Default: ci_build
|
||||
|
||||
.EXAMPLE
|
||||
.\Wait-VMReady.ps1 `
|
||||
-VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" `
|
||||
-IPAddress "192.168.11.101"
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $VMPath,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
||||
[string] $IPAddress,
|
||||
|
||||
[ValidateRange(3, 600)]
|
||||
[int] $TimeoutSeconds = 300,
|
||||
|
||||
[ValidateRange(1, 30)]
|
||||
[int] $PollIntervalSeconds = 5,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
||||
|
||||
# Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP
|
||||
# but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM)
|
||||
# are still checked.
|
||||
[switch] $SkipPing,
|
||||
|
||||
# Transport protocol for phase 2/3 checks.
|
||||
# WinRM (default) = Windows VM; SSH = Linux VM.
|
||||
[ValidateSet('WinRM', 'SSH')]
|
||||
[string] $Transport = 'WinRM',
|
||||
|
||||
# SSH private key path (used when Transport=SSH).
|
||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||
|
||||
# SSH username (used when Transport=SSH).
|
||||
[string] $SshUser = 'ci_build',
|
||||
|
||||
# Optional credential for WinRM authentication probe (Transport=WinRM).
|
||||
# When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds.
|
||||
[PSCredential] $Credential = $null
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
throw "vmrun.exe not found at: $VmrunPath"
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$attempt = 0
|
||||
$phase = 'vmrun-state' # tracks current phase for informative output
|
||||
$lastPhase = ''
|
||||
|
||||
Write-Host "[Wait-VMReady] Waiting for VM ready (timeout: ${TimeoutSeconds}s, IP: $IPAddress)..."
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$attempt++
|
||||
|
||||
# ── Phase 1: VM must be running ─────────────────────────────────────
|
||||
# vmrun has no getState. We previously used `getGuestIPAddress`, but that
|
||||
# requires VMware Tools to be up and responding inside the guest. In
|
||||
# headless boots Tools can take 30-60s to come online, during which the
|
||||
# call exits non-zero even though the VM is fully powered on. That caused
|
||||
# Phase 1 to appear "stuck" until a GUI was opened manually.
|
||||
# `vmrun list` reports actual power state without needing Tools.
|
||||
$vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue)
|
||||
if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath }
|
||||
$listOut = & $VmrunPath list 2>&1
|
||||
$isRunning = $false
|
||||
foreach ($line in $listOut) {
|
||||
if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$isRunning = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $isRunning) {
|
||||
if ($phase -ne 'vmrun-state') {
|
||||
$phase = 'vmrun-state'
|
||||
}
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 1/3: waiting for VM to enter running state..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
|
||||
# ── Phase 2: Network check ──────────────────────────────────────────
|
||||
if ($Transport -eq 'SSH') {
|
||||
$port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue
|
||||
if (-not $port22Open) {
|
||||
$phase = 'ssh-port'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
elseif (-not $SkipPing) {
|
||||
$pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $pingOk) {
|
||||
$phase = 'ping'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for network (ping $IPAddress)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
# ── Phase 3: Transport-specific login check ────────────────────────
|
||||
if ($Transport -eq 'SSH') {
|
||||
$sshArgs = @(
|
||||
'-i', $SshKeyPath,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-o', 'ConnectTimeout=5',
|
||||
'-o', 'BatchMode=yes',
|
||||
"$SshUser@$IPAddress",
|
||||
'echo ready'
|
||||
)
|
||||
# ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts")
|
||||
# to stderr. Under $ErrorActionPreference='Stop' (set at script top),
|
||||
# those stderr lines captured via 2>&1 are promoted to terminating errors
|
||||
# and abort the script. Wrap the call to demote stderr to non-terminating.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$sshOut = & ssh @sshArgs 2>&1
|
||||
$sshExit = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') {
|
||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
||||
return
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
$phase = 'ssh-echo'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
else {
|
||||
# ── WinRM port 5986 accepting TCP connections ─────────────────────
|
||||
# Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for
|
||||
# self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready.
|
||||
$winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue
|
||||
if ($winrmUp) {
|
||||
# Optional auth probe — ensures WinRM login is functional, not just TCP open
|
||||
if ($null -ne $Credential) {
|
||||
try {
|
||||
$so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
$null = Invoke-Command -ComputerName $IPAddress -Credential $Credential `
|
||||
-Authentication Basic -UseSSL -Port 5986 -SessionOption $so `
|
||||
-ScriptBlock { 'ready' } -ErrorAction Stop
|
||||
Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress."
|
||||
} catch {
|
||||
$phase = 'winrm-auth'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
||||
return
|
||||
}
|
||||
else {
|
||||
$phase = 'winrm'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
throw "[Wait-VMReady] Timed out after ${TimeoutSeconds}s waiting for VM at $IPAddress to become ready. Last phase: $phase"
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator wait-ready @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+16
-89
@@ -1,96 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook).
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB,
|
||||
writes a Warning entry to the Windows Application Event Log under source
|
||||
'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler
|
||||
marks the task as failed (visible in Task Scheduler history).
|
||||
|
||||
If WebhookUrl is provided, also POSTs a JSON payload to that URL
|
||||
(compatible with Discord and Gitea webhooks — content field only).
|
||||
|
||||
A full drive silently causes vmrun clone to fail with cryptic errors
|
||||
(not enough disk space for linked clone delta files). Alert early.
|
||||
|
||||
.PARAMETER MinFreeGB
|
||||
Alert threshold in GB. Default: 50
|
||||
|
||||
.PARAMETER DriveLetter
|
||||
Single letter (no colon) of the drive to monitor. Default: F
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual check
|
||||
.\Watch-DiskSpace.ps1 -MinFreeGB 50
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 2000)]
|
||||
[int] $MinFreeGB = 50,
|
||||
|
||||
[ValidatePattern('^[A-Za-z]$')]
|
||||
[string] $DriveLetter = 'F',
|
||||
|
||||
[string] $WebhookUrl = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue
|
||||
if (-not $drive) {
|
||||
Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found."
|
||||
exit 2
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
|
||||
$freeGB = [math]::Round($drive.Free / 1GB, 1)
|
||||
$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1)
|
||||
$pctFree = [math]::Round($freeGB / $totalGB * 100, 0)
|
||||
|
||||
if ($freeGB -ge $MinFreeGB) {
|
||||
Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Alert ─────────────────────────────────────────────────────────────────────
|
||||
$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " +
|
||||
"the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " +
|
||||
"Run Invoke-RetentionPolicy.ps1 or free disk space manually."
|
||||
|
||||
Write-Warning "[DiskSpace] $msg"
|
||||
|
||||
# Windows Event Log
|
||||
$logSource = 'CI-DiskSpaceAlert'
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId 1001 -EntryType Warning -Message $msg
|
||||
Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)."
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Could not write Event Log: $_"
|
||||
}
|
||||
|
||||
# Optional webhook (Discord / Gitea)
|
||||
if ($WebhookUrl) {
|
||||
$body = @{ content = "[WARNING] CI Disk Alert -- $msg" } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[DiskSpace] Webhook notified: $WebhookUrl"
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Webhook POST failed: $_"
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer)
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator monitor disk @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+16
-207
@@ -1,214 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Monitors the act_runner service and auto-restarts it if stopped.
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1 as CI-RunnerHealth).
|
||||
|
||||
Behaviour:
|
||||
- If act_runner is Running: exits 0, no action.
|
||||
- If not Running: writes a Warning to the Windows Application Event Log
|
||||
(source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
|
||||
- Restart attempts are rate-limited to MaxRestarts per hour using a JSON
|
||||
state file in StateDir. Once the limit is hit, further runs write an
|
||||
alert (EventId 1004) and exit 1 without restarting — Task Scheduler
|
||||
history shows the failure so the operator is alerted.
|
||||
- On successful restart: writes EventId 1003 (Information).
|
||||
- Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
|
||||
Gitea webhooks on every restart or limit-exceeded event.
|
||||
|
||||
.PARAMETER MaxRestarts
|
||||
Maximum number of auto-restart attempts allowed per rolling 60-minute
|
||||
window before giving up and requiring manual intervention. Default: 3
|
||||
|
||||
.PARAMETER ServiceName
|
||||
Windows service name to monitor. Default: act_runner
|
||||
|
||||
.PARAMETER StateDir
|
||||
Directory used for the cooldown state file (runner-restart-log.json).
|
||||
Default: F:\CI\State
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual health check
|
||||
.\Watch-RunnerHealth.ps1
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 10)]
|
||||
[int] $MaxRestarts = 3,
|
||||
|
||||
[string] $ServiceName = 'act_runner',
|
||||
|
||||
[string] $StateDir = 'F:\CI\State',
|
||||
|
||||
[string] $WebhookUrl = '',
|
||||
|
||||
# Optional Gitea API runner-online verification.
|
||||
# When $GiteaUrl is non-empty and the act_runner service is Running, this script
|
||||
# queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online.
|
||||
# Requires an admin Personal Access Token stored in Windows Credential Manager
|
||||
# under $GiteaCredentialTarget (same module: CredentialManager).
|
||||
# Silently skipped if GiteaUrl is empty or the credential is unavailable.
|
||||
[string] $GiteaUrl = '',
|
||||
|
||||
[string] $GiteaCredentialTarget = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$logSource = 'CI-RunnerHealth'
|
||||
|
||||
# ── Helper: write to Event Log ────────────────────────────────────────────────
|
||||
function Write-CIEvent {
|
||||
param([int] $EventId, [string] $EntryType, [string] $Message)
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId $EventId -EntryType $EntryType -Message $Message
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Could not write Event Log: $_"
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
# ── Helper: POST webhook ──────────────────────────────────────────────────────
|
||||
function Send-Webhook {
|
||||
param([string] $Content)
|
||||
if (-not $WebhookUrl) { return }
|
||||
$body = @{ content = $Content } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[RunnerHealth] Webhook notified."
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Webhook POST failed: $_"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Maintenance flag check ────────────────────────────────────────────────────
|
||||
# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic.
|
||||
# Create the flag before planned maintenance; delete it when done.
|
||||
$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag'
|
||||
if (Test-Path $maintenanceFlag) {
|
||||
Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic."
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Step 1: Check service ─────────────────────────────────────────────────────
|
||||
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
||||
if (-not $svc) {
|
||||
Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
|
||||
exit 2
|
||||
}
|
||||
|
||||
if ($svc.Status -eq 'Running') {
|
||||
Write-Host "[RunnerHealth] $ServiceName is Running — OK"
|
||||
|
||||
# ── Optional Gitea runner-online API check ────────────────────────────────
|
||||
if ($GiteaUrl -ne '') {
|
||||
$pat = $null
|
||||
if ($GiteaCredentialTarget -ne '') {
|
||||
try {
|
||||
$cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
|
||||
$pat = $cred.GetNetworkCredential().Password
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_"
|
||||
}
|
||||
}
|
||||
if ($pat) {
|
||||
try {
|
||||
$apiUrl = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners"
|
||||
$runners = Invoke-RestMethod -Uri $apiUrl `
|
||||
-Headers @{ Authorization = "token $pat" } `
|
||||
-Method Get -TimeoutSec 10 -ErrorAction Stop
|
||||
$online = @($runners | Where-Object { $_.online -eq $true })
|
||||
if ($online.Count -gt 0) {
|
||||
Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK"
|
||||
} else {
|
||||
$msg = "Gitea API ($apiUrl) reports 0 online runners. " +
|
||||
"$ServiceName service is Running — possible registration issue."
|
||||
Write-Warning "[RunnerHealth] $msg"
|
||||
Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg
|
||||
Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg"
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Gitea API check failed: $_"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Step 2: Service not running — load cooldown state ─────────────────────────
|
||||
if (-not (Test-Path $StateDir)) {
|
||||
New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
|
||||
$restartLog = @()
|
||||
|
||||
if (Test-Path $cooldownFile) {
|
||||
try {
|
||||
$raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
|
||||
if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
|
||||
} catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" }
|
||||
}
|
||||
|
||||
# Prune timestamps older than 1 hour
|
||||
$cutoff = (Get-Date).AddHours(-1)
|
||||
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
|
||||
$recentCount = $restartLog.Count
|
||||
|
||||
$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
|
||||
Write-Warning "[RunnerHealth] $stateMsg"
|
||||
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg
|
||||
|
||||
# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
|
||||
if ($recentCount -ge $MaxRestarts) {
|
||||
$limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
|
||||
"Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
|
||||
Write-Warning "[RunnerHealth] $limitMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
|
||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Step 4: Attempt restart ───────────────────────────────────────────────────
|
||||
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
|
||||
try {
|
||||
Restart-Service -Name $ServiceName -Force -ErrorAction Stop
|
||||
Start-Sleep -Seconds 5
|
||||
$newStatus = (Get-Service -Name $ServiceName).Status
|
||||
$restartMsg = "$ServiceName restarted — new status: $newStatus."
|
||||
Write-Host "[RunnerHealth] $restartMsg"
|
||||
|
||||
# Persist this restart in the cooldown log
|
||||
$restartLog += (Get-Date -Format 'o')
|
||||
$restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue
|
||||
|
||||
$evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
|
||||
Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg
|
||||
|
||||
$prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' }
|
||||
Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg"
|
||||
|
||||
exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
|
||||
}
|
||||
catch {
|
||||
$errMsg = "$ServiceName restart failed: $_"
|
||||
Write-Warning "[RunnerHealth] $errMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
|
||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg"
|
||||
exit 1
|
||||
}
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator monitor runner @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+21
-102
@@ -1,122 +1,41 @@
|
||||
"""CLI entry point.
|
||||
|
||||
Phase A1 ships the ``wait-ready`` PoC sub-command. Subsequent steps add
|
||||
``vm``, ``build``, ``artifacts``, ``monitor``, ``report``, and ``job``.
|
||||
Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor``
|
||||
(disk/runner) and ``report`` (job). Subsequent phases add ``vm new``,
|
||||
``build``, ``artifacts``, ``job``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
import time as time
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator import __version__
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
from ci_orchestrator.commands.monitor import monitor
|
||||
from ci_orchestrator.commands.report import report
|
||||
from ci_orchestrator.commands.vm import vm
|
||||
|
||||
# Re-export so legacy A1 tests that patched ``cli_module.<name>`` keep working.
|
||||
from ci_orchestrator.commands.wait import ( # noqa: F401 (intentional re-exports)
|
||||
KeyringCredentialStore,
|
||||
SshTransport,
|
||||
WinRmTransport,
|
||||
load_backend,
|
||||
wait_ready,
|
||||
)
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.version_option(__version__, prog_name="ci-orchestrator")
|
||||
def cli() -> None:
|
||||
"""Local CI/CD orchestrator (Phase A)."""
|
||||
"""Local CI/CD orchestrator."""
|
||||
|
||||
|
||||
@cli.command("wait-ready")
|
||||
@click.option("--vmx", required=True, help="Path to the guest VMX.")
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
type=click.Choice(["windows", "linux"]),
|
||||
default="windows",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--timeout", type=float, default=300.0, show_default=True)
|
||||
@click.option(
|
||||
"--poll-interval", type=float, default=5.0, show_default=True, help="Seconds between probes."
|
||||
)
|
||||
@click.option(
|
||||
"--credential-target",
|
||||
default=None,
|
||||
help="Override credential target name (defaults to config.guest_cred_target).",
|
||||
)
|
||||
@click.option("--ssh-user", default="ci_build", show_default=True)
|
||||
def wait_ready(
|
||||
vmx: str,
|
||||
guest_os: str,
|
||||
timeout: float,
|
||||
poll_interval: float,
|
||||
credential_target: str | None,
|
||||
ssh_user: str,
|
||||
) -> None:
|
||||
"""Poll a guest until WinRM (Windows) or SSH (Linux) is responsive."""
|
||||
config = load_config()
|
||||
backend = load_backend(config)
|
||||
handle = VmHandle(identifier=vmx)
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
# Step 1: wait until backend reports the VM as running.
|
||||
click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}")
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
break
|
||||
except BackendError as exc:
|
||||
click.echo(f"[wait-ready] backend probe error: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
else:
|
||||
click.echo("[wait-ready] timeout waiting for VM to be running.", err=True)
|
||||
sys.exit(2)
|
||||
|
||||
# Step 2: wait for an IP.
|
||||
ip: str | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
ip = backend.get_ip(handle)
|
||||
except BackendError:
|
||||
ip = None
|
||||
if ip:
|
||||
break
|
||||
time.sleep(poll_interval)
|
||||
if not ip:
|
||||
click.echo("[wait-ready] timeout waiting for guest IP.", err=True)
|
||||
sys.exit(3)
|
||||
click.echo(f"[wait-ready] guest IP: {ip}")
|
||||
|
||||
# Step 3: probe the transport layer.
|
||||
if guest_os == "windows":
|
||||
store = KeyringCredentialStore()
|
||||
target = credential_target or config.guest_cred_target
|
||||
cred = store.get(target)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with WinRmTransport(ip, cred.username, cred.password) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] WinRM ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
else:
|
||||
key_path = config.ssh_key_path
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with SshTransport(ip, username=ssh_user, key_path=key_path) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] SSH ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
|
||||
click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True)
|
||||
sys.exit(4)
|
||||
cli.add_command(wait_ready)
|
||||
cli.add_command(vm)
|
||||
cli.add_command(monitor)
|
||||
cli.add_command(report)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Sub-command modules for the ``ci_orchestrator`` CLI.
|
||||
|
||||
Each module exposes one or more :class:`click.Command` instances that the
|
||||
top-level :mod:`ci_orchestrator.__main__` registers on the root group.
|
||||
Phase A2 ports the leaf PowerShell scripts (Wait-VMReady, Remove-BuildVM,
|
||||
Cleanup-OrphanedBuildVMs, Watch-DiskSpace, Watch-RunnerHealth,
|
||||
Get-CIJobSummary).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,459 @@
|
||||
"""``monitor`` sub-commands.
|
||||
|
||||
Phase A2 ports the maintenance/monitoring leaf scripts:
|
||||
|
||||
* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1``
|
||||
* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1``
|
||||
|
||||
Both commands are designed to run from a scheduler (Windows Task Scheduler
|
||||
in Phase A, systemd timers in Phase B) and emit warnings via the platform
|
||||
event log when available, plus optional Discord/Gitea webhooks via stdlib
|
||||
``urllib`` (no extra dependency).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import click
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool:
|
||||
"""POST a Discord/Gitea-compatible JSON payload. Returns True on success."""
|
||||
if not url:
|
||||
return False
|
||||
payload = json.dumps({"content": content}).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=payload,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return bool(200 <= resp.status < 300)
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
logger.warning("Webhook POST failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def _write_windows_event(
|
||||
source: str, event_id: int, entry_type: str, message: str
|
||||
) -> bool:
|
||||
"""Write a Windows Application Event Log entry via ``eventcreate.exe``.
|
||||
|
||||
Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts.
|
||||
Returns True on success.
|
||||
"""
|
||||
if os.name != "nt":
|
||||
return False
|
||||
eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe")
|
||||
if eventcreate is None:
|
||||
return False
|
||||
type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"}
|
||||
cmd = [
|
||||
eventcreate,
|
||||
"/T", type_map.get(entry_type, "WARNING"),
|
||||
"/ID", str(event_id),
|
||||
"/L", "APPLICATION",
|
||||
"/SO", source,
|
||||
"/D", message,
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, check=False, timeout=10.0
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError) as exc:
|
||||
logger.warning("eventcreate failed: %s", exc)
|
||||
return False
|
||||
if result.returncode != 0:
|
||||
logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("monitor")
|
||||
def monitor() -> None:
|
||||
"""Host monitoring commands (disk, runner)."""
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────── disk
|
||||
|
||||
|
||||
def _resolve_drive_path(drive: str) -> str:
|
||||
"""Map a single-letter Windows drive (``F``) to a usable path.
|
||||
|
||||
On non-Windows the ``drive`` value is treated as a path itself.
|
||||
"""
|
||||
if os.name == "nt" and len(drive) == 1 and drive.isalpha():
|
||||
return f"{drive.upper()}:\\"
|
||||
return drive
|
||||
|
||||
|
||||
@monitor.command("disk")
|
||||
@click.option(
|
||||
"--min-free-gb",
|
||||
"min_free_gb",
|
||||
type=click.IntRange(1, 2000),
|
||||
default=50,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--drive-letter",
|
||||
"drive_letter",
|
||||
default="F" if os.name == "nt" else "/",
|
||||
show_default=True,
|
||||
help="Drive letter (Windows) or mount path (Linux).",
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.")
|
||||
@click.option(
|
||||
"--json",
|
||||
"as_json",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Emit a single-line JSON object instead of human-readable output.",
|
||||
)
|
||||
def monitor_disk(
|
||||
min_free_gb: int,
|
||||
drive_letter: str,
|
||||
webhook_url: str,
|
||||
as_json: bool,
|
||||
) -> None:
|
||||
"""Check free space on a CI host drive and alert if below threshold.
|
||||
|
||||
Exit codes mirror the original PowerShell script:
|
||||
|
||||
* 0 = drive OK
|
||||
* 1 = below threshold (alert raised)
|
||||
* 2 = drive not found
|
||||
"""
|
||||
target = _resolve_drive_path(drive_letter)
|
||||
try:
|
||||
usage = shutil.disk_usage(target)
|
||||
except (FileNotFoundError, OSError) as exc:
|
||||
click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True)
|
||||
sys.exit(2)
|
||||
|
||||
gb = 1024.0 ** 3
|
||||
free_gb = round(usage.free / gb, 1)
|
||||
total_gb = round(usage.total / gb, 1)
|
||||
pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0
|
||||
|
||||
payload = {
|
||||
"drive": drive_letter,
|
||||
"free_gb": free_gb,
|
||||
"total_gb": total_gb,
|
||||
"percent_free": pct_free,
|
||||
"threshold_gb": min_free_gb,
|
||||
"status": "ok" if free_gb >= min_free_gb else "alert",
|
||||
}
|
||||
|
||||
if free_gb >= min_free_gb:
|
||||
if as_json:
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(
|
||||
f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB "
|
||||
f"({pct_free}%) - OK (threshold: {min_free_gb} GB)"
|
||||
)
|
||||
return
|
||||
|
||||
msg = (
|
||||
f"CI host drive {drive_letter}: free space {free_gb} GB "
|
||||
f"({pct_free}%) is below the {min_free_gb} GB threshold. "
|
||||
"vmrun linked clones may fail silently."
|
||||
)
|
||||
if as_json:
|
||||
payload["message"] = msg
|
||||
click.echo(json.dumps(payload))
|
||||
else:
|
||||
click.echo(f"[disk] WARNING: {msg}", err=True)
|
||||
|
||||
_write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── runner
|
||||
|
||||
|
||||
def _service_status(service_name: str) -> str | None:
|
||||
"""Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[sc, "query", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
out = r.stdout.upper()
|
||||
if "RUNNING" in out:
|
||||
return "Running"
|
||||
if "STOPPED" in out:
|
||||
return "Stopped"
|
||||
return "Unknown"
|
||||
# Linux / systemd
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return None
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[systemctl, "is-active", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=10.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
state = r.stdout.strip().lower()
|
||||
if state == "active":
|
||||
return "Running"
|
||||
if state in {"inactive", "failed", "deactivating"}:
|
||||
return "Stopped"
|
||||
return None # unknown / not installed
|
||||
|
||||
|
||||
def _restart_service(service_name: str) -> bool:
|
||||
"""Attempt to (re)start ``service_name``. Returns True if now running."""
|
||||
if os.name == "nt":
|
||||
sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe"
|
||||
# `sc start` returns 1056 if already running; ignore.
|
||||
try:
|
||||
subprocess.run(
|
||||
[sc, "start", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
else:
|
||||
systemctl = shutil.which("systemctl")
|
||||
if systemctl is None:
|
||||
return False
|
||||
try:
|
||||
subprocess.run(
|
||||
[systemctl, "restart", service_name],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
timeout=30.0,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return False
|
||||
time.sleep(5.0)
|
||||
return _service_status(service_name) == "Running"
|
||||
|
||||
|
||||
def _load_restart_log(path: Path) -> list[str]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return []
|
||||
if not raw.strip():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
if isinstance(data, list):
|
||||
return [str(x) for x in data]
|
||||
return []
|
||||
|
||||
|
||||
def _save_restart_log(path: Path, entries: list[str]) -> None:
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(entries), encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.warning("Could not persist restart log: %s", exc)
|
||||
|
||||
|
||||
def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None:
|
||||
"""Return number of online runners reported by Gitea, or ``None`` on failure."""
|
||||
api = api_url.rstrip("/") + "/api/v1/admin/runners"
|
||||
req = urllib.request.Request(
|
||||
api, headers={"Authorization": f"token {token}"}
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("Gitea API call failed: %s", exc)
|
||||
return None
|
||||
if isinstance(data, list):
|
||||
return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True)
|
||||
return None
|
||||
|
||||
|
||||
@monitor.command("runner")
|
||||
@click.option(
|
||||
"--service-name",
|
||||
"service_name",
|
||||
default="act_runner",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--max-restarts",
|
||||
type=click.IntRange(1, 10),
|
||||
default=3,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--state-dir",
|
||||
"state_dir",
|
||||
default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--webhook-url", "webhook_url", default="")
|
||||
@click.option("--gitea-url", "gitea_url", default="")
|
||||
@click.option(
|
||||
"--gitea-credential-target",
|
||||
"gitea_credential_target",
|
||||
default="",
|
||||
help="Credential target name for the Gitea admin PAT.",
|
||||
)
|
||||
def monitor_runner(
|
||||
service_name: str,
|
||||
max_restarts: int,
|
||||
state_dir: str,
|
||||
webhook_url: str,
|
||||
gitea_url: str,
|
||||
gitea_credential_target: str,
|
||||
) -> None:
|
||||
"""Monitor act_runner; auto-restart up to ``--max-restarts`` per hour.
|
||||
|
||||
Exit codes:
|
||||
0 = healthy (or restart succeeded)
|
||||
1 = service down + restart failed/limit reached
|
||||
2 = service not installed
|
||||
"""
|
||||
state_dir_path = Path(state_dir)
|
||||
maintenance_flag = state_dir_path / "runner-maintenance.flag"
|
||||
if maintenance_flag.is_file():
|
||||
click.echo(
|
||||
f"[runner] maintenance mode active ({maintenance_flag}) - skipping."
|
||||
)
|
||||
return
|
||||
|
||||
status = _service_status(service_name)
|
||||
if status is None:
|
||||
click.echo(
|
||||
f"[runner] service {service_name!r} not found - is act_runner installed?",
|
||||
err=True,
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
if status == "Running":
|
||||
click.echo(f"[runner] {service_name} is Running - OK")
|
||||
if gitea_url and gitea_credential_target:
|
||||
try:
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
|
||||
cred = KeyringCredentialStore().get(gitea_credential_target)
|
||||
online = _gitea_runners_online(gitea_url, cred.password)
|
||||
except KeyError as exc:
|
||||
click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True)
|
||||
online = None
|
||||
if online is None:
|
||||
click.echo("[runner] Gitea API check skipped/failed.")
|
||||
elif online == 0:
|
||||
msg = (
|
||||
f"Gitea API ({gitea_url}) reports 0 online runners. "
|
||||
f"{service_name} is Running - possible registration issue."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1005, "Warning", msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}")
|
||||
else:
|
||||
click.echo(f"[runner] Gitea API: {online} runner(s) online - OK")
|
||||
return
|
||||
|
||||
# Not running → check rate limit and try to restart.
|
||||
cooldown_file = state_dir_path / "runner-restart-log.json"
|
||||
log = _load_restart_log(cooldown_file)
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=1)
|
||||
pruned: list[str] = []
|
||||
for entry in log:
|
||||
try:
|
||||
ts = datetime.fromisoformat(entry)
|
||||
except ValueError:
|
||||
continue
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=UTC)
|
||||
if ts > cutoff:
|
||||
pruned.append(entry)
|
||||
log = pruned
|
||||
|
||||
state_msg = (
|
||||
f"{service_name} service is '{status}'. "
|
||||
f"Auto-restarts in last hour: {len(log)} / {max_restarts}."
|
||||
)
|
||||
click.echo(f"[runner] WARNING: {state_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg)
|
||||
|
||||
if len(log) >= max_restarts:
|
||||
limit_msg = (
|
||||
f"{service_name} auto-restart limit ({max_restarts}/h) reached - "
|
||||
"manual intervention required."
|
||||
)
|
||||
click.echo(f"[runner] {limit_msg}", err=True)
|
||||
_write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg)
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}")
|
||||
sys.exit(1)
|
||||
|
||||
click.echo(
|
||||
f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..."
|
||||
)
|
||||
ok = _restart_service(service_name)
|
||||
log.append(datetime.now(tz=UTC).isoformat())
|
||||
_save_restart_log(cooldown_file, log)
|
||||
new_status = _service_status(service_name) or "Unknown"
|
||||
restart_msg = f"{service_name} restarted - new status: {new_status}."
|
||||
entry_type = "Information" if ok else "Warning"
|
||||
click.echo(f"[runner] {restart_msg}")
|
||||
_write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg)
|
||||
prefix = "[WARNING]" if ok else "[ALERT]"
|
||||
if webhook_url:
|
||||
_post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
def _ignored(_name: str, _value: Any) -> None:
|
||||
"""Marker so type checkers see we deliberately drop a value."""
|
||||
|
||||
|
||||
__all__ = ["monitor"]
|
||||
@@ -0,0 +1,258 @@
|
||||
"""``report`` sub-commands.
|
||||
|
||||
Phase A2 ports ``scripts/Get-CIJobSummary.ps1`` to ``report job``. Reads
|
||||
JSONL log files written by the (still-PowerShell) ``Invoke-CIJob.ps1``
|
||||
under ``F:\\CI\\Logs\\<jobId>\\invoke-ci.jsonl`` and prints either a
|
||||
summary table of recent jobs or a per-phase breakdown for one job.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Event:
|
||||
phase: str
|
||||
status: str
|
||||
ts: str
|
||||
elapsed_sec: int | None
|
||||
error: str | None
|
||||
raw: dict[str, object]
|
||||
|
||||
|
||||
def _read_events(jsonl_path: Path) -> list[_Event]:
|
||||
out: list[_Event] = []
|
||||
try:
|
||||
text = jsonl_path.read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return out
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
data = obj.get("data") if isinstance(obj.get("data"), dict) else {}
|
||||
elapsed: int | None = None
|
||||
if isinstance(data, dict):
|
||||
raw_elapsed = data.get("elapsedSec")
|
||||
if isinstance(raw_elapsed, int | float):
|
||||
elapsed = int(raw_elapsed)
|
||||
err: str | None = None
|
||||
if isinstance(data, dict) and isinstance(data.get("error"), str):
|
||||
err = str(data["error"])
|
||||
out.append(
|
||||
_Event(
|
||||
phase=str(obj.get("phase", "")),
|
||||
status=str(obj.get("status", "")),
|
||||
ts=str(obj.get("ts", "")),
|
||||
elapsed_sec=elapsed,
|
||||
error=err,
|
||||
raw=obj,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _format_hms(sec: int) -> str:
|
||||
if sec < 0:
|
||||
return "?"
|
||||
h, rem = divmod(sec, 3600)
|
||||
m, s = divmod(rem, 60)
|
||||
if h > 0:
|
||||
return f"{h}h{m:02d}m{s:02d}s"
|
||||
return f"{m}m{s:02d}s"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Row:
|
||||
job_id: str
|
||||
status: str
|
||||
elapsed: str
|
||||
started: str
|
||||
error: str
|
||||
|
||||
|
||||
def _summarise(events: list[_Event], default_id: str) -> _Row:
|
||||
job_start = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "start"), None
|
||||
)
|
||||
job_success = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "success"), None
|
||||
)
|
||||
job_failure = next(
|
||||
(e for e in events if e.phase == "job" and e.status == "failure"), None
|
||||
)
|
||||
raw_id = events[0].raw.get("jobId") if events else None
|
||||
job_id = str(raw_id) if isinstance(raw_id, str) and raw_id else default_id
|
||||
started = job_start.ts if job_start else ""
|
||||
if job_success:
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="success",
|
||||
elapsed=_format_hms(job_success.elapsed_sec or -1),
|
||||
started=started,
|
||||
error="",
|
||||
)
|
||||
if job_failure:
|
||||
err = job_failure.error or ""
|
||||
if len(err) > 60:
|
||||
err = err[:57] + "..."
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="FAILED",
|
||||
elapsed=_format_hms(job_failure.elapsed_sec or -1),
|
||||
started=started,
|
||||
error=err,
|
||||
)
|
||||
return _Row(
|
||||
job_id=job_id,
|
||||
status="in-progress",
|
||||
elapsed="?",
|
||||
started=started,
|
||||
error="",
|
||||
)
|
||||
|
||||
|
||||
@click.group("report")
|
||||
def report() -> None:
|
||||
"""Reporting commands (job summaries)."""
|
||||
|
||||
|
||||
@report.command("job")
|
||||
@click.option(
|
||||
"--log-dir",
|
||||
"log_dir",
|
||||
default=r"F:\CI\Logs",
|
||||
show_default=True,
|
||||
help="Base directory containing per-job log subdirectories.",
|
||||
)
|
||||
@click.option(
|
||||
"--last",
|
||||
type=click.IntRange(1, 1000),
|
||||
default=20,
|
||||
show_default=True,
|
||||
help="Show only the N most recent jobs.",
|
||||
)
|
||||
@click.option(
|
||||
"--job-id",
|
||||
"job_id",
|
||||
default="",
|
||||
help="Show a phase-by-phase breakdown for a single job.",
|
||||
)
|
||||
@click.option(
|
||||
"--failed",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Filter to failed jobs only.",
|
||||
)
|
||||
@click.option(
|
||||
"--json",
|
||||
"as_json",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Emit JSON instead of a human-readable table.",
|
||||
)
|
||||
def report_job(
|
||||
log_dir: str,
|
||||
last: int,
|
||||
job_id: str,
|
||||
failed: bool,
|
||||
as_json: bool,
|
||||
) -> None:
|
||||
"""Display CI job summaries parsed from invoke-ci.jsonl files."""
|
||||
base = Path(log_dir)
|
||||
if not base.is_dir():
|
||||
click.echo(f"Log directory not found: {log_dir}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
if job_id:
|
||||
jsonl = base / job_id / "invoke-ci.jsonl"
|
||||
if not jsonl.is_file():
|
||||
click.echo(f"No JSONL log found for job '{job_id}' at: {jsonl}", err=True)
|
||||
sys.exit(1)
|
||||
events = _read_events(jsonl)
|
||||
if not events:
|
||||
click.echo(f"JSONL file is empty or unreadable: {jsonl}", err=True)
|
||||
sys.exit(1)
|
||||
if as_json:
|
||||
click.echo(json.dumps([e.raw for e in events], default=str))
|
||||
return
|
||||
click.echo(f"\nJob: {job_id}")
|
||||
click.echo("=" * 60)
|
||||
click.echo(f"{'Phase':<35} {'Status':<10} {'Timestamp'}")
|
||||
click.echo(f"{'-' * 35:<35} {'-' * 10:<10} {'-' * 24}")
|
||||
for ev in events:
|
||||
click.echo(f"{ev.phase:<35} {ev.status:<10} {ev.ts}")
|
||||
fail = next(
|
||||
(e for e in reversed(events) if e.status == "failure"),
|
||||
None,
|
||||
)
|
||||
if fail and fail.error:
|
||||
click.echo(f"\nError: {fail.error}")
|
||||
return
|
||||
|
||||
files = sorted(
|
||||
base.rglob("invoke-ci.jsonl"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not files:
|
||||
click.echo(f"No invoke-ci.jsonl files found under {log_dir}")
|
||||
return
|
||||
|
||||
rows: list[_Row] = []
|
||||
for f in files:
|
||||
events = _read_events(f)
|
||||
if not events:
|
||||
continue
|
||||
default_id = f.parent.name
|
||||
rows.append(_summarise(events, default_id))
|
||||
|
||||
if failed:
|
||||
rows = [r for r in rows if r.status == "FAILED"]
|
||||
rows = rows[:last]
|
||||
if not rows:
|
||||
click.echo("No matching jobs found.")
|
||||
return
|
||||
|
||||
if as_json:
|
||||
click.echo(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"jobId": r.job_id,
|
||||
"status": r.status,
|
||||
"elapsed": r.elapsed,
|
||||
"started": r.started,
|
||||
"error": r.error,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
click.echo(f"\nCI Job Summary (last {len(rows)} jobs):")
|
||||
header = f"{'JobId':<40} {'Status':<12} {'Elapsed':<10} {'Started':<26} Error"
|
||||
click.echo(header)
|
||||
click.echo(
|
||||
f"{'-' * 40:<40} {'-' * 12:<12} {'-' * 10:<10} {'-' * 26:<26} {'-' * 30}"
|
||||
)
|
||||
for r in rows:
|
||||
click.echo(
|
||||
f"{r.job_id:<40} {r.status:<12} {r.elapsed:<10} {r.started:<26} {r.error}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["report"]
|
||||
@@ -0,0 +1,357 @@
|
||||
"""``vm`` sub-commands.
|
||||
|
||||
Phase A2 ports the leaf VM management scripts:
|
||||
|
||||
* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1``
|
||||
* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1``
|
||||
|
||||
Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend`
|
||||
dependency so an ESXi backend can later supply a folder-scoped VM list
|
||||
instead of scanning the local filesystem.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _make_backend(vmrun_path: str | None) -> VmBackend:
|
||||
"""Build a backend honouring an optional vmrun override."""
|
||||
config = load_config()
|
||||
if vmrun_path is not None:
|
||||
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||
|
||||
return WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||
return load_backend(config)
|
||||
|
||||
|
||||
def _try_remove_dir(path: Path) -> bool:
|
||||
"""Best-effort recursive directory removal. Returns True if path is gone."""
|
||||
if not path.exists():
|
||||
return True
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
return not path.exists()
|
||||
|
||||
|
||||
def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None:
|
||||
"""Soft-stop, then hard-stop on failure. Both errors are tolerated."""
|
||||
try:
|
||||
backend.stop(handle, hard=False)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] soft stop failed: {exc}", err=True)
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
click.echo("[vm remove] still running, forcing hard stop...")
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── group
|
||||
|
||||
|
||||
@click.group("vm")
|
||||
def vm() -> None:
|
||||
"""VM lifecycle commands."""
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────── remove
|
||||
|
||||
|
||||
@vm.command("remove")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
required=True,
|
||||
help="Path to the clone VMX file to destroy.",
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--graceful-timeout",
|
||||
"--graceful-timeout-seconds",
|
||||
"graceful_timeout",
|
||||
type=click.IntRange(1, 60),
|
||||
default=15,
|
||||
show_default=True,
|
||||
help="Seconds to wait for soft stop before forcing.",
|
||||
)
|
||||
@click.option(
|
||||
"--force",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Skip soft stop and go straight to hard stop.",
|
||||
)
|
||||
def vm_remove(
|
||||
vmx: str,
|
||||
vmrun_path: str | None,
|
||||
graceful_timeout: int,
|
||||
force: bool,
|
||||
) -> None:
|
||||
"""Stop and permanently delete an ephemeral build VM."""
|
||||
vmx_path = Path(vmx)
|
||||
clone_dir = vmx_path.parent
|
||||
click.echo(f"[vm remove] destroying VM: {vmx}")
|
||||
|
||||
if not vmx_path.is_file():
|
||||
click.echo(
|
||||
f"[vm remove] VMX file not found - nothing to remove: {vmx}",
|
||||
err=True,
|
||||
)
|
||||
# Still try to clean up a partial clone directory.
|
||||
if clone_dir.exists() and clone_dir != vmx_path:
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
try:
|
||||
backend = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
_try_remove_dir(clone_dir)
|
||||
return
|
||||
|
||||
handle = VmHandle(identifier=vmx)
|
||||
|
||||
if force:
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm remove] hard stop failed: {exc}", err=True)
|
||||
else:
|
||||
click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...")
|
||||
_stop_with_fallback(backend, handle)
|
||||
|
||||
# Brief settle: VMware may keep file locks for a moment after stop.
|
||||
deadline = time.monotonic() + 20.0
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if not backend.is_running(handle):
|
||||
break
|
||||
except BackendError:
|
||||
break
|
||||
time.sleep(2.0)
|
||||
|
||||
click.echo("[vm remove] deleting VM...")
|
||||
deleted = False
|
||||
last_err: str = ""
|
||||
for delay in (0, 3, 6):
|
||||
if delay > 0:
|
||||
click.echo(f"[vm remove] retrying deleteVM in {delay}s...")
|
||||
time.sleep(delay)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
deleted = True
|
||||
break
|
||||
except BackendError as exc:
|
||||
last_err = str(exc)
|
||||
|
||||
if not deleted:
|
||||
click.echo(
|
||||
f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True
|
||||
)
|
||||
click.echo("[vm remove] falling back to manual directory removal.")
|
||||
|
||||
if clone_dir.exists():
|
||||
click.echo(f"[vm remove] removing clone directory: {clone_dir}")
|
||||
if _try_remove_dir(clone_dir):
|
||||
click.echo("[vm remove] clone directory removed.")
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
click.echo("[vm remove] VM destruction complete.")
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────────── cleanup
|
||||
|
||||
|
||||
def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
"""Return orphan clone directories older than ``max_age_hours``."""
|
||||
if not clone_base.is_dir():
|
||||
return []
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
try:
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
if mtime < cutoff:
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def _find_vmx(clone_dir: Path) -> Path | None:
|
||||
for entry in clone_dir.iterdir():
|
||||
if entry.is_file() and entry.suffix.lower() == ".vmx":
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_orphans(
|
||||
clone_base: Path,
|
||||
max_age_hours: int,
|
||||
backend: VmBackend | None,
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
lock_file: Path | None = None,
|
||||
lock_max_age_minutes: int = 30,
|
||||
) -> int:
|
||||
"""Remove orphaned clone directories. Returns count removed.
|
||||
|
||||
Phase C hook: ``backend`` is used to stop+delete each orphan. If the
|
||||
backend is unavailable, directory removal is still attempted.
|
||||
"""
|
||||
orphans = _list_orphans(clone_base, max_age_hours)
|
||||
if not orphans:
|
||||
click.echo(
|
||||
f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)."
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] found {len(orphans)} orphaned director"
|
||||
f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h."
|
||||
)
|
||||
|
||||
removed = 0
|
||||
now = datetime.now(tz=UTC)
|
||||
for entry in orphans:
|
||||
age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600)
|
||||
if dry_run:
|
||||
click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)")
|
||||
continue
|
||||
click.echo(f"[vm cleanup] processing: {entry}")
|
||||
vmx = _find_vmx(entry)
|
||||
if vmx is not None and backend is not None:
|
||||
handle = VmHandle(identifier=str(vmx))
|
||||
try:
|
||||
backend.stop(handle, hard=True)
|
||||
except BackendError as exc:
|
||||
click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True)
|
||||
time.sleep(2.0)
|
||||
try:
|
||||
backend.delete(handle)
|
||||
except BackendError as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.",
|
||||
err=True,
|
||||
)
|
||||
elif vmx is None:
|
||||
click.echo(
|
||||
f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True
|
||||
)
|
||||
|
||||
if _try_remove_dir(entry):
|
||||
click.echo(f"[vm cleanup] removed: {entry}")
|
||||
removed += 1
|
||||
else:
|
||||
click.echo(
|
||||
f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.",
|
||||
err=True,
|
||||
)
|
||||
|
||||
# Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1).
|
||||
if lock_file is not None and lock_file.is_file():
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes)
|
||||
mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC)
|
||||
if mtime < cutoff and not dry_run:
|
||||
age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60)
|
||||
try:
|
||||
lock_file.unlink()
|
||||
click.echo(
|
||||
f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}"
|
||||
)
|
||||
except OSError as exc:
|
||||
click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True)
|
||||
|
||||
click.echo("[vm cleanup] done.")
|
||||
return removed
|
||||
|
||||
|
||||
@vm.command("cleanup")
|
||||
@click.option(
|
||||
"--clone-base-dir",
|
||||
"--clone-base",
|
||||
"clone_base_dir",
|
||||
default=None,
|
||||
help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).",
|
||||
)
|
||||
@click.option(
|
||||
"--max-age-hours",
|
||||
type=click.IntRange(0, 168),
|
||||
default=4,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--lock-file",
|
||||
default=None,
|
||||
help="Path to a stale vm-start lock file to remove if older than 30 minutes.",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
"--whatif",
|
||||
"what_if",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="List orphans without destroying them.",
|
||||
)
|
||||
def vm_cleanup(
|
||||
clone_base_dir: str | None,
|
||||
max_age_hours: int,
|
||||
vmrun_path: str | None,
|
||||
lock_file: str | None,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Destroy ephemeral build VMs left behind after a crash or timeout."""
|
||||
config = load_config()
|
||||
base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||
if not base_path.exists():
|
||||
click.echo(
|
||||
f"[vm cleanup] clone base dir not found: {base_path} - nothing to do."
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
backend: VmBackend | None = _make_backend(vmrun_path)
|
||||
except BackendNotAvailable as exc:
|
||||
click.echo(
|
||||
f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.",
|
||||
err=True,
|
||||
)
|
||||
backend = None
|
||||
|
||||
cleanup_orphans(
|
||||
clone_base=base_path,
|
||||
max_age_hours=max_age_hours,
|
||||
backend=backend,
|
||||
dry_run=what_if,
|
||||
lock_file=Path(lock_file) if lock_file else None,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["cleanup_orphans", "vm"]
|
||||
@@ -0,0 +1,202 @@
|
||||
"""``wait-ready`` sub-command.
|
||||
|
||||
Replaces ``scripts/Wait-VMReady.ps1``. Polls a guest VM until WinRM
|
||||
(Windows) or SSH (Linux) is responsive. Designed to be invoked from the
|
||||
PowerShell shim ``scripts/Wait-VMReady.ps1`` which translates PS-style
|
||||
PascalCase parameters (``-VMPath``, ``-IPAddress``, ``-Transport``,
|
||||
``-SshKeyPath``, ``-SshUser``) into click kebab-case options.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.backends import load_backend
|
||||
from ci_orchestrator.backends.errors import BackendError
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.config import load_config
|
||||
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||
from ci_orchestrator.transport.errors import TransportError
|
||||
from ci_orchestrator.transport.ssh import SshTransport
|
||||
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover
|
||||
from ci_orchestrator.backends.protocol import VmBackend
|
||||
|
||||
|
||||
# Exit codes (kept in sync with the PowerShell original where possible):
|
||||
# 0 ready, 2 timeout-running, 3 timeout-ip, 4 timeout-transport
|
||||
_EXIT_TIMEOUT_RUNNING = 2
|
||||
_EXIT_TIMEOUT_IP = 3
|
||||
_EXIT_TIMEOUT_TRANSPORT = 4
|
||||
|
||||
|
||||
def _normalise_guest_os(value: str) -> str:
|
||||
"""Map Transport/GuestOS aliases to ``windows``/``linux``."""
|
||||
v = value.strip().lower()
|
||||
if v in {"winrm", "windows", "win"}:
|
||||
return "windows"
|
||||
if v in {"ssh", "linux", "lnx"}:
|
||||
return "linux"
|
||||
raise click.BadParameter(f"Unknown guest-os/transport value: {value!r}")
|
||||
|
||||
|
||||
def _wait_running(
|
||||
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||
) -> bool:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if backend.is_running(handle):
|
||||
return True
|
||||
except BackendError as exc:
|
||||
click.echo(f"[wait-ready] backend probe error: {exc}", err=True)
|
||||
time.sleep(poll)
|
||||
return False
|
||||
|
||||
|
||||
def _wait_for_ip(
|
||||
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||
) -> str | None:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
ip = backend.get_ip(handle)
|
||||
except BackendError:
|
||||
ip = None
|
||||
if ip:
|
||||
return ip
|
||||
time.sleep(poll)
|
||||
return None
|
||||
|
||||
|
||||
@click.command("wait-ready")
|
||||
@click.option(
|
||||
"--vmx",
|
||||
"--vm-path",
|
||||
"vmx",
|
||||
required=True,
|
||||
help="Path to the guest VMX file.",
|
||||
)
|
||||
@click.option(
|
||||
"--ip-address",
|
||||
"ip_address",
|
||||
default=None,
|
||||
help="Guest IP address. If omitted the backend is polled for it.",
|
||||
)
|
||||
@click.option(
|
||||
"--guest-os",
|
||||
"--transport",
|
||||
"guest_os",
|
||||
default="windows",
|
||||
show_default=True,
|
||||
callback=lambda _ctx, _p, v: _normalise_guest_os(v),
|
||||
help="windows/WinRM or linux/SSH (case-insensitive).",
|
||||
)
|
||||
@click.option("--timeout", "--timeout-seconds", "timeout", type=float, default=300.0, show_default=True)
|
||||
@click.option(
|
||||
"--poll-interval",
|
||||
"--poll-interval-seconds",
|
||||
"poll_interval",
|
||||
type=float,
|
||||
default=5.0,
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.")
|
||||
@click.option(
|
||||
"--credential-target",
|
||||
default=None,
|
||||
help="Override credential target name (defaults to config.guest_cred_target).",
|
||||
)
|
||||
@click.option("--ssh-user", default="ci_build", show_default=True)
|
||||
@click.option(
|
||||
"--ssh-key-path",
|
||||
default=None,
|
||||
help="SSH private key (defaults to config.ssh_key_path).",
|
||||
)
|
||||
@click.option(
|
||||
"--skip-ping",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Accepted for PS compatibility; ignored (Python uses transport probe directly).",
|
||||
)
|
||||
def wait_ready(
|
||||
vmx: str,
|
||||
ip_address: str | None,
|
||||
guest_os: str,
|
||||
timeout: float,
|
||||
poll_interval: float,
|
||||
vmrun_path: str | None,
|
||||
credential_target: str | None,
|
||||
ssh_user: str,
|
||||
ssh_key_path: str | None,
|
||||
skip_ping: bool,
|
||||
) -> None:
|
||||
"""Poll a guest until WinRM (Windows) or SSH (Linux) is responsive."""
|
||||
del skip_ping # accepted for PS compatibility, no-op
|
||||
config = load_config()
|
||||
if vmrun_path is not None:
|
||||
# Re-load backend with override; cheap, no real connections.
|
||||
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||
|
||||
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||
else:
|
||||
backend = load_backend(config)
|
||||
handle = VmHandle(identifier=vmx)
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}")
|
||||
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||
click.echo("[wait-ready] timeout waiting for VM to be running.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_RUNNING)
|
||||
|
||||
if ip_address is None:
|
||||
ip = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||
if not ip:
|
||||
click.echo("[wait-ready] timeout waiting for guest IP.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_IP)
|
||||
click.echo(f"[wait-ready] guest IP: {ip}")
|
||||
else:
|
||||
ip = ip_address
|
||||
click.echo(f"[wait-ready] using provided IP: {ip}")
|
||||
|
||||
if guest_os == "windows":
|
||||
store = KeyringCredentialStore()
|
||||
target = credential_target or config.guest_cred_target
|
||||
cred = store.get(target)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with WinRmTransport(ip, cred.username, cred.password) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] WinRM ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
else:
|
||||
key_path: str | None = ssh_key_path
|
||||
if key_path is None and config.ssh_key_path is not None:
|
||||
key_path = str(config.ssh_key_path)
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with SshTransport(ip, username=ssh_user, key_path=key_path) as t:
|
||||
if t.is_ready():
|
||||
click.echo("[wait-ready] SSH ready.")
|
||||
return
|
||||
except TransportError as exc:
|
||||
click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True)
|
||||
time.sleep(poll_interval)
|
||||
|
||||
click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True)
|
||||
sys.exit(_EXIT_TIMEOUT_TRANSPORT)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KeyringCredentialStore",
|
||||
"SshTransport",
|
||||
"WinRmTransport",
|
||||
"load_backend",
|
||||
"wait_ready",
|
||||
]
|
||||
@@ -12,6 +12,7 @@ import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.__main__ as cli_module
|
||||
import ci_orchestrator.commands.wait as wait_module
|
||||
from ci_orchestrator import __version__
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.credentials import Credential
|
||||
@@ -54,11 +55,11 @@ class _FakeStore:
|
||||
|
||||
|
||||
def _patch_common(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend) -> None:
|
||||
monkeypatch.setattr(cli_module, "load_backend", lambda _cfg: backend)
|
||||
monkeypatch.setattr(cli_module, "KeyringCredentialStore", lambda: _FakeStore())
|
||||
monkeypatch.setattr(cli_module, "WinRmTransport", _FakeTransport)
|
||||
monkeypatch.setattr(cli_module, "SshTransport", _FakeTransport)
|
||||
monkeypatch.setattr(cli_module.time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend)
|
||||
monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore())
|
||||
monkeypatch.setattr(wait_module, "WinRmTransport", _FakeTransport)
|
||||
monkeypatch.setattr(wait_module, "SshTransport", _FakeTransport)
|
||||
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||
|
||||
|
||||
def test_help_lists_wait_ready() -> None:
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Tests for ``ci_orchestrator monitor disk`` and ``monitor runner``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.monitor as mon
|
||||
from ci_orchestrator.__main__ import cli
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_event_log_or_webhook(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(mon, "_write_windows_event", lambda *_a, **_kw: True)
|
||||
monkeypatch.setattr(mon, "_post_webhook", lambda *_a, **_kw: True)
|
||||
|
||||
|
||||
# ── monitor disk ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _fake_usage(free_gb: float, total_gb: float = 1000.0) -> shutil._ntuple_diskusage:
|
||||
gb = 1024 ** 3
|
||||
return shutil._ntuple_diskusage(int(total_gb * gb), int((total_gb - free_gb) * gb), int(free_gb * gb))
|
||||
|
||||
|
||||
def test_disk_ok_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "OK" in result.output
|
||||
|
||||
|
||||
def test_disk_alert_exits_one(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
||||
result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"])
|
||||
assert result.exit_code == 1
|
||||
assert "below" in result.output or "WARNING" in result.output
|
||||
|
||||
|
||||
def test_disk_drive_not_found(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def _raise(_p: Any) -> Any:
|
||||
raise FileNotFoundError("nope")
|
||||
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", _raise)
|
||||
result = CliRunner().invoke(cli, ["monitor", "disk", "--drive-letter", "Z"])
|
||||
assert result.exit_code == 2
|
||||
assert "not found" in result.output
|
||||
|
||||
|
||||
def test_disk_json_output_ok(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200))
|
||||
result = CliRunner().invoke(
|
||||
cli, ["monitor", "disk", "--min-free-gb", "50", "--json"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
payload = json.loads(result.output.strip())
|
||||
assert payload["status"] == "ok"
|
||||
assert payload["free_gb"] == 200.0
|
||||
|
||||
|
||||
def test_disk_webhook_invoked_on_alert(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[tuple[str, str]] = []
|
||||
monkeypatch.setattr(mon, "_post_webhook", lambda url, content, **_kw: calls.append((url, content)) or True)
|
||||
monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"monitor",
|
||||
"disk",
|
||||
"--min-free-gb",
|
||||
"50",
|
||||
"--webhook-url",
|
||||
"https://example.invalid/hook",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert calls and calls[0][0] == "https://example.invalid/hook"
|
||||
|
||||
|
||||
# ── monitor runner ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_runner_running_exits_zero(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setattr(mon, "_service_status", lambda _s: "Running")
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["monitor", "runner", "--state-dir", str(tmp_path)],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Running" in result.output
|
||||
|
||||
|
||||
def test_runner_not_installed_exits_two(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
monkeypatch.setattr(mon, "_service_status", lambda _s: None)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||
)
|
||||
assert result.exit_code == 2
|
||||
assert "not found" in result.output
|
||||
|
||||
|
||||
def test_runner_maintenance_flag_skips(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
flag = tmp_path / "runner-maintenance.flag"
|
||||
flag.write_text("")
|
||||
result = CliRunner().invoke(
|
||||
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "maintenance mode" in result.output
|
||||
|
||||
|
||||
def test_runner_restart_attempted_when_stopped(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
statuses = iter(["Stopped", "Running"])
|
||||
monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses))
|
||||
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["monitor", "runner", "--state-dir", str(tmp_path)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "restarted" in result.output
|
||||
assert (tmp_path / "runner-restart-log.json").is_file()
|
||||
|
||||
|
||||
def test_runner_rate_limit_exits_one(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
# Pre-fill the cooldown log with recent restarts up to the limit.
|
||||
recent = [datetime.now(tz=UTC).isoformat()] * 3
|
||||
(tmp_path / "runner-restart-log.json").write_text(json.dumps(recent))
|
||||
monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped")
|
||||
monkeypatch.setattr(mon, "_restart_service", lambda _s: True)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"monitor",
|
||||
"runner",
|
||||
"--state-dir",
|
||||
str(tmp_path),
|
||||
"--max-restarts",
|
||||
"3",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "limit" in result.output
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Tests for ``ci_orchestrator report job``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from ci_orchestrator.__main__ import cli
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, events: list[dict[str, object]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text("\n".join(json.dumps(e) for e in events) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _success_events(job_id: str = "run-1") -> list[dict[str, object]]:
|
||||
return [
|
||||
{"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T10:00:00Z"},
|
||||
{"jobId": job_id, "phase": "phase4.wait-ready", "status": "success", "ts": "2026-05-14T10:01:00Z"},
|
||||
{
|
||||
"jobId": job_id,
|
||||
"phase": "job",
|
||||
"status": "success",
|
||||
"ts": "2026-05-14T10:05:00Z",
|
||||
"data": {"elapsedSec": 305},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _failure_events(job_id: str = "run-2") -> list[dict[str, object]]:
|
||||
return [
|
||||
{"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T11:00:00Z"},
|
||||
{
|
||||
"jobId": job_id,
|
||||
"phase": "job",
|
||||
"status": "failure",
|
||||
"ts": "2026-05-14T11:00:30Z",
|
||||
"data": {"elapsedSec": 30, "error": "Clone failed: source not found"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_report_job_summary_table(tmp_path: Path) -> None:
|
||||
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "run-1" in result.output
|
||||
assert "run-2" in result.output
|
||||
assert "success" in result.output
|
||||
assert "FAILED" in result.output
|
||||
|
||||
|
||||
def test_report_job_failed_filter(tmp_path: Path) -> None:
|
||||
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||
result = CliRunner().invoke(
|
||||
cli, ["report", "job", "--log-dir", str(tmp_path), "--failed"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "run-1" not in result.output
|
||||
assert "run-2" in result.output
|
||||
|
||||
|
||||
def test_report_job_detail_view(tmp_path: Path) -> None:
|
||||
_write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2"))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["report", "job", "--log-dir", str(tmp_path), "--job-id", "run-2"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "Job: run-2" in result.output
|
||||
assert "Clone failed" in result.output
|
||||
|
||||
|
||||
def test_report_job_missing_log_dir(tmp_path: Path) -> None:
|
||||
missing = tmp_path / "nope"
|
||||
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(missing)])
|
||||
assert result.exit_code == 1
|
||||
assert "not found" in result.output
|
||||
|
||||
|
||||
def test_report_job_no_logs(tmp_path: Path) -> None:
|
||||
result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)])
|
||||
assert result.exit_code == 0
|
||||
assert "No invoke-ci.jsonl" in result.output
|
||||
|
||||
|
||||
def test_report_job_json_output(tmp_path: Path) -> None:
|
||||
_write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1"))
|
||||
result = CliRunner().invoke(
|
||||
cli, ["report", "job", "--log-dir", str(tmp_path), "--json"]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
payload = json.loads(result.output.strip())
|
||||
assert isinstance(payload, list)
|
||||
assert payload[0]["jobId"] == "run-1"
|
||||
assert payload[0]["status"] == "success"
|
||||
|
||||
|
||||
def test_report_job_unknown_id(tmp_path: Path) -> None:
|
||||
result = CliRunner().invoke(
|
||||
cli, ["report", "job", "--log-dir", str(tmp_path), "--job-id", "ghost"]
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "No JSONL log" in result.output
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Tests for ``ci_orchestrator vm remove`` and ``vm cleanup``.
|
||||
|
||||
Migrated from Pester ``tests/Remove-BuildVM.Tests.ps1`` (now removed).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.vm as vm_module
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
|
||||
|
||||
class _FakeBackend:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
running_after_stop: bool = False,
|
||||
delete_fails_first: int = 0,
|
||||
) -> None:
|
||||
self.calls: list[tuple[str, Any]] = []
|
||||
self._running = True
|
||||
self._running_after_stop = running_after_stop
|
||||
self._delete_fails_remaining = delete_fails_first
|
||||
|
||||
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||
self.calls.append(("stop", hard))
|
||||
if not hard or not self._running_after_stop:
|
||||
self._running = False
|
||||
|
||||
def is_running(self, _h: VmHandle) -> bool:
|
||||
return self._running
|
||||
|
||||
def delete(self, _h: VmHandle) -> None:
|
||||
self.calls.append(("delete", None))
|
||||
if self._delete_fails_remaining > 0:
|
||||
self._delete_fails_remaining -= 1
|
||||
raise BackendOperationFailed("deleteVM", 1, "transient")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(time, "sleep", lambda _s: None)
|
||||
monkeypatch.setattr(vm_module.time, "sleep", lambda _s: None)
|
||||
|
||||
|
||||
def _make_clone_dir(tmp_path: Path, name: str = "clone") -> tuple[Path, Path]:
|
||||
clone_dir = tmp_path / name
|
||||
clone_dir.mkdir()
|
||||
vmx = clone_dir / f"{name}.vmx"
|
||||
vmx.write_text('config.version = "8"', encoding="utf-8")
|
||||
return clone_dir, vmx
|
||||
|
||||
|
||||
# ── vm remove ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_vm_remove_missing_vmx_returns_without_error(tmp_path: Path) -> None:
|
||||
"""Pester migration: returns without error when VMX does not exist."""
|
||||
missing = tmp_path / "ghost" / "ghost.vmx"
|
||||
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(missing)])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "nothing to remove" in result.output
|
||||
|
||||
|
||||
def test_vm_remove_partial_clone_dir_is_removed(tmp_path: Path) -> None:
|
||||
"""Pester migration: partial clone dir without VMX is still removed."""
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "partial")
|
||||
vmx.unlink() # leave the directory but no VMX
|
||||
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||
assert result.exit_code == 0
|
||||
assert not clone_dir.exists()
|
||||
|
||||
|
||||
def test_vm_remove_happy_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
"""Pester migration: removes the clone dir when VMX exists and stop+delete succeed."""
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "live")
|
||||
backend = _FakeBackend()
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||
result = CliRunner().invoke(
|
||||
cli, ["vm", "remove", "--vmx", str(vmx), "--graceful-timeout", "1"]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert ("delete", None) in backend.calls
|
||||
assert not clone_dir.exists()
|
||||
|
||||
|
||||
def test_vm_remove_force_skips_soft_stop(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
_, vmx = _make_clone_dir(tmp_path, "force")
|
||||
backend = _FakeBackend()
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx), "--force"])
|
||||
assert result.exit_code == 0, result.output
|
||||
# No soft stop was issued.
|
||||
soft_stops = [c for c in backend.calls if c == ("stop", False)]
|
||||
assert soft_stops == []
|
||||
hard_stops = [c for c in backend.calls if c == ("stop", True)]
|
||||
assert hard_stops != []
|
||||
|
||||
|
||||
def test_vm_remove_delete_retry_succeeds(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
clone_dir, vmx = _make_clone_dir(tmp_path, "retry")
|
||||
backend = _FakeBackend(delete_fails_first=2)
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||
result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)])
|
||||
assert result.exit_code == 0
|
||||
delete_calls = [c for c in backend.calls if c[0] == "delete"]
|
||||
assert len(delete_calls) == 3 # 2 failures + 1 success
|
||||
assert not clone_dir.exists()
|
||||
|
||||
|
||||
# ── vm cleanup ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _age_dir(path: Path, hours: float) -> None:
|
||||
ts = (datetime.now() - timedelta(hours=hours)).timestamp()
|
||||
import os
|
||||
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
|
||||
def test_vm_cleanup_dry_run_lists_orphans(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
base = tmp_path / "build-vms"
|
||||
base.mkdir()
|
||||
fresh, _ = _make_clone_dir(base, "fresh")
|
||||
old, _ = _make_clone_dir(base, "old")
|
||||
_age_dir(old, hours=10)
|
||||
_age_dir(fresh, hours=0)
|
||||
# Make sure backend isn't constructed (vmrun absent on test host).
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"vm",
|
||||
"cleanup",
|
||||
"--clone-base-dir",
|
||||
str(base),
|
||||
"--max-age-hours",
|
||||
"4",
|
||||
"--what-if",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "would destroy" in result.output
|
||||
assert old.exists() # dry run did NOT delete
|
||||
assert fresh.exists()
|
||||
|
||||
|
||||
def test_vm_cleanup_destroys_old_dirs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
base = tmp_path / "build-vms"
|
||||
base.mkdir()
|
||||
old, _ = _make_clone_dir(base, "old")
|
||||
_age_dir(old, hours=10)
|
||||
|
||||
backend = _FakeBackend()
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"vm",
|
||||
"cleanup",
|
||||
"--clone-base-dir",
|
||||
str(base),
|
||||
"--max-age-hours",
|
||||
"4",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert not old.exists()
|
||||
# Backend was used to stop+delete.
|
||||
assert any(c[0] == "delete" for c in backend.calls)
|
||||
|
||||
|
||||
def test_vm_cleanup_handles_missing_base_dir(tmp_path: Path) -> None:
|
||||
missing = tmp_path / "does-not-exist"
|
||||
result = CliRunner().invoke(
|
||||
cli, ["vm", "cleanup", "--clone-base-dir", str(missing)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "nothing to do" in result.output
|
||||
|
||||
|
||||
def test_vm_cleanup_removes_stale_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
base = tmp_path / "bv"
|
||||
base.mkdir()
|
||||
lock = tmp_path / "vm-start.lock"
|
||||
lock.write_text("x")
|
||||
_age_dir(lock, hours=1)
|
||||
monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"vm",
|
||||
"cleanup",
|
||||
"--clone-base-dir",
|
||||
str(base),
|
||||
"--lock-file",
|
||||
str(lock),
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert not lock.exists()
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Tests for ``ci_orchestrator wait-ready``.
|
||||
|
||||
Migrated from Pester ``tests/Wait-VMReady.Tests.ps1`` (now removed).
|
||||
Preserves the negative cases:
|
||||
|
||||
* missing/invalid VMX → command surfaces a backend error path
|
||||
* never-becomes-ready → exit 2 with timeout message
|
||||
* IP override skips the get_ip polling step
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
import ci_orchestrator.commands.wait as wait_module
|
||||
from ci_orchestrator.__main__ import cli
|
||||
from ci_orchestrator.backends.protocol import VmHandle
|
||||
from ci_orchestrator.credentials import Credential
|
||||
|
||||
|
||||
class _FakeBackend:
|
||||
def __init__(self, *, running: bool = True, ip: str | None = "10.0.0.42") -> None:
|
||||
self._running = running
|
||||
self._ip = ip
|
||||
self.running_calls = 0
|
||||
|
||||
def is_running(self, _h: VmHandle) -> bool:
|
||||
self.running_calls += 1
|
||||
return self._running
|
||||
|
||||
def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None:
|
||||
return self._ip
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
def __init__(self, *_a: Any, **_kw: Any) -> None:
|
||||
pass
|
||||
|
||||
def __enter__(self) -> _FakeTransport:
|
||||
return self
|
||||
|
||||
def __exit__(self, *_e: object) -> None:
|
||||
pass
|
||||
|
||||
def is_ready(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class _BrokenTransport(_FakeTransport):
|
||||
def is_ready(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeStore:
|
||||
def get(self, _target: str) -> Credential:
|
||||
return Credential("user", "pwd")
|
||||
|
||||
|
||||
def _patch(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend, transport: type = _FakeTransport) -> None:
|
||||
monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend)
|
||||
monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore())
|
||||
monkeypatch.setattr(wait_module, "WinRmTransport", transport)
|
||||
monkeypatch.setattr(wait_module, "SshTransport", transport)
|
||||
monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None)
|
||||
|
||||
|
||||
# ── happy paths ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_wait_ready_windows_via_transport_alias(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``--transport WinRM`` is a PS-shim alias of ``--guest-os windows``."""
|
||||
_patch(monkeypatch, _FakeBackend())
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["wait-ready", "--vmx", "x.vmx", "--transport", "WinRM", "--timeout", "1"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "WinRM ready" in result.output
|
||||
|
||||
|
||||
def test_wait_ready_skip_ping_is_no_op(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend())
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["wait-ready", "--vmx", "x.vmx", "--skip-ping", "--timeout", "1"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
|
||||
|
||||
def test_wait_ready_ip_override_skips_get_ip(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""When --ip-address is provided, the backend is not polled for an IP."""
|
||||
backend = _FakeBackend(running=True, ip=None) # would time out at IP step
|
||||
_patch(monkeypatch, backend)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"wait-ready",
|
||||
"--vmx",
|
||||
"x.vmx",
|
||||
"--ip-address",
|
||||
"10.0.0.55",
|
||||
"--timeout",
|
||||
"1",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "10.0.0.55" in result.output
|
||||
|
||||
|
||||
def test_wait_ready_linux_via_ssh_alias(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend())
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["wait-ready", "--vmx", "x.vmx", "--transport", "SSH", "--timeout", "1"],
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "SSH ready" in result.output
|
||||
|
||||
|
||||
# ── failure paths (Pester migration) ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_wait_ready_timeout_when_vm_never_running(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend(running=False))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"wait-ready",
|
||||
"--vmx",
|
||||
"x.vmx",
|
||||
"--timeout",
|
||||
"0.01",
|
||||
"--poll-interval",
|
||||
"0.001",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 2
|
||||
assert "timeout waiting for VM" in result.output
|
||||
|
||||
|
||||
def test_wait_ready_timeout_no_ip(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend(running=True, ip=None))
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"wait-ready",
|
||||
"--vmx",
|
||||
"x.vmx",
|
||||
"--timeout",
|
||||
"0.01",
|
||||
"--poll-interval",
|
||||
"0.001",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 3
|
||||
assert "timeout waiting for guest IP" in result.output
|
||||
|
||||
|
||||
def test_wait_ready_timeout_transport_never_ready(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend(), transport=_BrokenTransport)
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
[
|
||||
"wait-ready",
|
||||
"--vmx",
|
||||
"x.vmx",
|
||||
"--ip-address",
|
||||
"10.0.0.55",
|
||||
"--timeout",
|
||||
"0.01",
|
||||
"--poll-interval",
|
||||
"0.001",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 4
|
||||
assert "transport to be ready" in result.output
|
||||
|
||||
|
||||
def test_wait_ready_invalid_transport_value(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_patch(monkeypatch, _FakeBackend())
|
||||
result = CliRunner().invoke(
|
||||
cli,
|
||||
["wait-ready", "--vmx", "x.vmx", "--transport", "bogus"],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "Unknown" in result.output or "bogus" in result.output
|
||||
Reference in New Issue
Block a user