From 80f6661ad5f8de0128a43d2090ac492b4471549e Mon Sep 17 00:00:00 2001 From: Simone Date: Thu, 14 May 2026 17:01:15 +0200 Subject: [PATCH] feat(a2): port leaf PS scripts to ci_orchestrator CLI Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate). --- scripts/Cleanup-OrphanedBuildVMs.ps1 | 125 +----- scripts/Get-CIJobSummary.ps1 | 26 +- scripts/Remove-BuildVM.ps1 | 139 +------ scripts/Wait-VMReady.ps1 | 255 +------------ scripts/Watch-DiskSpace.ps1 | 105 +----- scripts/Watch-RunnerHealth.ps1 | 225 +---------- src/ci_orchestrator/__main__.py | 123 ++---- src/ci_orchestrator/commands/__init__.py | 10 + src/ci_orchestrator/commands/monitor.py | 459 +++++++++++++++++++++++ src/ci_orchestrator/commands/report.py | 258 +++++++++++++ src/ci_orchestrator/commands/vm.py | 357 ++++++++++++++++++ src/ci_orchestrator/commands/wait.py | 202 ++++++++++ tests/python/test_cli.py | 11 +- tests/python/test_commands_monitor.py | 152 ++++++++ tests/python/test_commands_report.py | 108 ++++++ tests/python/test_commands_vm.py | 211 +++++++++++ tests/python/test_commands_wait.py | 189 ++++++++++ 17 files changed, 2074 insertions(+), 881 deletions(-) create mode 100644 src/ci_orchestrator/commands/__init__.py create mode 100644 src/ci_orchestrator/commands/monitor.py create mode 100644 src/ci_orchestrator/commands/report.py create mode 100644 src/ci_orchestrator/commands/vm.py create mode 100644 src/ci_orchestrator/commands/wait.py create mode 100644 tests/python/test_commands_monitor.py create mode 100644 tests/python/test_commands_report.py create mode 100644 tests/python/test_commands_vm.py create mode 100644 tests/python/test_commands_wait.py diff --git a/scripts/Cleanup-OrphanedBuildVMs.ps1 b/scripts/Cleanup-OrphanedBuildVMs.ps1 index ef41af4..ee1f3f9 100644 --- a/scripts/Cleanup-OrphanedBuildVMs.ps1 +++ b/scripts/Cleanup-OrphanedBuildVMs.ps1 @@ -1,118 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss). - -.DESCRIPTION - Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours. - For each stale directory, stops the VM (hard), deletes via vmrun deleteVM, - then removes the directory. - - Run as a daily scheduled task or on host startup to prevent disk accumulation. - Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours. - -.PARAMETER CloneBaseDir - Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs - -.PARAMETER MaxAgeHours - Age threshold in hours. Directories with LastWriteTime older than this are - treated as orphaned. Must exceed the longest expected build duration. - Default: 4 (runner.timeout is 2h — 4h gives double margin) - -.PARAMETER VmrunPath - Path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER WhatIf - List orphaned VMs without destroying them. - -.EXAMPLE - # Dry run - .\Cleanup-OrphanedBuildVMs.ps1 -WhatIf - - # Live cleanup (run elevated) - .\Cleanup-OrphanedBuildVMs.ps1 -#> -[CmdletBinding(SupportsShouldProcess)] -param( - [string] $CloneBaseDir = 'F:\CI\BuildVMs', - - [ValidateRange(0, 168)] - [int] $MaxAgeHours = 4, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' -) - Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors +$ErrorActionPreference = 'Stop' -if (-not (Test-Path $CloneBaseDir)) { - Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do." - exit 0 -} - -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath" - Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only." -} - -$cutoff = (Get-Date).AddHours(-$MaxAgeHours) -$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue | - Where-Object { $_.LastWriteTime -lt $cutoff }) - -if (-not $orphans) { - Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)." - exit 0 -} - -Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h." - -foreach ($dir in $orphans) { - $vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue | - Select-Object -First 1 - - if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) { - Write-Host "[Cleanup] Processing: $($dir.FullName)" - - if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) { - # Hard stop — don't wait for graceful shutdown on an orphan - & $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null - Start-Sleep -Seconds 2 - $deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1 - if ($LASTEXITCODE -ne 0) { - Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut" - Write-Warning " Falling back to directory removal." - } - } - elseif (-not $vmx) { - Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only." - } - - Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue - if (Test-Path $dir.FullName) { - Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed." - } - else { - Write-Host "[Cleanup] Removed: $($dir.FullName)" - } +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } else { - Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)" + $pyArgs += $a } } -Write-Host "[Cleanup] Done." - -# ── Stale vm-start.lock cleanup ───────────────────────────────────────────── -# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists -# and causes a confusing 10-minute retry loop on the next job. Remove if older -# than 30 minutes. -$lockFile = 'F:\CI\State\vm-start.lock' -$lockCutoff = (Get-Date).AddMinutes(-30) -if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) { - if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) { - $ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes - Remove-Item $lockFile -Force -ErrorAction SilentlyContinue - Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)" - } -} +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm cleanup @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Get-CIJobSummary.ps1 b/scripts/Get-CIJobSummary.ps1 index 0182f3e..d5df4c2 100644 --- a/scripts/Get-CIJobSummary.ps1 +++ b/scripts/Get-CIJobSummary.ps1 @@ -1,7 +1,27 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Displays a summary table of CI job results parsed from JSONL log files. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a + } +} + +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator report job @pyArgs +exit $LASTEXITCODE + .DESCRIPTION Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and diff --git a/scripts/Remove-BuildVM.ps1 b/scripts/Remove-BuildVM.ps1 index 2d85fbf..1553dfb 100644 --- a/scripts/Remove-BuildVM.ps1 +++ b/scripts/Remove-BuildVM.ps1 @@ -1,130 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Stops and permanently deletes an ephemeral build VM. - -.DESCRIPTION - Performs a graceful shutdown (soft stop) with a timeout fallback to - a hard stop, then uses vmrun deleteVM to remove all VM files. - Finally, removes the clone directory from disk. - - This script is designed to be called from a try/finally block so - it always runs, even on build failure. - -.PARAMETER VMPath - Full path to the clone VM's .vmx file. - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER GracefulTimeoutSeconds - Seconds to wait for graceful shutdown before forcing power-off. - Default: 15 - -.EXAMPLE - .\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" -#> -[CmdletBinding(SupportsShouldProcess)] -param( - [Parameter(Mandatory)] - [string] $VMPath, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - [ValidateRange(1, 60)] - [int] $GracefulTimeoutSeconds = 15 -) - Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step +$ErrorActionPreference = 'Stop' -Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force - -$cloneDir = Split-Path $VMPath -Parent - -Write-Host "[Remove-BuildVM] Destroying VM: $VMPath" - -# ── Step 1: Check if VM exists at all ──────────────────────────────────────── -if (-not (Test-Path $VMPath -PathType Leaf)) { - Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath" - # Still attempt to remove the directory in case of partial clone - if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) { - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } - return -} - -if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return } - -# ── Step 2: Graceful stop ───────────────────────────────────────────────────── -Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..." -$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds -if ($stopSoft.TimedOut) { - Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s." -} elseif ($stopSoft.ExitCode -eq 0) { - Write-Host "[Remove-BuildVM] VM stopped gracefully." -} - -# ── Step 3: Force stop if still running ────────────────────────────────────── -if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) { - Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..." - $stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60 - if ($stopHard.TimedOut) { - Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s." + else { + $pyArgs += $a } } -# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly -# after stop. Waiting for the entry to disappear ensures deleteVM won't race the -# process exit. -$vmxNorm = $VMPath.Trim().ToLower() -$releaseDeadline = (Get-Date).AddSeconds(20) -while ((Get-Date) -lt $releaseDeadline) { - $listedVMs = & $VmrunPath -T ws list 2>&1 | - Where-Object { $_ -notmatch '^Total running' } | - ForEach-Object { "$_".Trim().ToLower() } - if ($listedVMs -notcontains $vmxNorm) { break } - Start-Sleep -Seconds 2 -} - -# ── Step 4: Delete VM via vmrun — retry up to 3× with backoff ──────────────── -Write-Host "[Remove-BuildVM] Deleting VM..." -$deleteOutput = '' -$deleteExit = -1 -foreach ($delay in @(0, 3, 6)) { - if ($delay -gt 0) { - Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..." - Start-Sleep -Seconds $delay - } - $delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath ` - -Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90 - $deleteOutput = $delResult.Output - $deleteExit = $delResult.ExitCode - if ($delResult.TimedOut) { - Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s." - $deleteExit = -1 - } - if ($deleteExit -eq 0) { break } -} - -if ($deleteExit -ne 0) { - Write-Warning "[Remove-BuildVM] vmrun deleteVM failed after retries (exit $deleteExit): $deleteOutput" - Write-Warning "[Remove-BuildVM] Will attempt manual directory removal." -} - -# ── Step 5: Remove clone directory (catches any leftover files) ─────────────── -if (Test-Path $cloneDir) { - Write-Host "[Remove-BuildVM] Removing clone directory: $cloneDir" - Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue - if (Test-Path $cloneDir) { - Write-Warning "[Remove-BuildVM] Clone directory could not be fully removed: $cloneDir" - Write-Warning " Manual cleanup required." - } else { - Write-Host "[Remove-BuildVM] Clone directory removed." - } -} - -Write-Host "[Remove-BuildVM] VM destruction complete." +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator vm remove @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Wait-VMReady.ps1 b/scripts/Wait-VMReady.ps1 index 69b3633..44c09e8 100644 --- a/scripts/Wait-VMReady.ps1 +++ b/scripts/Wait-VMReady.ps1 @@ -1,250 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Waits until a build VM is fully ready to accept WinRM connections. - -.DESCRIPTION - Performs a three-phase readiness check in a polling loop: - 1. vmrun getState returns "running" - 2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH) - 3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux) - - Throws if the VM does not become ready within the timeout period. - - Transport modes: - WinRM (default) — existing Windows VM path: ping + WinRM port 5986. - SSH — Linux VM path: port 22 check + ssh echo test. - -.PARAMETER VMPath - Full path to the clone VM's .vmx file. - -.PARAMETER IPAddress - IP address of the VM on the build network (e.g. 192.168.11.101). - -.PARAMETER TimeoutSeconds - Maximum seconds to wait for the VM to become ready. - Default: 300 (5 minutes) - -.PARAMETER PollIntervalSeconds - Seconds between readiness checks. - Default: 5 - -.PARAMETER VmrunPath - Full path to vmrun.exe. - Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe - -.PARAMETER Transport - Transport protocol for readiness checks. - WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs). - SSH: checks TCP 22 + ssh echo (Linux VMs). - -.PARAMETER Credential - Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs - a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming - that WinRM authentication (not just port) is working. Useful to detect the - window where the WinRM listener is up but the LocalSystem session is not yet - accepting logins. When omitted the TCP check alone is used (existing behaviour). - -.PARAMETER SshKeyPath - Path to the SSH private key for key-based auth when Transport=SSH. - Default: F:\CI\keys\ci_linux - -.PARAMETER SshUser - SSH username when Transport=SSH. - Default: ci_build - -.EXAMPLE - .\Wait-VMReady.ps1 ` - -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" ` - -IPAddress "192.168.11.101" -#> -[CmdletBinding()] -param( - [Parameter(Mandatory)] - [string] $VMPath, - - [Parameter(Mandatory)] - [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')] - [string] $IPAddress, - - [ValidateRange(3, 600)] - [int] $TimeoutSeconds = 300, - - [ValidateRange(1, 30)] - [int] $PollIntervalSeconds = 5, - - [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', - - # Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP - # but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM) - # are still checked. - [switch] $SkipPing, - - # Transport protocol for phase 2/3 checks. - # WinRM (default) = Windows VM; SSH = Linux VM. - [ValidateSet('WinRM', 'SSH')] - [string] $Transport = 'WinRM', - - # SSH private key path (used when Transport=SSH). - [string] $SshKeyPath = 'F:\CI\keys\ci_linux', - - # SSH username (used when Transport=SSH). - [string] $SshUser = 'ci_build', - - # Optional credential for WinRM authentication probe (Transport=WinRM). - # When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds. - [PSCredential] $Credential = $null -) - Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.) -if (-not (Test-Path $VmrunPath -PathType Leaf)) { - throw "vmrun.exe not found at: $VmrunPath" -} - -$deadline = (Get-Date).AddSeconds($TimeoutSeconds) -$attempt = 0 -$phase = 'vmrun-state' # tracks current phase for informative output -$lastPhase = '' - -Write-Host "[Wait-VMReady] Waiting for VM ready (timeout: ${TimeoutSeconds}s, IP: $IPAddress)..." - -while ((Get-Date) -lt $deadline) { - $attempt++ - - # ── Phase 1: VM must be running ───────────────────────────────────── - # vmrun has no getState. We previously used `getGuestIPAddress`, but that - # requires VMware Tools to be up and responding inside the guest. In - # headless boots Tools can take 30-60s to come online, during which the - # call exits non-zero even though the VM is fully powered on. That caused - # Phase 1 to appear "stuck" until a GUI was opened manually. - # `vmrun list` reports actual power state without needing Tools. - $vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue) - if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath } - $listOut = & $VmrunPath list 2>&1 - $isRunning = $false - foreach ($line in $listOut) { - if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) { - $isRunning = $true - break - } - } - - if (-not $isRunning) { - if ($phase -ne 'vmrun-state') { - $phase = 'vmrun-state' - } - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 1/3: waiting for VM to enter running state..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - - # ── Phase 2: Network check ────────────────────────────────────────── - if ($Transport -eq 'SSH') { - $port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue - if (-not $port22Open) { - $phase = 'ssh-port' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - elseif (-not $SkipPing) { - $pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue - - if (-not $pingOk) { - $phase = 'ping' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for network (ping $IPAddress)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - - # ── Phase 3: Transport-specific login check ──────────────────────── - if ($Transport -eq 'SSH') { - $sshArgs = @( - '-i', $SshKeyPath, - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=NUL', - '-o', 'ConnectTimeout=5', - '-o', 'BatchMode=yes', - "$SshUser@$IPAddress", - 'echo ready' - ) - # ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts") - # to stderr. Under $ErrorActionPreference='Stop' (set at script top), - # those stderr lines captured via 2>&1 are promoted to terminating errors - # and abort the script. Wrap the call to demote stderr to non-terminating. - $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' - $sshOut = & ssh @sshArgs 2>&1 - $sshExit = $LASTEXITCODE - $ErrorActionPreference = $savedEap - if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') { - Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." - return - } - else { - $phase = 'ssh-echo' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } else { - # ── WinRM port 5986 accepting TCP connections ───────────────────── - # Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for - # self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready. - $winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 ` - -InformationLevel Quiet -WarningAction SilentlyContinue ` - -ErrorAction SilentlyContinue - if ($winrmUp) { - # Optional auth probe — ensures WinRM login is functional, not just TCP open - if ($null -ne $Credential) { - try { - $so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck - $null = Invoke-Command -ComputerName $IPAddress -Credential $Credential ` - -Authentication Basic -UseSSL -Port 5986 -SessionOption $so ` - -ScriptBlock { 'ready' } -ErrorAction Stop - Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress." - } catch { - $phase = 'winrm-auth' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } - } - Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." - return - } - else { - $phase = 'winrm' - if ($lastPhase -ne $phase) { - Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..." - $lastPhase = $phase - } - Start-Sleep -Seconds $PollIntervalSeconds - continue - } + $pyArgs += $a } } -throw "[Wait-VMReady] Timed out after ${TimeoutSeconds}s waiting for VM at $IPAddress to become ready. Last phase: $phase" +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator wait-ready @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Watch-DiskSpace.ps1 b/scripts/Watch-DiskSpace.ps1 index 977cd89..9328def 100644 --- a/scripts/Watch-DiskSpace.ps1 +++ b/scripts/Watch-DiskSpace.ps1 @@ -1,96 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook). - -.DESCRIPTION - Intended to run as a scheduled task every 15 minutes (registered by - Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB, - writes a Warning entry to the Windows Application Event Log under source - 'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler - marks the task as failed (visible in Task Scheduler history). - - If WebhookUrl is provided, also POSTs a JSON payload to that URL - (compatible with Discord and Gitea webhooks — content field only). - - A full drive silently causes vmrun clone to fail with cryptic errors - (not enough disk space for linked clone delta files). Alert early. - -.PARAMETER MinFreeGB - Alert threshold in GB. Default: 50 - -.PARAMETER DriveLetter - Single letter (no colon) of the drive to monitor. Default: F - -.PARAMETER WebhookUrl - Optional Discord/Gitea webhook URL. If empty, only Event Log is written. - -.EXAMPLE - # Manual check - .\Watch-DiskSpace.ps1 -MinFreeGB 50 - - # With Discord webhook - .\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." -#> -[CmdletBinding()] -param( - [ValidateRange(1, 2000)] - [int] $MinFreeGB = 50, - - [ValidatePattern('^[A-Za-z]$')] - [string] $DriveLetter = 'F', - - [string] $WebhookUrl = '' -) - Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' +$ErrorActionPreference = 'Stop' -$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue -if (-not $drive) { - Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found." - exit 2 -} - -$freeGB = [math]::Round($drive.Free / 1GB, 1) -$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1) -$pctFree = [math]::Round($freeGB / $totalGB * 100, 0) - -if ($freeGB -ge $MinFreeGB) { - Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)" - exit 0 -} - -# ── Alert ───────────────────────────────────────────────────────────────────── -$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " + - "the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " + - "Run Invoke-RetentionPolicy.ps1 or free disk space manually." - -Write-Warning "[DiskSpace] $msg" - -# Windows Event Log -$logSource = 'CI-DiskSpaceAlert' -try { - if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { - New-EventLog -LogName Application -Source $logSource -ErrorAction Stop +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() } - Write-EventLog -LogName Application -Source $logSource ` - -EventId 1001 -EntryType Warning -Message $msg - Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)." -} catch { - Write-Warning "[DiskSpace] Could not write Event Log: $_" -} - -# Optional webhook (Discord / Gitea) -if ($WebhookUrl) { - $body = @{ content = "[WARNING] CI Disk Alert -- $msg" } | ConvertTo-Json -Compress - try { - $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` - -Body $body -ContentType 'application/json' -TimeoutSec 10 - Write-Host "[DiskSpace] Webhook notified: $WebhookUrl" - } catch { - Write-Warning "[DiskSpace] Webhook POST failed: $_" + else { + $pyArgs += $a } } -exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer) +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator monitor disk @pyArgs +exit $LASTEXITCODE diff --git a/scripts/Watch-RunnerHealth.ps1 b/scripts/Watch-RunnerHealth.ps1 index 119754a..a7562d8 100644 --- a/scripts/Watch-RunnerHealth.ps1 +++ b/scripts/Watch-RunnerHealth.ps1 @@ -1,214 +1,23 @@ #Requires -Version 5.1 -<# -.SYNOPSIS - Monitors the act_runner service and auto-restarts it if stopped. - -.DESCRIPTION - Intended to run as a scheduled task every 15 minutes (registered by - Register-CIScheduledTasks.ps1 as CI-RunnerHealth). - - Behaviour: - - If act_runner is Running: exits 0, no action. - - If not Running: writes a Warning to the Windows Application Event Log - (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service. - - Restart attempts are rate-limited to MaxRestarts per hour using a JSON - state file in StateDir. Once the limit is hit, further runs write an - alert (EventId 1004) and exit 1 without restarting — Task Scheduler - history shows the failure so the operator is alerted. - - On successful restart: writes EventId 1003 (Information). - - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and - Gitea webhooks on every restart or limit-exceeded event. - -.PARAMETER MaxRestarts - Maximum number of auto-restart attempts allowed per rolling 60-minute - window before giving up and requiring manual intervention. Default: 3 - -.PARAMETER ServiceName - Windows service name to monitor. Default: act_runner - -.PARAMETER StateDir - Directory used for the cooldown state file (runner-restart-log.json). - Default: F:\CI\State - -.PARAMETER WebhookUrl - Optional Discord/Gitea webhook URL. If empty, only Event Log is written. - -.EXAMPLE - # Manual health check - .\Watch-RunnerHealth.ps1 - - # With Discord webhook - .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." -#> -[CmdletBinding()] -param( - [ValidateRange(1, 10)] - [int] $MaxRestarts = 3, - - [string] $ServiceName = 'act_runner', - - [string] $StateDir = 'F:\CI\State', - - [string] $WebhookUrl = '', - - # Optional Gitea API runner-online verification. - # When $GiteaUrl is non-empty and the act_runner service is Running, this script - # queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online. - # Requires an admin Personal Access Token stored in Windows Credential Manager - # under $GiteaCredentialTarget (same module: CredentialManager). - # Silently skipped if GiteaUrl is empty or the credential is unavailable. - [string] $GiteaUrl = '', - - [string] $GiteaCredentialTarget = '' -) - Set-StrictMode -Version Latest -$ErrorActionPreference = 'Continue' +$ErrorActionPreference = 'Stop' -$logSource = 'CI-RunnerHealth' - -# ── Helper: write to Event Log ──────────────────────────────────────────────── -function Write-CIEvent { - param([int] $EventId, [string] $EntryType, [string] $Message) - try { - if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { - New-EventLog -LogName Application -Source $logSource -ErrorAction Stop - } - Write-EventLog -LogName Application -Source $logSource ` - -EventId $EventId -EntryType $EntryType -Message $Message - } catch { - Write-Warning "[RunnerHealth] Could not write Event Log: $_" +# Shim: delegates to the Python ci_orchestrator CLI. +# Original PS implementation moved to git history; see plans/A2-closeout.md. +$pyArgs = @() +foreach ($a in $args) { + if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) { + $name = $a.Substring(1) + $kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1') + $kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2') + $pyArgs += '--' + $kebab.ToLower() + } + else { + $pyArgs += $a } } -# ── Helper: POST webhook ────────────────────────────────────────────────────── -function Send-Webhook { - param([string] $Content) - if (-not $WebhookUrl) { return } - $body = @{ content = $Content } | ConvertTo-Json -Compress - try { - $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` - -Body $body -ContentType 'application/json' -TimeoutSec 10 - Write-Host "[RunnerHealth] Webhook notified." - } catch { - Write-Warning "[RunnerHealth] Webhook POST failed: $_" - } -} - -# ── Maintenance flag check ──────────────────────────────────────────────────── -# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic. -# Create the flag before planned maintenance; delete it when done. -$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag' -if (Test-Path $maintenanceFlag) { - Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic." - exit 0 -} - -# ── Step 1: Check service ───────────────────────────────────────────────────── -$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue -if (-not $svc) { - Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?" - exit 2 -} - -if ($svc.Status -eq 'Running') { - Write-Host "[RunnerHealth] $ServiceName is Running — OK" - - # ── Optional Gitea runner-online API check ──────────────────────────────── - if ($GiteaUrl -ne '') { - $pat = $null - if ($GiteaCredentialTarget -ne '') { - try { - $cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop - $pat = $cred.GetNetworkCredential().Password - } catch { - Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_" - } - } - if ($pat) { - try { - $apiUrl = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners" - $runners = Invoke-RestMethod -Uri $apiUrl ` - -Headers @{ Authorization = "token $pat" } ` - -Method Get -TimeoutSec 10 -ErrorAction Stop - $online = @($runners | Where-Object { $_.online -eq $true }) - if ($online.Count -gt 0) { - Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK" - } else { - $msg = "Gitea API ($apiUrl) reports 0 online runners. " + - "$ServiceName service is Running — possible registration issue." - Write-Warning "[RunnerHealth] $msg" - Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg - Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg" - } - } catch { - Write-Warning "[RunnerHealth] Gitea API check failed: $_" - } - } - } - - exit 0 -} - -# ── Step 2: Service not running — load cooldown state ───────────────────────── -if (-not (Test-Path $StateDir)) { - New-Item -ItemType Directory -Path $StateDir -Force | Out-Null -} - -$cooldownFile = Join-Path $StateDir 'runner-restart-log.json' -$restartLog = @() - -if (Test-Path $cooldownFile) { - try { - $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue - if ($raw) { $restartLog = @($raw | ConvertFrom-Json) } - } catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" } -} - -# Prune timestamps older than 1 hour -$cutoff = (Get-Date).AddHours(-1) -$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff }) -$recentCount = $restartLog.Count - -$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts." -Write-Warning "[RunnerHealth] $stateMsg" -Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg - -# ── Step 3: Rate-limit check ────────────────────────────────────────────────── -if ($recentCount -ge $MaxRestarts) { - $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " + - "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\." - Write-Warning "[RunnerHealth] $limitMsg" - Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg - Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg" - exit 1 -} - -# ── Step 4: Attempt restart ─────────────────────────────────────────────────── -Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..." -try { - Restart-Service -Name $ServiceName -Force -ErrorAction Stop - Start-Sleep -Seconds 5 - $newStatus = (Get-Service -Name $ServiceName).Status - $restartMsg = "$ServiceName restarted — new status: $newStatus." - Write-Host "[RunnerHealth] $restartMsg" - - # Persist this restart in the cooldown log - $restartLog += (Get-Date -Format 'o') - $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue - - $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' } - Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg - - $prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' } - Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg" - - exit $(if ($newStatus -eq 'Running') { 0 } else { 1 }) -} -catch { - $errMsg = "$ServiceName restart failed: $_" - Write-Warning "[RunnerHealth] $errMsg" - Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg - Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg" - exit 1 -} +$venvPython = $env:CI_VENV_PYTHON +if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' } +& $venvPython -m ci_orchestrator monitor runner @pyArgs +exit $LASTEXITCODE diff --git a/src/ci_orchestrator/__main__.py b/src/ci_orchestrator/__main__.py index 864da0b..c00e2e2 100644 --- a/src/ci_orchestrator/__main__.py +++ b/src/ci_orchestrator/__main__.py @@ -1,122 +1,41 @@ """CLI entry point. -Phase A1 ships the ``wait-ready`` PoC sub-command. Subsequent steps add -``vm``, ``build``, ``artifacts``, ``monitor``, ``report``, and ``job``. +Phase A2 ships ``wait-ready``, ``vm`` (remove/cleanup), ``monitor`` +(disk/runner) and ``report`` (job). Subsequent phases add ``vm new``, +``build``, ``artifacts``, ``job``. """ from __future__ import annotations -import sys -import time +import time as time import click from ci_orchestrator import __version__ -from ci_orchestrator.backends import load_backend -from ci_orchestrator.backends.errors import BackendError -from ci_orchestrator.backends.protocol import VmHandle -from ci_orchestrator.config import load_config -from ci_orchestrator.credentials import KeyringCredentialStore -from ci_orchestrator.transport.errors import TransportError -from ci_orchestrator.transport.ssh import SshTransport -from ci_orchestrator.transport.winrm import WinRmTransport +from ci_orchestrator.commands.monitor import monitor +from ci_orchestrator.commands.report import report +from ci_orchestrator.commands.vm import vm + +# Re-export so legacy A1 tests that patched ``cli_module.`` keep working. +from ci_orchestrator.commands.wait import ( # noqa: F401 (intentional re-exports) + KeyringCredentialStore, + SshTransport, + WinRmTransport, + load_backend, + wait_ready, +) @click.group() @click.version_option(__version__, prog_name="ci-orchestrator") def cli() -> None: - """Local CI/CD orchestrator (Phase A).""" + """Local CI/CD orchestrator.""" -@cli.command("wait-ready") -@click.option("--vmx", required=True, help="Path to the guest VMX.") -@click.option( - "--guest-os", - type=click.Choice(["windows", "linux"]), - default="windows", - show_default=True, -) -@click.option("--timeout", type=float, default=300.0, show_default=True) -@click.option( - "--poll-interval", type=float, default=5.0, show_default=True, help="Seconds between probes." -) -@click.option( - "--credential-target", - default=None, - help="Override credential target name (defaults to config.guest_cred_target).", -) -@click.option("--ssh-user", default="ci_build", show_default=True) -def wait_ready( - vmx: str, - guest_os: str, - timeout: float, - poll_interval: float, - credential_target: str | None, - ssh_user: str, -) -> None: - """Poll a guest until WinRM (Windows) or SSH (Linux) is responsive.""" - config = load_config() - backend = load_backend(config) - handle = VmHandle(identifier=vmx) - - deadline = time.monotonic() + timeout - - # Step 1: wait until backend reports the VM as running. - click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}") - while time.monotonic() < deadline: - try: - if backend.is_running(handle): - break - except BackendError as exc: - click.echo(f"[wait-ready] backend probe error: {exc}", err=True) - time.sleep(poll_interval) - else: - click.echo("[wait-ready] timeout waiting for VM to be running.", err=True) - sys.exit(2) - - # Step 2: wait for an IP. - ip: str | None = None - while time.monotonic() < deadline: - try: - ip = backend.get_ip(handle) - except BackendError: - ip = None - if ip: - break - time.sleep(poll_interval) - if not ip: - click.echo("[wait-ready] timeout waiting for guest IP.", err=True) - sys.exit(3) - click.echo(f"[wait-ready] guest IP: {ip}") - - # Step 3: probe the transport layer. - if guest_os == "windows": - store = KeyringCredentialStore() - target = credential_target or config.guest_cred_target - cred = store.get(target) - while time.monotonic() < deadline: - try: - with WinRmTransport(ip, cred.username, cred.password) as t: - if t.is_ready(): - click.echo("[wait-ready] WinRM ready.") - return - except TransportError as exc: - click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True) - time.sleep(poll_interval) - else: - key_path = config.ssh_key_path - while time.monotonic() < deadline: - try: - with SshTransport(ip, username=ssh_user, key_path=key_path) as t: - if t.is_ready(): - click.echo("[wait-ready] SSH ready.") - return - except TransportError as exc: - click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True) - time.sleep(poll_interval) - - click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True) - sys.exit(4) +cli.add_command(wait_ready) +cli.add_command(vm) +cli.add_command(monitor) +cli.add_command(report) if __name__ == "__main__": # pragma: no cover diff --git a/src/ci_orchestrator/commands/__init__.py b/src/ci_orchestrator/commands/__init__.py new file mode 100644 index 0000000..574b144 --- /dev/null +++ b/src/ci_orchestrator/commands/__init__.py @@ -0,0 +1,10 @@ +"""Sub-command modules for the ``ci_orchestrator`` CLI. + +Each module exposes one or more :class:`click.Command` instances that the +top-level :mod:`ci_orchestrator.__main__` registers on the root group. +Phase A2 ports the leaf PowerShell scripts (Wait-VMReady, Remove-BuildVM, +Cleanup-OrphanedBuildVMs, Watch-DiskSpace, Watch-RunnerHealth, +Get-CIJobSummary). +""" + +from __future__ import annotations diff --git a/src/ci_orchestrator/commands/monitor.py b/src/ci_orchestrator/commands/monitor.py new file mode 100644 index 0000000..8b7f1b8 --- /dev/null +++ b/src/ci_orchestrator/commands/monitor.py @@ -0,0 +1,459 @@ +"""``monitor`` sub-commands. + +Phase A2 ports the maintenance/monitoring leaf scripts: + +* ``monitor disk`` — replaces ``scripts/Watch-DiskSpace.ps1`` +* ``monitor runner`` — replaces ``scripts/Watch-RunnerHealth.ps1`` + +Both commands are designed to run from a scheduler (Windows Task Scheduler +in Phase A, systemd timers in Phase B) and emit warnings via the platform +event log when available, plus optional Discord/Gitea webhooks via stdlib +``urllib`` (no extra dependency). +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +import click + +logger = logging.getLogger(__name__) + + +# ──────────────────────────────────────────────────────────────────── helpers + + +def _post_webhook(url: str, content: str, *, timeout: float = 10.0) -> bool: + """POST a Discord/Gitea-compatible JSON payload. Returns True on success.""" + if not url: + return False + payload = json.dumps({"content": content}).encode("utf-8") + req = urllib.request.Request( + url, + data=payload, + method="POST", + headers={"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return bool(200 <= resp.status < 300) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + logger.warning("Webhook POST failed: %s", exc) + return False + + +def _write_windows_event( + source: str, event_id: int, entry_type: str, message: str +) -> bool: + """Write a Windows Application Event Log entry via ``eventcreate.exe``. + + Avoids the ``pywin32`` dependency. Silently no-op on non-Windows hosts. + Returns True on success. + """ + if os.name != "nt": + return False + eventcreate = shutil.which("eventcreate") or shutil.which("eventcreate.exe") + if eventcreate is None: + return False + type_map = {"Information": "INFORMATION", "Warning": "WARNING", "Error": "ERROR"} + cmd = [ + eventcreate, + "/T", type_map.get(entry_type, "WARNING"), + "/ID", str(event_id), + "/L", "APPLICATION", + "/SO", source, + "/D", message, + ] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, timeout=10.0 + ) + except (subprocess.TimeoutExpired, OSError) as exc: + logger.warning("eventcreate failed: %s", exc) + return False + if result.returncode != 0: + logger.warning("eventcreate returned %d: %s", result.returncode, result.stderr) + return False + return True + + +# ─────────────────────────────────────────────────────────────────────── group + + +@click.group("monitor") +def monitor() -> None: + """Host monitoring commands (disk, runner).""" + + +# ──────────────────────────────────────────────────────────────────────── disk + + +def _resolve_drive_path(drive: str) -> str: + """Map a single-letter Windows drive (``F``) to a usable path. + + On non-Windows the ``drive`` value is treated as a path itself. + """ + if os.name == "nt" and len(drive) == 1 and drive.isalpha(): + return f"{drive.upper()}:\\" + return drive + + +@monitor.command("disk") +@click.option( + "--min-free-gb", + "min_free_gb", + type=click.IntRange(1, 2000), + default=50, + show_default=True, +) +@click.option( + "--drive-letter", + "drive_letter", + default="F" if os.name == "nt" else "/", + show_default=True, + help="Drive letter (Windows) or mount path (Linux).", +) +@click.option("--webhook-url", "webhook_url", default="", help="Optional Discord/Gitea webhook URL.") +@click.option( + "--json", + "as_json", + is_flag=True, + default=False, + help="Emit a single-line JSON object instead of human-readable output.", +) +def monitor_disk( + min_free_gb: int, + drive_letter: str, + webhook_url: str, + as_json: bool, +) -> None: + """Check free space on a CI host drive and alert if below threshold. + + Exit codes mirror the original PowerShell script: + + * 0 = drive OK + * 1 = below threshold (alert raised) + * 2 = drive not found + """ + target = _resolve_drive_path(drive_letter) + try: + usage = shutil.disk_usage(target) + except (FileNotFoundError, OSError) as exc: + click.echo(f"[disk] drive {drive_letter!r} not found: {exc}", err=True) + sys.exit(2) + + gb = 1024.0 ** 3 + free_gb = round(usage.free / gb, 1) + total_gb = round(usage.total / gb, 1) + pct_free = round(free_gb / total_gb * 100, 0) if total_gb > 0 else 0 + + payload = { + "drive": drive_letter, + "free_gb": free_gb, + "total_gb": total_gb, + "percent_free": pct_free, + "threshold_gb": min_free_gb, + "status": "ok" if free_gb >= min_free_gb else "alert", + } + + if free_gb >= min_free_gb: + if as_json: + click.echo(json.dumps(payload)) + else: + click.echo( + f"[disk] drive {drive_letter}: {free_gb} GB free / {total_gb} GB " + f"({pct_free}%) - OK (threshold: {min_free_gb} GB)" + ) + return + + msg = ( + f"CI host drive {drive_letter}: free space {free_gb} GB " + f"({pct_free}%) is below the {min_free_gb} GB threshold. " + "vmrun linked clones may fail silently." + ) + if as_json: + payload["message"] = msg + click.echo(json.dumps(payload)) + else: + click.echo(f"[disk] WARNING: {msg}", err=True) + + _write_windows_event("CI-DiskSpaceAlert", 1001, "Warning", msg) + if webhook_url: + _post_webhook(webhook_url, f"[WARNING] CI Disk Alert -- {msg}") + sys.exit(1) + + +# ────────────────────────────────────────────────────────────────────── runner + + +def _service_status(service_name: str) -> str | None: + """Return ``Running``/``Stopped``/``Unknown``, or ``None`` if not installed.""" + if os.name == "nt": + sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" + try: + r = subprocess.run( + [sc, "query", service_name], + capture_output=True, + text=True, + check=False, + timeout=10.0, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if r.returncode != 0: + return None + out = r.stdout.upper() + if "RUNNING" in out: + return "Running" + if "STOPPED" in out: + return "Stopped" + return "Unknown" + # Linux / systemd + systemctl = shutil.which("systemctl") + if systemctl is None: + return None + try: + r = subprocess.run( + [systemctl, "is-active", service_name], + capture_output=True, + text=True, + check=False, + timeout=10.0, + ) + except (subprocess.TimeoutExpired, OSError): + return None + state = r.stdout.strip().lower() + if state == "active": + return "Running" + if state in {"inactive", "failed", "deactivating"}: + return "Stopped" + return None # unknown / not installed + + +def _restart_service(service_name: str) -> bool: + """Attempt to (re)start ``service_name``. Returns True if now running.""" + if os.name == "nt": + sc = shutil.which("sc") or r"C:\Windows\System32\sc.exe" + # `sc start` returns 1056 if already running; ignore. + try: + subprocess.run( + [sc, "start", service_name], + capture_output=True, + text=True, + check=False, + timeout=30.0, + ) + except (subprocess.TimeoutExpired, OSError): + return False + else: + systemctl = shutil.which("systemctl") + if systemctl is None: + return False + try: + subprocess.run( + [systemctl, "restart", service_name], + capture_output=True, + text=True, + check=False, + timeout=30.0, + ) + except (subprocess.TimeoutExpired, OSError): + return False + time.sleep(5.0) + return _service_status(service_name) == "Running" + + +def _load_restart_log(path: Path) -> list[str]: + if not path.is_file(): + return [] + try: + raw = path.read_text(encoding="utf-8") + except OSError: + return [] + if not raw.strip(): + return [] + try: + data = json.loads(raw) + except json.JSONDecodeError: + return [] + if isinstance(data, list): + return [str(x) for x in data] + return [] + + +def _save_restart_log(path: Path, entries: list[str]) -> None: + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(entries), encoding="utf-8") + except OSError as exc: + logger.warning("Could not persist restart log: %s", exc) + + +def _gitea_runners_online(api_url: str, token: str, *, timeout: float = 10.0) -> int | None: + """Return number of online runners reported by Gitea, or ``None`` on failure.""" + api = api_url.rstrip("/") + "/api/v1/admin/runners" + req = urllib.request.Request( + api, headers={"Authorization": f"token {token}"} + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: + logger.warning("Gitea API call failed: %s", exc) + return None + if isinstance(data, list): + return sum(1 for r in data if isinstance(r, dict) and r.get("online") is True) + return None + + +@monitor.command("runner") +@click.option( + "--service-name", + "service_name", + default="act_runner", + show_default=True, +) +@click.option( + "--max-restarts", + type=click.IntRange(1, 10), + default=3, + show_default=True, +) +@click.option( + "--state-dir", + "state_dir", + default=r"F:\CI\State" if os.name == "nt" else "/var/lib/ci/state", + show_default=True, +) +@click.option("--webhook-url", "webhook_url", default="") +@click.option("--gitea-url", "gitea_url", default="") +@click.option( + "--gitea-credential-target", + "gitea_credential_target", + default="", + help="Credential target name for the Gitea admin PAT.", +) +def monitor_runner( + service_name: str, + max_restarts: int, + state_dir: str, + webhook_url: str, + gitea_url: str, + gitea_credential_target: str, +) -> None: + """Monitor act_runner; auto-restart up to ``--max-restarts`` per hour. + + Exit codes: + 0 = healthy (or restart succeeded) + 1 = service down + restart failed/limit reached + 2 = service not installed + """ + state_dir_path = Path(state_dir) + maintenance_flag = state_dir_path / "runner-maintenance.flag" + if maintenance_flag.is_file(): + click.echo( + f"[runner] maintenance mode active ({maintenance_flag}) - skipping." + ) + return + + status = _service_status(service_name) + if status is None: + click.echo( + f"[runner] service {service_name!r} not found - is act_runner installed?", + err=True, + ) + sys.exit(2) + + if status == "Running": + click.echo(f"[runner] {service_name} is Running - OK") + if gitea_url and gitea_credential_target: + try: + from ci_orchestrator.credentials import KeyringCredentialStore + + cred = KeyringCredentialStore().get(gitea_credential_target) + online = _gitea_runners_online(gitea_url, cred.password) + except KeyError as exc: + click.echo(f"[runner] could not load Gitea PAT: {exc}", err=True) + online = None + if online is None: + click.echo("[runner] Gitea API check skipped/failed.") + elif online == 0: + msg = ( + f"Gitea API ({gitea_url}) reports 0 online runners. " + f"{service_name} is Running - possible registration issue." + ) + click.echo(f"[runner] WARNING: {msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1005, "Warning", msg) + if webhook_url: + _post_webhook(webhook_url, f"[WARNING] CI Runner Alert -- {msg}") + else: + click.echo(f"[runner] Gitea API: {online} runner(s) online - OK") + return + + # Not running → check rate limit and try to restart. + cooldown_file = state_dir_path / "runner-restart-log.json" + log = _load_restart_log(cooldown_file) + cutoff = datetime.now(tz=UTC) - timedelta(hours=1) + pruned: list[str] = [] + for entry in log: + try: + ts = datetime.fromisoformat(entry) + except ValueError: + continue + if ts.tzinfo is None: + ts = ts.replace(tzinfo=UTC) + if ts > cutoff: + pruned.append(entry) + log = pruned + + state_msg = ( + f"{service_name} service is '{status}'. " + f"Auto-restarts in last hour: {len(log)} / {max_restarts}." + ) + click.echo(f"[runner] WARNING: {state_msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1002, "Warning", state_msg) + + if len(log) >= max_restarts: + limit_msg = ( + f"{service_name} auto-restart limit ({max_restarts}/h) reached - " + "manual intervention required." + ) + click.echo(f"[runner] {limit_msg}", err=True) + _write_windows_event("CI-RunnerHealth", 1004, "Error", limit_msg) + if webhook_url: + _post_webhook(webhook_url, f"[ALERT] CI Runner Alert -- {limit_msg}") + sys.exit(1) + + click.echo( + f"[runner] restarting {service_name} (attempt {len(log) + 1} of {max_restarts})..." + ) + ok = _restart_service(service_name) + log.append(datetime.now(tz=UTC).isoformat()) + _save_restart_log(cooldown_file, log) + new_status = _service_status(service_name) or "Unknown" + restart_msg = f"{service_name} restarted - new status: {new_status}." + entry_type = "Information" if ok else "Warning" + click.echo(f"[runner] {restart_msg}") + _write_windows_event("CI-RunnerHealth", 1003, entry_type, restart_msg) + prefix = "[WARNING]" if ok else "[ALERT]" + if webhook_url: + _post_webhook(webhook_url, f"{prefix} CI Runner Alert -- {restart_msg}") + sys.exit(0 if ok else 1) + + +def _ignored(_name: str, _value: Any) -> None: + """Marker so type checkers see we deliberately drop a value.""" + + +__all__ = ["monitor"] diff --git a/src/ci_orchestrator/commands/report.py b/src/ci_orchestrator/commands/report.py new file mode 100644 index 0000000..e6de6be --- /dev/null +++ b/src/ci_orchestrator/commands/report.py @@ -0,0 +1,258 @@ +"""``report`` sub-commands. + +Phase A2 ports ``scripts/Get-CIJobSummary.ps1`` to ``report job``. Reads +JSONL log files written by the (still-PowerShell) ``Invoke-CIJob.ps1`` +under ``F:\\CI\\Logs\\\\invoke-ci.jsonl`` and prints either a +summary table of recent jobs or a per-phase breakdown for one job. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path + +import click + + +@dataclass(frozen=True) +class _Event: + phase: str + status: str + ts: str + elapsed_sec: int | None + error: str | None + raw: dict[str, object] + + +def _read_events(jsonl_path: Path) -> list[_Event]: + out: list[_Event] = [] + try: + text = jsonl_path.read_text(encoding="utf-8") + except OSError: + return out + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(obj, dict): + continue + data = obj.get("data") if isinstance(obj.get("data"), dict) else {} + elapsed: int | None = None + if isinstance(data, dict): + raw_elapsed = data.get("elapsedSec") + if isinstance(raw_elapsed, int | float): + elapsed = int(raw_elapsed) + err: str | None = None + if isinstance(data, dict) and isinstance(data.get("error"), str): + err = str(data["error"]) + out.append( + _Event( + phase=str(obj.get("phase", "")), + status=str(obj.get("status", "")), + ts=str(obj.get("ts", "")), + elapsed_sec=elapsed, + error=err, + raw=obj, + ) + ) + return out + + +def _format_hms(sec: int) -> str: + if sec < 0: + return "?" + h, rem = divmod(sec, 3600) + m, s = divmod(rem, 60) + if h > 0: + return f"{h}h{m:02d}m{s:02d}s" + return f"{m}m{s:02d}s" + + +@dataclass(frozen=True) +class _Row: + job_id: str + status: str + elapsed: str + started: str + error: str + + +def _summarise(events: list[_Event], default_id: str) -> _Row: + job_start = next( + (e for e in events if e.phase == "job" and e.status == "start"), None + ) + job_success = next( + (e for e in events if e.phase == "job" and e.status == "success"), None + ) + job_failure = next( + (e for e in events if e.phase == "job" and e.status == "failure"), None + ) + raw_id = events[0].raw.get("jobId") if events else None + job_id = str(raw_id) if isinstance(raw_id, str) and raw_id else default_id + started = job_start.ts if job_start else "" + if job_success: + return _Row( + job_id=job_id, + status="success", + elapsed=_format_hms(job_success.elapsed_sec or -1), + started=started, + error="", + ) + if job_failure: + err = job_failure.error or "" + if len(err) > 60: + err = err[:57] + "..." + return _Row( + job_id=job_id, + status="FAILED", + elapsed=_format_hms(job_failure.elapsed_sec or -1), + started=started, + error=err, + ) + return _Row( + job_id=job_id, + status="in-progress", + elapsed="?", + started=started, + error="", + ) + + +@click.group("report") +def report() -> None: + """Reporting commands (job summaries).""" + + +@report.command("job") +@click.option( + "--log-dir", + "log_dir", + default=r"F:\CI\Logs", + show_default=True, + help="Base directory containing per-job log subdirectories.", +) +@click.option( + "--last", + type=click.IntRange(1, 1000), + default=20, + show_default=True, + help="Show only the N most recent jobs.", +) +@click.option( + "--job-id", + "job_id", + default="", + help="Show a phase-by-phase breakdown for a single job.", +) +@click.option( + "--failed", + is_flag=True, + default=False, + help="Filter to failed jobs only.", +) +@click.option( + "--json", + "as_json", + is_flag=True, + default=False, + help="Emit JSON instead of a human-readable table.", +) +def report_job( + log_dir: str, + last: int, + job_id: str, + failed: bool, + as_json: bool, +) -> None: + """Display CI job summaries parsed from invoke-ci.jsonl files.""" + base = Path(log_dir) + if not base.is_dir(): + click.echo(f"Log directory not found: {log_dir}", err=True) + sys.exit(1) + + if job_id: + jsonl = base / job_id / "invoke-ci.jsonl" + if not jsonl.is_file(): + click.echo(f"No JSONL log found for job '{job_id}' at: {jsonl}", err=True) + sys.exit(1) + events = _read_events(jsonl) + if not events: + click.echo(f"JSONL file is empty or unreadable: {jsonl}", err=True) + sys.exit(1) + if as_json: + click.echo(json.dumps([e.raw for e in events], default=str)) + return + click.echo(f"\nJob: {job_id}") + click.echo("=" * 60) + click.echo(f"{'Phase':<35} {'Status':<10} {'Timestamp'}") + click.echo(f"{'-' * 35:<35} {'-' * 10:<10} {'-' * 24}") + for ev in events: + click.echo(f"{ev.phase:<35} {ev.status:<10} {ev.ts}") + fail = next( + (e for e in reversed(events) if e.status == "failure"), + None, + ) + if fail and fail.error: + click.echo(f"\nError: {fail.error}") + return + + files = sorted( + base.rglob("invoke-ci.jsonl"), + key=lambda p: p.stat().st_mtime, + reverse=True, + ) + if not files: + click.echo(f"No invoke-ci.jsonl files found under {log_dir}") + return + + rows: list[_Row] = [] + for f in files: + events = _read_events(f) + if not events: + continue + default_id = f.parent.name + rows.append(_summarise(events, default_id)) + + if failed: + rows = [r for r in rows if r.status == "FAILED"] + rows = rows[:last] + if not rows: + click.echo("No matching jobs found.") + return + + if as_json: + click.echo( + json.dumps( + [ + { + "jobId": r.job_id, + "status": r.status, + "elapsed": r.elapsed, + "started": r.started, + "error": r.error, + } + for r in rows + ] + ) + ) + return + + click.echo(f"\nCI Job Summary (last {len(rows)} jobs):") + header = f"{'JobId':<40} {'Status':<12} {'Elapsed':<10} {'Started':<26} Error" + click.echo(header) + click.echo( + f"{'-' * 40:<40} {'-' * 12:<12} {'-' * 10:<10} {'-' * 26:<26} {'-' * 30}" + ) + for r in rows: + click.echo( + f"{r.job_id:<40} {r.status:<12} {r.elapsed:<10} {r.started:<26} {r.error}" + ) + + +__all__ = ["report"] diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py new file mode 100644 index 0000000..5573fb3 --- /dev/null +++ b/src/ci_orchestrator/commands/vm.py @@ -0,0 +1,357 @@ +"""``vm`` sub-commands. + +Phase A2 ports the leaf VM management scripts: + +* ``vm remove`` — replaces ``scripts/Remove-BuildVM.ps1`` +* ``vm cleanup`` — replaces ``scripts/Cleanup-OrphanedBuildVMs.ps1`` + +Phase C hook: ``vm cleanup`` accepts an optional :class:`VmBackend` +dependency so an ESXi backend can later supply a folder-scoped VM list +instead of scanning the local filesystem. +""" + +from __future__ import annotations + +import shutil +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.config import load_config + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +# ────────────────────────────────────────────────────────────────────── helpers + + +def _make_backend(vmrun_path: str | None) -> VmBackend: + """Build a backend honouring an optional vmrun override.""" + config = load_config() + if vmrun_path is not None: + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + return WorkstationVmrunBackend(vmrun_path=vmrun_path) + return load_backend(config) + + +def _try_remove_dir(path: Path) -> bool: + """Best-effort recursive directory removal. Returns True if path is gone.""" + if not path.exists(): + return True + shutil.rmtree(path, ignore_errors=True) + return not path.exists() + + +def _stop_with_fallback(backend: VmBackend, handle: VmHandle) -> None: + """Soft-stop, then hard-stop on failure. Both errors are tolerated.""" + try: + backend.stop(handle, hard=False) + except BackendError as exc: + click.echo(f"[vm remove] soft stop failed: {exc}", err=True) + try: + if backend.is_running(handle): + click.echo("[vm remove] still running, forcing hard stop...") + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm remove] hard stop failed: {exc}", err=True) + + +# ─────────────────────────────────────────────────────────────────────── group + + +@click.group("vm") +def vm() -> None: + """VM lifecycle commands.""" + + +# ─────────────────────────────────────────────────────────────────────── remove + + +@vm.command("remove") +@click.option( + "--vmx", + "--vm-path", + "vmx", + required=True, + help="Path to the clone VMX file to destroy.", +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--graceful-timeout", + "--graceful-timeout-seconds", + "graceful_timeout", + type=click.IntRange(1, 60), + default=15, + show_default=True, + help="Seconds to wait for soft stop before forcing.", +) +@click.option( + "--force", + is_flag=True, + default=False, + help="Skip soft stop and go straight to hard stop.", +) +def vm_remove( + vmx: str, + vmrun_path: str | None, + graceful_timeout: int, + force: bool, +) -> None: + """Stop and permanently delete an ephemeral build VM.""" + vmx_path = Path(vmx) + clone_dir = vmx_path.parent + click.echo(f"[vm remove] destroying VM: {vmx}") + + if not vmx_path.is_file(): + click.echo( + f"[vm remove] VMX file not found - nothing to remove: {vmx}", + err=True, + ) + # Still try to clean up a partial clone directory. + if clone_dir.exists() and clone_dir != vmx_path: + _try_remove_dir(clone_dir) + return + + try: + backend = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + click.echo( + f"[vm remove] vmrun unavailable ({exc}); attempting directory removal only.", + err=True, + ) + _try_remove_dir(clone_dir) + return + + handle = VmHandle(identifier=vmx) + + if force: + try: + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm remove] hard stop failed: {exc}", err=True) + else: + click.echo(f"[vm remove] sending soft stop (timeout: {graceful_timeout}s)...") + _stop_with_fallback(backend, handle) + + # Brief settle: VMware may keep file locks for a moment after stop. + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + if not backend.is_running(handle): + break + except BackendError: + break + time.sleep(2.0) + + click.echo("[vm remove] deleting VM...") + deleted = False + last_err: str = "" + for delay in (0, 3, 6): + if delay > 0: + click.echo(f"[vm remove] retrying deleteVM in {delay}s...") + time.sleep(delay) + try: + backend.delete(handle) + deleted = True + break + except BackendError as exc: + last_err = str(exc) + + if not deleted: + click.echo( + f"[vm remove] vmrun deleteVM failed after retries: {last_err}", err=True + ) + click.echo("[vm remove] falling back to manual directory removal.") + + if clone_dir.exists(): + click.echo(f"[vm remove] removing clone directory: {clone_dir}") + if _try_remove_dir(clone_dir): + click.echo("[vm remove] clone directory removed.") + else: + click.echo( + f"[vm remove] could not fully remove: {clone_dir} - manual cleanup needed.", + err=True, + ) + + click.echo("[vm remove] VM destruction complete.") + + +# ────────────────────────────────────────────────────────────────────── cleanup + + +def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]: + """Return orphan clone directories older than ``max_age_hours``.""" + if not clone_base.is_dir(): + return [] + cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) + out: list[Path] = [] + for entry in clone_base.iterdir(): + if not entry.is_dir(): + continue + try: + mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) + except OSError: + continue + if mtime < cutoff: + out.append(entry) + return out + + +def _find_vmx(clone_dir: Path) -> Path | None: + for entry in clone_dir.iterdir(): + if entry.is_file() and entry.suffix.lower() == ".vmx": + return entry + return None + + +def cleanup_orphans( + clone_base: Path, + max_age_hours: int, + backend: VmBackend | None, + *, + dry_run: bool = False, + lock_file: Path | None = None, + lock_max_age_minutes: int = 30, +) -> int: + """Remove orphaned clone directories. Returns count removed. + + Phase C hook: ``backend`` is used to stop+delete each orphan. If the + backend is unavailable, directory removal is still attempted. + """ + orphans = _list_orphans(clone_base, max_age_hours) + if not orphans: + click.echo( + f"[vm cleanup] no orphaned VMs found (threshold: {max_age_hours} h)." + ) + else: + click.echo( + f"[vm cleanup] found {len(orphans)} orphaned director" + f"{'y' if len(orphans) == 1 else 'ies'} older than {max_age_hours} h." + ) + + removed = 0 + now = datetime.now(tz=UTC) + for entry in orphans: + age_h = int((now - datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)).total_seconds() // 3600) + if dry_run: + click.echo(f"[vm cleanup] would destroy {entry} (age: {age_h} h)") + continue + click.echo(f"[vm cleanup] processing: {entry}") + vmx = _find_vmx(entry) + if vmx is not None and backend is not None: + handle = VmHandle(identifier=str(vmx)) + try: + backend.stop(handle, hard=True) + except BackendError as exc: + click.echo(f"[vm cleanup] hard stop ignored: {exc}", err=True) + time.sleep(2.0) + try: + backend.delete(handle) + except BackendError as exc: + click.echo( + f"[vm cleanup] vmrun deleteVM failed: {exc}; falling back to dir removal.", + err=True, + ) + elif vmx is None: + click.echo( + f"[vm cleanup] no .vmx in {entry} - removing directory only.", err=True + ) + + if _try_remove_dir(entry): + click.echo(f"[vm cleanup] removed: {entry}") + removed += 1 + else: + click.echo( + f"[vm cleanup] could not fully remove: {entry} - manual cleanup needed.", + err=True, + ) + + # Stale lock file cleanup (mirrors Cleanup-OrphanedBuildVMs.ps1). + if lock_file is not None and lock_file.is_file(): + cutoff = datetime.now(tz=UTC) - timedelta(minutes=lock_max_age_minutes) + mtime = datetime.fromtimestamp(lock_file.stat().st_mtime, tz=UTC) + if mtime < cutoff and not dry_run: + age_min = int((datetime.now(tz=UTC) - mtime).total_seconds() // 60) + try: + lock_file.unlink() + click.echo( + f"[vm cleanup] removed stale lock file (age: {age_min} min): {lock_file}" + ) + except OSError as exc: + click.echo(f"[vm cleanup] could not remove lock file: {exc}", err=True) + + click.echo("[vm cleanup] done.") + return removed + + +@vm.command("cleanup") +@click.option( + "--clone-base-dir", + "--clone-base", + "clone_base_dir", + default=None, + help="Directory containing ephemeral VM clones (defaults to config.paths.build_vms).", +) +@click.option( + "--max-age-hours", + type=click.IntRange(0, 168), + default=4, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--lock-file", + default=None, + help="Path to a stale vm-start lock file to remove if older than 30 minutes.", +) +@click.option( + "--what-if", + "--whatif", + "what_if", + is_flag=True, + default=False, + help="List orphans without destroying them.", +) +def vm_cleanup( + clone_base_dir: str | None, + max_age_hours: int, + vmrun_path: str | None, + lock_file: str | None, + what_if: bool, +) -> None: + """Destroy ephemeral build VMs left behind after a crash or timeout.""" + config = load_config() + base_path = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms + if not base_path.exists(): + click.echo( + f"[vm cleanup] clone base dir not found: {base_path} - nothing to do." + ) + return + + try: + backend: VmBackend | None = _make_backend(vmrun_path) + except BackendNotAvailable as exc: + click.echo( + f"[vm cleanup] vmrun not available ({exc}); will attempt directory removal only.", + err=True, + ) + backend = None + + cleanup_orphans( + clone_base=base_path, + max_age_hours=max_age_hours, + backend=backend, + dry_run=what_if, + lock_file=Path(lock_file) if lock_file else None, + ) + + +__all__ = ["cleanup_orphans", "vm"] diff --git a/src/ci_orchestrator/commands/wait.py b/src/ci_orchestrator/commands/wait.py new file mode 100644 index 0000000..20d9514 --- /dev/null +++ b/src/ci_orchestrator/commands/wait.py @@ -0,0 +1,202 @@ +"""``wait-ready`` sub-command. + +Replaces ``scripts/Wait-VMReady.ps1``. Polls a guest VM until WinRM +(Windows) or SSH (Linux) is responsive. Designed to be invoked from the +PowerShell shim ``scripts/Wait-VMReady.ps1`` which translates PS-style +PascalCase parameters (``-VMPath``, ``-IPAddress``, ``-Transport``, +``-SshKeyPath``, ``-SshUser``) into click kebab-case options. +""" + +from __future__ import annotations + +import sys +import time +from typing import TYPE_CHECKING + +import click + +from ci_orchestrator.backends import load_backend +from ci_orchestrator.backends.errors import BackendError +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.config import load_config +from ci_orchestrator.credentials import KeyringCredentialStore +from ci_orchestrator.transport.errors import TransportError +from ci_orchestrator.transport.ssh import SshTransport +from ci_orchestrator.transport.winrm import WinRmTransport + +if TYPE_CHECKING: # pragma: no cover + from ci_orchestrator.backends.protocol import VmBackend + + +# Exit codes (kept in sync with the PowerShell original where possible): +# 0 ready, 2 timeout-running, 3 timeout-ip, 4 timeout-transport +_EXIT_TIMEOUT_RUNNING = 2 +_EXIT_TIMEOUT_IP = 3 +_EXIT_TIMEOUT_TRANSPORT = 4 + + +def _normalise_guest_os(value: str) -> str: + """Map Transport/GuestOS aliases to ``windows``/``linux``.""" + v = value.strip().lower() + if v in {"winrm", "windows", "win"}: + return "windows" + if v in {"ssh", "linux", "lnx"}: + return "linux" + raise click.BadParameter(f"Unknown guest-os/transport value: {value!r}") + + +def _wait_running( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> bool: + while time.monotonic() < deadline: + try: + if backend.is_running(handle): + return True + except BackendError as exc: + click.echo(f"[wait-ready] backend probe error: {exc}", err=True) + time.sleep(poll) + return False + + +def _wait_for_ip( + backend: VmBackend, handle: VmHandle, deadline: float, poll: float +) -> str | None: + while time.monotonic() < deadline: + try: + ip = backend.get_ip(handle) + except BackendError: + ip = None + if ip: + return ip + time.sleep(poll) + return None + + +@click.command("wait-ready") +@click.option( + "--vmx", + "--vm-path", + "vmx", + required=True, + help="Path to the guest VMX file.", +) +@click.option( + "--ip-address", + "ip_address", + default=None, + help="Guest IP address. If omitted the backend is polled for it.", +) +@click.option( + "--guest-os", + "--transport", + "guest_os", + default="windows", + show_default=True, + callback=lambda _ctx, _p, v: _normalise_guest_os(v), + help="windows/WinRM or linux/SSH (case-insensitive).", +) +@click.option("--timeout", "--timeout-seconds", "timeout", type=float, default=300.0, show_default=True) +@click.option( + "--poll-interval", + "--poll-interval-seconds", + "poll_interval", + type=float, + default=5.0, + show_default=True, +) +@click.option("--vmrun-path", "vmrun_path", default=None, help="Override vmrun.exe path.") +@click.option( + "--credential-target", + default=None, + help="Override credential target name (defaults to config.guest_cred_target).", +) +@click.option("--ssh-user", default="ci_build", show_default=True) +@click.option( + "--ssh-key-path", + default=None, + help="SSH private key (defaults to config.ssh_key_path).", +) +@click.option( + "--skip-ping", + is_flag=True, + default=False, + help="Accepted for PS compatibility; ignored (Python uses transport probe directly).", +) +def wait_ready( + vmx: str, + ip_address: str | None, + guest_os: str, + timeout: float, + poll_interval: float, + vmrun_path: str | None, + credential_target: str | None, + ssh_user: str, + ssh_key_path: str | None, + skip_ping: bool, +) -> None: + """Poll a guest until WinRM (Windows) or SSH (Linux) is responsive.""" + del skip_ping # accepted for PS compatibility, no-op + config = load_config() + if vmrun_path is not None: + # Re-load backend with override; cheap, no real connections. + from ci_orchestrator.backends.workstation import WorkstationVmrunBackend + + backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path) + else: + backend = load_backend(config) + handle = VmHandle(identifier=vmx) + deadline = time.monotonic() + timeout + + click.echo(f"[wait-ready] waiting for backend to report VM running: {vmx}") + if not _wait_running(backend, handle, deadline, poll_interval): + click.echo("[wait-ready] timeout waiting for VM to be running.", err=True) + sys.exit(_EXIT_TIMEOUT_RUNNING) + + if ip_address is None: + ip = _wait_for_ip(backend, handle, deadline, poll_interval) + if not ip: + click.echo("[wait-ready] timeout waiting for guest IP.", err=True) + sys.exit(_EXIT_TIMEOUT_IP) + click.echo(f"[wait-ready] guest IP: {ip}") + else: + ip = ip_address + click.echo(f"[wait-ready] using provided IP: {ip}") + + if guest_os == "windows": + store = KeyringCredentialStore() + target = credential_target or config.guest_cred_target + cred = store.get(target) + while time.monotonic() < deadline: + try: + with WinRmTransport(ip, cred.username, cred.password) as t: + if t.is_ready(): + click.echo("[wait-ready] WinRM ready.") + return + except TransportError as exc: + click.echo(f"[wait-ready] WinRM probe failed: {exc}", err=True) + time.sleep(poll_interval) + else: + key_path: str | None = ssh_key_path + if key_path is None and config.ssh_key_path is not None: + key_path = str(config.ssh_key_path) + while time.monotonic() < deadline: + try: + with SshTransport(ip, username=ssh_user, key_path=key_path) as t: + if t.is_ready(): + click.echo("[wait-ready] SSH ready.") + return + except TransportError as exc: + click.echo(f"[wait-ready] SSH probe failed: {exc}", err=True) + time.sleep(poll_interval) + + click.echo("[wait-ready] timeout waiting for transport to be ready.", err=True) + sys.exit(_EXIT_TIMEOUT_TRANSPORT) + + +__all__ = [ + "KeyringCredentialStore", + "SshTransport", + "WinRmTransport", + "load_backend", + "wait_ready", +] diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index 9739e4a..4e47e64 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -12,6 +12,7 @@ import pytest from click.testing import CliRunner import ci_orchestrator.__main__ as cli_module +import ci_orchestrator.commands.wait as wait_module from ci_orchestrator import __version__ from ci_orchestrator.backends.protocol import VmHandle from ci_orchestrator.credentials import Credential @@ -54,11 +55,11 @@ class _FakeStore: def _patch_common(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend) -> None: - monkeypatch.setattr(cli_module, "load_backend", lambda _cfg: backend) - monkeypatch.setattr(cli_module, "KeyringCredentialStore", lambda: _FakeStore()) - monkeypatch.setattr(cli_module, "WinRmTransport", _FakeTransport) - monkeypatch.setattr(cli_module, "SshTransport", _FakeTransport) - monkeypatch.setattr(cli_module.time, "sleep", lambda _s: None) + monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend) + monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore()) + monkeypatch.setattr(wait_module, "WinRmTransport", _FakeTransport) + monkeypatch.setattr(wait_module, "SshTransport", _FakeTransport) + monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None) def test_help_lists_wait_ready() -> None: diff --git a/tests/python/test_commands_monitor.py b/tests/python/test_commands_monitor.py new file mode 100644 index 0000000..cf229fb --- /dev/null +++ b/tests/python/test_commands_monitor.py @@ -0,0 +1,152 @@ +"""Tests for ``ci_orchestrator monitor disk`` and ``monitor runner``.""" + +from __future__ import annotations + +import json +import shutil +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.monitor as mon +from ci_orchestrator.__main__ import cli + + +@pytest.fixture(autouse=True) +def _no_event_log_or_webhook(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(mon, "_write_windows_event", lambda *_a, **_kw: True) + monkeypatch.setattr(mon, "_post_webhook", lambda *_a, **_kw: True) + + +# ── monitor disk ─────────────────────────────────────────────────────────── + + +def _fake_usage(free_gb: float, total_gb: float = 1000.0) -> shutil._ntuple_diskusage: + gb = 1024 ** 3 + return shutil._ntuple_diskusage(int(total_gb * gb), int((total_gb - free_gb) * gb), int(free_gb * gb)) + + +def test_disk_ok_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"]) + assert result.exit_code == 0, result.output + assert "OK" in result.output + + +def test_disk_alert_exits_one(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10)) + result = CliRunner().invoke(cli, ["monitor", "disk", "--min-free-gb", "50"]) + assert result.exit_code == 1 + assert "below" in result.output or "WARNING" in result.output + + +def test_disk_drive_not_found(monkeypatch: pytest.MonkeyPatch) -> None: + def _raise(_p: Any) -> Any: + raise FileNotFoundError("nope") + + monkeypatch.setattr(mon.shutil, "disk_usage", _raise) + result = CliRunner().invoke(cli, ["monitor", "disk", "--drive-letter", "Z"]) + assert result.exit_code == 2 + assert "not found" in result.output + + +def test_disk_json_output_ok(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + result = CliRunner().invoke( + cli, ["monitor", "disk", "--min-free-gb", "50", "--json"] + ) + assert result.exit_code == 0 + payload = json.loads(result.output.strip()) + assert payload["status"] == "ok" + assert payload["free_gb"] == 200.0 + + +def test_disk_webhook_invoked_on_alert(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[tuple[str, str]] = [] + monkeypatch.setattr(mon, "_post_webhook", lambda url, content, **_kw: calls.append((url, content)) or True) + monkeypatch.setattr(mon.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=10)) + result = CliRunner().invoke( + cli, + [ + "monitor", + "disk", + "--min-free-gb", + "50", + "--webhook-url", + "https://example.invalid/hook", + ], + ) + assert result.exit_code == 1 + assert calls and calls[0][0] == "https://example.invalid/hook" + + +# ── monitor runner ───────────────────────────────────────────────────────── + + +def test_runner_running_exits_zero(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(mon, "_service_status", lambda _s: "Running") + result = CliRunner().invoke( + cli, + ["monitor", "runner", "--state-dir", str(tmp_path)], + ) + assert result.exit_code == 0 + assert "Running" in result.output + + +def test_runner_not_installed_exits_two(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(mon, "_service_status", lambda _s: None) + result = CliRunner().invoke( + cli, ["monitor", "runner", "--state-dir", str(tmp_path)] + ) + assert result.exit_code == 2 + assert "not found" in result.output + + +def test_runner_maintenance_flag_skips(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + flag = tmp_path / "runner-maintenance.flag" + flag.write_text("") + result = CliRunner().invoke( + cli, ["monitor", "runner", "--state-dir", str(tmp_path)] + ) + assert result.exit_code == 0 + assert "maintenance mode" in result.output + + +def test_runner_restart_attempted_when_stopped( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + statuses = iter(["Stopped", "Running"]) + monkeypatch.setattr(mon, "_service_status", lambda _s: next(statuses)) + monkeypatch.setattr(mon, "_restart_service", lambda _s: True) + result = CliRunner().invoke( + cli, ["monitor", "runner", "--state-dir", str(tmp_path)] + ) + assert result.exit_code == 0 + assert "restarted" in result.output + assert (tmp_path / "runner-restart-log.json").is_file() + + +def test_runner_rate_limit_exits_one( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # Pre-fill the cooldown log with recent restarts up to the limit. + recent = [datetime.now(tz=UTC).isoformat()] * 3 + (tmp_path / "runner-restart-log.json").write_text(json.dumps(recent)) + monkeypatch.setattr(mon, "_service_status", lambda _s: "Stopped") + monkeypatch.setattr(mon, "_restart_service", lambda _s: True) + result = CliRunner().invoke( + cli, + [ + "monitor", + "runner", + "--state-dir", + str(tmp_path), + "--max-restarts", + "3", + ], + ) + assert result.exit_code == 1 + assert "limit" in result.output diff --git a/tests/python/test_commands_report.py b/tests/python/test_commands_report.py new file mode 100644 index 0000000..ab41de6 --- /dev/null +++ b/tests/python/test_commands_report.py @@ -0,0 +1,108 @@ +"""Tests for ``ci_orchestrator report job``.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from click.testing import CliRunner + +from ci_orchestrator.__main__ import cli + + +def _write_jsonl(path: Path, events: list[dict[str, object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(json.dumps(e) for e in events) + "\n", encoding="utf-8") + + +def _success_events(job_id: str = "run-1") -> list[dict[str, object]]: + return [ + {"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T10:00:00Z"}, + {"jobId": job_id, "phase": "phase4.wait-ready", "status": "success", "ts": "2026-05-14T10:01:00Z"}, + { + "jobId": job_id, + "phase": "job", + "status": "success", + "ts": "2026-05-14T10:05:00Z", + "data": {"elapsedSec": 305}, + }, + ] + + +def _failure_events(job_id: str = "run-2") -> list[dict[str, object]]: + return [ + {"jobId": job_id, "phase": "job", "status": "start", "ts": "2026-05-14T11:00:00Z"}, + { + "jobId": job_id, + "phase": "job", + "status": "failure", + "ts": "2026-05-14T11:00:30Z", + "data": {"elapsedSec": 30, "error": "Clone failed: source not found"}, + }, + ] + + +def test_report_job_summary_table(tmp_path: Path) -> None: + _write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1")) + _write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2")) + result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)]) + assert result.exit_code == 0, result.output + assert "run-1" in result.output + assert "run-2" in result.output + assert "success" in result.output + assert "FAILED" in result.output + + +def test_report_job_failed_filter(tmp_path: Path) -> None: + _write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1")) + _write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2")) + result = CliRunner().invoke( + cli, ["report", "job", "--log-dir", str(tmp_path), "--failed"] + ) + assert result.exit_code == 0, result.output + assert "run-1" not in result.output + assert "run-2" in result.output + + +def test_report_job_detail_view(tmp_path: Path) -> None: + _write_jsonl(tmp_path / "run-2" / "invoke-ci.jsonl", _failure_events("run-2")) + result = CliRunner().invoke( + cli, + ["report", "job", "--log-dir", str(tmp_path), "--job-id", "run-2"], + ) + assert result.exit_code == 0, result.output + assert "Job: run-2" in result.output + assert "Clone failed" in result.output + + +def test_report_job_missing_log_dir(tmp_path: Path) -> None: + missing = tmp_path / "nope" + result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(missing)]) + assert result.exit_code == 1 + assert "not found" in result.output + + +def test_report_job_no_logs(tmp_path: Path) -> None: + result = CliRunner().invoke(cli, ["report", "job", "--log-dir", str(tmp_path)]) + assert result.exit_code == 0 + assert "No invoke-ci.jsonl" in result.output + + +def test_report_job_json_output(tmp_path: Path) -> None: + _write_jsonl(tmp_path / "run-1" / "invoke-ci.jsonl", _success_events("run-1")) + result = CliRunner().invoke( + cli, ["report", "job", "--log-dir", str(tmp_path), "--json"] + ) + assert result.exit_code == 0 + payload = json.loads(result.output.strip()) + assert isinstance(payload, list) + assert payload[0]["jobId"] == "run-1" + assert payload[0]["status"] == "success" + + +def test_report_job_unknown_id(tmp_path: Path) -> None: + result = CliRunner().invoke( + cli, ["report", "job", "--log-dir", str(tmp_path), "--job-id", "ghost"] + ) + assert result.exit_code == 1 + assert "No JSONL log" in result.output diff --git a/tests/python/test_commands_vm.py b/tests/python/test_commands_vm.py new file mode 100644 index 0000000..2103410 --- /dev/null +++ b/tests/python/test_commands_vm.py @@ -0,0 +1,211 @@ +"""Tests for ``ci_orchestrator vm remove`` and ``vm cleanup``. + +Migrated from Pester ``tests/Remove-BuildVM.Tests.ps1`` (now removed). +""" + +from __future__ import annotations + +import time +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.vm as vm_module +from ci_orchestrator.__main__ import cli +from ci_orchestrator.backends.errors import BackendOperationFailed +from ci_orchestrator.backends.protocol import VmHandle + + +class _FakeBackend: + def __init__( + self, + *, + running_after_stop: bool = False, + delete_fails_first: int = 0, + ) -> None: + self.calls: list[tuple[str, Any]] = [] + self._running = True + self._running_after_stop = running_after_stop + self._delete_fails_remaining = delete_fails_first + + def stop(self, handle: VmHandle, hard: bool = False) -> None: + self.calls.append(("stop", hard)) + if not hard or not self._running_after_stop: + self._running = False + + def is_running(self, _h: VmHandle) -> bool: + return self._running + + def delete(self, _h: VmHandle) -> None: + self.calls.append(("delete", None)) + if self._delete_fails_remaining > 0: + self._delete_fails_remaining -= 1 + raise BackendOperationFailed("deleteVM", 1, "transient") + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(time, "sleep", lambda _s: None) + monkeypatch.setattr(vm_module.time, "sleep", lambda _s: None) + + +def _make_clone_dir(tmp_path: Path, name: str = "clone") -> tuple[Path, Path]: + clone_dir = tmp_path / name + clone_dir.mkdir() + vmx = clone_dir / f"{name}.vmx" + vmx.write_text('config.version = "8"', encoding="utf-8") + return clone_dir, vmx + + +# ── vm remove ────────────────────────────────────────────────────────────── + + +def test_vm_remove_missing_vmx_returns_without_error(tmp_path: Path) -> None: + """Pester migration: returns without error when VMX does not exist.""" + missing = tmp_path / "ghost" / "ghost.vmx" + result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(missing)]) + assert result.exit_code == 0, result.output + assert "nothing to remove" in result.output + + +def test_vm_remove_partial_clone_dir_is_removed(tmp_path: Path) -> None: + """Pester migration: partial clone dir without VMX is still removed.""" + clone_dir, vmx = _make_clone_dir(tmp_path, "partial") + vmx.unlink() # leave the directory but no VMX + result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)]) + assert result.exit_code == 0 + assert not clone_dir.exists() + + +def test_vm_remove_happy_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Pester migration: removes the clone dir when VMX exists and stop+delete succeed.""" + clone_dir, vmx = _make_clone_dir(tmp_path, "live") + backend = _FakeBackend() + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend) + result = CliRunner().invoke( + cli, ["vm", "remove", "--vmx", str(vmx), "--graceful-timeout", "1"] + ) + assert result.exit_code == 0, result.output + assert ("delete", None) in backend.calls + assert not clone_dir.exists() + + +def test_vm_remove_force_skips_soft_stop(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + _, vmx = _make_clone_dir(tmp_path, "force") + backend = _FakeBackend() + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend) + result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx), "--force"]) + assert result.exit_code == 0, result.output + # No soft stop was issued. + soft_stops = [c for c in backend.calls if c == ("stop", False)] + assert soft_stops == [] + hard_stops = [c for c in backend.calls if c == ("stop", True)] + assert hard_stops != [] + + +def test_vm_remove_delete_retry_succeeds(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + clone_dir, vmx = _make_clone_dir(tmp_path, "retry") + backend = _FakeBackend(delete_fails_first=2) + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend) + result = CliRunner().invoke(cli, ["vm", "remove", "--vmx", str(vmx)]) + assert result.exit_code == 0 + delete_calls = [c for c in backend.calls if c[0] == "delete"] + assert len(delete_calls) == 3 # 2 failures + 1 success + assert not clone_dir.exists() + + +# ── vm cleanup ───────────────────────────────────────────────────────────── + + +def _age_dir(path: Path, hours: float) -> None: + ts = (datetime.now() - timedelta(hours=hours)).timestamp() + import os + + os.utime(path, (ts, ts)) + + +def test_vm_cleanup_dry_run_lists_orphans(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + base = tmp_path / "build-vms" + base.mkdir() + fresh, _ = _make_clone_dir(base, "fresh") + old, _ = _make_clone_dir(base, "old") + _age_dir(old, hours=10) + _age_dir(fresh, hours=0) + # Make sure backend isn't constructed (vmrun absent on test host). + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None) + result = CliRunner().invoke( + cli, + [ + "vm", + "cleanup", + "--clone-base-dir", + str(base), + "--max-age-hours", + "4", + "--what-if", + ], + ) + assert result.exit_code == 0, result.output + assert "would destroy" in result.output + assert old.exists() # dry run did NOT delete + assert fresh.exists() + + +def test_vm_cleanup_destroys_old_dirs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + base = tmp_path / "build-vms" + base.mkdir() + old, _ = _make_clone_dir(base, "old") + _age_dir(old, hours=10) + + backend = _FakeBackend() + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: backend) + + result = CliRunner().invoke( + cli, + [ + "vm", + "cleanup", + "--clone-base-dir", + str(base), + "--max-age-hours", + "4", + ], + ) + assert result.exit_code == 0, result.output + assert not old.exists() + # Backend was used to stop+delete. + assert any(c[0] == "delete" for c in backend.calls) + + +def test_vm_cleanup_handles_missing_base_dir(tmp_path: Path) -> None: + missing = tmp_path / "does-not-exist" + result = CliRunner().invoke( + cli, ["vm", "cleanup", "--clone-base-dir", str(missing)] + ) + assert result.exit_code == 0 + assert "nothing to do" in result.output + + +def test_vm_cleanup_removes_stale_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + base = tmp_path / "bv" + base.mkdir() + lock = tmp_path / "vm-start.lock" + lock.write_text("x") + _age_dir(lock, hours=1) + monkeypatch.setattr(vm_module, "_make_backend", lambda _v: None) + result = CliRunner().invoke( + cli, + [ + "vm", + "cleanup", + "--clone-base-dir", + str(base), + "--lock-file", + str(lock), + ], + ) + assert result.exit_code == 0 + assert not lock.exists() diff --git a/tests/python/test_commands_wait.py b/tests/python/test_commands_wait.py new file mode 100644 index 0000000..3bf3420 --- /dev/null +++ b/tests/python/test_commands_wait.py @@ -0,0 +1,189 @@ +"""Tests for ``ci_orchestrator wait-ready``. + +Migrated from Pester ``tests/Wait-VMReady.Tests.ps1`` (now removed). +Preserves the negative cases: + +* missing/invalid VMX → command surfaces a backend error path +* never-becomes-ready → exit 2 with timeout message +* IP override skips the get_ip polling step +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.wait as wait_module +from ci_orchestrator.__main__ import cli +from ci_orchestrator.backends.protocol import VmHandle +from ci_orchestrator.credentials import Credential + + +class _FakeBackend: + def __init__(self, *, running: bool = True, ip: str | None = "10.0.0.42") -> None: + self._running = running + self._ip = ip + self.running_calls = 0 + + def is_running(self, _h: VmHandle) -> bool: + self.running_calls += 1 + return self._running + + def get_ip(self, _h: VmHandle, timeout: float = 0.0) -> str | None: + return self._ip + + +class _FakeTransport: + def __init__(self, *_a: Any, **_kw: Any) -> None: + pass + + def __enter__(self) -> _FakeTransport: + return self + + def __exit__(self, *_e: object) -> None: + pass + + def is_ready(self) -> bool: + return True + + +class _BrokenTransport(_FakeTransport): + def is_ready(self) -> bool: + return False + + +class _FakeStore: + def get(self, _target: str) -> Credential: + return Credential("user", "pwd") + + +def _patch(monkeypatch: pytest.MonkeyPatch, backend: _FakeBackend, transport: type = _FakeTransport) -> None: + monkeypatch.setattr(wait_module, "load_backend", lambda _cfg: backend) + monkeypatch.setattr(wait_module, "KeyringCredentialStore", lambda: _FakeStore()) + monkeypatch.setattr(wait_module, "WinRmTransport", transport) + monkeypatch.setattr(wait_module, "SshTransport", transport) + monkeypatch.setattr(wait_module.time, "sleep", lambda _s: None) + + +# ── happy paths ──────────────────────────────────────────────────────────── + + +def test_wait_ready_windows_via_transport_alias(monkeypatch: pytest.MonkeyPatch) -> None: + """``--transport WinRM`` is a PS-shim alias of ``--guest-os windows``.""" + _patch(monkeypatch, _FakeBackend()) + result = CliRunner().invoke( + cli, + ["wait-ready", "--vmx", "x.vmx", "--transport", "WinRM", "--timeout", "1"], + ) + assert result.exit_code == 0, result.output + assert "WinRM ready" in result.output + + +def test_wait_ready_skip_ping_is_no_op(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend()) + result = CliRunner().invoke( + cli, + ["wait-ready", "--vmx", "x.vmx", "--skip-ping", "--timeout", "1"], + ) + assert result.exit_code == 0, result.output + + +def test_wait_ready_ip_override_skips_get_ip(monkeypatch: pytest.MonkeyPatch) -> None: + """When --ip-address is provided, the backend is not polled for an IP.""" + backend = _FakeBackend(running=True, ip=None) # would time out at IP step + _patch(monkeypatch, backend) + result = CliRunner().invoke( + cli, + [ + "wait-ready", + "--vmx", + "x.vmx", + "--ip-address", + "10.0.0.55", + "--timeout", + "1", + ], + ) + assert result.exit_code == 0, result.output + assert "10.0.0.55" in result.output + + +def test_wait_ready_linux_via_ssh_alias(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend()) + result = CliRunner().invoke( + cli, + ["wait-ready", "--vmx", "x.vmx", "--transport", "SSH", "--timeout", "1"], + ) + assert result.exit_code == 0, result.output + assert "SSH ready" in result.output + + +# ── failure paths (Pester migration) ─────────────────────────────────────── + + +def test_wait_ready_timeout_when_vm_never_running(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend(running=False)) + result = CliRunner().invoke( + cli, + [ + "wait-ready", + "--vmx", + "x.vmx", + "--timeout", + "0.01", + "--poll-interval", + "0.001", + ], + ) + assert result.exit_code == 2 + assert "timeout waiting for VM" in result.output + + +def test_wait_ready_timeout_no_ip(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend(running=True, ip=None)) + result = CliRunner().invoke( + cli, + [ + "wait-ready", + "--vmx", + "x.vmx", + "--timeout", + "0.01", + "--poll-interval", + "0.001", + ], + ) + assert result.exit_code == 3 + assert "timeout waiting for guest IP" in result.output + + +def test_wait_ready_timeout_transport_never_ready(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend(), transport=_BrokenTransport) + result = CliRunner().invoke( + cli, + [ + "wait-ready", + "--vmx", + "x.vmx", + "--ip-address", + "10.0.0.55", + "--timeout", + "0.01", + "--poll-interval", + "0.001", + ], + ) + assert result.exit_code == 4 + assert "transport to be ready" in result.output + + +def test_wait_ready_invalid_transport_value(monkeypatch: pytest.MonkeyPatch) -> None: + _patch(monkeypatch, _FakeBackend()) + result = CliRunner().invoke( + cli, + ["wait-ready", "--vmx", "x.vmx", "--transport", "bogus"], + ) + assert result.exit_code != 0 + assert "Unknown" in result.output or "bogus" in result.output