feat(a2): port leaf PS scripts to ci_orchestrator CLI
Implements Phase A2 of plans/implementation-plan-A-B.md: - commands/wait.py -> wait-ready (replaces Wait-VMReady.ps1) - commands/vm.py -> vm remove + vm cleanup (replaces Remove-BuildVM.ps1, Cleanup-OrphanedBuildVMs.ps1); cleanup accepts injected VmBackend (Phase C ESXi hook preserved) - commands/monitor.py -> monitor disk + monitor runner (replaces Watch-DiskSpace.ps1, Watch-RunnerHealth.ps1) - commands/report.py -> report job (replaces Get-CIJobSummary.ps1) Each PS script reduced to a 3-line shim that delegates to the Python CLI and preserves \0. Tests: 69 pytest cases across new test_commands_*.py modules; ruff clean, mypy --strict clean, coverage 74.5% (>=70 gate).
This commit is contained in:
@@ -1,118 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Destroys ephemeral build VMs that were not cleaned up (e.g. after host power loss).
|
||||
|
||||
.DESCRIPTION
|
||||
Scans F:\CI\BuildVMs\ for clone directories older than -MaxAgeHours.
|
||||
For each stale directory, stops the VM (hard), deletes via vmrun deleteVM,
|
||||
then removes the directory.
|
||||
|
||||
Run as a daily scheduled task or on host startup to prevent disk accumulation.
|
||||
Safe to run while CI jobs are active — only touches VMs older than MaxAgeHours.
|
||||
|
||||
.PARAMETER CloneBaseDir
|
||||
Directory containing ephemeral VM clones. Default: F:\CI\BuildVMs
|
||||
|
||||
.PARAMETER MaxAgeHours
|
||||
Age threshold in hours. Directories with LastWriteTime older than this are
|
||||
treated as orphaned. Must exceed the longest expected build duration.
|
||||
Default: 4 (runner.timeout is 2h — 4h gives double margin)
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER WhatIf
|
||||
List orphaned VMs without destroying them.
|
||||
|
||||
.EXAMPLE
|
||||
# Dry run
|
||||
.\Cleanup-OrphanedBuildVMs.ps1 -WhatIf
|
||||
|
||||
# Live cleanup (run elevated)
|
||||
.\Cleanup-OrphanedBuildVMs.ps1
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[string] $CloneBaseDir = 'F:\CI\BuildVMs',
|
||||
|
||||
[ValidateRange(0, 168)]
|
||||
[int] $MaxAgeHours = 4,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't abort on individual VM errors
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if (-not (Test-Path $CloneBaseDir)) {
|
||||
Write-Host "[Cleanup] Clone base dir not found: $CloneBaseDir — nothing to do."
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
Write-Warning "[Cleanup] vmrun.exe not found at: $VmrunPath"
|
||||
Write-Warning " Skipping vmrun stop/delete — will attempt directory removal only."
|
||||
}
|
||||
|
||||
$cutoff = (Get-Date).AddHours(-$MaxAgeHours)
|
||||
$orphans = @(Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.LastWriteTime -lt $cutoff })
|
||||
|
||||
if (-not $orphans) {
|
||||
Write-Host "[Cleanup] No orphaned VMs found (threshold: $MaxAgeHours h)."
|
||||
exit 0
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Found $($orphans.Count) orphaned VM director$(if ($orphans.Count -eq 1) {'y'} else {'ies'}) older than $MaxAgeHours h."
|
||||
|
||||
foreach ($dir in $orphans) {
|
||||
$vmx = Get-ChildItem -Path $dir.FullName -Filter '*.vmx' -ErrorAction SilentlyContinue |
|
||||
Select-Object -First 1
|
||||
|
||||
if ($PSCmdlet.ShouldProcess($dir.FullName, "Destroy orphaned VM")) {
|
||||
Write-Host "[Cleanup] Processing: $($dir.FullName)"
|
||||
|
||||
if ($vmx -and (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
# Hard stop — don't wait for graceful shutdown on an orphan
|
||||
& $VmrunPath -T ws stop $vmx.FullName hard 2>&1 | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$deleteOut = & $VmrunPath -T ws deleteVM $vmx.FullName 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Warning "[Cleanup] vmrun deleteVM failed (exit $LASTEXITCODE): $deleteOut"
|
||||
Write-Warning " Falling back to directory removal."
|
||||
}
|
||||
}
|
||||
elseif (-not $vmx) {
|
||||
Write-Warning "[Cleanup] No .vmx found in $($dir.FullName) — removing directory only."
|
||||
}
|
||||
|
||||
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path $dir.FullName) {
|
||||
Write-Warning "[Cleanup] Could not fully remove: $($dir.FullName) — manual cleanup needed."
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] Removed: $($dir.FullName)"
|
||||
}
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
Write-Host "[Cleanup] WhatIf: would destroy $($dir.FullName) (age: $([int]((Get-Date) - $dir.LastWriteTime).TotalHours) h)"
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[Cleanup] Done."
|
||||
|
||||
# ── Stale vm-start.lock cleanup ─────────────────────────────────────────────
|
||||
# The lock is released by Invoke-CIJob.ps1 normally. On host crash it persists
|
||||
# and causes a confusing 10-minute retry loop on the next job. Remove if older
|
||||
# than 30 minutes.
|
||||
$lockFile = 'F:\CI\State\vm-start.lock'
|
||||
$lockCutoff = (Get-Date).AddMinutes(-30)
|
||||
if ((Test-Path $lockFile) -and (Get-Item $lockFile).LastWriteTime -lt $lockCutoff) {
|
||||
if ($PSCmdlet.ShouldProcess($lockFile, 'Remove stale vm-start.lock')) {
|
||||
$ageMin = [int]((Get-Date) - (Get-Item $lockFile).LastWriteTime).TotalMinutes
|
||||
Remove-Item $lockFile -Force -ErrorAction SilentlyContinue
|
||||
Write-Host "[Cleanup] Removed stale vm-start.lock (age: ${ageMin} min)"
|
||||
}
|
||||
}
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator vm cleanup @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
@@ -1,7 +1,27 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Displays a summary table of CI job results parsed from JSONL log files.
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator report job @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
|
||||
.DESCRIPTION
|
||||
Scans F:\CI\Logs\ (or a custom path) for invoke-ci.jsonl files and
|
||||
|
||||
+16
-123
@@ -1,130 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Stops and permanently deletes an ephemeral build VM.
|
||||
|
||||
.DESCRIPTION
|
||||
Performs a graceful shutdown (soft stop) with a timeout fallback to
|
||||
a hard stop, then uses vmrun deleteVM to remove all VM files.
|
||||
Finally, removes the clone directory from disk.
|
||||
|
||||
This script is designed to be called from a try/finally block so
|
||||
it always runs, even on build failure.
|
||||
|
||||
.PARAMETER VMPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER GracefulTimeoutSeconds
|
||||
Seconds to wait for graceful shutdown before forcing power-off.
|
||||
Default: 15
|
||||
|
||||
.EXAMPLE
|
||||
.\Remove-BuildVM.ps1 -VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx"
|
||||
#>
|
||||
[CmdletBinding(SupportsShouldProcess)]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $VMPath,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
||||
|
||||
[ValidateRange(1, 60)]
|
||||
[int] $GracefulTimeoutSeconds = 15
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue' # Don't stop on errors — always try next step
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
Import-Module (Join-Path $PSScriptRoot '_Common.psm1') -Force
|
||||
|
||||
$cloneDir = Split-Path $VMPath -Parent
|
||||
|
||||
Write-Host "[Remove-BuildVM] Destroying VM: $VMPath"
|
||||
|
||||
# ── Step 1: Check if VM exists at all ────────────────────────────────────────
|
||||
if (-not (Test-Path $VMPath -PathType Leaf)) {
|
||||
Write-Warning "[Remove-BuildVM] VMX file not found — nothing to remove: $VMPath"
|
||||
# Still attempt to remove the directory in case of partial clone
|
||||
if ((Test-Path $cloneDir) -and $PSCmdlet.ShouldProcess($cloneDir, 'Remove partial clone directory')) {
|
||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if (-not $PSCmdlet.ShouldProcess($VMPath, 'Stop and delete build VM')) { return }
|
||||
|
||||
# ── Step 2: Graceful stop ─────────────────────────────────────────────────────
|
||||
Write-Host "[Remove-BuildVM] Sending soft stop (timeout: ${GracefulTimeoutSeconds}s)..."
|
||||
$stopSoft = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'soft') -TimeoutSeconds $GracefulTimeoutSeconds
|
||||
if ($stopSoft.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Soft stop timed out after ${GracefulTimeoutSeconds}s."
|
||||
} elseif ($stopSoft.ExitCode -eq 0) {
|
||||
Write-Host "[Remove-BuildVM] VM stopped gracefully."
|
||||
}
|
||||
|
||||
# ── Step 3: Force stop if still running ──────────────────────────────────────
|
||||
if ($stopSoft.TimedOut -or $stopSoft.ExitCode -ne 0) {
|
||||
Write-Host "[Remove-BuildVM] Graceful stop did not succeed — forcing hard stop..."
|
||||
$stopHard = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation stop -Arguments @($VMPath, 'hard') -TimeoutSeconds 60
|
||||
if ($stopHard.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] Hard stop timed out after 60s."
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
# Poll vmrun list until VMX is gone — vmware-vmx.exe may hold file locks briefly
|
||||
# after stop. Waiting for the entry to disappear ensures deleteVM won't race the
|
||||
# process exit.
|
||||
$vmxNorm = $VMPath.Trim().ToLower()
|
||||
$releaseDeadline = (Get-Date).AddSeconds(20)
|
||||
while ((Get-Date) -lt $releaseDeadline) {
|
||||
$listedVMs = & $VmrunPath -T ws list 2>&1 |
|
||||
Where-Object { $_ -notmatch '^Total running' } |
|
||||
ForEach-Object { "$_".Trim().ToLower() }
|
||||
if ($listedVMs -notcontains $vmxNorm) { break }
|
||||
Start-Sleep -Seconds 2
|
||||
}
|
||||
|
||||
# ── Step 4: Delete VM via vmrun — retry up to 3× with backoff ────────────────
|
||||
Write-Host "[Remove-BuildVM] Deleting VM..."
|
||||
$deleteOutput = ''
|
||||
$deleteExit = -1
|
||||
foreach ($delay in @(0, 3, 6)) {
|
||||
if ($delay -gt 0) {
|
||||
Write-Host "[Remove-BuildVM] Retrying deleteVM in ${delay}s..."
|
||||
Start-Sleep -Seconds $delay
|
||||
}
|
||||
$delResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
||||
-Operation deleteVM -Arguments @($VMPath) -TimeoutSeconds 90
|
||||
$deleteOutput = $delResult.Output
|
||||
$deleteExit = $delResult.ExitCode
|
||||
if ($delResult.TimedOut) {
|
||||
Write-Warning "[Remove-BuildVM] deleteVM timed out after 90s."
|
||||
$deleteExit = -1
|
||||
}
|
||||
if ($deleteExit -eq 0) { break }
|
||||
}
|
||||
|
||||
if ($deleteExit -ne 0) {
|
||||
Write-Warning "[Remove-BuildVM] vmrun deleteVM failed after retries (exit $deleteExit): $deleteOutput"
|
||||
Write-Warning "[Remove-BuildVM] Will attempt manual directory removal."
|
||||
}
|
||||
|
||||
# ── Step 5: Remove clone directory (catches any leftover files) ───────────────
|
||||
if (Test-Path $cloneDir) {
|
||||
Write-Host "[Remove-BuildVM] Removing clone directory: $cloneDir"
|
||||
Remove-Item $cloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path $cloneDir) {
|
||||
Write-Warning "[Remove-BuildVM] Clone directory could not be fully removed: $cloneDir"
|
||||
Write-Warning " Manual cleanup required."
|
||||
} else {
|
||||
Write-Host "[Remove-BuildVM] Clone directory removed."
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "[Remove-BuildVM] VM destruction complete."
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator vm remove @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+14
-241
@@ -1,250 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Waits until a build VM is fully ready to accept WinRM connections.
|
||||
|
||||
.DESCRIPTION
|
||||
Performs a three-phase readiness check in a polling loop:
|
||||
1. vmrun getState returns "running"
|
||||
2. Network reachability (ICMP ping for WinRM; TCP port 22 check for SSH)
|
||||
3. Transport-specific login: WinRM port 5986 (Windows) or SSH echo (Linux)
|
||||
|
||||
Throws if the VM does not become ready within the timeout period.
|
||||
|
||||
Transport modes:
|
||||
WinRM (default) — existing Windows VM path: ping + WinRM port 5986.
|
||||
SSH — Linux VM path: port 22 check + ssh echo test.
|
||||
|
||||
.PARAMETER VMPath
|
||||
Full path to the clone VM's .vmx file.
|
||||
|
||||
.PARAMETER IPAddress
|
||||
IP address of the VM on the build network (e.g. 192.168.11.101).
|
||||
|
||||
.PARAMETER TimeoutSeconds
|
||||
Maximum seconds to wait for the VM to become ready.
|
||||
Default: 300 (5 minutes)
|
||||
|
||||
.PARAMETER PollIntervalSeconds
|
||||
Seconds between readiness checks.
|
||||
Default: 5
|
||||
|
||||
.PARAMETER VmrunPath
|
||||
Full path to vmrun.exe.
|
||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
||||
|
||||
.PARAMETER Transport
|
||||
Transport protocol for readiness checks.
|
||||
WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs).
|
||||
SSH: checks TCP 22 + ssh echo (Linux VMs).
|
||||
|
||||
.PARAMETER Credential
|
||||
Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs
|
||||
a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming
|
||||
that WinRM authentication (not just port) is working. Useful to detect the
|
||||
window where the WinRM listener is up but the LocalSystem session is not yet
|
||||
accepting logins. When omitted the TCP check alone is used (existing behaviour).
|
||||
|
||||
.PARAMETER SshKeyPath
|
||||
Path to the SSH private key for key-based auth when Transport=SSH.
|
||||
Default: F:\CI\keys\ci_linux
|
||||
|
||||
.PARAMETER SshUser
|
||||
SSH username when Transport=SSH.
|
||||
Default: ci_build
|
||||
|
||||
.EXAMPLE
|
||||
.\Wait-VMReady.ps1 `
|
||||
-VMPath "F:\CI\BuildVMs\Clone_run-42_20260508_143022\Clone_run-42_20260508_143022.vmx" `
|
||||
-IPAddress "192.168.11.101"
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory)]
|
||||
[string] $VMPath,
|
||||
|
||||
[Parameter(Mandatory)]
|
||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
||||
[string] $IPAddress,
|
||||
|
||||
[ValidateRange(3, 600)]
|
||||
[int] $TimeoutSeconds = 300,
|
||||
|
||||
[ValidateRange(1, 30)]
|
||||
[int] $PollIntervalSeconds = 5,
|
||||
|
||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
||||
|
||||
# Skip ICMP ping phase (phase 2). Useful when the guest firewall blocks ICMP
|
||||
# but WinRM (TCP 5986) is open. Phase 1 (vmrun state) and phase 3 (WinRM)
|
||||
# are still checked.
|
||||
[switch] $SkipPing,
|
||||
|
||||
# Transport protocol for phase 2/3 checks.
|
||||
# WinRM (default) = Windows VM; SSH = Linux VM.
|
||||
[ValidateSet('WinRM', 'SSH')]
|
||||
[string] $Transport = 'WinRM',
|
||||
|
||||
# SSH private key path (used when Transport=SSH).
|
||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
||||
|
||||
# SSH username (used when Transport=SSH).
|
||||
[string] $SshUser = 'ci_build',
|
||||
|
||||
# Optional credential for WinRM authentication probe (Transport=WinRM).
|
||||
# When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds.
|
||||
[PSCredential] $Credential = $null
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$ProgressPreference = 'SilentlyContinue' # suppress Write-Progress (Test-NetConnection, IWR, etc.)
|
||||
|
||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
||||
throw "vmrun.exe not found at: $VmrunPath"
|
||||
}
|
||||
|
||||
$deadline = (Get-Date).AddSeconds($TimeoutSeconds)
|
||||
$attempt = 0
|
||||
$phase = 'vmrun-state' # tracks current phase for informative output
|
||||
$lastPhase = ''
|
||||
|
||||
Write-Host "[Wait-VMReady] Waiting for VM ready (timeout: ${TimeoutSeconds}s, IP: $IPAddress)..."
|
||||
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
$attempt++
|
||||
|
||||
# ── Phase 1: VM must be running ─────────────────────────────────────
|
||||
# vmrun has no getState. We previously used `getGuestIPAddress`, but that
|
||||
# requires VMware Tools to be up and responding inside the guest. In
|
||||
# headless boots Tools can take 30-60s to come online, during which the
|
||||
# call exits non-zero even though the VM is fully powered on. That caused
|
||||
# Phase 1 to appear "stuck" until a GUI was opened manually.
|
||||
# `vmrun list` reports actual power state without needing Tools.
|
||||
$vmxNorm = (Resolve-Path -LiteralPath $VMPath -ErrorAction SilentlyContinue)
|
||||
if ($null -ne $vmxNorm) { $vmxNorm = $vmxNorm.Path } else { $vmxNorm = $VMPath }
|
||||
$listOut = & $VmrunPath list 2>&1
|
||||
$isRunning = $false
|
||||
foreach ($line in $listOut) {
|
||||
if ([string]::Equals($line.Trim(), $vmxNorm, [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$isRunning = $true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $isRunning) {
|
||||
if ($phase -ne 'vmrun-state') {
|
||||
$phase = 'vmrun-state'
|
||||
}
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 1/3: waiting for VM to enter running state..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
|
||||
# ── Phase 2: Network check ──────────────────────────────────────────
|
||||
if ($Transport -eq 'SSH') {
|
||||
$port22Open = Test-NetConnection -ComputerName $IPAddress -Port 22 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue
|
||||
if (-not $port22Open) {
|
||||
$phase = 'ssh-port'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for SSH port 22 at $IPAddress..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
elseif (-not $SkipPing) {
|
||||
$pingOk = Test-Connection -ComputerName $IPAddress -Count 1 -Quiet -ErrorAction SilentlyContinue
|
||||
|
||||
if (-not $pingOk) {
|
||||
$phase = 'ping'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 2/3: VM running, waiting for network (ping $IPAddress)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
# ── Phase 3: Transport-specific login check ────────────────────────
|
||||
if ($Transport -eq 'SSH') {
|
||||
$sshArgs = @(
|
||||
'-i', $SshKeyPath,
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=NUL',
|
||||
'-o', 'ConnectTimeout=5',
|
||||
'-o', 'BatchMode=yes',
|
||||
"$SshUser@$IPAddress",
|
||||
'echo ready'
|
||||
)
|
||||
# ssh writes diagnostic warnings (e.g. "Permanently added ... to known hosts")
|
||||
# to stderr. Under $ErrorActionPreference='Stop' (set at script top),
|
||||
# those stderr lines captured via 2>&1 are promoted to terminating errors
|
||||
# and abort the script. Wrap the call to demote stderr to non-terminating.
|
||||
$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue'
|
||||
$sshOut = & ssh @sshArgs 2>&1
|
||||
$sshExit = $LASTEXITCODE
|
||||
$ErrorActionPreference = $savedEap
|
||||
if ($sshExit -eq 0 -and ($sshOut -join '') -like '*ready*') {
|
||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
||||
return
|
||||
}
|
||||
else {
|
||||
$phase = 'ssh-echo'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: SSH port open, waiting for login ($SshUser@$IPAddress)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
# ── WinRM port 5986 accepting TCP connections ─────────────────────
|
||||
# Test-WSMan has no -SessionOption in PS 5.1 and fails cert validation for
|
||||
# self-signed certs. TCP open on 5986 = HTTPS listener up = VM ready.
|
||||
$winrmUp = Test-NetConnection -ComputerName $IPAddress -Port 5986 `
|
||||
-InformationLevel Quiet -WarningAction SilentlyContinue `
|
||||
-ErrorAction SilentlyContinue
|
||||
if ($winrmUp) {
|
||||
# Optional auth probe — ensures WinRM login is functional, not just TCP open
|
||||
if ($null -ne $Credential) {
|
||||
try {
|
||||
$so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
$null = Invoke-Command -ComputerName $IPAddress -Credential $Credential `
|
||||
-Authentication Basic -UseSSL -Port 5986 -SessionOption $so `
|
||||
-ScriptBlock { 'ready' } -ErrorAction Stop
|
||||
Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress."
|
||||
} catch {
|
||||
$phase = 'winrm-auth'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
}
|
||||
Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)."
|
||||
return
|
||||
}
|
||||
else {
|
||||
$phase = 'winrm'
|
||||
if ($lastPhase -ne $phase) {
|
||||
Write-Host "[Wait-VMReady] Phase 3/3: ping OK, waiting for WinRM on port 5986 (HTTPS)..."
|
||||
$lastPhase = $phase
|
||||
}
|
||||
Start-Sleep -Seconds $PollIntervalSeconds
|
||||
continue
|
||||
}
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
throw "[Wait-VMReady] Timed out after ${TimeoutSeconds}s waiting for VM at $IPAddress to become ready. Last phase: $phase"
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator wait-ready @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+16
-89
@@ -1,96 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook).
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB,
|
||||
writes a Warning entry to the Windows Application Event Log under source
|
||||
'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler
|
||||
marks the task as failed (visible in Task Scheduler history).
|
||||
|
||||
If WebhookUrl is provided, also POSTs a JSON payload to that URL
|
||||
(compatible with Discord and Gitea webhooks — content field only).
|
||||
|
||||
A full drive silently causes vmrun clone to fail with cryptic errors
|
||||
(not enough disk space for linked clone delta files). Alert early.
|
||||
|
||||
.PARAMETER MinFreeGB
|
||||
Alert threshold in GB. Default: 50
|
||||
|
||||
.PARAMETER DriveLetter
|
||||
Single letter (no colon) of the drive to monitor. Default: F
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual check
|
||||
.\Watch-DiskSpace.ps1 -MinFreeGB 50
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 2000)]
|
||||
[int] $MinFreeGB = 50,
|
||||
|
||||
[ValidatePattern('^[A-Za-z]$')]
|
||||
[string] $DriveLetter = 'F',
|
||||
|
||||
[string] $WebhookUrl = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue
|
||||
if (-not $drive) {
|
||||
Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found."
|
||||
exit 2
|
||||
}
|
||||
|
||||
$freeGB = [math]::Round($drive.Free / 1GB, 1)
|
||||
$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1)
|
||||
$pctFree = [math]::Round($freeGB / $totalGB * 100, 0)
|
||||
|
||||
if ($freeGB -ge $MinFreeGB) {
|
||||
Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Alert ─────────────────────────────────────────────────────────────────────
|
||||
$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " +
|
||||
"the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " +
|
||||
"Run Invoke-RetentionPolicy.ps1 or free disk space manually."
|
||||
|
||||
Write-Warning "[DiskSpace] $msg"
|
||||
|
||||
# Windows Event Log
|
||||
$logSource = 'CI-DiskSpaceAlert'
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId 1001 -EntryType Warning -Message $msg
|
||||
Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)."
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Could not write Event Log: $_"
|
||||
}
|
||||
|
||||
# Optional webhook (Discord / Gitea)
|
||||
if ($WebhookUrl) {
|
||||
$body = @{ content = "[WARNING] CI Disk Alert -- $msg" } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[DiskSpace] Webhook notified: $WebhookUrl"
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Webhook POST failed: $_"
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer)
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator monitor disk @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+17
-208
@@ -1,214 +1,23 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Monitors the act_runner service and auto-restarts it if stopped.
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1 as CI-RunnerHealth).
|
||||
|
||||
Behaviour:
|
||||
- If act_runner is Running: exits 0, no action.
|
||||
- If not Running: writes a Warning to the Windows Application Event Log
|
||||
(source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
|
||||
- Restart attempts are rate-limited to MaxRestarts per hour using a JSON
|
||||
state file in StateDir. Once the limit is hit, further runs write an
|
||||
alert (EventId 1004) and exit 1 without restarting — Task Scheduler
|
||||
history shows the failure so the operator is alerted.
|
||||
- On successful restart: writes EventId 1003 (Information).
|
||||
- Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
|
||||
Gitea webhooks on every restart or limit-exceeded event.
|
||||
|
||||
.PARAMETER MaxRestarts
|
||||
Maximum number of auto-restart attempts allowed per rolling 60-minute
|
||||
window before giving up and requiring manual intervention. Default: 3
|
||||
|
||||
.PARAMETER ServiceName
|
||||
Windows service name to monitor. Default: act_runner
|
||||
|
||||
.PARAMETER StateDir
|
||||
Directory used for the cooldown state file (runner-restart-log.json).
|
||||
Default: F:\CI\State
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual health check
|
||||
.\Watch-RunnerHealth.ps1
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 10)]
|
||||
[int] $MaxRestarts = 3,
|
||||
|
||||
[string] $ServiceName = 'act_runner',
|
||||
|
||||
[string] $StateDir = 'F:\CI\State',
|
||||
|
||||
[string] $WebhookUrl = '',
|
||||
|
||||
# Optional Gitea API runner-online verification.
|
||||
# When $GiteaUrl is non-empty and the act_runner service is Running, this script
|
||||
# queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online.
|
||||
# Requires an admin Personal Access Token stored in Windows Credential Manager
|
||||
# under $GiteaCredentialTarget (same module: CredentialManager).
|
||||
# Silently skipped if GiteaUrl is empty or the credential is unavailable.
|
||||
[string] $GiteaUrl = '',
|
||||
|
||||
[string] $GiteaCredentialTarget = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$logSource = 'CI-RunnerHealth'
|
||||
|
||||
# ── Helper: write to Event Log ────────────────────────────────────────────────
|
||||
function Write-CIEvent {
|
||||
param([int] $EventId, [string] $EntryType, [string] $Message)
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId $EventId -EntryType $EntryType -Message $Message
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Could not write Event Log: $_"
|
||||
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||
# Original PS implementation moved to git history; see plans/A2-closeout.md.
|
||||
$pyArgs = @()
|
||||
foreach ($a in $args) {
|
||||
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||
$name = $a.Substring(1)
|
||||
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||
$pyArgs += '--' + $kebab.ToLower()
|
||||
}
|
||||
else {
|
||||
$pyArgs += $a
|
||||
}
|
||||
}
|
||||
|
||||
# ── Helper: POST webhook ──────────────────────────────────────────────────────
|
||||
function Send-Webhook {
|
||||
param([string] $Content)
|
||||
if (-not $WebhookUrl) { return }
|
||||
$body = @{ content = $Content } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[RunnerHealth] Webhook notified."
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Webhook POST failed: $_"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Maintenance flag check ────────────────────────────────────────────────────
|
||||
# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic.
|
||||
# Create the flag before planned maintenance; delete it when done.
|
||||
$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag'
|
||||
if (Test-Path $maintenanceFlag) {
|
||||
Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic."
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Step 1: Check service ─────────────────────────────────────────────────────
|
||||
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
||||
if (-not $svc) {
|
||||
Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
|
||||
exit 2
|
||||
}
|
||||
|
||||
if ($svc.Status -eq 'Running') {
|
||||
Write-Host "[RunnerHealth] $ServiceName is Running — OK"
|
||||
|
||||
# ── Optional Gitea runner-online API check ────────────────────────────────
|
||||
if ($GiteaUrl -ne '') {
|
||||
$pat = $null
|
||||
if ($GiteaCredentialTarget -ne '') {
|
||||
try {
|
||||
$cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
|
||||
$pat = $cred.GetNetworkCredential().Password
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_"
|
||||
}
|
||||
}
|
||||
if ($pat) {
|
||||
try {
|
||||
$apiUrl = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners"
|
||||
$runners = Invoke-RestMethod -Uri $apiUrl `
|
||||
-Headers @{ Authorization = "token $pat" } `
|
||||
-Method Get -TimeoutSec 10 -ErrorAction Stop
|
||||
$online = @($runners | Where-Object { $_.online -eq $true })
|
||||
if ($online.Count -gt 0) {
|
||||
Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK"
|
||||
} else {
|
||||
$msg = "Gitea API ($apiUrl) reports 0 online runners. " +
|
||||
"$ServiceName service is Running — possible registration issue."
|
||||
Write-Warning "[RunnerHealth] $msg"
|
||||
Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg
|
||||
Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg"
|
||||
}
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Gitea API check failed: $_"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Step 2: Service not running — load cooldown state ─────────────────────────
|
||||
if (-not (Test-Path $StateDir)) {
|
||||
New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
|
||||
$restartLog = @()
|
||||
|
||||
if (Test-Path $cooldownFile) {
|
||||
try {
|
||||
$raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
|
||||
if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
|
||||
} catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" }
|
||||
}
|
||||
|
||||
# Prune timestamps older than 1 hour
|
||||
$cutoff = (Get-Date).AddHours(-1)
|
||||
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
|
||||
$recentCount = $restartLog.Count
|
||||
|
||||
$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
|
||||
Write-Warning "[RunnerHealth] $stateMsg"
|
||||
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg
|
||||
|
||||
# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
|
||||
if ($recentCount -ge $MaxRestarts) {
|
||||
$limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
|
||||
"Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
|
||||
Write-Warning "[RunnerHealth] $limitMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
|
||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Step 4: Attempt restart ───────────────────────────────────────────────────
|
||||
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
|
||||
try {
|
||||
Restart-Service -Name $ServiceName -Force -ErrorAction Stop
|
||||
Start-Sleep -Seconds 5
|
||||
$newStatus = (Get-Service -Name $ServiceName).Status
|
||||
$restartMsg = "$ServiceName restarted — new status: $newStatus."
|
||||
Write-Host "[RunnerHealth] $restartMsg"
|
||||
|
||||
# Persist this restart in the cooldown log
|
||||
$restartLog += (Get-Date -Format 'o')
|
||||
$restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue
|
||||
|
||||
$evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
|
||||
Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg
|
||||
|
||||
$prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' }
|
||||
Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg"
|
||||
|
||||
exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
|
||||
}
|
||||
catch {
|
||||
$errMsg = "$ServiceName restart failed: $_"
|
||||
Write-Warning "[RunnerHealth] $errMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
|
||||
Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg"
|
||||
exit 1
|
||||
}
|
||||
$venvPython = $env:CI_VENV_PYTHON
|
||||
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||
& $venvPython -m ci_orchestrator monitor runner @pyArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
Reference in New Issue
Block a user