#Requires -Version 5.1 <# .SYNOPSIS Monitors the act_runner service and auto-restarts it if stopped. .DESCRIPTION Intended to run as a scheduled task every 15 minutes (registered by Register-CIScheduledTasks.ps1 as CI-RunnerHealth). Behaviour: - If act_runner is Running: exits 0, no action. - If not Running: writes a Warning to the Windows Application Event Log (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service. - Restart attempts are rate-limited to MaxRestarts per hour using a JSON state file in StateDir. Once the limit is hit, further runs write an alert (EventId 1004) and exit 1 without restarting — Task Scheduler history shows the failure so the operator is alerted. - On successful restart: writes EventId 1003 (Information). - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and Gitea webhooks on every restart or limit-exceeded event. .PARAMETER MaxRestarts Maximum number of auto-restart attempts allowed per rolling 60-minute window before giving up and requiring manual intervention. Default: 3 .PARAMETER ServiceName Windows service name to monitor. Default: act_runner .PARAMETER StateDir Directory used for the cooldown state file (runner-restart-log.json). Default: F:\CI\State .PARAMETER WebhookUrl Optional Discord/Gitea webhook URL. If empty, only Event Log is written. .EXAMPLE # Manual health check .\Watch-RunnerHealth.ps1 # With Discord webhook .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..." #> [CmdletBinding()] param( [ValidateRange(1, 10)] [int] $MaxRestarts = 3, [string] $ServiceName = 'act_runner', [string] $StateDir = 'F:\CI\State', [string] $WebhookUrl = '' ) Set-StrictMode -Version Latest $ErrorActionPreference = 'Continue' $logSource = 'CI-RunnerHealth' # ── Helper: write to Event Log ──────────────────────────────────────────────── function Write-CIEvent { param([int] $EventId, [string] $EntryType, [string] $Message) try { if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) { New-EventLog -LogName Application -Source $logSource -ErrorAction Stop } Write-EventLog -LogName Application -Source $logSource ` -EventId $EventId -EntryType $EntryType -Message $Message } catch { Write-Warning "[RunnerHealth] Could not write Event Log: $_" } } # ── Helper: POST webhook ────────────────────────────────────────────────────── function Send-Webhook { param([string] $Content) if (-not $WebhookUrl) { return } $body = @{ content = $Content } | ConvertTo-Json -Compress try { $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post ` -Body $body -ContentType 'application/json' -TimeoutSec 10 Write-Host "[RunnerHealth] Webhook notified." } catch { Write-Warning "[RunnerHealth] Webhook POST failed: $_" } } # ── Step 1: Check service ───────────────────────────────────────────────────── $svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue if (-not $svc) { Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?" exit 2 } if ($svc.Status -eq 'Running') { Write-Host "[RunnerHealth] $ServiceName is Running — OK" exit 0 } # ── Step 2: Service not running — load cooldown state ───────────────────────── if (-not (Test-Path $StateDir)) { New-Item -ItemType Directory -Path $StateDir -Force | Out-Null } $cooldownFile = Join-Path $StateDir 'runner-restart-log.json' $restartLog = @() if (Test-Path $cooldownFile) { try { $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue if ($raw) { $restartLog = @($raw | ConvertFrom-Json) } } catch { } } # Prune timestamps older than 1 hour $cutoff = (Get-Date).AddHours(-1) $restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff }) $recentCount = $restartLog.Count $stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts." Write-Warning "[RunnerHealth] $stateMsg" Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg # ── Step 3: Rate-limit check ────────────────────────────────────────────────── if ($recentCount -ge $MaxRestarts) { $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " + "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\." Write-Warning "[RunnerHealth] $limitMsg" Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg Send-Webhook -Content ":sos: **CI Runner Alert** — $limitMsg" exit 1 } # ── Step 4: Attempt restart ─────────────────────────────────────────────────── Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..." try { Restart-Service -Name $ServiceName -Force -ErrorAction Stop Start-Sleep -Seconds 5 $newStatus = (Get-Service -Name $ServiceName).Status $restartMsg = "$ServiceName restarted — new status: $newStatus." Write-Host "[RunnerHealth] $restartMsg" # Persist this restart in the cooldown log $restartLog += (Get-Date -Format 'o') $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' } Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg $icon = if ($newStatus -eq 'Running') { ':warning:' } else { ':sos:' } Send-Webhook -Content "$icon **CI Runner Alert** — $restartMsg" exit $(if ($newStatus -eq 'Running') { 0 } else { 1 }) } catch { $errMsg = "$ServiceName restart failed: $_" Write-Warning "[RunnerHealth] $errMsg" Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg Send-Webhook -Content ":sos: **CI Runner Alert** — $errMsg" exit 1 }