local-ci-cd-system/scripts/Watch-RunnerHealth.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Monitors the act_runner service and auto-restarts it if stopped.

.DESCRIPTION
    Intended to run as a scheduled task every 15 minutes (registered by
    Register-CIScheduledTasks.ps1 as CI-RunnerHealth).

    Behaviour:
    - If act_runner is Running: exits 0, no action.
    - If not Running: writes a Warning to the Windows Application Event Log
      (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
    - Restart attempts are rate-limited to MaxRestarts per hour using a JSON
      state file in StateDir. Once the limit is hit, further runs write an
      alert (EventId 1004) and exit 1 without restarting — Task Scheduler
      history shows the failure so the operator is alerted.
    - On successful restart: writes EventId 1003 (Information).
    - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
      Gitea webhooks on every restart or limit-exceeded event.

.PARAMETER MaxRestarts
    Maximum number of auto-restart attempts allowed per rolling 60-minute
    window before giving up and requiring manual intervention. Default: 3

.PARAMETER ServiceName
    Windows service name to monitor. Default: act_runner

.PARAMETER StateDir
    Directory used for the cooldown state file (runner-restart-log.json).
    Default: F:\CI\State

.PARAMETER WebhookUrl
    Optional Discord/Gitea webhook URL. If empty, only Event Log is written.

.EXAMPLE
    # Manual health check
    .\Watch-RunnerHealth.ps1

    # With Discord webhook
    .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
#>
[CmdletBinding()]
param(
    [ValidateRange(1, 10)]
    [int] $MaxRestarts = 3,

    [string] $ServiceName = 'act_runner',

    [string] $StateDir = 'F:\CI\State',

    [string] $WebhookUrl = ''
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Continue'

$logSource = 'CI-RunnerHealth'

# ── Helper: write to Event Log ────────────────────────────────────────────────
function Write-CIEvent {
    param([int] $EventId, [string] $EntryType, [string] $Message)
    try {
        if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
            New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
        }
        Write-EventLog -LogName Application -Source $logSource `
            -EventId $EventId -EntryType $EntryType -Message $Message
    } catch {
        Write-Warning "[RunnerHealth] Could not write Event Log: $_"
    }
}

# ── Helper: POST webhook ──────────────────────────────────────────────────────
function Send-Webhook {
    param([string] $Content)
    if (-not $WebhookUrl) { return }
    $body = @{ content = $Content } | ConvertTo-Json -Compress
    try {
        $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
            -Body $body -ContentType 'application/json' -TimeoutSec 10
        Write-Host "[RunnerHealth] Webhook notified."
    } catch {
        Write-Warning "[RunnerHealth] Webhook POST failed: $_"
    }
}

# ── Maintenance flag check ────────────────────────────────────────────────────
# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic.
# Create the flag before planned maintenance; delete it when done.
$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag'
if (Test-Path $maintenanceFlag) {
    Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic."
    exit 0
}

# ── Step 1: Check service ─────────────────────────────────────────────────────
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
if (-not $svc) {
    Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
    exit 2
}

if ($svc.Status -eq 'Running') {
    Write-Host "[RunnerHealth] $ServiceName is Running — OK"
    exit 0
}

# ── Step 2: Service not running — load cooldown state ─────────────────────────
if (-not (Test-Path $StateDir)) {
    New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
}

$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
$restartLog   = @()

if (Test-Path $cooldownFile) {
    try {
        $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
        if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
    } catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" }
}

# Prune timestamps older than 1 hour
$cutoff     = (Get-Date).AddHours(-1)
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
$recentCount = $restartLog.Count

$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
Write-Warning "[RunnerHealth] $stateMsg"
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg

# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
if ($recentCount -ge $MaxRestarts) {
    $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
                "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
    Write-Warning "[RunnerHealth] $limitMsg"
    Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
    Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg"
    exit 1
}

# ── Step 4: Attempt restart ───────────────────────────────────────────────────
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
try {
    Restart-Service -Name $ServiceName -Force -ErrorAction Stop
    Start-Sleep -Seconds 5
    $newStatus  = (Get-Service -Name $ServiceName).Status
    $restartMsg = "$ServiceName restarted — new status: $newStatus."
    Write-Host "[RunnerHealth] $restartMsg"

    # Persist this restart in the cooldown log
    $restartLog += (Get-Date -Format 'o')
    $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue

    $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
    Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg

    $prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' }
    Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg"

    exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
}
catch {
    $errMsg = "$ServiceName restart failed: $_"
    Write-Warning "[RunnerHealth] $errMsg"
    Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
    Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg"
    exit 1
}