local-ci-cd-system/scripts/Watch-RunnerHealth.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Monitors the act_runner service and auto-restarts it if stopped.

.DESCRIPTION
    Intended to run as a scheduled task every 15 minutes (registered by
    Register-CIScheduledTasks.ps1 as CI-RunnerHealth).

    Behaviour:
    - If act_runner is Running: exits 0, no action.
    - If not Running: writes a Warning to the Windows Application Event Log
      (source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
    - Restart attempts are rate-limited to MaxRestarts per hour using a JSON
      state file in StateDir. Once the limit is hit, further runs write an
      alert (EventId 1004) and exit 1 without restarting — Task Scheduler
      history shows the failure so the operator is alerted.
    - On successful restart: writes EventId 1003 (Information).
    - Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
      Gitea webhooks on every restart or limit-exceeded event.

.PARAMETER MaxRestarts
    Maximum number of auto-restart attempts allowed per rolling 60-minute
    window before giving up and requiring manual intervention. Default: 3

.PARAMETER ServiceName
    Windows service name to monitor. Default: act_runner

.PARAMETER StateDir
    Directory used for the cooldown state file (runner-restart-log.json).
    Default: F:\CI\State

.PARAMETER WebhookUrl
    Optional Discord/Gitea webhook URL. If empty, only Event Log is written.

.EXAMPLE
    # Manual health check
    .\Watch-RunnerHealth.ps1

    # With Discord webhook
    .\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
#>
[CmdletBinding()]
param(
    [ValidateRange(1, 10)]
    [int] $MaxRestarts = 3,

    [string] $ServiceName = 'act_runner',

    [string] $StateDir = 'F:\CI\State',

    [string] $WebhookUrl = '',

    # Optional Gitea API runner-online verification.
    # When $GiteaUrl is non-empty and the act_runner service is Running, this script
    # queries GET $GiteaUrl/api/v1/admin/runners and warns if zero runners are online.
    # Requires an admin Personal Access Token stored in Windows Credential Manager
    # under $GiteaCredentialTarget (same module: CredentialManager).
    # Silently skipped if GiteaUrl is empty or the credential is unavailable.
    [string] $GiteaUrl = '',

    [string] $GiteaCredentialTarget = ''
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Continue'

$logSource = 'CI-RunnerHealth'

# ── Helper: write to Event Log ────────────────────────────────────────────────
function Write-CIEvent {
    param([int] $EventId, [string] $EntryType, [string] $Message)
    try {
        if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
            New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
        }
        Write-EventLog -LogName Application -Source $logSource `
            -EventId $EventId -EntryType $EntryType -Message $Message
    } catch {
        Write-Warning "[RunnerHealth] Could not write Event Log: $_"
    }
}

# ── Helper: POST webhook ──────────────────────────────────────────────────────
function Send-Webhook {
    param([string] $Content)
    if (-not $WebhookUrl) { return }
    $body = @{ content = $Content } | ConvertTo-Json -Compress
    try {
        $null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
            -Body $body -ContentType 'application/json' -TimeoutSec 10
        Write-Host "[RunnerHealth] Webhook notified."
    } catch {
        Write-Warning "[RunnerHealth] Webhook POST failed: $_"
    }
}

# ── Maintenance flag check ────────────────────────────────────────────────────
# If F:\CI\State\runner-maintenance.flag exists, skip all restart logic.
# Create the flag before planned maintenance; delete it when done.
$maintenanceFlag = Join-Path $StateDir 'runner-maintenance.flag'
if (Test-Path $maintenanceFlag) {
    Write-Host "[RunnerHealth] Maintenance mode active (flag: $maintenanceFlag) — skipping restart logic."
    exit 0
}

# ── Step 1: Check service ─────────────────────────────────────────────────────
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
if (-not $svc) {
    Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
    exit 2
}

if ($svc.Status -eq 'Running') {
    Write-Host "[RunnerHealth] $ServiceName is Running — OK"

    # ── Optional Gitea runner-online API check ────────────────────────────────
    if ($GiteaUrl -ne '') {
        $pat = $null
        if ($GiteaCredentialTarget -ne '') {
            try {
                $cred = Get-StoredCredential -Target $GiteaCredentialTarget -ErrorAction Stop
                $pat  = $cred.GetNetworkCredential().Password
            } catch {
                Write-Warning "[RunnerHealth] Could not load Gitea PAT from '$GiteaCredentialTarget': $_"
            }
        }
        if ($pat) {
            try {
                $apiUrl  = "$($GiteaUrl.TrimEnd('/'))/api/v1/admin/runners"
                $runners = Invoke-RestMethod -Uri $apiUrl `
                               -Headers @{ Authorization = "token $pat" } `
                               -Method Get -TimeoutSec 10 -ErrorAction Stop
                $online  = @($runners | Where-Object { $_.online -eq $true })
                if ($online.Count -gt 0) {
                    Write-Host "[RunnerHealth] Gitea API: $($online.Count) runner(s) online — OK"
                } else {
                    $msg = "Gitea API ($apiUrl) reports 0 online runners. " +
                           "$ServiceName service is Running — possible registration issue."
                    Write-Warning "[RunnerHealth] $msg"
                    Write-CIEvent -EventId 1005 -EntryType Warning -Message $msg
                    Send-Webhook -Content "[WARNING] CI Runner Alert -- $msg"
                }
            } catch {
                Write-Warning "[RunnerHealth] Gitea API check failed: $_"
            }
        }
    }

    exit 0
}

# ── Step 2: Service not running — load cooldown state ─────────────────────────
if (-not (Test-Path $StateDir)) {
    New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
}

$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
$restartLog   = @()

if (Test-Path $cooldownFile) {
    try {
        $raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
        if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
    } catch { Write-Debug "[Watch-RunnerHealth] Restart log unreadable — starting fresh: $_" }
}

# Prune timestamps older than 1 hour
$cutoff     = (Get-Date).AddHours(-1)
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
$recentCount = $restartLog.Count

$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
Write-Warning "[RunnerHealth] $stateMsg"
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg

# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
if ($recentCount -ge $MaxRestarts) {
    $limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
                "Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
    Write-Warning "[RunnerHealth] $limitMsg"
    Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
    Send-Webhook -Content "[ALERT] CI Runner Alert -- $limitMsg"
    exit 1
}

# ── Step 4: Attempt restart ───────────────────────────────────────────────────
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
try {
    Restart-Service -Name $ServiceName -Force -ErrorAction Stop
    Start-Sleep -Seconds 5
    $newStatus  = (Get-Service -Name $ServiceName).Status
    $restartMsg = "$ServiceName restarted — new status: $newStatus."
    Write-Host "[RunnerHealth] $restartMsg"

    # Persist this restart in the cooldown log
    $restartLog += (Get-Date -Format 'o')
    $restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue

    $evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
    Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg

    $prefix = if ($newStatus -eq 'Running') { '[WARNING]' } else { '[ALERT]' }
    Send-Webhook -Content "$prefix CI Runner Alert -- $restartMsg"

    exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
}
catch {
    $errMsg = "$ServiceName restart failed: $_"
    Write-Warning "[RunnerHealth] $errMsg"
    Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
    Send-Webhook -Content "[ALERT] CI Runner Alert -- $errMsg"
    exit 1
}