feat(sprint4-6): quality, reliability, perf baseline
Sprint 4 — qualità codice: - scripts/_Common.psm1: New-CISessionOption, Resolve-VmrunPath, Invoke-Vmrun - PSScriptAnalyzerSettings.psd1: project-wide PSSA rules (security + formatting) - Remove-BuildVM.ps1: SupportsShouldProcess (-WhatIf support) - Invoke-RemoteBuild.ps1, Get-BuildArtifacts.ps1: use New-CISessionOption from module - New-BuildVM.ps1: use Resolve-VmrunPath + Invoke-Vmrun from module Sprint 5 — affidabilità: - Backup-CITemplate.ps1: stop runner, timestamped VMDK backup, prune old, restart - Watch-RunnerHealth.ps1: auto-restart act_runner, cooldown 3/h, EventLog + webhook - Register-CIScheduledTasks.ps1: add CI-RunnerHealth task (every 15 min) Sprint 6 — perf baseline: - Set-TemplateSharedFolders.ps1: idempotent VMX editor for NuGet/pip shared folders - Invoke-RemoteBuild.ps1: -UseSharedCache injects NUGET_PACKAGES + PIP_CACHE_DIR in guest - Measure-CIBenchmark.ps1: times clone/start/IP/WinRM/destroy, appends to benchmark.jsonl Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
@@ -0,0 +1,161 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Monitors the act_runner service and auto-restarts it if stopped.
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1 as CI-RunnerHealth).
|
||||
|
||||
Behaviour:
|
||||
- If act_runner is Running: exits 0, no action.
|
||||
- If not Running: writes a Warning to the Windows Application Event Log
|
||||
(source CI-RunnerHealth, EventId 1002), then attempts Restart-Service.
|
||||
- Restart attempts are rate-limited to MaxRestarts per hour using a JSON
|
||||
state file in StateDir. Once the limit is hit, further runs write an
|
||||
alert (EventId 1004) and exit 1 without restarting — Task Scheduler
|
||||
history shows the failure so the operator is alerted.
|
||||
- On successful restart: writes EventId 1003 (Information).
|
||||
- Optional WebhookUrl: POSTs a JSON payload compatible with Discord and
|
||||
Gitea webhooks on every restart or limit-exceeded event.
|
||||
|
||||
.PARAMETER MaxRestarts
|
||||
Maximum number of auto-restart attempts allowed per rolling 60-minute
|
||||
window before giving up and requiring manual intervention. Default: 3
|
||||
|
||||
.PARAMETER ServiceName
|
||||
Windows service name to monitor. Default: act_runner
|
||||
|
||||
.PARAMETER StateDir
|
||||
Directory used for the cooldown state file (runner-restart-log.json).
|
||||
Default: F:\CI\State
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual health check
|
||||
.\Watch-RunnerHealth.ps1
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-RunnerHealth.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 10)]
|
||||
[int] $MaxRestarts = 3,
|
||||
|
||||
[string] $ServiceName = 'act_runner',
|
||||
|
||||
[string] $StateDir = 'F:\CI\State',
|
||||
|
||||
[string] $WebhookUrl = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
|
||||
$logSource = 'CI-RunnerHealth'
|
||||
|
||||
# ── Helper: write to Event Log ────────────────────────────────────────────────
|
||||
function Write-CIEvent {
|
||||
param([int] $EventId, [string] $EntryType, [string] $Message)
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId $EventId -EntryType $EntryType -Message $Message
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Could not write Event Log: $_"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Helper: POST webhook ──────────────────────────────────────────────────────
|
||||
function Send-Webhook {
|
||||
param([string] $Content)
|
||||
if (-not $WebhookUrl) { return }
|
||||
$body = @{ content = $Content } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[RunnerHealth] Webhook notified."
|
||||
} catch {
|
||||
Write-Warning "[RunnerHealth] Webhook POST failed: $_"
|
||||
}
|
||||
}
|
||||
|
||||
# ── Step 1: Check service ─────────────────────────────────────────────────────
|
||||
$svc = Get-Service -Name $ServiceName -ErrorAction SilentlyContinue
|
||||
if (-not $svc) {
|
||||
Write-Warning "[RunnerHealth] Service '$ServiceName' not found — is act_runner installed?"
|
||||
exit 2
|
||||
}
|
||||
|
||||
if ($svc.Status -eq 'Running') {
|
||||
Write-Host "[RunnerHealth] $ServiceName is Running — OK"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Step 2: Service not running — load cooldown state ─────────────────────────
|
||||
if (-not (Test-Path $StateDir)) {
|
||||
New-Item -ItemType Directory -Path $StateDir -Force | Out-Null
|
||||
}
|
||||
|
||||
$cooldownFile = Join-Path $StateDir 'runner-restart-log.json'
|
||||
$restartLog = @()
|
||||
|
||||
if (Test-Path $cooldownFile) {
|
||||
try {
|
||||
$raw = Get-Content $cooldownFile -Raw -ErrorAction SilentlyContinue
|
||||
if ($raw) { $restartLog = @($raw | ConvertFrom-Json) }
|
||||
} catch { }
|
||||
}
|
||||
|
||||
# Prune timestamps older than 1 hour
|
||||
$cutoff = (Get-Date).AddHours(-1)
|
||||
$restartLog = @($restartLog | Where-Object { [datetime]$_ -gt $cutoff })
|
||||
$recentCount = $restartLog.Count
|
||||
|
||||
$stateMsg = "$ServiceName service is '$($svc.Status)'. Auto-restarts in last hour: $recentCount / $MaxRestarts."
|
||||
Write-Warning "[RunnerHealth] $stateMsg"
|
||||
Write-CIEvent -EventId 1002 -EntryType Warning -Message $stateMsg
|
||||
|
||||
# ── Step 3: Rate-limit check ──────────────────────────────────────────────────
|
||||
if ($recentCount -ge $MaxRestarts) {
|
||||
$limitMsg = "$ServiceName auto-restart limit ($MaxRestarts/h) reached — manual intervention required. " +
|
||||
"Check Event Viewer → Application (source: $logSource) and F:\CI\act_runner\logs\."
|
||||
Write-Warning "[RunnerHealth] $limitMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $limitMsg
|
||||
Send-Webhook -Content ":sos: **CI Runner Alert** — $limitMsg"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# ── Step 4: Attempt restart ───────────────────────────────────────────────────
|
||||
Write-Host "[RunnerHealth] Restarting $ServiceName (attempt $($recentCount + 1) of $MaxRestarts this hour)..."
|
||||
try {
|
||||
Restart-Service -Name $ServiceName -Force -ErrorAction Stop
|
||||
Start-Sleep -Seconds 5
|
||||
$newStatus = (Get-Service -Name $ServiceName).Status
|
||||
$restartMsg = "$ServiceName restarted — new status: $newStatus."
|
||||
Write-Host "[RunnerHealth] $restartMsg"
|
||||
|
||||
# Persist this restart in the cooldown log
|
||||
$restartLog += (Get-Date -Format 'o')
|
||||
$restartLog | ConvertTo-Json | Set-Content $cooldownFile -Encoding UTF8 -ErrorAction SilentlyContinue
|
||||
|
||||
$evtType = if ($newStatus -eq 'Running') { 'Information' } else { 'Warning' }
|
||||
Write-CIEvent -EventId 1003 -EntryType $evtType -Message $restartMsg
|
||||
|
||||
$icon = if ($newStatus -eq 'Running') { ':warning:' } else { ':sos:' }
|
||||
Send-Webhook -Content "$icon **CI Runner Alert** — $restartMsg"
|
||||
|
||||
exit $(if ($newStatus -eq 'Running') { 0 } else { 1 })
|
||||
}
|
||||
catch {
|
||||
$errMsg = "$ServiceName restart failed: $_"
|
||||
Write-Warning "[RunnerHealth] $errMsg"
|
||||
Write-CIEvent -EventId 1004 -EntryType Error -Message $errMsg
|
||||
Send-Webhook -Content ":sos: **CI Runner Alert** — $errMsg"
|
||||
exit 1
|
||||
}
|
||||
Reference in New Issue
Block a user