feat(sprint3): §4.1/4.3/4.4 — JSONL logging, disk alert, runbook
§4.1 — Structured JSONL log (Invoke-CIJob.ps1)
Each job now emits a parallel invoke-ci.jsonl alongside the transcript.
Write-JobEvent appends one JSON line per phase transition (ts, jobId,
phase, status, data). Events emitted at: job.start, phase1.clone-repo
start/success, phase2-3b.vm-start start/success, phase4.wait-ready
start/success, phase5.build start/success, phase6.artifacts start/success,
job.success/failure (with elapsedSec and error string).
Silent no-op if $jsonLog is null (log dir setup failed). Errors in
Write-JobEvent are swallowed — logging never breaks the build.
Enables post-hoc per-phase duration analysis with jq or ConvertFrom-Json.
§4.3 — Disk space alert (scripts/Watch-DiskSpace.ps1)
Checks drive free space every 15 min (scheduled by Register-CIScheduledTasks).
Below MinFreeGB (default 50 GB): writes Warning to Windows Application Event Log
(source CI-DiskAlert, EventId 1001) and exits 1 so Task Scheduler flags the run.
Optional -WebhookUrl for Discord/Gitea webhook notification.
Register-CIScheduledTasks.ps1 updated with Task 3: CI-DiskSpaceAlert (every 15 min).
§4.4 — Incident runbook (docs/RUNBOOK.md)
Four scenarios with symptom / triage commands / fix / escalation:
1. Runner offline in Gitea UI
2. All builds fail in Phase 2 (clone/start/IP)
3. Builds are slow (per-phase JSONL analysis, disk/CPU/NAT checks)
4. Template VMX corrupt after host crash (lock removal + backup restore)
Quick-reference table at the end.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
@@ -159,14 +159,16 @@ $jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
|
||||
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
|
||||
$cloneVmxPath = $null
|
||||
$leaseFile = $null # §2.1 IP lease file path — released in finally
|
||||
$jsonLog = $null # §4.1 JSONL structured log path
|
||||
$startTime = Get-Date
|
||||
|
||||
# ── Start transcript ──────────────────────────────────────────────────────────
|
||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
||||
$transcriptStarted = $false
|
||||
try {
|
||||
$jobLogDir = Join-Path $LogDir $JobId
|
||||
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
|
||||
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
|
||||
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
|
||||
# -Append allows starting a new transcript even when VS Code (or another host)
|
||||
# has already opened a transcript on this PowerShell session.
|
||||
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
|
||||
@@ -176,6 +178,29 @@ catch {
|
||||
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
|
||||
}
|
||||
|
||||
# ── §4.1 Structured event emitter ────────────────────────────────────────────
|
||||
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
|
||||
# Silent no-op if $jsonLog is null (transcript setup failed).
|
||||
function Write-JobEvent {
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $Phase,
|
||||
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
|
||||
[hashtable] $Data = @{}
|
||||
)
|
||||
if (-not $script:jsonLog) { return }
|
||||
$event = [ordered]@{
|
||||
ts = (Get-Date -Format 'o')
|
||||
jobId = $script:JobId
|
||||
phase = $Phase
|
||||
status = $Status
|
||||
data = $Data
|
||||
}
|
||||
try {
|
||||
$event | ConvertTo-Json -Compress -Depth 5 |
|
||||
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
|
||||
} catch { } # never let logging break the build
|
||||
}
|
||||
|
||||
Write-Host "============================================================"
|
||||
Write-Host "[Invoke-CIJob] Job : $JobId"
|
||||
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
|
||||
@@ -187,6 +212,14 @@ Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
|
||||
Write-Host "[Invoke-CIJob] Started : $startTime"
|
||||
Write-Host "============================================================"
|
||||
|
||||
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
|
||||
repo = $RepoUrl
|
||||
branch = $Branch
|
||||
commit = $Commit
|
||||
template = $TemplatePath
|
||||
snapshot = $SnapshotName
|
||||
}
|
||||
|
||||
$exitCode = 0
|
||||
|
||||
try {
|
||||
@@ -195,6 +228,7 @@ try {
|
||||
# still cloned here and copied via WinRM zip to avoid injecting a PAT
|
||||
# into the VM environment.
|
||||
Write-Host "`n[Phase 1/6] Cloning repository on host..."
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
|
||||
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
|
||||
$gitArgs = @('clone', '--depth', '1', '--branch', $Branch)
|
||||
if ($Submodules) { $gitArgs += '--recurse-submodules' }
|
||||
@@ -224,6 +258,7 @@ try {
|
||||
}
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
||||
|
||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
||||
@@ -237,6 +272,7 @@ try {
|
||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
||||
|
||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
||||
$lockStream = $null
|
||||
$lockDeadline = (Get-Date).AddMinutes(10)
|
||||
while ($true) {
|
||||
@@ -296,6 +332,7 @@ try {
|
||||
}
|
||||
[System.IO.File]::WriteAllText($leaseFile, $JobId)
|
||||
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
|
||||
|
||||
} finally {
|
||||
# Release lock — next waiting job can now enter clone→start→IP phase
|
||||
@@ -310,13 +347,16 @@ try {
|
||||
|
||||
# ── Phase 4: Wait for readiness ───────────────────────────────────────
|
||||
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
|
||||
& "$scriptDir\Wait-VMReady.ps1" `
|
||||
-VMPath $cloneVmxPath `
|
||||
-IPAddress $VMIPAddress `
|
||||
-VmrunPath $VmrunPath
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
||||
|
||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'start'
|
||||
$remoteBuildParams = @{
|
||||
IPAddress = $VMIPAddress
|
||||
Credential = $credential
|
||||
@@ -326,9 +366,11 @@ try {
|
||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
||||
|
||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
|
||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
||||
-IPAddress $VMIPAddress `
|
||||
-Credential $credential `
|
||||
@@ -337,6 +379,8 @@ try {
|
||||
-IncludeLogs
|
||||
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
||||
@@ -345,6 +389,10 @@ try {
|
||||
catch {
|
||||
$exitCode = 1
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
|
||||
elapsedSec = [int]$elapsed.TotalSeconds
|
||||
error = "$_"
|
||||
}
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "Error: $_"
|
||||
|
||||
@@ -123,4 +123,36 @@ if (-not (Test-Path $retentionScript -PathType Leaf)) {
|
||||
}
|
||||
}
|
||||
|
||||
# ── Task 3: CI-DiskSpaceAlert ─────────────────────────────────────────────────
|
||||
$diskScript = Join-Path $ScriptRoot 'Watch-DiskSpace.ps1'
|
||||
if (-not (Test-Path $diskScript -PathType Leaf)) {
|
||||
Write-Warning "Script not found — skipping CI-DiskSpaceAlert: $diskScript"
|
||||
} else {
|
||||
$diskAction = New-ScheduledTaskAction `
|
||||
-Execute $psExe `
|
||||
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$diskScript`""
|
||||
|
||||
# Every 15 minutes indefinitely
|
||||
$disk15min = New-ScheduledTaskTrigger -Once -At '00:00' `
|
||||
-RepetitionInterval (New-TimeSpan -Minutes 15)
|
||||
|
||||
$diskSettings = New-ScheduledTaskSettingsSet `
|
||||
-ExecutionTimeLimit (New-TimeSpan -Minutes 5) `
|
||||
-MultipleInstances IgnoreNew `
|
||||
-StartWhenAvailable
|
||||
|
||||
if ($PSCmdlet.ShouldProcess('CI-DiskSpaceAlert', 'Register/update scheduled task')) {
|
||||
Register-ScheduledTask `
|
||||
-TaskName 'CI-DiskSpaceAlert' `
|
||||
-TaskPath $taskPath `
|
||||
-Action $diskAction `
|
||||
-Trigger $disk15min `
|
||||
-Principal $principal `
|
||||
-Settings $diskSettings `
|
||||
-Description 'Alerts via Event Log when CI drive free space drops below 50 GB (Local CI/CD System)' `
|
||||
-Force | Out-Null
|
||||
Write-Host "[Register] CI-DiskSpaceAlert registered (every 15 min)." -ForegroundColor Green
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host "`n[Register] Done. Verify with: Get-ScheduledTask -TaskPath '\CI\' | Format-Table TaskName, State" -ForegroundColor Cyan
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
#Requires -Version 5.1
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Checks CI host drive free space and alerts via Windows Event Log (and optionally a webhook).
|
||||
|
||||
.DESCRIPTION
|
||||
Intended to run as a scheduled task every 15 minutes (registered by
|
||||
Register-CIScheduledTasks.ps1). If the monitored drive is below MinFreeGB,
|
||||
writes a Warning entry to the Windows Application Event Log under source
|
||||
'CI-DiskAlert' (Event ID 1001) and exits with code 1 so Task Scheduler
|
||||
marks the task as failed (visible in Task Scheduler history).
|
||||
|
||||
If WebhookUrl is provided, also POSTs a JSON payload to that URL
|
||||
(compatible with Discord and Gitea webhooks — content field only).
|
||||
|
||||
A full drive silently causes vmrun clone to fail with cryptic errors
|
||||
(not enough disk space for linked clone delta files). Alert early.
|
||||
|
||||
.PARAMETER MinFreeGB
|
||||
Alert threshold in GB. Default: 50
|
||||
|
||||
.PARAMETER DriveLetter
|
||||
Single letter (no colon) of the drive to monitor. Default: F
|
||||
|
||||
.PARAMETER WebhookUrl
|
||||
Optional Discord/Gitea webhook URL. If empty, only Event Log is written.
|
||||
|
||||
.EXAMPLE
|
||||
# Manual check
|
||||
.\Watch-DiskSpace.ps1 -MinFreeGB 50
|
||||
|
||||
# With Discord webhook
|
||||
.\Watch-DiskSpace.ps1 -WebhookUrl "https://discord.com/api/webhooks/..."
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateRange(1, 2000)]
|
||||
[int] $MinFreeGB = 50,
|
||||
|
||||
[ValidatePattern('^[A-Za-z]$')]
|
||||
[string] $DriveLetter = 'F',
|
||||
|
||||
[string] $WebhookUrl = ''
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Continue'
|
||||
|
||||
$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue
|
||||
if (-not $drive) {
|
||||
Write-Warning "[DiskSpace] Drive ${DriveLetter}: not found."
|
||||
exit 2
|
||||
}
|
||||
|
||||
$freeGB = [math]::Round($drive.Free / 1GB, 1)
|
||||
$totalGB = [math]::Round(($drive.Free + $drive.Used) / 1GB, 1)
|
||||
$pctFree = [math]::Round($freeGB / $totalGB * 100, 0)
|
||||
|
||||
if ($freeGB -ge $MinFreeGB) {
|
||||
Write-Host "[DiskSpace] Drive ${DriveLetter}: ${freeGB} GB free / ${totalGB} GB (${pctFree}%) — OK (threshold: ${MinFreeGB} GB)"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# ── Alert ─────────────────────────────────────────────────────────────────────
|
||||
$msg = "CI host drive ${DriveLetter}: free space ${freeGB} GB (${pctFree}%) is below " +
|
||||
"the ${MinFreeGB} GB threshold. vmrun linked clones may fail silently. " +
|
||||
"Run Invoke-RetentionPolicy.ps1 or free disk space manually."
|
||||
|
||||
Write-Warning "[DiskSpace] $msg"
|
||||
|
||||
# Windows Event Log
|
||||
$logSource = 'CI-DiskAlert'
|
||||
try {
|
||||
if (-not [System.Diagnostics.EventLog]::SourceExists($logSource)) {
|
||||
New-EventLog -LogName Application -Source $logSource -ErrorAction Stop
|
||||
}
|
||||
Write-EventLog -LogName Application -Source $logSource `
|
||||
-EventId 1001 -EntryType Warning -Message $msg
|
||||
Write-Host "[DiskSpace] Event Log entry written (Application/$logSource, EventId 1001)."
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Could not write Event Log: $_"
|
||||
}
|
||||
|
||||
# Optional webhook (Discord / Gitea)
|
||||
if ($WebhookUrl) {
|
||||
$body = @{ content = ":warning: **CI Disk Alert** — $msg" } | ConvertTo-Json -Compress
|
||||
try {
|
||||
$null = Invoke-RestMethod -Uri $WebhookUrl -Method Post `
|
||||
-Body $body -ContentType 'application/json' -TimeoutSec 10
|
||||
Write-Host "[DiskSpace] Webhook notified: $WebhookUrl"
|
||||
} catch {
|
||||
Write-Warning "[DiskSpace] Webhook POST failed: $_"
|
||||
}
|
||||
}
|
||||
|
||||
exit 1 # non-zero → Task Scheduler marks run as failed (visible in history/Event Viewer)
|
||||
Reference in New Issue
Block a user