feat(sprint3): §4.1/4.3/4.4 — JSONL logging, disk alert, runbook

§4.1 — Structured JSONL log (Invoke-CIJob.ps1)
  Each job now emits a parallel invoke-ci.jsonl alongside the transcript.
  Write-JobEvent appends one JSON line per phase transition (ts, jobId,
  phase, status, data). Events emitted at: job.start, phase1.clone-repo
  start/success, phase2-3b.vm-start start/success, phase4.wait-ready
  start/success, phase5.build start/success, phase6.artifacts start/success,
  job.success/failure (with elapsedSec and error string).
  Silent no-op if $jsonLog is null (log dir setup failed). Errors in
  Write-JobEvent are swallowed — logging never breaks the build.
  Enables post-hoc per-phase duration analysis with jq or ConvertFrom-Json.

§4.3 — Disk space alert (scripts/Watch-DiskSpace.ps1)
  Checks drive free space every 15 min (scheduled by Register-CIScheduledTasks).
  Below MinFreeGB (default 50 GB): writes Warning to Windows Application Event Log
  (source CI-DiskAlert, EventId 1001) and exits 1 so Task Scheduler flags the run.
  Optional -WebhookUrl for Discord/Gitea webhook notification.
  Register-CIScheduledTasks.ps1 updated with Task 3: CI-DiskSpaceAlert (every 15 min).

§4.4 — Incident runbook (docs/RUNBOOK.md)
  Four scenarios with symptom / triage commands / fix / escalation:
    1. Runner offline in Gitea UI
    2. All builds fail in Phase 2 (clone/start/IP)
    3. Builds are slow (per-phase JSONL analysis, disk/CPU/NAT checks)
    4. Template VMX corrupt after host crash (lock removal + backup restore)
  Quick-reference table at the end.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-10 19:07:48 +02:00
parent bc894a8058
commit 1962c977ba
5 changed files with 394 additions and 6 deletions
+49 -1
View File
@@ -159,14 +159,16 @@ $jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
$cloneVmxPath = $null
$leaseFile = $null # §2.1 IP lease file path — released in finally
$jsonLog = $null # §4.1 JSONL structured log path
$startTime = Get-Date
# ── Start transcript ──────────────────────────────────────────────────────────
# ── Start transcript + JSONL log ──────────────────────────────────────────────
$transcriptStarted = $false
try {
$jobLogDir = Join-Path $LogDir $JobId
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
# -Append allows starting a new transcript even when VS Code (or another host)
# has already opened a transcript on this PowerShell session.
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
@@ -176,6 +178,29 @@ catch {
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
}
# ── §4.1 Structured event emitter ────────────────────────────────────────────
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
# Silent no-op if $jsonLog is null (transcript setup failed).
function Write-JobEvent {
param(
[Parameter(Mandatory)] [string] $Phase,
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
[hashtable] $Data = @{}
)
if (-not $script:jsonLog) { return }
$event = [ordered]@{
ts = (Get-Date -Format 'o')
jobId = $script:JobId
phase = $Phase
status = $Status
data = $Data
}
try {
$event | ConvertTo-Json -Compress -Depth 5 |
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
} catch { } # never let logging break the build
}
Write-Host "============================================================"
Write-Host "[Invoke-CIJob] Job : $JobId"
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
@@ -187,6 +212,14 @@ Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
Write-Host "[Invoke-CIJob] Started : $startTime"
Write-Host "============================================================"
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
repo = $RepoUrl
branch = $Branch
commit = $Commit
template = $TemplatePath
snapshot = $SnapshotName
}
$exitCode = 0
try {
@@ -195,6 +228,7 @@ try {
# still cloned here and copied via WinRM zip to avoid injecting a PAT
# into the VM environment.
Write-Host "`n[Phase 1/6] Cloning repository on host..."
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
$gitArgs = @('clone', '--depth', '1', '--branch', $Branch)
if ($Submodules) { $gitArgs += '--recurse-submodules' }
@@ -224,6 +258,7 @@ try {
}
}
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
@@ -237,6 +272,7 @@ try {
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
$lockStream = $null
$lockDeadline = (Get-Date).AddMinutes(10)
while ($true) {
@@ -296,6 +332,7 @@ try {
}
[System.IO.File]::WriteAllText($leaseFile, $JobId)
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
} finally {
# Release lock — next waiting job can now enter clone→start→IP phase
@@ -310,13 +347,16 @@ try {
# ── Phase 4: Wait for readiness ───────────────────────────────────────
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
& "$scriptDir\Wait-VMReady.ps1" `
-VMPath $cloneVmxPath `
-IPAddress $VMIPAddress `
-VmrunPath $VmrunPath
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
Write-Host "`n[Phase 5/6] Invoking remote build..."
Write-JobEvent -Phase 'phase5.build' -Status 'start'
$remoteBuildParams = @{
IPAddress = $VMIPAddress
Credential = $credential
@@ -326,9 +366,11 @@ try {
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
Write-JobEvent -Phase 'phase5.build' -Status 'success'
# ── Phase 6: Collect artifacts ────────────────────────────────────────
Write-Host "`n[Phase 6/6] Collecting artifacts..."
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
& "$scriptDir\Get-BuildArtifacts.ps1" `
-IPAddress $VMIPAddress `
-Credential $credential `
@@ -337,6 +379,8 @@ try {
-IncludeLogs
$elapsed = (Get-Date) - $startTime
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
Write-Host "`n============================================================"
Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
@@ -345,6 +389,10 @@ try {
catch {
$exitCode = 1
$elapsed = (Get-Date) - $startTime
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
elapsedSec = [int]$elapsed.TotalSeconds
error = "$_"
}
Write-Host "`n============================================================"
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
Write-Host "Error: $_"