feat(sprint3): §4.1/4.3/4.4 — JSONL logging, disk alert, runbook
§4.1 — Structured JSONL log (Invoke-CIJob.ps1)
Each job now emits a parallel invoke-ci.jsonl alongside the transcript.
Write-JobEvent appends one JSON line per phase transition (ts, jobId,
phase, status, data). Events emitted at: job.start, phase1.clone-repo
start/success, phase2-3b.vm-start start/success, phase4.wait-ready
start/success, phase5.build start/success, phase6.artifacts start/success,
job.success/failure (with elapsedSec and error string).
Silent no-op if $jsonLog is null (log dir setup failed). Errors in
Write-JobEvent are swallowed — logging never breaks the build.
Enables post-hoc per-phase duration analysis with jq or ConvertFrom-Json.
§4.3 — Disk space alert (scripts/Watch-DiskSpace.ps1)
Checks drive free space every 15 min (scheduled by Register-CIScheduledTasks).
Below MinFreeGB (default 50 GB): writes Warning to Windows Application Event Log
(source CI-DiskAlert, EventId 1001) and exits 1 so Task Scheduler flags the run.
Optional -WebhookUrl for Discord/Gitea webhook notification.
Register-CIScheduledTasks.ps1 updated with Task 3: CI-DiskSpaceAlert (every 15 min).
§4.4 — Incident runbook (docs/RUNBOOK.md)
Four scenarios with symptom / triage commands / fix / escalation:
1. Runner offline in Gitea UI
2. All builds fail in Phase 2 (clone/start/IP)
3. Builds are slow (per-phase JSONL analysis, disk/CPU/NAT checks)
4. Template VMX corrupt after host crash (lock removal + backup restore)
Quick-reference table at the end.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -159,14 +159,16 @@ $jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
|
||||
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
|
||||
$cloneVmxPath = $null
|
||||
$leaseFile = $null # §2.1 IP lease file path — released in finally
|
||||
$jsonLog = $null # §4.1 JSONL structured log path
|
||||
$startTime = Get-Date
|
||||
|
||||
# ── Start transcript ──────────────────────────────────────────────────────────
|
||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
||||
$transcriptStarted = $false
|
||||
try {
|
||||
$jobLogDir = Join-Path $LogDir $JobId
|
||||
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
|
||||
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
|
||||
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
|
||||
# -Append allows starting a new transcript even when VS Code (or another host)
|
||||
# has already opened a transcript on this PowerShell session.
|
||||
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
|
||||
@@ -176,6 +178,29 @@ catch {
|
||||
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
|
||||
}
|
||||
|
||||
# ── §4.1 Structured event emitter ────────────────────────────────────────────
|
||||
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
|
||||
# Silent no-op if $jsonLog is null (transcript setup failed).
|
||||
function Write-JobEvent {
|
||||
param(
|
||||
[Parameter(Mandatory)] [string] $Phase,
|
||||
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
|
||||
[hashtable] $Data = @{}
|
||||
)
|
||||
if (-not $script:jsonLog) { return }
|
||||
$event = [ordered]@{
|
||||
ts = (Get-Date -Format 'o')
|
||||
jobId = $script:JobId
|
||||
phase = $Phase
|
||||
status = $Status
|
||||
data = $Data
|
||||
}
|
||||
try {
|
||||
$event | ConvertTo-Json -Compress -Depth 5 |
|
||||
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
|
||||
} catch { } # never let logging break the build
|
||||
}
|
||||
|
||||
Write-Host "============================================================"
|
||||
Write-Host "[Invoke-CIJob] Job : $JobId"
|
||||
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
|
||||
@@ -187,6 +212,14 @@ Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
|
||||
Write-Host "[Invoke-CIJob] Started : $startTime"
|
||||
Write-Host "============================================================"
|
||||
|
||||
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
|
||||
repo = $RepoUrl
|
||||
branch = $Branch
|
||||
commit = $Commit
|
||||
template = $TemplatePath
|
||||
snapshot = $SnapshotName
|
||||
}
|
||||
|
||||
$exitCode = 0
|
||||
|
||||
try {
|
||||
@@ -195,6 +228,7 @@ try {
|
||||
# still cloned here and copied via WinRM zip to avoid injecting a PAT
|
||||
# into the VM environment.
|
||||
Write-Host "`n[Phase 1/6] Cloning repository on host..."
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
|
||||
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
|
||||
$gitArgs = @('clone', '--depth', '1', '--branch', $Branch)
|
||||
if ($Submodules) { $gitArgs += '--recurse-submodules' }
|
||||
@@ -224,6 +258,7 @@ try {
|
||||
}
|
||||
}
|
||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
||||
|
||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
||||
@@ -237,6 +272,7 @@ try {
|
||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
||||
|
||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
||||
$lockStream = $null
|
||||
$lockDeadline = (Get-Date).AddMinutes(10)
|
||||
while ($true) {
|
||||
@@ -296,6 +332,7 @@ try {
|
||||
}
|
||||
[System.IO.File]::WriteAllText($leaseFile, $JobId)
|
||||
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
|
||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
|
||||
|
||||
} finally {
|
||||
# Release lock — next waiting job can now enter clone→start→IP phase
|
||||
@@ -310,13 +347,16 @@ try {
|
||||
|
||||
# ── Phase 4: Wait for readiness ───────────────────────────────────────
|
||||
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
|
||||
& "$scriptDir\Wait-VMReady.ps1" `
|
||||
-VMPath $cloneVmxPath `
|
||||
-IPAddress $VMIPAddress `
|
||||
-VmrunPath $VmrunPath
|
||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
||||
|
||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'start'
|
||||
$remoteBuildParams = @{
|
||||
IPAddress = $VMIPAddress
|
||||
Credential = $credential
|
||||
@@ -326,9 +366,11 @@ try {
|
||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
||||
|
||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
|
||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
||||
-IPAddress $VMIPAddress `
|
||||
-Credential $credential `
|
||||
@@ -337,6 +379,8 @@ try {
|
||||
-IncludeLogs
|
||||
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
||||
@@ -345,6 +389,10 @@ try {
|
||||
catch {
|
||||
$exitCode = 1
|
||||
$elapsed = (Get-Date) - $startTime
|
||||
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
|
||||
elapsedSec = [int]$elapsed.TotalSeconds
|
||||
error = "$_"
|
||||
}
|
||||
Write-Host "`n============================================================"
|
||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
||||
Write-Host "Error: $_"
|
||||
|
||||
Reference in New Issue
Block a user