local-ci-cd-system/scripts/Invoke-CIJob.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Main CI orchestrator: creates an ephemeral VM, runs a build, collects
    artifacts, and destroys the VM — in a guaranteed try/finally pattern.

.DESCRIPTION
    This script is called by act_runner as the primary build step.
    It coordinates all sub-scripts:
      1. New-BuildVM.ps1    — linked clone from template
      2. vmrun start        — start the clone
      3. Wait-VMReady.ps1   — poll until WinRM is ready
      4. Invoke-RemoteBuild.ps1 — execute build inside VM
      5. Get-BuildArtifacts.ps1 — copy artifacts to host
      6. Remove-BuildVM.ps1 — destroy VM (always runs via finally)

    Exit codes:
      0  = success
      1  = build or orchestration failure (VM destroyed, artifacts may be partial)

.PARAMETER JobId
    Unique job identifier (use ${{ github.run_id }} from Gitea Actions).

.PARAMETER RepoUrl
    Git URL of the repository to build (reachable from the build VM).

.PARAMETER Branch
    Branch to build.

.PARAMETER Commit
    Optional specific commit SHA to check out.

.PARAMETER Configuration
    Build configuration. Default: Release

.PARAMETER TemplatePath
    Full path to the template VM .vmx file.
    Can also be set via env var GITEA_CI_TEMPLATE_PATH.

.PARAMETER SnapshotName
    Template snapshot to clone from. Default: BaseClean

.PARAMETER CloneBaseDir
    Base directory for clone VMs.
    Can also be set via env var GITEA_CI_CLONE_BASE_DIR.
    Default: F:\CI\BuildVMs

.PARAMETER ArtifactBaseDir
    Host directory where artifacts are stored per job.
    Default: F:\CI\Artifacts

.PARAMETER VMIPAddress
    IP address to assign/expect for the build VM.
    Must be unique per concurrent build.
    Can be passed explicitly or derived from a registry/file-based IP allocator.

.PARAMETER GuestCredentialTarget
    Target name in Windows Credential Manager for guest credentials.
    Default: BuildVMGuest

.PARAMETER VmrunPath
    Path to vmrun.exe.
    Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe

.EXAMPLE
    # Called from Gitea Actions workflow:
    .\scripts\Invoke-CIJob.ps1 `
        -JobId       "${{ github.run_id }}" `
        -RepoUrl     "http://gitea.local/org/myrepo.git" `
        -Branch      "${{ github.ref_name }}" `
        -Commit      "${{ github.sha }}" `
        -VMIPAddress "192.168.11.101"
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory)]
    [ValidatePattern('^[A-Za-z0-9._-]+$')]
    [string] $JobId,

    [Parameter(Mandatory)]
    [string] $RepoUrl,

    [Parameter(Mandatory)]
    [string] $Branch,

    [string] $Commit = '',

    [string] $Configuration = 'Release',

    # Pass -Submodules to clone with --recurse-submodules
    [switch] $Submodules,

    # Custom build command to run inside the VM (empty = dotnet build)
    [string] $BuildCommand = '',

    # Artifact source pattern relative to GuestWorkDir (used with BuildCommand)
    [string] $GuestArtifactSource = 'dist',

    # Clone repo IN the VM instead of on host (requires Git in template §6.6)
    # Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone.
    # Prerequisite: Git for Windows installed in template.
    [switch] $UseGitClone,

    # Use VMware shared folders for NuGet/pip cache (§SharedCache).
    # Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
    [switch] $UseSharedCache,

    # Skip artifact packaging (Invoke-RemoteBuild) and collection (Phase 6).
    # Use for jobs that intentionally produce no output (lint, test-only, etc.).
    # When absent, the build must produce GuestArtifactSource or the job fails.
    [switch] $SkipArtifact,

    # Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
    # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
    # Ignored when -UseGitClone is not set.
    [string] $GiteaCredentialTarget = 'GiteaPAT',

    [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,

    [string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),

    [string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),

    [string] $ArtifactBaseDir = 'F:\CI\Artifacts',

    [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
    [string] $VMIPAddress = '',  # if empty, auto-detected via vmrun getGuestIPAddress

    [string] $GuestCredentialTarget = 'BuildVMGuest',

    [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',

    # Directory for job execution logs. Each job gets its own sub-directory:
    # $LogDir\$JobId\invoke-ci.log
    [string] $LogDir = 'F:\CI\Logs',

    # Days to retain job logs. Directories older than this are purged at job end.
    # Set to 0 to disable retention/purge.
    [int] $LogRetentionDays = 30,

    # Guest OS type — determines transport (WinRM for Windows, SSH for Linux).
    # When 'Auto', detected from the VMX guestOS field after clone.
    [ValidateSet('Windows', 'Linux', 'Auto')]
    [string] $GuestOS = 'Auto',

    [string] $SshKeyPath = 'F:\CI\keys\ci_linux',
    [string] $SshUser    = 'ci_build',

    # Persistent known_hosts file used for CI-job SSH calls.
    # Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL).
    # Ephemeral CI VMs on a local NAT network do not need strict host-key verification.
    # Pass a file path only if you need accept-new mode for a specific deployment.
    [string] $SshKnownHostsFile = '',

    # Extra environment variables injected into the guest VM before the build command (§6.5).
    # Keys must be valid environment variable names. Values must be plain strings.
    # Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
    [hashtable] $ExtraGuestEnv = @{},

    # Optional webhook URL (Discord/Gitea). When set, a background job fires a
    # [WARNING] once the job has been running for 90 minutes.
    [string] $WebhookUrl = '',

    # Optional per-job VMX CPU and RAM override (default 0 = use template value).
    # Applied to the linked clone after New-BuildVM.ps1, before vmrun start.
    # The template VMX is never modified.
    # Example: -GuestCPU 4 -GuestMemoryMB 8192
    [ValidateRange(0, 32)]
    [int] $GuestCPU = 0,

    [ValidateRange(0, 131072)]
    [int] $GuestMemoryMB = 0
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'

# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
$scriptDir = $PSScriptRoot

# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force

# ── Validate required config ──────────────────────────────────────────────────
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
    Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
    exit 1
}
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
    Write-Error "Template VMX not found: $TemplatePath"
    exit 1
}
# Sanity check: warn if template is outside the expected CI tree
if ($TemplatePath -notlike 'F:\CI\Templates\*') {
    Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional."
}
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
    Write-Error "vmrun.exe not found: $VmrunPath"
    exit 1
}

# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
foreach ($key in $ExtraGuestEnv.Keys) {
    if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
        Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
        exit 1
    }
}

# ── Load guest credentials from Windows Credential Manager ───────────────────
# Requires the CredentialManager module: Install-Module CredentialManager
# In a lab setup you can fall back to prompting, but never hardcode credentials.
$credential = $null
try {
    $credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop
    Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)."
}
catch {
    Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)."
    Write-Warning "             Using interactive prompt as fallback. Store credentials with:"
    Write-Warning "             cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password"
    $credential = Get-Credential -Message "Enter guest VM credentials for job $JobId"
}

# ── Setup paths ───────────────────────────────────────────────────────────────
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
$hostCloneDir   = Join-Path $env:TEMP "ci-src-$JobId"
$cloneVmxPath   = $null
$leaseFile      = $null   # §2.1 IP lease file path — released in finally
$jsonLog        = $null   # §4.1 JSONL structured log path
$startTime      = Get-Date

# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
$phaseTimings   = [System.Collections.Generic.List[PSCustomObject]]::new()
$phaseStart     = $startTime

# ── Start transcript + JSONL log ──────────────────────────────────────────────
$transcriptStarted = $false
try {
    $jobLogDir = Join-Path $LogDir $JobId
    if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
    $transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
    $jsonLog        = Join-Path $jobLogDir 'invoke-ci.jsonl'
    # -Append allows starting a new transcript even when VS Code (or another host)
    # has already opened a transcript on this PowerShell session.
    Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
    $transcriptStarted = $true
}
catch {
    Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
}

# ── §4.1 Structured event emitter ────────────────────────────────────────────
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
# Silent no-op if $jsonLog is null (transcript setup failed).
function Write-JobEvent {
    param(
        [Parameter(Mandatory)] [string] $Phase,
        [Parameter(Mandatory)] [string] $Status,   # start | success | failure | info
        [hashtable] $Data = @{}
    )
    if (-not $script:jsonLog) { return }
    $logEntry = [ordered]@{
        ts     = (Get-Date -Format 'o')
        jobId  = $script:JobId
        phase  = $Phase
        status = $Status
        data   = $Data
    }
    try {
        $logEntry | ConvertTo-Json -Compress -Depth 5 |
            Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
    } catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" }  # never let logging fail the build
}

Write-Host "============================================================"
Write-Host "[Invoke-CIJob] Job          : $JobId"
Write-Host "[Invoke-CIJob] Repository   : $RepoUrl"
Write-Host "[Invoke-CIJob] Branch       : $Branch"
if ($Commit) { Write-Host "[Invoke-CIJob] Commit       : $Commit" }
Write-Host "[Invoke-CIJob] Build VM IP  : $VMIPAddress"
Write-Host "[Invoke-CIJob] Template     : $TemplatePath"
Write-Host "[Invoke-CIJob] Artifacts    : $jobArtifactDir"
Write-Host "[Invoke-CIJob] Started      : $startTime"
Write-Host "============================================================"

Write-JobEvent -Phase 'job' -Status 'start' -Data @{
    repo     = $RepoUrl
    branch   = $Branch
    commit   = $Commit
    template = $TemplatePath
    snapshot = $SnapshotName
}

$exitCode        = 0
$durationWarnJob = $null

try {
    # ── 90-minute duration warning (fires once via background job) ───────────────
    if ($WebhookUrl -ne '') {
        $warnMsg  = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck."
        $warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress)
        $durationWarnJob = Start-Job -ScriptBlock {
            param($url, $body)
            Start-Sleep -Seconds 5400   # 90 minutes
            try {
                Invoke-RestMethod -Uri $url -Method Post -Body $body `
                    -ContentType 'application/json' -TimeoutSec 10
            } catch { }
        } -ArgumentList $WebhookUrl, $warnBody
    }

    # ── Phase 1: Clone repo (HOST or GUEST) ───────────────────────────────
    if ($UseGitClone) {
        # §3.3: Clone in guest VM instead of host. Skip host clone phase.
        # PAT will be injected at runtime (Phase 5) from Credential Manager.
        Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..."
        Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' }
    }
    else {
        # Default: Clone on host, zip, transfer to guest via WinRM.
        Write-Host "`n[Phase 1/6] Cloning repository on host..."
        Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
        if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
        $gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch)
        if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') }
        $gitArgs += @($RepoUrl, $hostCloneDir)
        Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')"
        $ErrorActionPreference = 'Continue'
        # Stream git stdout/stderr live (otherwise heavy clones with submodules
        # look frozen for a long time). Tee for diagnostic in case of failure.
        $gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host "  [git] $_"; "$_" }
        $gitExit = $LASTEXITCODE
        $ErrorActionPreference = 'Stop'
        if ($gitExit -ne 0) {
            throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")"
        }
        if ($Commit -ne '') {
            $ErrorActionPreference = 'Continue'
            $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
            $gitExit = $LASTEXITCODE
            if ($gitExit -ne 0) {
                # Commit not in shallow history (e.g. manual run on older SHA).
                # Fetch full history and retry.
                Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..."
                & git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null
                $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
                $gitExit = $LASTEXITCODE
            }
            $ErrorActionPreference = 'Stop'
            if ($gitExit -ne 0) {
                throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")"
            }
        }
        Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
        Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
    }
    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
    $phaseStart = Get-Date

    # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
    # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
    # and produce two VMs with the same DHCP lease. A file-based mutex ensures
    # only one job is in the clone→start→IP phase at a time. After the IP is
    # acquired and written to a lease file the lock is released, so build phases
    # (the slow part) run fully in parallel. Lease is released in finally.
    $stateDir  = 'F:\CI\State'
    $leasesDir = Join-Path $stateDir 'ip-leases'
    $lockPath  = Join-Path $stateDir 'vm-start.lock'
    if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }

    # ── Pre-clone disk space gate ─────────────────────────────────────────
    $cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
    $cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
    if ($null -ne $cloneVol) {
        $freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
        if ($freeGB -lt 20) {
            throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
        }
        Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
    }

    Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
    Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
    $lockStream   = $null
    $lockDeadline = (Get-Date).AddMinutes(10)
    while ($true) {
        try {
            $lockStream = [System.IO.File]::Open(
                $lockPath,
                [System.IO.FileMode]::OpenOrCreate,
                [System.IO.FileAccess]::ReadWrite,
                [System.IO.FileShare]::None)
            break
        } catch [System.IO.IOException] {
            if ((Get-Date) -ge $lockDeadline) {
                throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
            }
            Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
            Start-Sleep -Seconds 5
        }
    }
    Write-Host "[Invoke-CIJob] VM-start lock acquired."

    try {
        # ── Phase 2: Clone VM ─────────────────────────────────────────────
        Write-Host "[Phase 2/6] Cloning VM..."
        $cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
            -TemplatePath $TemplatePath `
            -SnapshotName $SnapshotName `
            -CloneBaseDir $CloneBaseDir `
            -JobId        $JobId `
            -VmrunPath    $VmrunPath

        # ── Optional VMX CPU/RAM override (post-clone, pre-start) ─────────────
        if ($GuestCPU -gt 0 -or $GuestMemoryMB -gt 0) {
            Write-Host "[Invoke-CIJob] Applying VMX override: CPU=$GuestCPU, RAM=${GuestMemoryMB}MB"
            $vmxLines = Get-Content -LiteralPath $cloneVmxPath
            if ($GuestCPU -gt 0) {
                if ($vmxLines -imatch '^numvcpus\s*=') {
                    $vmxLines = $vmxLines -replace '(?i)^numvcpus\s*=.*', "numvcpus = `"$GuestCPU`""
                } else {
                    $vmxLines = $vmxLines + "numvcpus = `"$GuestCPU`""
                }
            }
            if ($GuestMemoryMB -gt 0) {
                if ($vmxLines -imatch '^memsize\s*=') {
                    $vmxLines = $vmxLines -replace '(?i)^memsize\s*=.*', "memsize = `"$GuestMemoryMB`""
                } else {
                    $vmxLines = $vmxLines + "memsize = `"$GuestMemoryMB`""
                }
            }
            Set-Content -LiteralPath $cloneVmxPath -Value $vmxLines -Encoding UTF8
            Write-Host "[Invoke-CIJob] VMX updated: $cloneVmxPath"
        }

        # ── Phase 3: Start VM ─────────────────────────────────────────────
        Write-Host "`n[Phase 3/6] Starting VM..."
        $startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
                           -Operation start `
                           -Arguments @($cloneVmxPath, 'nogui') `
                           -TimeoutSeconds 180
        if ($startResult.TimedOut) {
            throw "vmrun start timed out after 180s for: $cloneVmxPath"
        }
        if ($startResult.ExitCode -ne 0) {
            throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
        }
        Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."

        # ── Phase 3b: Resolve VM IP ───────────────────────────────────────
        if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
            $detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
                              -VmxPath $cloneVmxPath -TimeoutSeconds 120
            if ($detectedIP -eq '') {
                throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
            }
            $VMIPAddress = $detectedIP
            Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
        }

        # ── Lease IP ──────────────────────────────────────────────────────
        # If two VMs somehow got the same IP (DHCP edge case), fail fast here
        # rather than letting both jobs write to the same WinRM target.
        $leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
        if (Test-Path $leaseFile) {
            $holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
            throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
        }
        [System.IO.File]::WriteAllText($leaseFile, $JobId)
        Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
        Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }

    } finally {
        # Release lock — next waiting job can now enter clone→start→IP phase
        if ($lockStream) {
            $lockStream.Close()
            $lockStream.Dispose()
            $lockStream = $null
        }
        Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
        Write-Host "[Invoke-CIJob] VM-start lock released."
    }
    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
    $phaseStart = Get-Date
    # ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
    if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
        $vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
        if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') {
            $detectedOS = $Matches[1]
            $GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' }
            Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)"
        } else {
            $GuestOS = 'Windows'
            Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows"
        }
    }
    # ── Phase 4: Wait for readiness ───────────────────────────────────────
    Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
    Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
    $waitParams = @{
        VMPath    = $cloneVmxPath
        IPAddress = $VMIPAddress
        VmrunPath = $VmrunPath
    }
    if ($GuestOS -eq 'Linux') {
        $waitParams['Transport']  = 'SSH'
        $waitParams['SshKeyPath'] = $SshKeyPath
        $waitParams['SshUser']    = $SshUser
    }
    & "$scriptDir\Wait-VMReady.ps1" @waitParams
    Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
    $phaseStart = Get-Date

    # ── Phase 5: Remote build (copy source + build inside VM) ─────────────
    Write-Host "`n[Phase 5/6] Invoking remote build..."
    Write-JobEvent -Phase 'phase5.build' -Status 'start'
    $remoteBuildParams = @{
        IPAddress     = $VMIPAddress
        Credential    = $credential
        Configuration = $Configuration
    }
    if ($GuestOS -eq 'Linux') {
        $remoteBuildParams['GuestOS']           = 'Linux'
        $remoteBuildParams['SshKeyPath']        = $SshKeyPath
        $remoteBuildParams['SshUser']           = $SshUser
        $remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
        $remoteBuildParams.Remove('Credential')
        if ($UseGitClone) {
            # In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
            $remoteBuildParams['CloneUrl']              = $RepoUrl
            $remoteBuildParams['CloneBranch']           = $Branch
            $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
            if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
            if ($Submodules)    { $remoteBuildParams['CloneSubmodules'] = $true }
        } else {
            # Default: host-cloned source shipped to guest via tar+scp
            $remoteBuildParams['HostSourceDir'] = $hostCloneDir
        }
    } else {
        if ($UseGitClone) {
            # §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir
            $remoteBuildParams['CloneUrl']              = $RepoUrl
            $remoteBuildParams['CloneBranch']           = $Branch
            $remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
            if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
            if ($Submodules)    { $remoteBuildParams['CloneSubmodules'] = $true }
        }
        else {
            # Default: Use pre-cloned source from host
            $remoteBuildParams['HostSourceDir'] = $hostCloneDir
        }
    }
    if ($BuildCommand -ne '')        { $remoteBuildParams['BuildCommand']        = $BuildCommand }
    if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
    if ($ExtraGuestEnv.Count -gt 0)  { $remoteBuildParams['ExtraGuestEnv']       = $ExtraGuestEnv }
    if ($UseSharedCache)             { $remoteBuildParams['UseSharedCache']       = $true }
    if ($SkipArtifact)               { $remoteBuildParams['SkipArtifact']         = $true }
    & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
    Write-JobEvent -Phase 'phase5.build' -Status 'success'
    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
    $phaseStart = Get-Date

    # ── Phase 6: Collect artifacts ────────────────────────────────────────
    if ($SkipArtifact) {
        Write-Host "`n[Phase 6/6] Artifact collection skipped (-SkipArtifact)."
        Write-JobEvent -Phase 'phase6.artifacts' -Status 'skipped'
    } else {
        Write-Host "`n[Phase 6/6] Collecting artifacts..."
        Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
        if ($GuestOS -eq 'Linux') {
            & "$scriptDir\Get-BuildArtifacts.ps1" `
                -IPAddress         $VMIPAddress `
                -GuestOS           'Linux' `
                -SshKeyPath        $SshKeyPath `
                -SshUser           $SshUser `
                -SshKnownHostsFile $SshKnownHostsFile `
                -GuestArtifactPath '/opt/ci/output' `
                -HostArtifactDir   $jobArtifactDir `
                -JobId             $JobId `
                -Commit            $Commit `
                -IncludeLogs
        } else {
            & "$scriptDir\Get-BuildArtifacts.ps1" `
                -IPAddress         $VMIPAddress `
                -Credential        $credential `
                -GuestArtifactPath 'C:\CI\output\artifacts.zip' `
                -HostArtifactDir   $jobArtifactDir `
                -JobId             $JobId `
                -Commit            $Commit `
                -IncludeLogs
        }
        Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
    }
    $elapsed = (Get-Date) - $startTime
    Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
    $phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
    Write-Host "`n============================================================"
    Write-Host "[Invoke-CIJob] Phase durations:"
    foreach ($pt in $phaseTimings) {
        Write-Host ("  {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
    }
    Write-Host ("  {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds)
    Write-Host "============================================================"
    Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
    Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
    Write-Host "============================================================"
}
catch {
    $exitCode = 1
    $elapsed  = (Get-Date) - $startTime
    Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
        elapsedSec = [int]$elapsed.TotalSeconds
        error      = "$_"
    }

    # ── Best-effort guest diagnostics (skip if VM was never started) ─────────
    # Only collect when a VM IP is known (Phase 3b completed). Never throws.
    if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
        try {
            $diagDir    = Join-Path $jobArtifactDir 'diagnostics'
            $diagParams = @{
                IPAddress      = $VMIPAddress
                GuestOS        = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
                DestinationDir = $diagDir
            }
            if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
                $diagParams['Credential'] = $credential
            }
            if ($diagParams.GuestOS -eq 'Linux') {
                $diagParams['SshKeyPath'] = $SshKeyPath
                $diagParams['SshUser']    = $SshUser
            }
            Get-GuestDiagnostics @diagParams
        } catch {
            Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
        }
    }

    Write-Host "`n============================================================"
    Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
    Write-Host "Error: $_"
    Write-Host "============================================================"
}
finally {
    # ── Release IP lease (§2.1) ───────────────────────────────────────────
    if ($leaseFile -and (Test-Path $leaseFile)) {
        Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
        Write-Host "[Cleanup] IP lease released: $VMIPAddress"
    }

    # ── Always destroy the VM ─────────────────────────────────────────────
    if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
        Write-Host "`n[Cleanup] Destroying ephemeral VM..."
        & "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath
    }
    else {
        Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)."
    }
    # ── Clean up host-side git clone (unless in-VM clone was used) ────────
    # When -UseGitClone, $hostCloneDir is never created
    if (-not $UseGitClone -and (Test-Path $hostCloneDir)) {
        Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..."
        Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue
    }

    # ── Cancel 90-minute warning job (build finished before it fired) ──────────
    if ($null -ne $durationWarnJob) {
        Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue
        $durationWarnJob = $null
    }

    # ── Stop transcript ───────────────────────────────────────────────────
    if ($transcriptStarted) {
        try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" }
    }

    # ── Log retention: purge directories older than $LogRetentionDays days ─
    if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) {
        $cutoff = (Get-Date).AddDays(-$LogRetentionDays)
        Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue |
            Where-Object { $_.LastWriteTime -lt $cutoff } |
            ForEach-Object {
                Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
                Write-Host "[Cleanup] Purged old log dir: $($_.FullName)"
            }
    }
}

exit $exitCode