local-ci-cd-system/scripts/Invoke-CIJob.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Main CI orchestrator: creates an ephemeral VM, runs a build, collects
    artifacts, and destroys the VM — in a guaranteed try/finally pattern.

.DESCRIPTION
    This script is called by act_runner as the primary build step.
    It coordinates all sub-scripts:
      1. New-BuildVM.ps1    — linked clone from template
      2. vmrun start        — start the clone
      3. Wait-VMReady.ps1   — poll until WinRM is ready
      4. Invoke-RemoteBuild.ps1 — execute build inside VM
      5. Get-BuildArtifacts.ps1 — copy artifacts to host
      6. Remove-BuildVM.ps1 — destroy VM (always runs via finally)

    Exit codes:
      0  = success
      1  = build or orchestration failure (VM destroyed, artifacts may be partial)

.PARAMETER JobId
    Unique job identifier (use ${{ github.run_id }} from Gitea Actions).

.PARAMETER RepoUrl
    Git URL of the repository to build (reachable from the build VM).

.PARAMETER Branch
    Branch to build.

.PARAMETER Commit
    Optional specific commit SHA to check out.

.PARAMETER Configuration
    Build configuration. Default: Release

.PARAMETER TemplatePath
    Full path to the template VM .vmx file.
    Can also be set via env var GITEA_CI_TEMPLATE_PATH.

.PARAMETER SnapshotName
    Template snapshot to clone from. Default: BaseClean

.PARAMETER CloneBaseDir
    Base directory for clone VMs.
    Can also be set via env var GITEA_CI_CLONE_BASE_DIR.
    Default: F:\CI\BuildVMs

.PARAMETER ArtifactBaseDir
    Host directory where artifacts are stored per job.
    Default: F:\CI\Artifacts

.PARAMETER VMIPAddress
    IP address to assign/expect for the build VM.
    Must be unique per concurrent build.
    Can be passed explicitly or derived from a registry/file-based IP allocator.

.PARAMETER GuestCredentialTarget
    Target name in Windows Credential Manager for guest credentials.
    Default: BuildVMGuest

.PARAMETER VmrunPath
    Path to vmrun.exe.
    Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe

.EXAMPLE
    # Called from Gitea Actions workflow:
    .\scripts\Invoke-CIJob.ps1 `
        -JobId       "${{ github.run_id }}" `
        -RepoUrl     "http://gitea.local/org/myrepo.git" `
        -Branch      "${{ github.ref_name }}" `
        -Commit      "${{ github.sha }}" `
        -VMIPAddress "192.168.11.101"
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory)]
    [string] $JobId,

    [Parameter(Mandatory)]
    [string] $RepoUrl,

    [Parameter(Mandatory)]
    [string] $Branch,

    [string] $Commit = '',

    [string] $Configuration = 'Release',

    # Pass -Submodules to clone with --recurse-submodules
    [switch] $Submodules,

    # Custom build command to run inside the VM (empty = dotnet build)
    [string] $BuildCommand = '',

    # Artifact source pattern relative to GuestWorkDir (used with BuildCommand)
    [string] $GuestArtifactSource = 'dist',

    [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,

    [string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),

    [string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),

    [string] $ArtifactBaseDir = 'F:\CI\Artifacts',

    [ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
    [string] $VMIPAddress = '',  # if empty, auto-detected via vmrun getGuestIPAddress

    [string] $GuestCredentialTarget = 'BuildVMGuest',

    [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',

    # Directory for job execution logs. Each job gets its own sub-directory:
    # $LogDir\$JobId\invoke-ci.log
    [string] $LogDir = 'F:\CI\Logs',

    # Days to retain job logs. Directories older than this are purged at job end.
    # Set to 0 to disable retention/purge.
    [int] $LogRetentionDays = 30
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'

# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
$scriptDir = $PSScriptRoot

# ── Validate required config ──────────────────────────────────────────────────
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
    Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
    exit 1
}
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
    Write-Error "Template VMX not found: $TemplatePath"
    exit 1
}
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
    Write-Error "vmrun.exe not found: $VmrunPath"
    exit 1
}

# ── Load guest credentials from Windows Credential Manager ───────────────────
# Requires the CredentialManager module: Install-Module CredentialManager
# In a lab setup you can fall back to prompting, but never hardcode credentials.
$credential = $null
try {
    $credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop
    Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)."
}
catch {
    Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)."
    Write-Warning "             Using interactive prompt as fallback. Store credentials with:"
    Write-Warning "             cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password"
    $credential = Get-Credential -Message "Enter guest VM credentials for job $JobId"
}

# ── Setup paths ───────────────────────────────────────────────────────────────
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
$hostCloneDir   = Join-Path $env:TEMP "ci-src-$JobId"
$cloneVmxPath   = $null
$leaseFile      = $null   # §2.1 IP lease file path — released in finally
$jsonLog        = $null   # §4.1 JSONL structured log path
$startTime      = Get-Date

# ── Start transcript + JSONL log ──────────────────────────────────────────────
$transcriptStarted = $false
try {
    $jobLogDir = Join-Path $LogDir $JobId
    if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
    $transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
    $jsonLog        = Join-Path $jobLogDir 'invoke-ci.jsonl'
    # -Append allows starting a new transcript even when VS Code (or another host)
    # has already opened a transcript on this PowerShell session.
    Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
    $transcriptStarted = $true
}
catch {
    Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
}

# ── §4.1 Structured event emitter ────────────────────────────────────────────
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
# Silent no-op if $jsonLog is null (transcript setup failed).
function Write-JobEvent {
    param(
        [Parameter(Mandatory)] [string] $Phase,
        [Parameter(Mandatory)] [string] $Status,   # start | success | failure | info
        [hashtable] $Data = @{}
    )
    if (-not $script:jsonLog) { return }
    $event = [ordered]@{
        ts     = (Get-Date -Format 'o')
        jobId  = $script:JobId
        phase  = $Phase
        status = $Status
        data   = $Data
    }
    try {
        $event | ConvertTo-Json -Compress -Depth 5 |
            Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
    } catch { }  # never let logging break the build
}

Write-Host "============================================================"
Write-Host "[Invoke-CIJob] Job          : $JobId"
Write-Host "[Invoke-CIJob] Repository   : $RepoUrl"
Write-Host "[Invoke-CIJob] Branch       : $Branch"
if ($Commit) { Write-Host "[Invoke-CIJob] Commit       : $Commit" }
Write-Host "[Invoke-CIJob] Build VM IP  : $VMIPAddress"
Write-Host "[Invoke-CIJob] Template     : $TemplatePath"
Write-Host "[Invoke-CIJob] Artifacts    : $jobArtifactDir"
Write-Host "[Invoke-CIJob] Started      : $startTime"
Write-Host "============================================================"

Write-JobEvent -Phase 'job' -Status 'start' -Data @{
    repo     = $RepoUrl
    branch   = $Branch
    commit   = $Commit
    template = $TemplatePath
    snapshot = $SnapshotName
}

$exitCode = 0

try {
    # ── Phase 1: Clone repo on HOST ───────────────────────────────────────
    # Build VM is on VMnet8 NAT (internet OK for builds), but source is
    # still cloned here and copied via WinRM zip to avoid injecting a PAT
    # into the VM environment.
    Write-Host "`n[Phase 1/6] Cloning repository on host..."
    Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
    if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
    $gitArgs = @('clone', '--depth', '1', '--branch', $Branch)
    if ($Submodules) { $gitArgs += '--recurse-submodules' }
    $gitArgs += @($RepoUrl, $hostCloneDir)
    $ErrorActionPreference = 'Continue'
    $gitOutput = & git @gitArgs 2>&1 | ForEach-Object { "$_" }
    $gitExit = $LASTEXITCODE
    $ErrorActionPreference = 'Stop'
    if ($gitExit -ne 0) {
        throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")"
    }
    if ($Commit -ne '') {
        $ErrorActionPreference = 'Continue'
        $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
        $gitExit = $LASTEXITCODE
        if ($gitExit -ne 0) {
            # Commit not in shallow history (e.g. manual run on older SHA).
            # Fetch full history and retry.
            Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..."
            & git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null
            $gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
            $gitExit = $LASTEXITCODE
        }
        $ErrorActionPreference = 'Stop'
        if ($gitExit -ne 0) {
            throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")"
        }
    }
    Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
    Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }

    # ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
    # §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
    # and produce two VMs with the same DHCP lease. A file-based mutex ensures
    # only one job is in the clone→start→IP phase at a time. After the IP is
    # acquired and written to a lease file the lock is released, so build phases
    # (the slow part) run fully in parallel. Lease is released in finally.
    $stateDir  = 'F:\CI\State'
    $leasesDir = Join-Path $stateDir 'ip-leases'
    $lockPath  = Join-Path $stateDir 'vm-start.lock'
    if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }

    Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
    Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
    $lockStream   = $null
    $lockDeadline = (Get-Date).AddMinutes(10)
    while ($true) {
        try {
            $lockStream = [System.IO.File]::Open(
                $lockPath,
                [System.IO.FileMode]::OpenOrCreate,
                [System.IO.FileAccess]::ReadWrite,
                [System.IO.FileShare]::None)
            break
        } catch [System.IO.IOException] {
            if ((Get-Date) -ge $lockDeadline) {
                throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
            }
            Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
            Start-Sleep -Seconds 5
        }
    }
    Write-Host "[Invoke-CIJob] VM-start lock acquired."

    try {
        # ── Phase 2: Clone VM ─────────────────────────────────────────────
        Write-Host "[Phase 2/6] Cloning VM..."
        $cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
            -TemplatePath $TemplatePath `
            -SnapshotName $SnapshotName `
            -CloneBaseDir $CloneBaseDir `
            -JobId        $JobId `
            -VmrunPath    $VmrunPath

        # ── Phase 3: Start VM ─────────────────────────────────────────────
        Write-Host "`n[Phase 3/6] Starting VM..."
        $startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
        if ($LASTEXITCODE -ne 0) {
            throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
        }
        Write-Host "[Invoke-CIJob] VM started."

        # ── Phase 3b: Resolve VM IP ───────────────────────────────────────
        if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
            Write-Host "[Invoke-CIJob] Auto-detecting VM IP via vmrun getGuestIPAddress -wait..."
            $detectedIP = & $VmrunPath getGuestIPAddress $cloneVmxPath -wait 2>&1
            if ($LASTEXITCODE -ne 0 -or $detectedIP -notmatch '^(\d{1,3}\.){3}\d{1,3}$') {
                throw "Could not detect VM IP address: $detectedIP"
            }
            $VMIPAddress = $detectedIP.Trim()
            Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
        }

        # ── Lease IP ──────────────────────────────────────────────────────
        # If two VMs somehow got the same IP (DHCP edge case), fail fast here
        # rather than letting both jobs write to the same WinRM target.
        $leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
        if (Test-Path $leaseFile) {
            $holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
            throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
        }
        [System.IO.File]::WriteAllText($leaseFile, $JobId)
        Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
        Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }

    } finally {
        # Release lock — next waiting job can now enter clone→start→IP phase
        if ($lockStream) {
            $lockStream.Close()
            $lockStream.Dispose()
            $lockStream = $null
        }
        Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
        Write-Host "[Invoke-CIJob] VM-start lock released."
    }

    # ── Phase 4: Wait for readiness ───────────────────────────────────────
    Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
    Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
    & "$scriptDir\Wait-VMReady.ps1" `
        -VMPath    $cloneVmxPath `
        -IPAddress $VMIPAddress `
        -VmrunPath $VmrunPath
    Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'

    # ── Phase 5: Remote build (copy source + build inside VM) ─────────────
    Write-Host "`n[Phase 5/6] Invoking remote build..."
    Write-JobEvent -Phase 'phase5.build' -Status 'start'
    $remoteBuildParams = @{
        IPAddress     = $VMIPAddress
        Credential    = $credential
        HostSourceDir = $hostCloneDir
        Configuration = $Configuration
    }
    if ($BuildCommand -ne '')        { $remoteBuildParams['BuildCommand']        = $BuildCommand }
    if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
    & "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
    Write-JobEvent -Phase 'phase5.build' -Status 'success'

    # ── Phase 6: Collect artifacts ────────────────────────────────────────
    Write-Host "`n[Phase 6/6] Collecting artifacts..."
    Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
    & "$scriptDir\Get-BuildArtifacts.ps1" `
        -IPAddress         $VMIPAddress `
        -Credential        $credential `
        -GuestArtifactPath 'C:\CI\output\artifacts.zip' `
        -HostArtifactDir   $jobArtifactDir `
        -IncludeLogs

    $elapsed = (Get-Date) - $startTime
    Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
    Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
    Write-Host "`n============================================================"
    Write-Host "[Invoke-CIJob] SUCCESS — Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
    Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
    Write-Host "============================================================"
}
catch {
    $exitCode = 1
    $elapsed  = (Get-Date) - $startTime
    Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
        elapsedSec = [int]$elapsed.TotalSeconds
        error      = "$_"
    }
    Write-Host "`n============================================================"
    Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
    Write-Host "Error: $_"
    Write-Host "============================================================"
}
finally {
    # ── Release IP lease (§2.1) ───────────────────────────────────────────
    if ($leaseFile -and (Test-Path $leaseFile)) {
        Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
        Write-Host "[Cleanup] IP lease released: $VMIPAddress"
    }

    # ── Always destroy the VM ─────────────────────────────────────────────
    if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
        Write-Host "`n[Cleanup] Destroying ephemeral VM..."
        & "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath
    }
    else {
        Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)."
    }
    # ── Always clean up host-side git clone ──────────────────────────────
    if (Test-Path $hostCloneDir) {
        Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..."
        Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue
    }

    # ── Stop transcript ───────────────────────────────────────────────────
    if ($transcriptStarted) {
        try { Stop-Transcript -ErrorAction SilentlyContinue } catch {}
    }

    # ── Log retention: purge directories older than $LogRetentionDays days ─
    if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) {
        $cutoff = (Get-Date).AddDays(-$LogRetentionDays)
        Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue |
            Where-Object { $_.LastWriteTime -lt $cutoff } |
            ForEach-Object {
                Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
                Write-Host "[Cleanup] Purged old log dir: $($_.FullName)"
            }
    }
}

exit $exitCode