local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1

#Requires -Version 5.1
<#
.SYNOPSIS
    Concurrency burn-in harness: runs N simultaneous Invoke-CIJob.ps1 jobs
    for R rounds and asserts correct cleanup after each round.

.DESCRIPTION
    Launches $Parallelism child PowerShell processes (via Start-Job) all
    running Invoke-CIJob.ps1 concurrently. After each round verifies that:
      - All job exit codes were 0
      - No clone directories for burn-in jobs remain in CloneBaseDir
      - No stale vm-start.lock file remains (older than 5 min)

    Exits 0 only when ALL rounds and assertions pass.

.PARAMETER RepoUrl
    Git URL to pass to Invoke-CIJob.ps1 as -RepoUrl.

.PARAMETER Branch
    Branch to build. Default: main

.PARAMETER IPList
    Optional array of IP addresses, one per concurrent job slot.
    When provided, each job receives the corresponding -VMIPAddress (useful when
    VMs have static IPs or per-slot DHCP reservations).
    When omitted, -VMIPAddress is not passed to Invoke-CIJob.ps1, which then
    auto-detects the IP via Get-GuestIPAddress (requires VMware Tools / ci-report-ip.service).
    When provided, must have at least $Parallelism entries.

.PARAMETER TemplatePath
    VMX template path. Falls back to GITEA_CI_TEMPLATE_PATH env var.

.PARAMETER SnapshotName
    Snapshot to clone from. Falls back to GITEA_CI_SNAPSHOT_NAME env var.
    Default: BaseClean

.PARAMETER CloneBaseDir
    Clone VM base directory. Falls back to GITEA_CI_CLONE_BASE_DIR env var.
    Default: F:\CI\BuildVMs

.PARAMETER ArtifactBaseDir
    Artifact output directory. Default: F:\CI\Artifacts

.PARAMETER GuestCredentialTarget
    Windows Credential Manager target for guest credentials.
    Default: BuildVMGuest

.PARAMETER VmrunPath
    Path to vmrun.exe.
    Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe

.PARAMETER GuestOS
    Guest OS transport. Default: Auto (detected from VMX).

.PARAMETER BuildCommand
    Custom build command forwarded to Invoke-CIJob.ps1.
    Default: empty (uses dotnet build).

.PARAMETER Parallelism
    Number of concurrent jobs per round. Default: 4.

.PARAMETER Rounds
    Number of sequential rounds. Default: 3.

.PARAMETER RoundTimeoutMinutes
    Maximum wait time per round before force-stopping remaining jobs.
    Default: 60.

.PARAMETER ScriptDir
    Directory containing sibling CI scripts. Default: directory of this script.

.EXAMPLE
    .\Test-CapacityBurnIn.ps1 `
        -RepoUrl 'http://gitea.local/ci/hello-world.git' `
        -IPList '192.168.79.101','192.168.79.102','192.168.79.103','192.168.79.104' `
        -Rounds 3 -Parallelism 4

.EXAMPLE
    # Quick smoke round (1 round, 2 jobs) — auto-detect IPs via VMware Tools
    .\Test-CapacityBurnIn.ps1 `
        -RepoUrl      'http://gitea.local/ci/hello-world.git' `
        -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
        -Parallelism 2 -Rounds 1

.EXAMPLE
    # With explicit static IPs (DHCP reservations configured)
    .\Test-CapacityBurnIn.ps1 `
        -RepoUrl      'http://gitea.local/ci/hello-world.git' `
        -IPList       '192.168.79.101','192.168.79.102' `
        -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' `
        -Parallelism 2 -Rounds 1
#>
[CmdletBinding()]
param(
    [Parameter(Mandatory)]
    [string] $RepoUrl,

    [string[]] $IPList = @(),

    [string] $Branch = 'main',

    [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,

    [string] $SnapshotName = $(
        if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }
    ),

    [string] $CloneBaseDir = $(
        if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }
    ),

    [string] $ArtifactBaseDir = 'F:\CI\Artifacts',

    [string] $GuestCredentialTarget = 'BuildVMGuest',

    # Credential Manager target for the Gitea PAT (used by Invoke-CIJob -UseGitClone).
    # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
    [string] $GiteaCredentialTarget = 'GiteaPAT',

    [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',

    [ValidateSet('Windows', 'Linux', 'Auto')]
    [string] $GuestOS = 'Auto',

    [string] $BuildCommand = '',

    # Skip artifact packaging and collection in every child job.
    # Set this for burn-in and smoke runs that use delay-only build commands.
    [switch] $SkipArtifact,

    [ValidateRange(1, 10)]
    [int] $Parallelism = 4,

    [ValidateRange(1, 10)]
    [int] $Rounds = 3,

    [ValidateRange(5, 240)]
    [int] $RoundTimeoutMinutes = 60,

    [string] $ScriptDir = $PSScriptRoot
)

Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'

# ── Validate inputs ───────────────────────────────────────────────────────────
if ($IPList.Count -gt 0 -and $IPList.Count -lt $Parallelism) {
    throw "IPList must contain at least $Parallelism entries (got $($IPList.Count))."
}
if (-not $TemplatePath) {
    throw "TemplatePath is required. Pass -TemplatePath or set GITEA_CI_TEMPLATE_PATH."
}
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
    throw "TemplatePath not found: $TemplatePath"
}

# Resolve script dir — $PSScriptRoot may be empty when dot-sourced interactively
if (-not $ScriptDir) {
    $ScriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent
}
$invokeCIJobPath = Join-Path $ScriptDir 'Invoke-CIJob.ps1'
if (-not (Test-Path $invokeCIJobPath -PathType Leaf)) {
    throw "Invoke-CIJob.ps1 not found at: $invokeCIJobPath"
}

# ── State trackers ────────────────────────────────────────────────────────────
$roundResults = [System.Collections.Generic.List[PSCustomObject]]::new()
$totalPass    = 0
$totalFail    = 0
$roundsFailed = 0

Write-Host '============================================================'
Write-Host '[Test-CapacityBurnIn] Burn-in configuration:'
Write-Host "  Parallelism    : $Parallelism"
Write-Host "  Rounds         : $Rounds"
Write-Host "  Round timeout  : ${RoundTimeoutMinutes}m"
Write-Host "  Template       : $TemplatePath"
Write-Host "  Snapshot       : $SnapshotName"
Write-Host "  CloneBaseDir   : $CloneBaseDir"
  $ipDisplay = if ($IPList.Count -ge $Parallelism) { $IPList[0..($Parallelism - 1)] -join ', ' } else { '(auto-detect via VMware Tools)' }
  Write-Host "  IPs            : $ipDisplay"
Write-Host '============================================================'

for ($r = 1; $r -le $Rounds; $r++) {

    Write-Host "`n[Round $r/$Rounds] Launching $Parallelism concurrent jobs..."
    $roundStart = Get-Date

    # ── Launch jobs ───────────────────────────────────────────────────────
    $jobEntries = [System.Collections.Generic.List[hashtable]]::new()

    for ($j = 1; $j -le $Parallelism; $j++) {
        $jId    = "burnin-r${r}-j${j}-$(Get-Date -Format 'HHmmss')"
        $ipAddr = if ($IPList.Count -ge $j) { $IPList[$j - 1] } else { '' }

        # Build a flat string array for powershell.exe -File
        $argList = [System.Collections.Generic.List[string]]::new()
        $argList.Add('-NonInteractive')
        $argList.Add('-ExecutionPolicy'); $argList.Add('Bypass')
        $argList.Add('-File');            $argList.Add($invokeCIJobPath)
        $argList.Add('-JobId');           $argList.Add($jId)
        $argList.Add('-RepoUrl');         $argList.Add($RepoUrl)
        $argList.Add('-Branch');          $argList.Add($Branch)
        $argList.Add('-TemplatePath');    $argList.Add($TemplatePath)
        $argList.Add('-SnapshotName');    $argList.Add($SnapshotName)
        $argList.Add('-CloneBaseDir');    $argList.Add($CloneBaseDir)
        $argList.Add('-ArtifactBaseDir'); $argList.Add($ArtifactBaseDir)
        if ($ipAddr -ne '') {
            $argList.Add('-VMIPAddress'); $argList.Add($ipAddr)
        }
        $argList.Add('-GuestCredentialTarget');  $argList.Add($GuestCredentialTarget)
        $argList.Add('-GiteaCredentialTarget'); $argList.Add($GiteaCredentialTarget)
        $argList.Add('-VmrunPath');            $argList.Add($VmrunPath)
        $argList.Add('-GuestOS');         $argList.Add($GuestOS)
        if ($BuildCommand -ne '') {
            $argList.Add('-BuildCommand'); $argList.Add($BuildCommand)
        }
        if ($SkipArtifact) {
            $argList.Add('-SkipArtifact')
        }

        $argArray = $argList.ToArray()

        # Start-Job runs a new PS session; call powershell.exe as child process
        # so exit $exitCode from Invoke-CIJob.ps1 is captured in $LASTEXITCODE.
        # NOTE: do NOT name the parameter $Args — @Args is a PS reserved splatting
        # alias for the automatic $Args variable (unbound arguments), which would
        # be empty here since the parameter binding consumed all positional args.
        $psJob = Start-Job -ScriptBlock {
            param([string[]]$ChildArgs)
            $out = & powershell.exe @ChildArgs 2>&1
            [PSCustomObject]@{
                ExitCode = $LASTEXITCODE
                Output   = ($out -join "`n")
            }
        } -ArgumentList (, $argArray)

        $jobEntries.Add(@{
            PSJob = $psJob
            JobId = $jId
            IP    = $ipAddr
            Slot  = $j
        })

        $ipDisplay2 = if ($ipAddr -ne '') { $ipAddr } else { '(auto)' }
        Write-Host "  Slot $j — JobId=$jId  IP=$ipDisplay2  PSJob=$($psJob.Id)"
    }

    # ── Wait for round to complete ────────────────────────────────────────
    Write-Host "  Waiting (timeout: ${RoundTimeoutMinutes}m)..."
    $psJobArray = @($jobEntries | ForEach-Object { $_.PSJob })
    $null = Wait-Job -Job $psJobArray -Timeout ($RoundTimeoutMinutes * 60)

    # Force-stop anything still running (timeout exceeded)
    foreach ($entry in $jobEntries) {
        if ($entry.PSJob.State -eq 'Running') {
            Write-Warning "  [Round $r] Slot $($entry.Slot) ($($entry.JobId)) still running after timeout — stopping."
            Stop-Job $entry.PSJob
        }
    }

    $roundElapsed = (Get-Date) - $roundStart
    $roundPass = 0
    $roundFail = 0

    # ── Print per-job results ─────────────────────────────────────────────
    Write-Host ''
    Write-Host ('  ' + ('{0,-44} {1,-16} {2,-8} {3}' -f 'JobId', 'IP', 'Status', 'ExitCode'))
    Write-Host ('  ' + ('{0,-44} {1,-16} {2,-8} {3}' -f ('-' * 44), ('-' * 16), ('-' * 8), ('-' * 8)))

    foreach ($entry in $jobEntries) {
        $savedState = $entry.PSJob.State
        $result     = Receive-Job $entry.PSJob -ErrorAction SilentlyContinue
        Remove-Job  $entry.PSJob -Force

        if ($savedState -eq 'Stopped') {
            $status = 'TIMEOUT'; $ec = -1
        } elseif ($null -eq $result) {
            $status = 'NO-RESULT'; $ec = -1
        } else {
            $ec     = $result.ExitCode
            $status = if ($ec -eq 0) { 'PASS' } else { 'FAIL' }
        }

        Write-Host ('  ' + ('{0,-44} {1,-16} {2,-8} {3}' -f $entry.JobId, $entry.IP, $status, $ec))

        # On failure, print the last 20 lines of output to aid diagnosis
        if ($status -ne 'PASS' -and $null -ne $result -and $result.Output -ne '') {
            $lines = $result.Output -split "`n"
            $tail  = if ($lines.Count -gt 20) { $lines[-20..-1] } else { $lines }
            Write-Host '  --- last output ---'
            foreach ($line in $tail) { Write-Host "  $line" }
            Write-Host '  ---'
        }

        if ($status -eq 'PASS') { $roundPass++ } else { $roundFail++ }
    }

    # ── Assert: no orphaned clone dirs ────────────────────────────────────
    $orphaned = [System.Collections.Generic.List[string]]::new()
    if (Test-Path $CloneBaseDir) {
        foreach ($entry in $jobEntries) {
            $dirs = Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue |
                    Where-Object { $_.Name -like "Clone_$($entry.JobId)_*" }
            if ($dirs) {
                foreach ($d in $dirs) { $orphaned.Add($d.FullName) }
            }
        }
    }

    Write-Host ''
    if ($orphaned.Count -eq 0) {
        Write-Host "  Clone cleanup  : OK (0 orphaned)"
    } else {
        Write-Host "  Clone cleanup  : FAIL ($($orphaned.Count) orphaned directories)"
        foreach ($p in $orphaned) { Write-Host "    $p" }
        $roundFail++
    }

    # ── Assert: no stale lock file ────────────────────────────────────────
    $lockPath  = Join-Path $CloneBaseDir 'vm-start.lock'
    $staleLock = $false
    if (Test-Path $lockPath) {
        $ageMin = ((Get-Date) - (Get-Item $lockPath).LastWriteTime).TotalMinutes
        if ($ageMin -gt 5) {
            $staleLock = $true
        }
    }

    if (-not $staleLock) {
        Write-Host "  Lock cleanup   : OK"
    } else {
        Write-Host "  Lock cleanup   : FAIL (stale vm-start.lock present)"
        $roundFail++
    }

    $roundStatus = if ($roundFail -eq 0) { 'PASS' } else { 'FAIL' }
    Write-Host ("  [Round $r] Elapsed: {0:mm\:ss}  --  $roundStatus ($roundPass PASS, $roundFail FAIL)" -f $roundElapsed)

    $totalPass += $roundPass
    $totalFail += $roundFail
    if ($roundFail -gt 0) { $roundsFailed++ }

    $roundResults.Add([PSCustomObject]@{
        Round     = $r
        Pass      = $roundPass
        Fail      = $roundFail
        Status    = $roundStatus
        ElapsedSec = [int]$roundElapsed.TotalSeconds
    })
}

# ── Overall summary ───────────────────────────────────────────────────────────
Write-Host "`n============================================================"
Write-Host '[Test-CapacityBurnIn] SUMMARY'
Write-Host ('  {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f 'Round', 'Pass', 'Fail', 'Elapsed', 'Status')
Write-Host ('  {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f ('-'*8), ('-'*6), ('-'*6), ('-'*10), ('-'*8))
foreach ($rr in $roundResults) {
    Write-Host ('  {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f $rr.Round, $rr.Pass, $rr.Fail, "$($rr.ElapsedSec)s", $rr.Status)
}
Write-Host ''
Write-Host "  Total jobs : $($Rounds * $Parallelism)"
Write-Host "  Passed     : $totalPass"
Write-Host "  Failed     : $totalFail"
Write-Host "  Rounds OK  : $($Rounds - $roundsFailed) / $Rounds"
$overallStatus = if ($totalFail -eq 0) { 'PASS' } else { 'FAIL' }
Write-Host "`n  OVERALL: $overallStatus"
Write-Host '============================================================'

if ($totalFail -gt 0) {
    exit 1
}
exit 0