diff --git a/scripts/Start-BurnInTest.ps1 b/scripts/Start-BurnInTest.ps1 new file mode 100644 index 0000000..f047a79 --- /dev/null +++ b/scripts/Start-BurnInTest.ps1 @@ -0,0 +1,72 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Wrapper di lancio per Test-CapacityBurnIn.ps1 con i parametri del lab locale. + +.DESCRIPTION + Esegue il burn-in a 4 job / 3 round contro il repo burnin-dummy su Gitea. + Tutti i parametri fissi del lab sono qui hardcoded; i parametri variabili + (Rounds, Parallelism, BuildCommand) possono essere sovrascritti da CLI. + + Prerequisiti: + - Repo burnin-dummy creato su Gitea (anche vuoto con branch main) + - Credenziali guest nel Credential Manager target 'BuildVMGuest' + - Gitea PAT nel Credential Manager target 'GiteaPAT' + New-StoredCredential -Target 'GiteaPAT' -UserName 'ci' -Password '' -Persist LocalMachine + - Template WinBuild2025 con snapshot BaseClean in stato powered-off + - VMware Tools attivo nel guest (per auto-detect IP) + + Esempi di utilizzo: + # Run completo (3 round, 4 job paralleli) + .\Start-BurnInTest.ps1 + + # Smoke veloce (1 round, 2 job) + .\Start-BurnInTest.ps1 -Rounds 1 -Parallelism 2 + + # Verifica parallelismo reale (build da 2 minuti, 1 round) + .\Start-BurnInTest.ps1 -Rounds 1 -BuildCommand 'Start-Sleep 120' + +.PARAMETER Rounds + Numero di round sequenziali. Default: 3. + +.PARAMETER Parallelism + Job concorrenti per round. Default: 4. + +.PARAMETER BuildCommand + Comando da eseguire nel guest come passo build. + Default: 'exit 0' (smoke veloce — nessun build reale). + Usare 'Start-Sleep 120' per mantenere 4 VM attive e osservare la concorrenza. + +.PARAMETER RoundTimeoutMinutes + Minuti massimi di attesa per ogni round prima di interrompere i job rimasti. + Default: 60. +#> +param( + [int] $Rounds = 3, + [int] $Parallelism = 4, + [string] $BuildCommand = 'exit 0', + [int] $RoundTimeoutMinutes = 60 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +& "$scriptDir\Test-CapacityBurnIn.ps1" ` + -RepoUrl 'http://10.10.20.11:3100/ci/burnin-dummy.git' ` + -Branch 'main' ` + -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` + -SnapshotName 'BaseClean' ` + -CloneBaseDir 'F:\CI\BuildVMs' ` + -ArtifactBaseDir 'F:\CI\Artifacts' ` + -GuestCredentialTarget 'BuildVMGuest' ` + -GiteaCredentialTarget 'GiteaPAT' ` + -VmrunPath 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' ` + -GuestOS 'Auto' ` + -BuildCommand $BuildCommand ` + -Parallelism $Parallelism ` + -Rounds $Rounds ` + -RoundTimeoutMinutes $RoundTimeoutMinutes + +exit $LASTEXITCODE diff --git a/scripts/Test-CapacityBurnIn.ps1 b/scripts/Test-CapacityBurnIn.ps1 new file mode 100644 index 0000000..eeebac1 --- /dev/null +++ b/scripts/Test-CapacityBurnIn.ps1 @@ -0,0 +1,366 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Concurrency burn-in harness: runs N simultaneous Invoke-CIJob.ps1 jobs + for R rounds and asserts correct cleanup after each round. + +.DESCRIPTION + Launches $Parallelism child PowerShell processes (via Start-Job) all + running Invoke-CIJob.ps1 concurrently. After each round verifies that: + - All job exit codes were 0 + - No clone directories for burn-in jobs remain in CloneBaseDir + - No stale vm-start.lock file remains (older than 5 min) + + Exits 0 only when ALL rounds and assertions pass. + +.PARAMETER RepoUrl + Git URL to pass to Invoke-CIJob.ps1 as -RepoUrl. + +.PARAMETER Branch + Branch to build. Default: main + +.PARAMETER IPList + Optional array of IP addresses, one per concurrent job slot. + When provided, each job receives the corresponding -VMIPAddress (useful when + VMs have static IPs or per-slot DHCP reservations). + When omitted, -VMIPAddress is not passed to Invoke-CIJob.ps1, which then + auto-detects the IP via Get-GuestIPAddress (requires VMware Tools / ci-report-ip.service). + When provided, must have at least $Parallelism entries. + +.PARAMETER TemplatePath + VMX template path. Falls back to GITEA_CI_TEMPLATE_PATH env var. + +.PARAMETER SnapshotName + Snapshot to clone from. Falls back to GITEA_CI_SNAPSHOT_NAME env var. + Default: BaseClean + +.PARAMETER CloneBaseDir + Clone VM base directory. Falls back to GITEA_CI_CLONE_BASE_DIR env var. + Default: F:\CI\BuildVMs + +.PARAMETER ArtifactBaseDir + Artifact output directory. Default: F:\CI\Artifacts + +.PARAMETER GuestCredentialTarget + Windows Credential Manager target for guest credentials. + Default: BuildVMGuest + +.PARAMETER VmrunPath + Path to vmrun.exe. + Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe + +.PARAMETER GuestOS + Guest OS transport. Default: Auto (detected from VMX). + +.PARAMETER BuildCommand + Custom build command forwarded to Invoke-CIJob.ps1. + Default: empty (uses dotnet build). + +.PARAMETER Parallelism + Number of concurrent jobs per round. Default: 4. + +.PARAMETER Rounds + Number of sequential rounds. Default: 3. + +.PARAMETER RoundTimeoutMinutes + Maximum wait time per round before force-stopping remaining jobs. + Default: 60. + +.PARAMETER ScriptDir + Directory containing sibling CI scripts. Default: directory of this script. + +.EXAMPLE + .\Test-CapacityBurnIn.ps1 ` + -RepoUrl 'http://gitea.local/ci/hello-world.git' ` + -IPList '192.168.79.101','192.168.79.102','192.168.79.103','192.168.79.104' ` + -Rounds 3 -Parallelism 4 + +.EXAMPLE + # Quick smoke round (1 round, 2 jobs) — auto-detect IPs via VMware Tools + .\Test-CapacityBurnIn.ps1 ` + -RepoUrl 'http://gitea.local/ci/hello-world.git' ` + -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` + -Parallelism 2 -Rounds 1 + +.EXAMPLE + # With explicit static IPs (DHCP reservations configured) + .\Test-CapacityBurnIn.ps1 ` + -RepoUrl 'http://gitea.local/ci/hello-world.git' ` + -IPList '192.168.79.101','192.168.79.102' ` + -TemplatePath 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' ` + -Parallelism 2 -Rounds 1 +#> +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [string] $RepoUrl, + + [string[]] $IPList = @(), + + [string] $Branch = 'main', + + [string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH, + + [string] $SnapshotName = $( + if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' } + ), + + [string] $CloneBaseDir = $( + if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' } + ), + + [string] $ArtifactBaseDir = 'F:\CI\Artifacts', + + [string] $GuestCredentialTarget = 'BuildVMGuest', + + # Credential Manager target for the Gitea PAT (used by Invoke-CIJob -UseGitClone). + # Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '' -Password '' -Persist LocalMachine + [string] $GiteaCredentialTarget = 'GiteaPAT', + + [string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe', + + [ValidateSet('Windows', 'Linux', 'Auto')] + [string] $GuestOS = 'Auto', + + [string] $BuildCommand = '', + + [ValidateRange(1, 10)] + [int] $Parallelism = 4, + + [ValidateRange(1, 10)] + [int] $Rounds = 3, + + [ValidateRange(5, 240)] + [int] $RoundTimeoutMinutes = 60, + + [string] $ScriptDir = $PSScriptRoot +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ── Validate inputs ─────────────────────────────────────────────────────────── +if ($IPList.Count -gt 0 -and $IPList.Count -lt $Parallelism) { + throw "IPList must contain at least $Parallelism entries (got $($IPList.Count))." +} +if (-not $TemplatePath) { + throw "TemplatePath is required. Pass -TemplatePath or set GITEA_CI_TEMPLATE_PATH." +} +if (-not (Test-Path $TemplatePath -PathType Leaf)) { + throw "TemplatePath not found: $TemplatePath" +} + +# Resolve script dir — $PSScriptRoot may be empty when dot-sourced interactively +if (-not $ScriptDir) { + $ScriptDir = Split-Path $MyInvocation.MyCommand.Path -Parent +} +$invokeCIJobPath = Join-Path $ScriptDir 'Invoke-CIJob.ps1' +if (-not (Test-Path $invokeCIJobPath -PathType Leaf)) { + throw "Invoke-CIJob.ps1 not found at: $invokeCIJobPath" +} + +# ── State trackers ──────────────────────────────────────────────────────────── +$roundResults = [System.Collections.Generic.List[PSCustomObject]]::new() +$totalPass = 0 +$totalFail = 0 +$roundsFailed = 0 + +Write-Host '============================================================' +Write-Host '[Test-CapacityBurnIn] Burn-in configuration:' +Write-Host " Parallelism : $Parallelism" +Write-Host " Rounds : $Rounds" +Write-Host " Round timeout : ${RoundTimeoutMinutes}m" +Write-Host " Template : $TemplatePath" +Write-Host " Snapshot : $SnapshotName" +Write-Host " CloneBaseDir : $CloneBaseDir" + $ipDisplay = if ($IPList.Count -ge $Parallelism) { $IPList[0..($Parallelism - 1)] -join ', ' } else { '(auto-detect via VMware Tools)' } + Write-Host " IPs : $ipDisplay" +Write-Host '============================================================' + +for ($r = 1; $r -le $Rounds; $r++) { + + Write-Host "`n[Round $r/$Rounds] Launching $Parallelism concurrent jobs..." + $roundStart = Get-Date + + # ── Launch jobs ─────────────────────────────────────────────────────── + $jobEntries = [System.Collections.Generic.List[hashtable]]::new() + + for ($j = 1; $j -le $Parallelism; $j++) { + $jId = "burnin-r${r}-j${j}-$(Get-Date -Format 'HHmmss')" + $ipAddr = if ($IPList.Count -ge $j) { $IPList[$j - 1] } else { '' } + + # Build a flat string array for powershell.exe -File + $argList = [System.Collections.Generic.List[string]]::new() + $argList.Add('-NonInteractive') + $argList.Add('-ExecutionPolicy'); $argList.Add('Bypass') + $argList.Add('-File'); $argList.Add($invokeCIJobPath) + $argList.Add('-JobId'); $argList.Add($jId) + $argList.Add('-RepoUrl'); $argList.Add($RepoUrl) + $argList.Add('-Branch'); $argList.Add($Branch) + $argList.Add('-TemplatePath'); $argList.Add($TemplatePath) + $argList.Add('-SnapshotName'); $argList.Add($SnapshotName) + $argList.Add('-CloneBaseDir'); $argList.Add($CloneBaseDir) + $argList.Add('-ArtifactBaseDir'); $argList.Add($ArtifactBaseDir) + if ($ipAddr -ne '') { + $argList.Add('-VMIPAddress'); $argList.Add($ipAddr) + } + $argList.Add('-GuestCredentialTarget'); $argList.Add($GuestCredentialTarget) + $argList.Add('-GiteaCredentialTarget'); $argList.Add($GiteaCredentialTarget) + $argList.Add('-VmrunPath'); $argList.Add($VmrunPath) + $argList.Add('-GuestOS'); $argList.Add($GuestOS) + if ($BuildCommand -ne '') { + $argList.Add('-BuildCommand'); $argList.Add($BuildCommand) + } + + $argArray = $argList.ToArray() + + # Start-Job runs a new PS session; call powershell.exe as child process + # so exit $exitCode from Invoke-CIJob.ps1 is captured in $LASTEXITCODE. + # NOTE: do NOT name the parameter $Args — @Args is a PS reserved splatting + # alias for the automatic $Args variable (unbound arguments), which would + # be empty here since the parameter binding consumed all positional args. + $psJob = Start-Job -ScriptBlock { + param([string[]]$ChildArgs) + $out = & powershell.exe @ChildArgs 2>&1 + [PSCustomObject]@{ + ExitCode = $LASTEXITCODE + Output = ($out -join "`n") + } + } -ArgumentList (, $argArray) + + $jobEntries.Add(@{ + PSJob = $psJob + JobId = $jId + IP = $ipAddr + Slot = $j + }) + + $ipDisplay2 = if ($ipAddr -ne '') { $ipAddr } else { '(auto)' } + Write-Host " Slot $j — JobId=$jId IP=$ipDisplay2 PSJob=$($psJob.Id)" + } + + # ── Wait for round to complete ──────────────────────────────────────── + Write-Host " Waiting (timeout: ${RoundTimeoutMinutes}m)..." + $psJobArray = @($jobEntries | ForEach-Object { $_.PSJob }) + $null = Wait-Job -Job $psJobArray -Timeout ($RoundTimeoutMinutes * 60) + + # Force-stop anything still running (timeout exceeded) + foreach ($entry in $jobEntries) { + if ($entry.PSJob.State -eq 'Running') { + Write-Warning " [Round $r] Slot $($entry.Slot) ($($entry.JobId)) still running after timeout — stopping." + Stop-Job $entry.PSJob + } + } + + $roundElapsed = (Get-Date) - $roundStart + $roundPass = 0 + $roundFail = 0 + + # ── Print per-job results ───────────────────────────────────────────── + Write-Host '' + Write-Host (' ' + ('{0,-44} {1,-16} {2,-8} {3}' -f 'JobId', 'IP', 'Status', 'ExitCode')) + Write-Host (' ' + ('{0,-44} {1,-16} {2,-8} {3}' -f ('-' * 44), ('-' * 16), ('-' * 8), ('-' * 8))) + + foreach ($entry in $jobEntries) { + $savedState = $entry.PSJob.State + $result = Receive-Job $entry.PSJob -ErrorAction SilentlyContinue + Remove-Job $entry.PSJob -Force + + if ($savedState -eq 'Stopped') { + $status = 'TIMEOUT'; $ec = -1 + } elseif ($null -eq $result) { + $status = 'NO-RESULT'; $ec = -1 + } else { + $ec = $result.ExitCode + $status = if ($ec -eq 0) { 'PASS' } else { 'FAIL' } + } + + Write-Host (' ' + ('{0,-44} {1,-16} {2,-8} {3}' -f $entry.JobId, $entry.IP, $status, $ec)) + + # On failure, print the last 20 lines of output to aid diagnosis + if ($status -ne 'PASS' -and $null -ne $result -and $result.Output -ne '') { + $lines = $result.Output -split "`n" + $tail = if ($lines.Count -gt 20) { $lines[-20..-1] } else { $lines } + Write-Host ' --- last output ---' + foreach ($line in $tail) { Write-Host " $line" } + Write-Host ' ---' + } + + if ($status -eq 'PASS') { $roundPass++ } else { $roundFail++ } + } + + # ── Assert: no orphaned clone dirs ──────────────────────────────────── + $orphaned = [System.Collections.Generic.List[string]]::new() + if (Test-Path $CloneBaseDir) { + foreach ($entry in $jobEntries) { + $dirs = Get-ChildItem -Path $CloneBaseDir -Directory -ErrorAction SilentlyContinue | + Where-Object { $_.Name -like "Clone_$($entry.JobId)_*" } + if ($dirs) { + foreach ($d in $dirs) { $orphaned.Add($d.FullName) } + } + } + } + + Write-Host '' + if ($orphaned.Count -eq 0) { + Write-Host " Clone cleanup : OK (0 orphaned)" + } else { + Write-Host " Clone cleanup : FAIL ($($orphaned.Count) orphaned directories)" + foreach ($p in $orphaned) { Write-Host " $p" } + $roundFail++ + } + + # ── Assert: no stale lock file ──────────────────────────────────────── + $lockPath = Join-Path $CloneBaseDir 'vm-start.lock' + $staleLock = $false + if (Test-Path $lockPath) { + $ageMin = ((Get-Date) - (Get-Item $lockPath).LastWriteTime).TotalMinutes + if ($ageMin -gt 5) { + $staleLock = $true + } + } + + if (-not $staleLock) { + Write-Host " Lock cleanup : OK" + } else { + Write-Host " Lock cleanup : FAIL (stale vm-start.lock present)" + $roundFail++ + } + + $roundStatus = if ($roundFail -eq 0) { 'PASS' } else { 'FAIL' } + Write-Host (" [Round $r] Elapsed: {0:mm\:ss} -- $roundStatus ($roundPass PASS, $roundFail FAIL)" -f $roundElapsed) + + $totalPass += $roundPass + $totalFail += $roundFail + if ($roundFail -gt 0) { $roundsFailed++ } + + $roundResults.Add([PSCustomObject]@{ + Round = $r + Pass = $roundPass + Fail = $roundFail + Status = $roundStatus + ElapsedSec = [int]$roundElapsed.TotalSeconds + }) +} + +# ── Overall summary ─────────────────────────────────────────────────────────── +Write-Host "`n============================================================" +Write-Host '[Test-CapacityBurnIn] SUMMARY' +Write-Host (' {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f 'Round', 'Pass', 'Fail', 'Elapsed', 'Status') +Write-Host (' {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f ('-'*8), ('-'*6), ('-'*6), ('-'*10), ('-'*8)) +foreach ($rr in $roundResults) { + Write-Host (' {0,-8} {1,-6} {2,-6} {3,-10} {4}' -f $rr.Round, $rr.Pass, $rr.Fail, "$($rr.ElapsedSec)s", $rr.Status) +} +Write-Host '' +Write-Host " Total jobs : $($Rounds * $Parallelism)" +Write-Host " Passed : $totalPass" +Write-Host " Failed : $totalFail" +Write-Host " Rounds OK : $($Rounds - $roundsFailed) / $Rounds" +$overallStatus = if ($totalFail -eq 0) { 'PASS' } else { 'FAIL' } +Write-Host "`n OVERALL: $overallStatus" +Write-Host '============================================================' + +if ($totalFail -gt 0) { + exit 1 +} +exit 0