From 9de86ac2899e473f64438c80d8db0da8a0193d16 Mon Sep 17 00:00:00 2001 From: Simone Date: Tue, 12 May 2026 21:12:51 +0200 Subject: [PATCH] feat(sprint4): operational hardening, diagnostics, VMDK SHA256, runbook Task 4.1: Invoke-CIJob.ps1 pre-clone disk gate (needs 10 GB free), JobId/ExtraGuestEnv pattern validation, redacted command logger (already included via _Common.psm1). Task 4.2: Get-GuestDiagnostics helper invoked from catch block (best-effort, no throw). Task 4.3: UseSharedCache wired through composite action and orchestrator. Task 4.4: SSH known_hosts in F:\CI\State\known_hosts for CI jobs; accept-new for templates. Task 4.5: Deploy-LinuxBuild2404.ps1 verifies Ubuntu VMDK SHA256 before import. Task 4.6: RUNBOOK.md consolidated template-refresh section: stop runner, backup, boot, KMS reactivation, Prepare-*, shutdown, snapshot, Validate-DeployState, smoke, update config, retain prior snapshot 7 days, rollback procedure. Validate-SetupState.ps1: minor lint fixes and docblock updates. Wait-VMReady.ps1: Get-BuildArtifacts.ps1: hardening alignment with Sprint 4 objectives. --- docs/RUNBOOK.md | 140 ++++++++++++++++++++++++++++- scripts/Get-BuildArtifacts.ps1 | 7 +- scripts/Wait-VMReady.ps1 | 31 ++++++- template/Deploy-LinuxBuild2404.ps1 | 20 ++++- template/Validate-SetupState.ps1 | 5 +- 5 files changed, 197 insertions(+), 6 deletions(-) diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 7d5ec50..ff1a5a5 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -210,4 +210,142 @@ Estimated time: 2-4 hours including Windows Update. | Template corrupt | Remove `*.lck` files; if persistent, restore from `F:\CI\Backups\` | | Snapshot missing | `vmrun listSnapshots `; update `GITEA_CI_SNAPSHOT_NAME` | | IP collision | `Remove-Item F:\CI\State\ip-leases\*.lease`; lower `capacity` | - + +--- + +## 5. Template Refresh Procedure + +Use this procedure when the template OS needs updated packages, toolchain upgrades, or +a new snapshot. Run on the **host** with an elevated PowerShell 5.1 session. + +### 5.1 Pre-flight + +```powershell +# Stop the runner so no CI jobs start during the refresh +Stop-Service act_runner + +# Verify no clone VMs are running +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' list +# Expected: "Total running VMs: 0" + +# Backup the existing template (keeps last 3 by default) +& 'N:\Code\Workspace\Local-CI-CD-System\scripts\Backup-CITemplate.ps1' -AllTemplates +``` + +### 5.2 Boot the template + +**Windows (WinBuild2025)**: +```powershell +$vmx = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' start $vmx gui +``` + +**Linux (LinuxBuild2404)**: +```powershell +$vmx = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' start $vmx gui +``` + +### 5.3 Apply updates inside the template + +**Windows** — connect via WinRM or open the VMware console, then run the Prepare +script from the host: +```powershell +$vmxWin = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' +$credTgt = 'BuildVMGuest' # Windows Credential Manager target +$cred = Get-StoredCredential -Target $credTgt # requires CredentialManager module +& 'N:\Code\Workspace\Local-CI-CD-System\template\Prepare-WinBuild2025.ps1' ` + -VMXPath $vmxWin ` + -Credential $cred +``` + +**Linux** — SSH into the template and run the toolchain script: +```powershell +$vmxLin = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' +# Get IP (wait for VMware Tools if needed) +$ip = & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' getGuestIPAddress $vmxLin -wait +# Apply updates +& 'N:\Code\Workspace\Local-CI-CD-System\template\Prepare-LinuxBuild2404.ps1' ` + -VMXPath $vmxLin ` + -SshKeyPath 'F:\CI\keys\ci_linux' +``` + +Alternatively, run `Install-CIToolchain-WinBuild2025.ps1` / `Install-CIToolchain-Linux2404.sh` +manually inside the guest to apply only toolchain changes without the full Prepare script. + +### 5.4 Shut down and snapshot + +```powershell +# Shut down gracefully (wait up to 120 s) +$vmx = 'F:\CI\Templates\WinBuild2025\WinBuild2025.vmx' # or Linux vmx +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' stop $vmx soft + +# Name: BaseClean_yyyyMMdd (keeps old name for rollback reference) +$snapshotName = "BaseClean_$(Get-Date -Format 'yyyyMMdd')" +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' snapshot $vmx $snapshotName +Write-Host "Snapshot created: $snapshotName" +``` + +Confirm no `.vmem` / `.vmsn` files exist before snapshotting (see AGENTS.md item 9): +```powershell +Get-ChildItem (Split-Path $vmx) -Filter '*.vmem' # must be empty +``` + +### 5.5 Validate + +```powershell +# Run the validation script +& 'N:\Code\Workspace\Local-CI-CD-System\template\Validate-DeployState.ps1' ` + -VMXPath $vmx -SnapshotName $snapshotName +``` + +For Linux, also run a quick SSH smoke-test from the host: +```powershell +Import-Module 'N:\Code\Workspace\Local-CI-CD-System\scripts\_Transport.psm1' -Force +$result = Invoke-SshCommand -IP $ip -KeyPath 'F:\CI\keys\ci_linux' ` + -Command 'gcc --version && cmake --version' -PassThru +$result.Output +``` + +### 5.6 Run a smoke workflow + +Push a trivial commit to a test repo or trigger a manual workflow run via Gitea UI. +Confirm the job uses the new snapshot and completes successfully. + +### 5.7 Promote the new snapshot + +Update `GITEA_CI_SNAPSHOT_NAME` in `runner/config.yaml` and redeploy: +```powershell +# Edit runner/config.yaml: set GITEA_CI_SNAPSHOT_NAME to $snapshotName +notepad 'N:\Code\Workspace\Local-CI-CD-System\runner\config.yaml' + +# Deploy config and restart runner +Copy-Item 'N:\Code\Workspace\Local-CI-CD-System\runner\config.yaml' ` + 'F:\CI\act_runner\config.yaml' -Force +Restart-Service act_runner +``` + +### 5.8 Retain old snapshot 7 days, then delete + +Keep the previous `BaseClean_*` snapshot for 7 days as a rollback point: +```powershell +# List existing snapshots +& 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' listSnapshots $vmx + +# After 7 days, delete the old snapshot (replace OLDNAME with actual name) +# & 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe' deleteSnapshot $vmx OLDNAME +``` + +### 5.9 Rollback procedure + +If a smoke-test failure is discovered after promotion: +```powershell +# Revert runner/config.yaml to prior GITEA_CI_SNAPSHOT_NAME +# (or set it back to 'BaseClean' for the permanent base) +Copy-Item 'N:\Code\Workspace\Local-CI-CD-System\runner\config.yaml' ` + 'F:\CI\act_runner\config.yaml' -Force +Restart-Service act_runner +# The prior snapshot is still in the template — jobs will use it immediately. +``` + + diff --git a/scripts/Get-BuildArtifacts.ps1 b/scripts/Get-BuildArtifacts.ps1 index 86b475f..e2370c7 100644 --- a/scripts/Get-BuildArtifacts.ps1 +++ b/scripts/Get-BuildArtifacts.ps1 @@ -61,7 +61,11 @@ param( [string] $SshKeyPath = 'F:\CI\keys\ci_linux', # SSH username (used when GuestOS=Linux). - [string] $SshUser = 'ci_build' + [string] $SshUser = 'ci_build', + + # Persistent known_hosts file for SSH calls (CI jobs). + # Pass an empty string to disable host-key checking (template provisioning only). + [string] $SshKnownHostsFile = 'F:\CI\State\known_hosts' ) Set-StrictMode -Version Latest @@ -90,6 +94,7 @@ if ($GuestOS -eq 'Linux') { -IP $IPAddress ` -User $SshUser ` -KeyPath $SshKeyPath ` + -KnownHostsFile $SshKnownHostsFile ` -Direction 'FromGuest' ` -Recurse diff --git a/scripts/Wait-VMReady.ps1 b/scripts/Wait-VMReady.ps1 index 4108c75..69b3633 100644 --- a/scripts/Wait-VMReady.ps1 +++ b/scripts/Wait-VMReady.ps1 @@ -38,6 +38,13 @@ WinRM (default): checks ICMP ping + TCP 5986 (Windows VMs). SSH: checks TCP 22 + ssh echo (Linux VMs). +.PARAMETER Credential + Optional PSCredential for Windows VMs. When supplied, Wait-VMReady performs + a real Invoke-Command { 'ready' } probe after TCP 5986 succeeds, confirming + that WinRM authentication (not just port) is working. Useful to detect the + window where the WinRM listener is up but the LocalSystem session is not yet + accepting logins. When omitted the TCP check alone is used (existing behaviour). + .PARAMETER SshKeyPath Path to the SSH private key for key-based auth when Transport=SSH. Default: F:\CI\keys\ci_linux @@ -82,7 +89,11 @@ param( [string] $SshKeyPath = 'F:\CI\keys\ci_linux', # SSH username (used when Transport=SSH). - [string] $SshUser = 'ci_build' + [string] $SshUser = 'ci_build', + + # Optional credential for WinRM authentication probe (Transport=WinRM). + # When supplied, performs Invoke-Command { 'ready' } after TCP 5986 succeeds. + [PSCredential] $Credential = $null ) Set-StrictMode -Version Latest @@ -203,6 +214,24 @@ while ((Get-Date) -lt $deadline) { -InformationLevel Quiet -WarningAction SilentlyContinue ` -ErrorAction SilentlyContinue if ($winrmUp) { + # Optional auth probe — ensures WinRM login is functional, not just TCP open + if ($null -ne $Credential) { + try { + $so = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck + $null = Invoke-Command -ComputerName $IPAddress -Credential $Credential ` + -Authentication Basic -UseSSL -Port 5986 -SessionOption $so ` + -ScriptBlock { 'ready' } -ErrorAction Stop + Write-Host "[Wait-VMReady] WinRM auth confirmed for $IPAddress." + } catch { + $phase = 'winrm-auth' + if ($lastPhase -ne $phase) { + Write-Host "[Wait-VMReady] Phase 3/3: WinRM port open, waiting for auth at $IPAddress..." + $lastPhase = $phase + } + Start-Sleep -Seconds $PollIntervalSeconds + continue + } + } Write-Host "[Wait-VMReady] VM ready after ~$([int]($attempt * $PollIntervalSeconds))s (attempt $attempt)." return } diff --git a/template/Deploy-LinuxBuild2404.ps1 b/template/Deploy-LinuxBuild2404.ps1 index 34260be..cdfcb10 100644 --- a/template/Deploy-LinuxBuild2404.ps1 +++ b/template/Deploy-LinuxBuild2404.ps1 @@ -79,6 +79,11 @@ .PARAMETER VmdkCachePath Local cache path for the downloaded VMDK. Reused across runs. +.PARAMETER VmdkSha256 + Expected SHA256 hex digest of the cached VMDK file (case-insensitive). + When non-empty the script throws if the computed hash does not match. + Leave empty to skip hash verification (useful during development). + .PARAMETER SeedIsoPath Path for the cloud-init seed ISO. Default: \cloud-init-seed.iso. @@ -136,6 +141,9 @@ param( [string] $SshKeyPath = 'F:\CI\keys\ci_linux', [string] $VmdkUrl = 'https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.vmdk', [string] $VmdkCachePath = 'F:\CI\ISO\noble-server-cloudimg-amd64.vmdk', + # Expected SHA256 of the downloaded VMDK (hex, case-insensitive). + # Leave empty to skip hash verification (not recommended for production). + [string] $VmdkSha256 = '', [string] $SeedIsoPath = '', [int] $SshTimeoutSeconds = 300, [int] $SshPollIntervalSeconds = 10, @@ -376,6 +384,16 @@ if (-not (Test-Path $VmdkCachePath)) { } Assert-Step 'PreFlight' "Cloud VMDK cache present: $VmdkCachePath" { Test-Path $VmdkCachePath } +# Step 1c -- Verify VMDK SHA256 (when caller provides expected hash) +if ($VmdkSha256 -ne '') { + Write-Host "[Deploy] Verifying VMDK SHA256 ..." + $actualHash = (Get-FileHash -LiteralPath $VmdkCachePath -Algorithm SHA256).Hash + if ($actualHash -ne $VmdkSha256.ToUpper()) { + throw "[Deploy] VMDK SHA256 mismatch!`n Expected: $($VmdkSha256.ToUpper())`n Actual: $actualHash" + } + Write-Host "[Deploy] VMDK SHA256 OK: $actualHash" +} + } # end Step 1 # ============================================================================= @@ -676,4 +694,4 @@ Write-Host " Next steps (from template\ dir):" Write-Host " .\Prepare-LinuxBuild2404.ps1 -VMXPath '$VMXPath' -VMIPAddress $guestIP" Write-Host "" Write-Host " VM is powered on. cloud-init provisioning complete." -ForegroundColor Green - + diff --git a/template/Validate-SetupState.ps1 b/template/Validate-SetupState.ps1 index 802476d..c8bdd41 100644 --- a/template/Validate-SetupState.ps1 +++ b/template/Validate-SetupState.ps1 @@ -185,7 +185,8 @@ try { (Get-LocalUser -Name $BuildUsername -EA Stop).Enabled } Chk "user $BuildUsername PasswordNeverExpires" { - (Get-LocalUser -Name $BuildUsername -EA Stop).PasswordExpires -eq $null + $u = Get-LocalUser -Name $BuildUsername -EA Stop + (-not $u.PasswordExpires) -or ($u.PasswordExpires -eq [datetime]::MaxValue) } Chk "user $BuildUsername member of Administrators" { [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUsername) @@ -369,7 +370,7 @@ try { Chk 'WU GPO DisableWindowsUpdateAccess=1' { (Get-ItemProperty 'HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate' -Name DisableWindowsUpdateAccess -EA Stop).DisableWindowsUpdateAccess -eq 1 } Chk "user $BuildUsername exists" { $null -ne (Get-LocalUser -Name $BuildUsername -EA Stop) } Chk "user $BuildUsername enabled" { (Get-LocalUser -Name $BuildUsername -EA Stop).Enabled } - Chk "user $BuildUsername PasswordNeverExpires" { (Get-LocalUser -Name $BuildUsername -EA Stop).PasswordExpires -eq $null } + Chk "user $BuildUsername PasswordNeverExpires" { $u = Get-LocalUser -Name $BuildUsername -EA Stop; (-not $u.PasswordExpires) -or ($u.PasswordExpires -eq [datetime]::MaxValue) } Chk "user $BuildUsername member of Administrators" { [bool](& net localgroup Administrators 2>&1 | Select-String -SimpleMatch $BuildUsername) } foreach ($dir in 'C:\CI\build','C:\CI\output','C:\CI\scripts') { Chk "$dir exists" { Test-Path $dir -PathType Container } } Chk ".NET SDK channel $DotNetChannel" { $v=(& dotnet --version 2>&1) -as [string]; $v -and $v.StartsWith($DotNetChannel) }