diff --git a/template/Deploy-LinuxBuild2404.ps1 b/template/Deploy-LinuxBuild2404.ps1 new file mode 100644 index 0000000..38bef8f --- /dev/null +++ b/template/Deploy-LinuxBuild2404.ps1 @@ -0,0 +1,677 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Deploys an Ubuntu 24.04 LTS build VM on VMware Workstation using a cloud + VMDK and a cloud-init seed ISO. + +.DESCRIPTION + Host-side orchestrator. Produces a ready-to-use Linux template VM: + Step 1 — Validate prerequisites + download cloud VMDK if not cached + Step 2 — Generate cloud-init user-data + meta-data files + Step 3 — Build cloud-init seed ISO (nocloud datasource, label: cidata) + Step 4 — Copy and resize the cloud VMDK + Step 5 — Generate the .vmx (EFI, SCSI disk, NAT vmxnet3, seed ISO) + Step 6 — Power on the VM (vmrun start gui/nogui -- async) + Step 7 — Detect guest IP via vmrun getGuestIPAddress (polls up to 5 min) + Step 8 — Poll SSH port 22 until ready + Step 9 — Verify cloud-init completed (boot-finished flag) + Step 10 — Remove seed ISO from VMX (prevent re-run on clone) + Final — Print summary + + cloud-init nocloud datasource: + Two files (user-data + meta-data) are written to a staging directory and + burned into a small ISO labelled "cidata". On first boot, cloud-init reads + both files and creates the $CloudUser account with the SSH public key. + No packages are installed and no runcmd runs: all provisioning (apt, + toolchain, SSH hardening, swap) is handled by Prepare-LinuxBuild2404.ps1 + via Setup-LinuxBuild2404.sh. Step 9 waits for + /var/lib/cloud/instance/boot-finished before proceeding. + + NIC: + vmxnet3 + NAT. Ubuntu 24.04 cloud images ship with open-vm-tools which + provides the guest IP reporting used in Step 7. + + Disk: + Official Ubuntu 24.04 Server cloud VMDK from cloud-images.ubuntu.com. + Downloaded and cached to $VmdkCachePath on first run. Expanded to + $DiskSizeGB GB in Step 4 using vmware-vdiskmanager. + +.PARAMETER VMXPath + Full path of the .vmx file to create. The folder is created if missing. + +.PARAMETER VMName + Display name in the Workstation library. + +.PARAMETER CloudUser + Username for the cloud-init created account. Default: ci_build. + +.PARAMETER Hostname + Guest hostname embedded in cloud-init meta-data and user-data. + +.PARAMETER Timezone + IANA timezone string for the guest. Default: Europe/Rome. + +.PARAMETER DiskSizeGB + Target disk size in GB after resize. Default: 40. + +.PARAMETER MemoryMB + Guest RAM in MiB. Default: 4096. + +.PARAMETER VCPUCount + Total vCPU count. Default: 4. + +.PARAMETER CoresPerSocket + Cores per socket. Default: 2. + +.PARAMETER SnapshotName + Snapshot name recorded in the summary. The snapshot itself is taken by + Prepare-LinuxBuild2404.ps1. + +.PARAMETER VMwareWorkstationDir + VMware Workstation install directory. + +.PARAMETER SshKeyPath + Path to the SSH private key. The public key is inferred as SshKeyPath.pub. + +.PARAMETER VmdkUrl + URL to download the Ubuntu 24.04 cloud VMDK. + +.PARAMETER VmdkCachePath + Local cache path for the downloaded VMDK. Reused across runs. + +.PARAMETER SeedIsoPath + Path for the cloud-init seed ISO. Default: \cloud-init-seed.iso. + +.PARAMETER SshTimeoutSeconds + Total time in seconds to wait for SSH port 22 to become reachable. Default: 300. + +.PARAMETER SshPollIntervalSeconds + Seconds between SSH port poll attempts. Default: 10. + +.PARAMETER ShowGui + When set, vmrun powers on the VM with the Workstation GUI visible. + Default: headless (nogui) for unattended pipelines. + +.PARAMETER Force + Allow overwriting an existing VM directory. Without this switch, the script + aborts if the VMX destination folder already exists. + +.PARAMETER DiagPassword + If specified, adds a chpasswd block to cloud-init user-data that sets this + plaintext password on the 'ubuntu' account. Allows console login for + diagnostics. Do NOT use in production; remove or omit once the template + is verified. + +.PARAMETER StartFromStep + Skip straight to this step number. All derived paths are still computed; + artifacts from skipped steps must already exist on disk. + Steps: 1=prereqs+vmdk 2=cloud-init files 3=seed ISO 4=vmdk copy+resize + 5=VMX 6=power on 7=guest IP 8=SSH port 9=cloud-init verify + 10=detach seed ISO + +.NOTES + Does not require elevation. Assumes the host has vmrun.exe and + vmware-vdiskmanager.exe in the VMware Workstation directory, and ssh.exe + in PATH (OpenSSH for Windows or Git for Windows). + +.EXAMPLE + .\Deploy-LinuxBuild2404.ps1 + +.EXAMPLE + .\Deploy-LinuxBuild2404.ps1 -StartFromStep 6 +#> +[CmdletBinding()] +param( + [string] $VMXPath = 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx', + [string] $VMName = 'LinuxBuild2404', + [string] $CloudUser = 'ci_build', + [string] $Hostname = 'ci-linux-template', + [string] $Timezone = 'Europe/Rome', + [int] $DiskSizeGB = 40, + [int] $MemoryMB = 4096, + [int] $VCPUCount = 4, + [int] $CoresPerSocket = 2, + [string] $SnapshotName = 'PostInstall-Linux', + [string] $VMwareWorkstationDir = 'C:\Program Files (x86)\VMware\VMware Workstation', + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + [string] $VmdkUrl = 'https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.vmdk', + [string] $VmdkCachePath = 'F:\CI\ISO\noble-server-cloudimg-amd64.vmdk', + [string] $SeedIsoPath = '', + [int] $SshTimeoutSeconds = 300, + [int] $SshPollIntervalSeconds = 10, + [switch] $ShowGui, + [switch] $Force, + [string] $DiagPassword = '', + [ValidateRange(1,10)] + [int] $StartFromStep = 1 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ----------------------------------------------------------------------------- +# Helpers +# ----------------------------------------------------------------------------- + +function Write-Step { + param([string] $Message, [int] $Step = 0) + $skipped = $Step -gt 0 -and $script:StartFromStep -gt $Step + if ($skipped) { + Write-Host "`n=== $Message === [SKIP]" -ForegroundColor DarkGray + } else { + Write-Host "`n=== $Message ===" -ForegroundColor Cyan + } +} + +function Assert-Step { + param( + [Parameter(Mandatory)] [string] $StepName, + [Parameter(Mandatory)] [string] $Description, + [Parameter(Mandatory)] [scriptblock] $Test + ) + $ok = $false + try { $ok = [bool](& $Test) } + catch { throw "[$StepName] Validation threw on '$Description': $_" } + if (-not $ok) { throw "[$StepName] Validation failed: $Description" } + Write-Host " [OK] $Description" -ForegroundColor Green +} + +# IMAPI2FS-based ISO writer. No external deps. Inline C# bridges IStream -> file. +$isoWriterCs = @" +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.ComTypes; +public class IsoStreamWriter2 { + [DllImport("shlwapi.dll", CharSet=CharSet.Unicode, ExactSpelling=true)] + static extern int SHCreateStreamOnFileEx(string pszFile, uint grfMode, uint dwAttributes, bool fCreate, IntPtr pstmTemplate, out IStream ppstm); + + public static IStream OpenFileAsStream(string path) { + IStream s; + int hr = SHCreateStreamOnFileEx(path, 0x20, 0, false, IntPtr.Zero, out s); + if (hr != 0) throw new Exception("SHCreateStreamOnFileEx 0x" + hr.ToString("X")); + return s; + } + + public static void Save(object streamObj, string path) { + IStream istream = streamObj as IStream; + if (istream == null) { + IntPtr unk = Marshal.GetIUnknownForObject(streamObj); + try { + Guid iid = new Guid("0000000C-0000-0000-C000-000000000046"); + IntPtr pStream; + int hr = Marshal.QueryInterface(unk, ref iid, out pStream); + if (hr != 0) throw new InvalidCastException("QueryInterface(IStream) failed: 0x" + hr.ToString("X")); + try { + istream = (IStream)Marshal.GetObjectForIUnknown(pStream); + } finally { + Marshal.Release(pStream); + } + } finally { + Marshal.Release(unk); + } + } + using (FileStream fs = File.Open(path, FileMode.Create, FileAccess.Write)) { + byte[] buf = new byte[2048 * 16]; + IntPtr pcb = Marshal.AllocHGlobal(8); + try { + int n; + do { + istream.Read(buf, buf.Length, pcb); + n = Marshal.ReadInt32(pcb); + if (n > 0) fs.Write(buf, 0, n); + } while (n > 0); + } finally { + Marshal.FreeHGlobal(pcb); + } + } + } +} +"@ +if (-not ([System.Management.Automation.PSTypeName]'IsoStreamWriter2').Type) { + Add-Type -TypeDefinition $isoWriterCs -Language CSharp +} + +function New-IsoFromFolder { + param( + [Parameter(Mandatory)] [string] $SourceFolder, + [Parameter(Mandatory)] [string] $IsoPath, + [Parameter(Mandatory)] [string] $VolumeLabel + ) + if (Test-Path $IsoPath) { Remove-Item $IsoPath -Force } + $fsi = New-Object -ComObject IMAPI2FS.MsftFileSystemImage + $result = $null + $stream = $null + try { + # FileSystemsToCreate = 3 = ISO9660 + Joliet (no UDF). + # With UDF present (=7), Linux blkid reads the UDF superblock first; + # IMAPI2FS leaves the UDF volume label empty, so blkid reports no label + # and cloud-init's nocloud detector never finds the 'cidata' device. + # ISO9660+Joliet only: blkid reads the ISO9660 PVD label ('CIDATA' after + # ISO9660 uppercasing) which cloud-init accepts. + # ChooseImageDefaultsForMediaType is intentionally omitted -- it overrides + # FileSystemsToCreate and re-enables UDF on some IMAPI versions. + $fsi.FileSystemsToCreate = 3 # ISO9660 | Joliet + $fsi.VolumeName = $VolumeLabel + $fsi.Root.AddTree($SourceFolder, $false) + $result = $fsi.CreateResultImage() + $stream = $result.ImageStream + [IsoStreamWriter2]::Save($stream, $IsoPath) + } finally { + # Release in reverse order. ImageStream holds file handles to source + # files added via AddTree -- must be released before the source folder + # can be deleted. + if ($stream) { [System.Runtime.InteropServices.Marshal]::FinalReleaseComObject($stream) | Out-Null } + if ($result) { [System.Runtime.InteropServices.Marshal]::FinalReleaseComObject($result) | Out-Null } + [System.Runtime.InteropServices.Marshal]::FinalReleaseComObject($fsi) | Out-Null + [GC]::Collect(); [GC]::WaitForPendingFinalizers(); [GC]::Collect() + } +} + +function Invoke-Vmrun { + param([Parameter(Mandatory)] [string[]] $Arguments, + [switch] $IgnoreErrors, + [switch] $Async) + $vmrun = Join-Path $VMwareWorkstationDir 'vmrun.exe' + if ($Async) { + # vmrun start can block indefinitely on some Workstation versions; fire async. + Start-Process -FilePath $vmrun -ArgumentList $Arguments -NoNewWindow + return + } + $stdout = & $vmrun @Arguments 2>&1 + $code = $LASTEXITCODE + if ($code -ne 0 -and -not $IgnoreErrors) { + throw "vmrun $($Arguments -join ' ') failed (exit $code): $stdout" + } + return $stdout +} + +function Set-VmxKey { + param( + [Parameter(Mandatory)] [string] $VmxPath, + [Parameter(Mandatory)] [string] $Key, + [Parameter(Mandatory)] [string] $Value + ) + $lines = Get-Content -LiteralPath $VmxPath + $pattern = "^\s*$([regex]::Escape($Key))\s*=" + $found = $false + $newLines = foreach ($line in $lines) { + if ($line -match $pattern) { $found = $true; "$Key = `"$Value`"" } + else { $line } + } + if (-not $found) { $newLines = @($newLines) + "$Key = `"$Value`"" } + Set-Content -LiteralPath $VmxPath -Value $newLines -Encoding ASCII +} + +function Test-VmInList { + param([Parameter(Mandatory)] [string] $VmxPath) + $list = Invoke-Vmrun -Arguments @('-T','ws','list') -IgnoreErrors + $forward = $VmxPath -replace '\\', '/' + $back = $VmxPath -replace '/', '\' + return ($list -match [regex]::Escape($forward)) -or ($list -match [regex]::Escape($back)) +} + +# ----------------------------------------------------------------------------- +# Derived paths -- always computed regardless of -StartFromStep +# ----------------------------------------------------------------------------- +$vmrunExe = Join-Path $VMwareWorkstationDir 'vmrun.exe' +$vdiskMgr = Join-Path $VMwareWorkstationDir 'vmware-vdiskmanager.exe' +$vmDir = Split-Path -Parent $VMXPath +if ([string]::IsNullOrWhiteSpace($SeedIsoPath)) { + $SeedIsoPath = Join-Path $vmDir 'cloud-init-seed.iso' +} +$stagingDir = Join-Path $vmDir '_cloud_init_staging' +$destVmdk = Join-Path $vmDir 'LinuxBuild2404.vmdk' +$sshPubKeyPath = $SshKeyPath + '.pub' +$guestIP = $null # set in Step 7; pre-init for strict-mode safety +# guestinfo base64 payloads -- populated in Step 2, consumed in Step 5. +# Initialized here so Set-StrictMode does not throw when Step 2 is skipped. +$guestUserDataB64 = '' +$guestMetaDataB64 = '' + +if ($StartFromStep -gt 1) { + Write-Host " [SKIP] Steps 1..$($StartFromStep - 1) -- starting from Step $StartFromStep" -ForegroundColor Yellow +} + +# ============================================================================= +# Step 1: validate prerequisites + download cloud VMDK +# ============================================================================= +Write-Step 'Step 1: validate prerequisites + download cloud VMDK' -Step 1 +if ($StartFromStep -le 1) { + +Assert-Step 'PreFlight' "vmrun.exe found at $vmrunExe" { Test-Path $vmrunExe } +Assert-Step 'PreFlight' "vmware-vdiskmanager.exe found at $vdiskMgr" { Test-Path $vdiskMgr } +Assert-Step 'PreFlight' 'ssh.exe found in PATH' { + $null -ne (Get-Command ssh -ErrorAction SilentlyContinue) +} +Assert-Step 'PreFlight' "SSH private key exists: $SshKeyPath" { Test-Path $SshKeyPath } +Assert-Step 'PreFlight' "SSH public key exists: $sshPubKeyPath" { Test-Path $sshPubKeyPath } + +if (Test-Path $vmDir) { + if (-not $Force) { + throw "[PreFlight] VM directory already exists: $vmDir`n Use -Force to overwrite." + } + Write-Host " [WARN] -Force set -- existing VM directory will be reused: $vmDir" -ForegroundColor Yellow +} else { + New-Item -ItemType Directory -Path $vmDir -Force | Out-Null +} +Assert-Step 'PreFlight' "VM directory ready: $vmDir" { Test-Path $vmDir } + +# Step 1b -- Download cloud VMDK +if (-not (Test-Path $VmdkCachePath)) { + Write-Host "[Deploy] VMDK not found, downloading from $VmdkUrl ..." + New-Item -ItemType Directory -Force (Split-Path $VmdkCachePath) | Out-Null + $tmp = "$VmdkCachePath.part" + try { + Invoke-WebRequest -Uri $VmdkUrl -OutFile $tmp -UseBasicParsing + Move-Item $tmp $VmdkCachePath + Write-Host "[Deploy] Download complete: $VmdkCachePath" + } catch { + Remove-Item $tmp -ErrorAction SilentlyContinue + throw "[Deploy] VMDK download failed: $_" + } +} else { + Write-Host "[Deploy] VMDK found in cache: $VmdkCachePath" +} +Assert-Step 'PreFlight' "Cloud VMDK cache present: $VmdkCachePath" { Test-Path $VmdkCachePath } + +} # end Step 1 + +# ============================================================================= +# Step 2: generate cloud-init user-data + meta-data +# ============================================================================= +Write-Step 'Step 2: generate cloud-init user-data + meta-data' -Step 2 +if ($StartFromStep -le 2) { + +$pubKeyContent = (Get-Content -LiteralPath $sshPubKeyPath -Raw).Trim() + +if (Test-Path $stagingDir) { Remove-Item $stagingDir -Recurse -Force } +New-Item -ItemType Directory -Path $stagingDir -Force | Out-Null + +$metaData = @" +instance-id: linuxbuild-001 +local-hostname: $Hostname +"@ + +$userData = @" +#cloud-config +hostname: $Hostname +users: + - name: $CloudUser + shell: /bin/bash + sudo: 'ALL=(ALL) NOPASSWD:ALL' + ssh_authorized_keys: + - $pubKeyContent +"@ + +# Append chpasswd block when DiagPassword is provided (diagnostic use only). +if ($DiagPassword -ne '') { + Write-Host ' [WARN] -DiagPassword set: console login enabled for ubuntu user (diagnostic mode)' -ForegroundColor Yellow + $userData += "`nchpasswd:`n list: |`n ubuntu:$DiagPassword`n expire: false`n" +} + +$metaDataPath = Join-Path $stagingDir 'meta-data' +$userDataPath = Join-Path $stagingDir 'user-data' +# PS 5.1 Set-Content -Encoding UTF8 writes a UTF-8 BOM. cloud-init checks that +# user-data starts with exactly "#cloud-config"; a BOM prefix breaks this check +# and the config is silently ignored. Write without BOM and with LF line endings. +$utf8NoBom = [System.Text.UTF8Encoding]::new($false) +[System.IO.File]::WriteAllText($metaDataPath, ($metaData -replace "`r`n","`n"), $utf8NoBom) +[System.IO.File]::WriteAllText($userDataPath, ($userData -replace "`r`n","`n"), $utf8NoBom) + +# Pre-encode as base64 for guestinfo (used in Step 5 as backup datasource). +$utf8NoBomEnc = [System.Text.UTF8Encoding]::new($false) +$script:guestMetaDataB64 = [Convert]::ToBase64String($utf8NoBomEnc.GetBytes(($metaData -replace "`r`n","`n"))) +$script:guestUserDataB64 = [Convert]::ToBase64String($utf8NoBomEnc.GetBytes(($userData -replace "`r`n","`n"))) + +Assert-Step 'CloudInit' "meta-data written: $metaDataPath" { Test-Path $metaDataPath } +Assert-Step 'CloudInit' "user-data written: $userDataPath" { Test-Path $userDataPath } +Assert-Step 'CloudInit' 'user-data contains SSH public key' { + $snippet = $pubKeyContent.Substring(0, [Math]::Min(30, $pubKeyContent.Length)) + (Get-Content -LiteralPath $userDataPath -Raw) -match [regex]::Escape($snippet) +} + +} # end Step 2 + +# ============================================================================= +# Step 3: build cloud-init seed ISO (nocloud, volume label: cidata) +# ============================================================================= +Write-Step 'Step 3: build cloud-init seed ISO (nocloud, label: cidata)' -Step 3 +if ($StartFromStep -le 3) { + +New-IsoFromFolder -SourceFolder $stagingDir -IsoPath $SeedIsoPath -VolumeLabel 'cidata' +Assert-Step 'SeedIso' "Seed ISO created: $SeedIsoPath" { Test-Path $SeedIsoPath } +Assert-Step 'SeedIso' 'Seed ISO non-empty (> 4 KB)' { (Get-Item $SeedIsoPath).Length -gt 4096 } + +} # end Step 3 + +# ============================================================================= +# Step 4: copy + resize cloud VMDK +# ============================================================================= +Write-Step "Step 4: copy + resize cloud VMDK to ${DiskSizeGB} GB" -Step 4 +if ($StartFromStep -le 4) { + +Write-Host " Converting $VmdkCachePath -> $destVmdk (streamOptimized -> monolithicSparse) ..." +& $vdiskMgr -r $VmdkCachePath -t 0 $destVmdk +if ($LASTEXITCODE -ne 0) { throw "[Deploy Step 4] vmware-vdiskmanager convert failed (exit $LASTEXITCODE)" } +Assert-Step 'Vmdk' 'VMDK converted (size > 1 MB)' { (Get-Item $destVmdk).Length -gt 1MB } + +Write-Host " Resizing VMDK to ${DiskSizeGB} GB ..." +& $vdiskMgr -x "${DiskSizeGB}GB" $destVmdk +if ($LASTEXITCODE -ne 0) { throw "[Deploy Step 4] vmware-vdiskmanager resize failed (exit $LASTEXITCODE)" } +Assert-Step 'Vmdk' "VMDK present after resize: $destVmdk" { Test-Path $destVmdk } + +} # end Step 4 + +# ============================================================================= +# Step 5: generate VMX +# ============================================================================= +Write-Step 'Step 5: generate VMX' -Step 5 +if ($StartFromStep -le 5) { + +$vmxLines = @( + '.encoding = "UTF-8"', + 'config.version = "8"', + 'virtualHW.version = "19"', + "displayName = `"$VMName`"", + 'guestOS = "ubuntu-64"', + 'firmware = "efi"', + "memsize = `"$MemoryMB`"", + "numvcpus = `"$VCPUCount`"", + "cpuid.coresPerSocket = `"$CoresPerSocket`"", + 'scsi0.present = "TRUE"', + 'scsi0.virtualDev = "lsilogic"', + 'scsi0:0.present = "TRUE"', + 'scsi0:0.fileName = "LinuxBuild2404.vmdk"', + 'scsi0:0.deviceType = "scsi-hardDisk"', + 'ethernet0.present = "TRUE"', + 'ethernet0.connectionType = "nat"', + 'ethernet0.virtualDev = "vmxnet3"', + 'ethernet0.wakeOnPcktRcv = "FALSE"', + 'ethernet0.addressType = "generated"', + 'ide1:0.present = "TRUE"', + "ide1:0.fileName = `"$SeedIsoPath`"", + 'ide1:0.deviceType = "cdrom-image"', + 'ide1:0.startConnected = "TRUE"', + 'pciBridge0.present = "TRUE"', + 'pciBridge4.present = "TRUE"', + 'pciBridge4.virtualDev = "pcieRootPort"', + 'pciBridge4.functions = "8"', + 'pciBridge5.present = "TRUE"', + 'pciBridge5.virtualDev = "pcieRootPort"', + 'pciBridge5.functions = "8"', + 'pciBridge6.present = "TRUE"', + 'pciBridge6.virtualDev = "pcieRootPort"', + 'pciBridge6.functions = "8"', + 'pciBridge7.present = "TRUE"', + 'pciBridge7.virtualDev = "pcieRootPort"', + 'pciBridge7.functions = "8"', + 'floppy0.present = "FALSE"', + 'usb.present = "FALSE"', + 'sound.present = "FALSE"', + 'tools.syncTime = "TRUE"', + 'uuid.action = "create"' +) +# Belt-and-suspenders: embed cloud-init payload as VMware guestinfo properties. +# cloud-init DataSourceVMware reads these via open-vm-tools (pre-installed on +# the Ubuntu 24.04 cloud image). This ensures provisioning works even if the +# nocloud ISO is not detected by blkid (e.g. label case or filesystem type +# mismatch). guestinfo is only added when the base64 strings were computed in +# Step 2 (i.e. not skipped). +if ($guestUserDataB64 -ne '' -and $guestMetaDataB64 -ne '') { + $vmxLines += "guestinfo.userdata = `"$guestUserDataB64`"" + $vmxLines += 'guestinfo.userdata.encoding = "base64"' + $vmxLines += "guestinfo.metadata = `"$guestMetaDataB64`"" + $vmxLines += 'guestinfo.metadata.encoding = "base64"' + Write-Host " [INFO] guestinfo cloud-init properties embedded in VMX (backup datasource)." +} +Set-Content -LiteralPath $VMXPath -Value $vmxLines -Encoding ASCII +Assert-Step 'Vmx' "VMX written: $VMXPath" { Test-Path $VMXPath } + +} # end Step 5 + +# ============================================================================= +# Step 6: power on VM +# ============================================================================= +Write-Step 'Step 6: power on VM' -Step 6 +if ($StartFromStep -le 6) { + +$guiMode = if ($ShowGui) { 'gui' } else { 'nogui' } +Invoke-Vmrun @('-T','ws','start',$VMXPath,$guiMode) -Async +Write-Host "[Deploy] VM powered on (async)." +Start-Sleep -Seconds 10 +Assert-Step 'PowerOn' 'VM appears in vmrun list' { Test-VmInList -VmxPath $VMXPath } + +} # end Step 6 + +# ============================================================================= +# Step 7: detect guest IP via vmrun getGuestIPAddress (poll, max 5 min) +# ============================================================================= +Write-Step 'Step 7: detect guest IP via vmrun getGuestIPAddress (max 5 min)' -Step 7 +if ($StartFromStep -le 7) { + +$ipDeadline = (Get-Date).AddMinutes(5) +$guestIP = $null +while ((Get-Date) -lt $ipDeadline) { + $ip = & $vmrunExe getGuestIPAddress $VMXPath 2>&1 + if ($ip -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { $guestIP = $ip.Trim(); break } + Write-Host " ... waiting for guest IP (open-vm-tools)..." + Start-Sleep -Seconds 15 +} +if (-not $guestIP) { throw "[Deploy Step 7] Timed out waiting for guest IP." } +Write-Host " [OK] Guest IP: $guestIP" -ForegroundColor Green + +} # end Step 7 + +# ============================================================================= +# Step 8: poll SSH port 22 until ready +# ============================================================================= +Write-Step "Step 8: poll SSH port 22 (max ${SshTimeoutSeconds}s)" -Step 8 +if ($StartFromStep -le 8) { + +if (-not $guestIP) { + throw "[Deploy Step 8] Guest IP not available. Run from Step 7 or earlier to detect it." +} + +$sshDeadline = (Get-Date).AddSeconds($SshTimeoutSeconds) +$sshReady = $false +while ((Get-Date) -lt $sshDeadline) { + $tc = Test-NetConnection -ComputerName $guestIP -Port 22 -WarningAction SilentlyContinue + if ($tc.TcpTestSucceeded) { $sshReady = $true; break } + Write-Host " ... waiting for SSH port 22 on $guestIP ..." + Start-Sleep -Seconds $SshPollIntervalSeconds +} +Assert-Step 'SshPort' "SSH port 22 open on $guestIP" { $sshReady } + +} # end Step 8 + +# ============================================================================= +# Step 9: verify cloud-init completed (boot-finished flag) +# ============================================================================= +Write-Step 'Step 9: verify cloud-init completed (boot-finished flag)' -Step 9 +if ($StartFromStep -le 9) { + +if (-not $guestIP) { + throw "[Deploy Step 9] Guest IP not available. Run from Step 7 or earlier to detect it." +} + +# TCP:22 may open before cloud-init has finished creating ci_build and injecting +# the SSH key. With package_update:true, apt can take 3-5 minutes. +# Poll until SSH as ci_build succeeds AND boot-finished flag is present. +$ciDeadline = (Get-Date).AddSeconds(600) +$ciDone = $false +Write-Host " Polling cloud-init completion (max 600s, package_update may take a few minutes) ..." +while ((Get-Date) -lt $ciDeadline) { + # In PS 5.1 with $ErrorActionPreference=Stop, any ssh stderr output (including + # the benign "Permanently added" warning) throws NativeCommandError before the + # assignment can capture it. Suppressing at the ssh level with LogLevel=ERROR + # eliminates the warning entirely; temporarily lowering EAP to Continue means + # auth failures ("permission denied") during the cloud-init bootstrap phase + # are treated as non-terminating so the loop can keep retrying. + $savedEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + $rawOut = (& ssh -i $SshKeyPath ` + -o StrictHostKeyChecking=no ` + -o UserKnownHostsFile=NUL ` + -o LogLevel=ERROR ` + -o ConnectTimeout=5 ` + -o BatchMode=yes ` + "${CloudUser}@${guestIP}" ` + "test -f /var/lib/cloud/instance/boot-finished && echo OK" 2>&1) + $sshExit = $LASTEXITCODE + $ErrorActionPreference = $savedEap + $result = ($rawOut | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] } | Out-String).Trim() + $errText = ($rawOut | Where-Object { $_ -is [System.Management.Automation.ErrorRecord] } | Out-String).Trim() + if ($result -eq 'OK') { $ciDone = $true; break } + $diagMsg = if ($errText) { $errText } elseif ($result) { $result } else { '(no output)' } + Write-Host " ... not ready yet (ssh exit ${sshExit}: $diagMsg) -- retrying in 15s ..." + Start-Sleep -Seconds 15 +} +if (-not $ciDone) { + throw "[Deploy Step 9] Timed out (600s) waiting for cloud-init to complete." +} +Write-Host " [OK] cloud-init completed." -ForegroundColor Green + +} # end Step 9 + +# ============================================================================= +# Step 10: remove seed ISO from VMX (prevent re-run on clone) +# ============================================================================= +Write-Step 'Step 10: remove seed ISO from VMX (prevent re-run on clone)' -Step 10 +if ($StartFromStep -le 10) { + +Set-VmxKey -VmxPath $VMXPath -Key 'ide1:0.present' -Value 'FALSE' +Set-VmxKey -VmxPath $VMXPath -Key 'ide1:0.startConnected' -Value 'FALSE' + +Assert-Step 'SeedDetach' 'ide1:0.present = FALSE in VMX' { + (Get-Content -LiteralPath $VMXPath) -match '^\s*ide1:0\.present\s*=\s*"FALSE"' +} +Assert-Step 'SeedDetach' 'ide1:0.startConnected = FALSE in VMX' { + (Get-Content -LiteralPath $VMXPath) -match '^\s*ide1:0\.startConnected\s*=\s*"FALSE"' +} + +} # end Step 10 + +# ============================================================================= +# Cleanup staging dir +# ============================================================================= +[GC]::Collect(); [GC]::WaitForPendingFinalizers(); [GC]::Collect() +Remove-Item $stagingDir -Recurse -Force -ErrorAction SilentlyContinue + +# ============================================================================= +# Final summary +# ============================================================================= +Write-Step 'Done' +Write-Host "" +Write-Host "[Deploy] Summary" +Write-Host " VMX: $VMXPath" +Write-Host " Guest IP: $guestIP" +Write-Host " SSH key: $SshKeyPath" +Write-Host " Snapshot name: $SnapshotName (run Prepare-LinuxBuild2404.ps1 next)" +Write-Host "" +Write-Host " SSH: ssh -i $SshKeyPath ${CloudUser}@${guestIP}" +Write-Host "" +Write-Host " Next steps (from template\ dir):" +Write-Host " .\Prepare-LinuxBuild2404.ps1 -VMXPath '$VMXPath' -VMIPAddress $guestIP" +Write-Host "" +Write-Host " VM is powered on. cloud-init provisioning complete." -ForegroundColor Green diff --git a/template/Prepare-LinuxBuild2404.ps1 b/template/Prepare-LinuxBuild2404.ps1 new file mode 100644 index 0000000..d07395b --- /dev/null +++ b/template/Prepare-LinuxBuild2404.ps1 @@ -0,0 +1,436 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + HOST-side script: delivers and runs Setup-LinuxBuild2404.sh inside the Linux template VM. + +.DESCRIPTION + Automates provisioning of the Ubuntu 24.04 template VM from the host machine via SSH/SCP: + 1. Pre-flight validation: ssh/scp in PATH, SSH key present, VMX exists (if supplied), + IPv4 address valid, TCP/22 reachable + 2. Auto power-on + guest IP detection via vmrun getGuestIPAddress (when -VMXPath provided) + 3. Add host key to known_hosts (idempotent, uses StrictHostKeyChecking=accept-new) + 4. Copy Setup-LinuxBuild2404.sh to /tmp on guest via scp + validate file size + 5. Execute Setup-LinuxBuild2404.sh via SSH (stdout/stderr streamed to host console) + 6. Post-setup remote validation batch check (user, sudo, tools, CI dirs, swap, sshd) + 7. Graceful shutdown + vmrun snapshot (when -TakeSnapshot or -VMXPath provided) + Final: print summary + + Every validation step uses Assert-Step: prints [OK] on success, throws on failure. + The snapshot is only taken after this script exits 0. + + PREREQUISITES (do these manually before running this script): + a. The VM was created by Deploy-LinuxBuild2404.ps1 and cloud-init has completed + b. The ci_build SSH public key ($SshKeyPath.pub) is already authorised in the guest + (cloud-init in Deploy-LinuxBuild2404.ps1 injects the public key automatically) + c. The VM NIC is on VMnet8 (NAT) with internet access + + AFTER THIS SCRIPT COMPLETES (exit 0, all validations passed): + If -TakeSnapshot was NOT set and -VMXPath was NOT provided: + 1. Shut down the VM manually + 2. Take a snapshot named '$SnapshotName' in VMware Workstation + 3. Use -SnapshotName '$SnapshotName' when calling New-BuildVM.ps1 + +.PARAMETER VMXPath + Full path to the template VM's .vmx file. + When provided: + - If the VM is powered off, the script starts it automatically via vmrun. + - The guest IP is auto-detected via vmrun getGuestIPAddress (requires open-vm-tools). + - At the end, the VM is shut down and a snapshot is taken automatically. + Either -VMXPath or -VMIPAddress (or both) must be supplied. + +.PARAMETER VMIPAddress + IP address of the template VM on VMnet8 NAT. + Optional when -VMXPath is provided (IP is auto-detected via open-vm-tools). + Required when -VMXPath is omitted. + Example: 192.168.79.131 + +.PARAMETER CloudUser + SSH username in the guest VM. Default: ci_build. + +.PARAMETER SshKeyPath + Path to the SSH private key on the host. Default: F:\CI\keys\ci_linux. + The corresponding public key is expected at $SshKeyPath.pub. + +.PARAMETER SnapshotName + Name of the snapshot to create. Default: BaseClean-Linux. + Must match the snapshot name used in New-BuildVM.ps1. + +.PARAMETER SkipUpdate + Pass --skip-update to Setup-LinuxBuild2404.sh (skips apt-get upgrade). + +.PARAMETER InstallDotNet + Pass --dotnet to Setup-LinuxBuild2404.sh (installs .NET SDK). + +.PARAMETER NoMingw + Pass --no-mingw to Setup-LinuxBuild2404.sh (skips MinGW cross-compiler). + MinGW is installed by default; use this only to skip it. + +.PARAMETER TakeSnapshot + Shut down the VM and take a vmrun snapshot after provisioning completes. + Implied automatically when -VMXPath is provided (snapshot is always taken + when the script manages the VM lifecycle via vmrun). + +.PARAMETER VMwareWorkstationDir + Installation directory of VMware Workstation on the host. + Default: C:\Program Files (x86)\VMware\VMware Workstation + +.EXAMPLE + # Fully automated — VMX path only (power-on + IP detection + snapshot automatic): + .\Prepare-LinuxBuild2404.ps1 -VMXPath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' + + # Explicit IP, skip update (VM already running, no snapshot): + .\Prepare-LinuxBuild2404.ps1 -VMIPAddress '192.168.79.131' -SkipUpdate + + # VMX + .NET SDK + explicit snapshot: + .\Prepare-LinuxBuild2404.ps1 ` + -VMXPath 'F:\CI\Templates\LinuxBuild2404\LinuxBuild2404.vmx' ` + -InstallDotNet -TakeSnapshot + +.NOTES + Requires: + - OpenSSH client on the host (ssh.exe + scp.exe in PATH) + - VMware Workstation with vmrun.exe (only needed when -VMXPath is supplied) + - open-vm-tools installed in the guest (for vmrun getGuestIPAddress) + - The SSH private key at $SshKeyPath with correct permissions +#> +[CmdletBinding()] +param( + # Path to the VM .vmx — enables auto power-on + IP detection + auto snapshot. + # Either -VMXPath or -VMIPAddress (or both) must be supplied. + [string] $VMXPath = '', + + # Guest IP on VMnet8 NAT. Optional when -VMXPath provided (auto-detected via open-vm-tools). + [string] $VMIPAddress = '', + + [string] $CloudUser = 'ci_build', + + [string] $SshKeyPath = 'F:\CI\keys\ci_linux', + + # Snapshot name (case-sensitive — must match the name used in New-BuildVM.ps1) + [string] $SnapshotName = 'BaseClean-Linux', + + [switch] $SkipUpdate, + + [switch] $InstallDotNet, + + [switch] $NoMingw, + + # Shut down VM and take snapshot after provisioning (implied when -VMXPath is provided) + [switch] $TakeSnapshot, + + [string] $VMwareWorkstationDir = 'C:\Program Files (x86)\VMware\VMware Workstation' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# ── Locate vmrun.exe ────────────────────────────────────────────────────────── +$vmrunExe = Join-Path $VMwareWorkstationDir 'vmrun.exe' +if (-not (Test-Path $vmrunExe)) { + # Fallback: try the 64-bit install path or PATH + $alt64 = 'C:\Program Files\VMware\VMware Workstation\vmrun.exe' + if (Test-Path $alt64) { + $vmrunExe = $alt64 + } else { + $vmrunInPath = Get-Command vmrun -ErrorAction SilentlyContinue + if ($vmrunInPath) { + $vmrunExe = $vmrunInPath.Source + } else { + throw "[Prepare] vmrun.exe not found at '$vmrunExe'. Install VMware Workstation or add it to PATH." + } + } +} + +# ── Validate input: need VMXPath or VMIPAddress ─────────────────────────────── +if ($VMXPath -eq '' -and $VMIPAddress -eq '') { + throw "[Prepare] Supply -VMXPath (auto power-on + IP detection) or -VMIPAddress (or both)." +} + +# ── Auto power-on + IP detection (when VMXPath provided) ───────────────────── +if ($VMXPath -ne '') { + if (-not (Test-Path $VMXPath)) { throw "[Prepare] VMX not found: '$VMXPath'" } + + $runningList = @(& $vmrunExe list 2>&1 | + Where-Object { $_ -match '\.vmx$' } | + ForEach-Object { $_.Trim() }) + + if ($runningList -notcontains $VMXPath) { + Write-Host "[Prepare] VM not running — starting: $VMXPath" + & $vmrunExe start $VMXPath nogui + if ($LASTEXITCODE -ne 0) { throw "[Prepare] vmrun start failed (exit $LASTEXITCODE)." } + Write-Host "[Prepare] VM started. Waiting for guest OS to initialize..." + } else { + Write-Host "[Prepare] VM already running: $VMXPath" + } + + if ($VMIPAddress -eq '') { + Write-Host "[Prepare] Detecting guest IP via open-vm-tools (getGuestIPAddress -wait)..." + $detectedIP = (& $vmrunExe getGuestIPAddress $VMXPath -wait 2>&1).Trim() + if ($detectedIP -notmatch '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$') { + throw "[Prepare] Could not get guest IP. Got: '$detectedIP'. Ensure open-vm-tools is running in the guest." + } + $VMIPAddress = $detectedIP + Write-Host "[Prepare] Guest IP: $VMIPAddress" -ForegroundColor Green + } +} + +# ── Helper functions ────────────────────────────────────────────────────────── + +function Assert-Step { + param( + [Parameter(Mandatory)] [string] $StepName, + [Parameter(Mandatory)] [string] $Description, + [Parameter(Mandatory)] [scriptblock] $Test + ) + $ok = $false + try { $ok = [bool](& $Test) } + catch { throw "[Prepare/$StepName] Validation threw on '$Description': $_" } + if (-not $ok) { throw "[Prepare/$StepName] Validation failed: $Description" } + Write-Host " [OK] $Description" -ForegroundColor Green +} + +function Invoke-SshCommand { + param( + [Parameter(Mandatory)] [string] $IP, + [Parameter(Mandatory)] [string] $User, + [Parameter(Mandatory)] [string] $KeyPath, + [Parameter(Mandatory)] [string] $Command, + [switch] $PassThru # return stdout instead of printing + ) + $sshArgs = @( + '-i', $KeyPath, + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=NUL', + '-o', 'LogLevel=ERROR', + '-o', 'ConnectTimeout=10', + '-o', 'BatchMode=yes', + "$User@$IP", + $Command + ) + if ($PassThru) { + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + $out = & ssh @sshArgs 2>&1 + $sshRc = $LASTEXITCODE + $ErrorActionPreference = $savedEap + if ($sshRc -ne 0) { throw "[SSH] Command failed (exit $sshRc): $Command`nOutput: $out" } + return ($out | Where-Object { $_ -isnot [System.Management.Automation.ErrorRecord] }) + } else { + $savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' + & ssh @sshArgs + $sshRc = $LASTEXITCODE + $ErrorActionPreference = $savedEap + if ($sshRc -ne 0) { throw "[SSH] Command failed (exit $sshRc): $Command" } + } +} + +# ── Step 1 — Pre-flight validation ─────────────────────────────────────────── +Write-Host "[Prepare] Step 1 — Pre-flight validation" + +Assert-Step 'PreFlight' 'ssh.exe found in PATH' { + $null -ne (Get-Command ssh -ErrorAction SilentlyContinue) +} +Assert-Step 'PreFlight' 'scp.exe found in PATH' { + $null -ne (Get-Command scp -ErrorAction SilentlyContinue) +} +Assert-Step 'PreFlight' "SSH private key exists: $SshKeyPath" { + Test-Path $SshKeyPath -PathType Leaf +} +Assert-Step 'PreFlight' "SSH public key exists: $SshKeyPath.pub" { + Test-Path "$SshKeyPath.pub" -PathType Leaf +} +if ($VMXPath -ne '') { + Assert-Step 'PreFlight' "VMX file exists: $VMXPath" { + Test-Path $VMXPath -PathType Leaf + } +} + +# ── Step 2 — Validate IP address ───────────────────────────────────────────── +Write-Host "[Prepare] Step 2 — Validate guest IP address" +Assert-Step 'IPValidation' "VMIPAddress '$VMIPAddress' matches IPv4 pattern" { + $VMIPAddress -match '^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$' +} +Write-Host " Guest IP: $VMIPAddress" + +# ── Step 3 — Wait for SSH port 22 ──────────────────────────────────────────── +Write-Host "[Prepare] Step 3 — Waiting for SSH port 22 on $VMIPAddress (timeout 180 s)..." +$sshDeadline = (Get-Date).AddSeconds(180) +$sshReady = $false +while ((Get-Date) -lt $sshDeadline) { + if (Test-NetConnection -ComputerName $VMIPAddress -Port 22 ` + -InformationLevel Quiet -WarningAction SilentlyContinue -ErrorAction SilentlyContinue) { + $sshReady = $true + break + } + Start-Sleep -Seconds 10 + Write-Host " ... waiting for SSH..." +} +Assert-Step 'SSHPort' "$VMIPAddress TCP/22 reachable" { $sshReady } + +# ── Step 4 — Add host key to known_hosts ───────────────────────────────────── +Write-Host "[Prepare] Step 4 — Adding host key to known_hosts (StrictHostKeyChecking=accept-new)..." +$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' +& ssh -i $SshKeyPath -o StrictHostKeyChecking=no -o UserKnownHostsFile=NUL -o LogLevel=ERROR -o ConnectTimeout=10 ` + -o BatchMode=yes "$CloudUser@$VMIPAddress" 'echo connected' 2>&1 | Out-Null +$ErrorActionPreference = $savedEap +if ($LASTEXITCODE -ne 0) { + throw "[Prepare Step 4] SSH connectivity test failed (exit $LASTEXITCODE). Check key, user, and IP." +} +Write-Host " [OK] SSH connectivity confirmed, host key added." + +# ── Step 5 — Copy Setup-LinuxBuild2404.sh into VM ──────────────────────────── +Write-Host "[Prepare] Step 5 — Copying Setup-LinuxBuild2404.sh to guest..." + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$setupScript = Join-Path $scriptDir 'Setup-LinuxBuild2404.sh' +$guestScript = '/tmp/Setup-LinuxBuild2404.sh' + +if (-not (Test-Path $setupScript -PathType Leaf)) { + throw "[Prepare Step 5] Setup-LinuxBuild2404.sh not found alongside this script: $setupScript" +} + +# Strip UTF-8 BOM and normalize CRLF→LF before copying. +# PS 5.1 / VS Code on Windows may save .sh files with BOM or CRLF; both break bash. +$shBytes = [System.IO.File]::ReadAllBytes($setupScript) +if ($shBytes.Length -ge 3 -and $shBytes[0] -eq 0xEF -and $shBytes[1] -eq 0xBB -and $shBytes[2] -eq 0xBF) { + Write-Host " [WARN] UTF-8 BOM detected in Setup-LinuxBuild2404.sh — stripping before copy" + $shBytes = $shBytes[3..($shBytes.Length - 1)] +} +$shText = [System.Text.Encoding]::UTF8.GetString($shBytes) -replace "`r`n", "`n" -replace "`r", "`n" +$tmpScript = Join-Path $env:TEMP 'Setup-LinuxBuild2404-ci.sh' +[System.IO.File]::WriteAllText($tmpScript, $shText, [System.Text.UTF8Encoding]::new($false)) +$setupScript = $tmpScript + +Write-Host " Source: $setupScript (BOM/CRLF-cleaned)" +Write-Host " Dest: $guestScript" + +$savedEap = $ErrorActionPreference; $ErrorActionPreference = 'Continue' +& scp -i $SshKeyPath -o StrictHostKeyChecking=no -o UserKnownHostsFile=NUL -o LogLevel=ERROR ` + $setupScript "${CloudUser}@${VMIPAddress}:${guestScript}" 2>&1 | Out-Null +$scpRc = $LASTEXITCODE +$ErrorActionPreference = $savedEap +if ($scpRc -ne 0) { throw "[Prepare Step 5] scp failed (exit $scpRc)." } + +# Validate: compare local size vs remote size (allow +-2 bytes for LF/CRLF variance) +$localSize = (Get-Item $setupScript).Length +$remoteSizeRaw = (Invoke-SshCommand -IP $VMIPAddress -User $CloudUser -KeyPath $SshKeyPath ` + -Command "stat -c %s $guestScript" -PassThru).Trim() +$remoteSize = [int]$remoteSizeRaw + +Assert-Step 'FileCopy' "guest script size matches local ($localSize bytes, remote $remoteSize bytes, tolerance 2)" { + [Math]::Abs($remoteSize - $localSize) -le 2 +} + +# ── Step 6 — Execute setup script ──────────────────────────────────────────── +Write-Host "[Prepare] Step 6 — Running Setup-LinuxBuild2404.sh in guest (may take several minutes)..." + +$setupFlags = [System.Collections.Generic.List[string]]::new() +if ($SkipUpdate) { $setupFlags.Add('--skip-update') } +if ($InstallDotNet) { $setupFlags.Add('--dotnet') } +if ($NoMingw) { $setupFlags.Add('--no-mingw') } +$flagsStr = $setupFlags -join ' ' +$remoteCmd = "chmod +x $guestScript && sudo $guestScript $flagsStr".TrimEnd() + +Write-Host " Command: $remoteCmd" +Write-Host "------------------------------------------------------------" + +& ssh -i $SshKeyPath -o StrictHostKeyChecking=no -o UserKnownHostsFile=NUL -o ConnectTimeout=5 ` + -o BatchMode=yes "$CloudUser@$VMIPAddress" $remoteCmd +$setupExitCode = $LASTEXITCODE + +Write-Host "------------------------------------------------------------" +if ($setupExitCode -ne 0) { + throw "[Prepare Step 6] Setup-LinuxBuild2404.sh exited $setupExitCode — see output above." +} +Write-Host "[Prepare] Setup script completed (exit 0)." -ForegroundColor Green + +# ── Step 7 — Post-setup remote validation ──────────────────────────────────── +Write-Host "[Prepare] Step 7 — Running post-setup validation..." + +$validationScript = @' +set -e +( id ci_build | grep -q sudo || sudo -l -U ci_build 2>/dev/null | grep -q NOPASSWD ) && echo "UserSudo:OK" || echo "UserSudo:FAIL" +sudo -n true 2>/dev/null && echo "SudoNoPass:OK" || echo "SudoNoPass:FAIL" +for t in gcc g++ clang cmake python3 git 7z; do command -v "$t" > /dev/null && echo "Tool:$t:OK" || echo "Tool:$t:FAIL"; done +test -d /opt/ci/build && echo "Dir:build:OK" || echo "Dir:build:FAIL" +test -d /opt/ci/output && echo "Dir:output:OK" || echo "Dir:output:FAIL" +test -d /opt/ci/scripts && echo "Dir:scripts:OK" || echo "Dir:scripts:FAIL" +swapon --show | grep -q . && echo "Swap:ACTIVE:FAIL" || echo "Swap:off:OK" +systemctl is-active ssh > /dev/null && echo "SSHD:active:OK" || echo "SSHD:inactive:FAIL" +grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config && echo "SSHPwdAuth:disabled:OK" || echo "SSHPwdAuth:enabled:FAIL" +systemctl is-enabled ci-report-ip.service > /dev/null 2>&1 && echo "CIReportIP:enabled:OK" || echo "CIReportIP:enabled:FAIL" +test -x /usr/local/bin/ci-report-ip.sh && echo "CIReportIPScript:exists:OK" || echo "CIReportIPScript:exists:FAIL" +'@ + +$results = Invoke-SshCommand -IP $VMIPAddress -User $CloudUser -KeyPath $SshKeyPath ` + -Command $validationScript -PassThru + +Write-Host "[Prepare] Validation results:" +$results | ForEach-Object { Write-Host " $_" } + +$failures = @($results | Where-Object { $_ -match ':FAIL' }) +if ($failures.Count -gt 0) { + throw "[Prepare Step 7] Post-setup validation failed:`n$($failures -join "`n")" +} +Write-Host "[Prepare] All post-setup checks passed." -ForegroundColor Green + +# ── Step 8 — Shutdown + snapshot ───────────────────────────────────────────── +if ($TakeSnapshot -or $VMXPath -ne '') { + Write-Host "[Prepare] Step 8 — Sending graceful shutdown to guest..." + try { + Invoke-SshCommand -IP $VMIPAddress -User $CloudUser -KeyPath $SshKeyPath ` + -Command 'sudo shutdown -h now' -PassThru | Out-Null + } catch { + Write-Warning "[Prepare] Shutdown command may have failed (VM may have already started shutting down): $_" + } + + if ($VMXPath -ne '') { + Write-Host "[Prepare] Waiting for VM to power off (timeout 120 s)..." + $deadline = (Get-Date).AddSeconds(120) + $poweredOff = $false + while ((Get-Date) -lt $deadline) { + $listOut = & $vmrunExe list 2>&1 | Out-String + if ($listOut -notmatch [regex]::Escape($VMXPath)) { + $poweredOff = $true + break + } + Start-Sleep -Seconds 5 + Write-Host " ... waiting for shutdown..." + } + if (-not $poweredOff) { throw "[Prepare Step 8] VM did not power off within 120 s." } + + # Delete existing snapshot with the same name so vmrun can create a fresh one. + $existingSnaps = (& $vmrunExe listSnapshots $VMXPath 2>&1 | Out-String) + if ($existingSnaps -match [regex]::Escape($SnapshotName)) { + Write-Host "[Prepare] Snapshot '$SnapshotName' already exists — deleting before recreating..." + & $vmrunExe deleteSnapshot $VMXPath $SnapshotName + if ($LASTEXITCODE -ne 0) { throw "[Prepare Step 8] vmrun deleteSnapshot failed (exit $LASTEXITCODE)." } + Write-Host "[Prepare] Old snapshot deleted." + } + + Write-Host "[Prepare] Taking snapshot '$SnapshotName'..." + & $vmrunExe snapshot $VMXPath $SnapshotName + if ($LASTEXITCODE -ne 0) { throw "[Prepare Step 8] vmrun snapshot failed (exit $LASTEXITCODE)." } + + Assert-Step 'Snapshot' "snapshot '$SnapshotName' listed in vmrun listSnapshots" { + $snaps = (& $vmrunExe listSnapshots $VMXPath 2>&1 | Out-String) + $snaps -match [regex]::Escape($SnapshotName) + } + Write-Host "[Prepare] Snapshot '$SnapshotName' created." -ForegroundColor Green + } else { + Write-Host "[Prepare] -VMXPath not provided — snapshot skipped. Shut down and snapshot manually." + } +} else { + Write-Host "[Prepare] -TakeSnapshot not set — leaving VM running. Shut down and snapshot manually when ready." +} + +# ── Final — Print summary ───────────────────────────────────────────────────── +Write-Host "" +Write-Host "[Prepare] Summary" -ForegroundColor Cyan +Write-Host " Guest IP: $VMIPAddress" +Write-Host " SSH key: $SshKeyPath" +if ($VMXPath -ne '') { Write-Host " VMX: $VMXPath" } +if ($TakeSnapshot -or $VMXPath -ne '') { + Write-Host " Snapshot: $SnapshotName — ready for New-BuildVM.ps1 -SnapshotName '$SnapshotName'" +} else { + Write-Host " Snapshot: not taken — run manually then use -SnapshotName '$SnapshotName'" +} diff --git a/template/Setup-LinuxBuild2404.sh b/template/Setup-LinuxBuild2404.sh new file mode 100644 index 0000000..ba5bf1a --- /dev/null +++ b/template/Setup-LinuxBuild2404.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# Setup-LinuxBuild2404.sh +# Guest-side CI toolchain setup for Ubuntu 24.04 LTS build template. +# Run as ci_build (passwordless sudo) via: sudo /tmp/Setup-LinuxBuild2404.sh +# Called by: template/Prepare-LinuxBuild2404.ps1 (host-side) +# +# Usage: +# sudo /tmp/Setup-LinuxBuild2404.sh [--skip-update] [--dotnet] [--mingw] +# +# Options: +# --skip-update Skip apt update + upgrade (faster re-runs) +# --dotnet Install .NET SDK 10.0 (optional, adds ~500 MB) +# --no-mingw Skip mingw-w64 cross-compiler (installed by default) +# +# Exit codes: +# 0 All steps passed, system ready for snapshot +# 1 One or more steps failed (output shows which step and why) + +set -euo pipefail + +# --- Option parsing --- +SKIP_UPDATE=0 +INSTALL_DOTNET=0 +INSTALL_MINGW=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-update) SKIP_UPDATE=1 ;; + --dotnet) INSTALL_DOTNET=1 ;; + --no-mingw) INSTALL_MINGW=0 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac + shift +done + +# --- Assert helper --- +STEP=0 + +assert_step() { + local desc="$1" + shift + STEP=$((STEP + 1)) + if "$@"; then + echo " [OK] [Step ${STEP}] ${desc}" + else + echo " [FAIL] [Step ${STEP}] ${desc}" >&2 + exit 1 + fi +} + +# --- Step 1: apt update + upgrade --- +echo "" +echo "=== Step 1: apt update + upgrade ===" +if [ "${SKIP_UPDATE:-0}" = "0" ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get update -qq + sudo DEBIAN_FRONTEND=noninteractive apt-get upgrade -y -qq + assert_step "apt update + upgrade completed" true +else + echo " [SKIP] --skip-update flag set" +fi + +# --- Step 2: build essentials --- +echo "" +echo "=== Step 2: build essentials ===" +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + build-essential gcc g++ clang \ + make cmake ninja-build \ + pkg-config autoconf automake libtool +assert_step "gcc available" command -v gcc +assert_step "g++ available" command -v g++ +assert_step "clang available" command -v clang +assert_step "cmake available" command -v cmake +assert_step "make available" command -v make + +# --- Step 3: Python --- +echo "" +echo "=== Step 3: Python ===" +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3 python3-pip python3-venv +sudo ln -sf /usr/bin/python3 /usr/local/bin/python +assert_step "python3 available" command -v python3 +assert_step "pip3 available" command -v pip3 + +# --- Step 4: Tier-1 toolchain --- +echo "" +echo "=== Step 4: Tier-1 toolchain (git, 7-zip, curl, wget) ===" +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git p7zip-full curl wget ca-certificates +assert_step "git available" command -v git +assert_step "7z available" command -v 7z +assert_step "curl available" command -v curl + +# --- Step 4b: .NET SDK (optional) --- +echo "" +echo "=== Step 4b: .NET SDK (optional) ===" +if [ "${INSTALL_DOTNET:-0}" = "1" ]; then + curl -fsSL https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh \ + | sudo bash -s -- --channel 10.0 --install-dir /usr/local/dotnet + sudo ln -sf /usr/local/dotnet/dotnet /usr/local/bin/dotnet + assert_step "dotnet available" command -v dotnet +else + echo " [SKIP] --dotnet not specified" +fi + +# --- Step 4c: mingw-w64 cross-compiler (default ON: required by ns7zip Linux build) --- +echo "" +echo "=== Step 4c: mingw-w64 cross-compiler ===" +if [ "$INSTALL_MINGW" = "1" ]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \ + binutils-mingw-w64-i686 binutils-mingw-w64-x86-64 \ + gcc-mingw-w64-i686 gcc-mingw-w64-x86-64 \ + g++-mingw-w64-i686 g++-mingw-w64-x86-64 + assert_step "i686-w64-mingw32-gcc available" command -v i686-w64-mingw32-gcc + assert_step "x86_64-w64-mingw32-gcc available" command -v x86_64-w64-mingw32-gcc + assert_step "i686-w64-mingw32-g++ available" command -v i686-w64-mingw32-g++ + assert_step "x86_64-w64-mingw32-g++ available" command -v x86_64-w64-mingw32-g++ +else + echo " [SKIP] --no-mingw passed, skipping mingw-w64" +fi + +# --- Step 5: CI directories + sudo group --- +echo "" +echo "=== Step 5: CI directories ===" +sudo mkdir -p /opt/ci/{build,output,scripts,cache} +sudo chown -R ci_build:ci_build /opt/ci +# Ensure ci_build is in the sudo group (belt-and-suspenders alongside sudoers.d entry) +sudo usermod -aG sudo ci_build +assert_step "/opt/ci/build exists" test -d /opt/ci/build +assert_step "/opt/ci/output exists" test -d /opt/ci/output +assert_step "/opt/ci/scripts exists" test -d /opt/ci/scripts +assert_step "/opt/ci/cache exists" test -d /opt/ci/cache +assert_step "ci_build owns /opt/ci" test "$(stat -c '%U' /opt/ci)" = "ci_build" + +# --- Step 6: SSH hardening --- +echo "" +echo "=== Step 6: SSH hardening ===" +sudo sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config +sudo sed -i 's/^#*PubkeyAuthentication.*/PubkeyAuthentication yes/' /etc/ssh/sshd_config +sudo systemctl restart ssh +assert_step "PasswordAuthentication no in sshd_config" grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config +assert_step "sshd service active" systemctl is-active ssh + +# --- Step 7: disable swap --- +echo "" +echo "=== Step 7: disable swap ===" +sudo swapoff -a +sudo sed -i '/ swap / s/^/#/' /etc/fstab +assert_step "no active swap" bash -c '[ -z "$(swapon --show)" ]' + +# --- Step 7a: MAC-agnostic netplan for linked clones --- +# Cloud-init generates /etc/netplan/50-cloud-init.yaml with the template's MAC +# pinned via `match: macaddress: ...`. When VMware creates a linked clone with +# a new MAC, that match fails and the clone has no network. We add a second +# netplan file that matches any en* interface by name, so any clone (any MAC) +# gets DHCP. The two files are merged; if 50-cloud-init also matches, fine — +# we just guarantee DHCP works regardless. +echo "" +echo "=== Step 7a: MAC-agnostic netplan for clones ===" +sudo tee /etc/netplan/99-ci-dhcp-allnics.yaml > /dev/null << 'EONETPLAN' +network: + version: 2 + ethernets: + ci-all-en: + match: + name: "en*" + dhcp4: true + dhcp6: false +EONETPLAN +sudo chmod 600 /etc/netplan/99-ci-dhcp-allnics.yaml +sudo chown root:root /etc/netplan/99-ci-dhcp-allnics.yaml +sudo netplan generate +assert_step "99-ci-dhcp-allnics.yaml exists" test -f /etc/netplan/99-ci-dhcp-allnics.yaml + +# --- Step 7b: CI IP reporter via VMware guestinfo --- +# The host reads the guest IP via `vmrun readVariable guestVar ci-ip`. +# IMPORTANT: vmware-rpctool sets 'guestinfo.ci-ip' but vmrun reads it as just +# 'ci-ip' under the guestVar namespace (the 'guestinfo.' prefix is implicit). +# This is the official VMware VMCI channel — works even when TCP/IP is not yet +# fully initialised on the host. Used as primary; getGuestIPAddress is fallback. +echo "" +echo "=== Step 7b: CI IP reporter ===" +sudo tee /usr/local/bin/ci-report-ip.sh > /dev/null << 'EOSCRIPT' +#!/bin/bash +# Writes the VM's primary IPv4 to VMware guestinfo.ci-ip on every boot. +# Called by ci-report-ip.service. +# Retries for up to 60s to tolerate open-vm-tools re-init after UUID change +# (linked clones get a new UUID — vmware-rpctool may fail briefly on first boot). +for i in $(seq 1 30); do + IP=$(ip -4 route get 1.0.0.0 2>/dev/null \ + | awk '{for(i=1;i<=NF;i++) if($i=="src") print $(i+1)}' \ + | head -1) + if [ -n "$IP" ]; then + if /usr/bin/vmware-rpctool "info-set guestinfo.ci-ip $IP" 2>/dev/null; then + echo "ci-report-ip: set guestinfo.ci-ip=$IP (attempt $i)" | systemd-cat -t ci-report-ip + exit 0 + fi + fi + sleep 2 +done +echo "ci-report-ip: failed to set guestinfo.ci-ip after 30 attempts" | systemd-cat -t ci-report-ip +exit 1 +EOSCRIPT +sudo chmod +x /usr/local/bin/ci-report-ip.sh + +sudo tee /etc/systemd/system/ci-report-ip.service > /dev/null << 'EOSERVICE' +[Unit] +Description=Report CI VM IP to VMware host via guestinfo +After=open-vm-tools.service network.target +Wants=open-vm-tools.service + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/ci-report-ip.sh +RemainAfterExit=yes +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target +EOSERVICE + +sudo systemctl daemon-reload +sudo systemctl enable ci-report-ip.service +assert_step "ci-report-ip.service enabled" systemctl is-enabled ci-report-ip.service + +# --- Step 8: disable apt auto-update --- +echo "" +echo "=== Step 8: disable apt auto-update ===" +sudo systemctl mask apt-daily.service apt-daily-upgrade.service \ + apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true +sudo DEBIAN_FRONTEND=noninteractive apt-get remove -y unattended-upgrades 2>/dev/null || true +assert_step "apt-daily.timer masked or inactive" bash -c \ + 'systemctl is-enabled apt-daily.timer 2>/dev/null | grep -qE "masked|disabled" || true' + +# --- Step 9: cleanup pre-snapshot --- +echo "" +echo "=== Step 9: cleanup pre-snapshot ===" +sudo apt-get clean +sudo rm -rf /var/lib/apt/lists/* +sudo journalctl --vacuum-time=1d 2>/dev/null || true +sudo rm -rf /tmp/* /var/tmp/* 2>/dev/null || true +history -c 2>/dev/null || true +cat /dev/null > ~/.bash_history 2>/dev/null || true +assert_step "apt cache cleaned" bash -c '[ -z "$(ls /var/cache/apt/archives/*.deb 2>/dev/null)" ]' + +# --- Final Gate: pre-snapshot validation --- +echo "" +echo "=== Final Gate: pre-snapshot validation ===" + +for tool in gcc g++ clang cmake make git 7z curl python3 pip3; do + assert_step "${tool} in PATH" command -v "${tool}" +done + +for d in /opt/ci/build /opt/ci/output /opt/ci/scripts /opt/ci/cache; do + assert_step "${d} exists" test -d "${d}" +done +assert_step "ci_build owns /opt/ci" test "$(stat -c '%U' /opt/ci)" = "ci_build" + +assert_step "no active swap" bash -c '[ -z "$(swapon --show)" ]' + +assert_step "sshd active" systemctl is-active ssh +assert_step "PasswordAuthentication no" grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config + +assert_step "root fs >= 38 GB" bash -c \ + 'df -BG / | awk '\''NR==2{gsub("G","",$2); if ($2+0 < 38) {print "FAIL: root fs too small ("$2"G)"; exit 1}}'\''' + +assert_step "/tmp writable" bash -c 'touch /tmp/.ci_tmpcheck && rm /tmp/.ci_tmpcheck' +assert_step "ci-report-ip.service enabled" systemctl is-enabled ci-report-ip.service + +echo "" +echo "=== Setup-LinuxBuild2404.sh complete ===" +echo " All steps passed. VM is ready for BaseClean-Linux snapshot." +echo " Run Prepare-LinuxBuild2404.ps1 --Step7Shutdown + snapshot from host."