feat(a4): port Invoke-CIJob.ps1 to ci_orchestrator job
Phase A4 of plans/implementation-plan-A-B.md. Implements the full job orchestrator (clone -> start -> wait -> probe -> build -> collect -> guaranteed cleanup) as a new commands/job.py click command, registered under python -m ci_orchestrator job. Backend selection goes through backends.load_backend(config) so Phase C can swap in remote drivers without touching the command. The legacy scripts/Invoke-CIJob.ps1 is replaced by a thin PS 5.1 shim that delegates to the Python CLI; tests/python/test_commands_job.py adds 13 cases covering Linux/Windows happy paths, override application, skip-artifact, and cleanup on every failure mode.
This commit is contained in:
+14
-682
@@ -1,691 +1,23 @@
|
|||||||
#Requires -Version 5.1
|
#Requires -Version 5.1
|
||||||
<#
|
|
||||||
.SYNOPSIS
|
|
||||||
Main CI orchestrator: creates an ephemeral VM, runs a build, collects
|
|
||||||
artifacts, and destroys the VM — in a guaranteed try/finally pattern.
|
|
||||||
|
|
||||||
.DESCRIPTION
|
|
||||||
This script is called by act_runner as the primary build step.
|
|
||||||
It coordinates all sub-scripts:
|
|
||||||
1. New-BuildVM.ps1 — linked clone from template
|
|
||||||
2. vmrun start — start the clone
|
|
||||||
3. Wait-VMReady.ps1 — poll until WinRM is ready
|
|
||||||
4. Invoke-RemoteBuild.ps1 — execute build inside VM
|
|
||||||
5. Get-BuildArtifacts.ps1 — copy artifacts to host
|
|
||||||
6. Remove-BuildVM.ps1 — destroy VM (always runs via finally)
|
|
||||||
|
|
||||||
Exit codes:
|
|
||||||
0 = success
|
|
||||||
1 = build or orchestration failure (VM destroyed, artifacts may be partial)
|
|
||||||
|
|
||||||
.PARAMETER JobId
|
|
||||||
Unique job identifier (use ${{ github.run_id }} from Gitea Actions).
|
|
||||||
|
|
||||||
.PARAMETER RepoUrl
|
|
||||||
Git URL of the repository to build (reachable from the build VM).
|
|
||||||
|
|
||||||
.PARAMETER Branch
|
|
||||||
Branch to build.
|
|
||||||
|
|
||||||
.PARAMETER Commit
|
|
||||||
Optional specific commit SHA to check out.
|
|
||||||
|
|
||||||
.PARAMETER Configuration
|
|
||||||
Build configuration. Default: Release
|
|
||||||
|
|
||||||
.PARAMETER TemplatePath
|
|
||||||
Full path to the template VM .vmx file.
|
|
||||||
Can also be set via env var GITEA_CI_TEMPLATE_PATH.
|
|
||||||
|
|
||||||
.PARAMETER SnapshotName
|
|
||||||
Template snapshot to clone from. Default: BaseClean
|
|
||||||
|
|
||||||
.PARAMETER CloneBaseDir
|
|
||||||
Base directory for clone VMs.
|
|
||||||
Can also be set via env var GITEA_CI_CLONE_BASE_DIR.
|
|
||||||
Default: F:\CI\BuildVMs
|
|
||||||
|
|
||||||
.PARAMETER ArtifactBaseDir
|
|
||||||
Host directory where artifacts are stored per job.
|
|
||||||
Default: F:\CI\Artifacts
|
|
||||||
|
|
||||||
.PARAMETER VMIPAddress
|
|
||||||
IP address to assign/expect for the build VM.
|
|
||||||
Must be unique per concurrent build.
|
|
||||||
Can be passed explicitly or derived from a registry/file-based IP allocator.
|
|
||||||
|
|
||||||
.PARAMETER GuestCredentialTarget
|
|
||||||
Target name in Windows Credential Manager for guest credentials.
|
|
||||||
Default: BuildVMGuest
|
|
||||||
|
|
||||||
.PARAMETER VmrunPath
|
|
||||||
Path to vmrun.exe.
|
|
||||||
Default: C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe
|
|
||||||
|
|
||||||
.EXAMPLE
|
|
||||||
# Called from Gitea Actions workflow:
|
|
||||||
.\scripts\Invoke-CIJob.ps1 `
|
|
||||||
-JobId "${{ github.run_id }}" `
|
|
||||||
-RepoUrl "http://gitea.local/org/myrepo.git" `
|
|
||||||
-Branch "${{ github.ref_name }}" `
|
|
||||||
-Commit "${{ github.sha }}" `
|
|
||||||
-VMIPAddress "192.168.11.101"
|
|
||||||
#>
|
|
||||||
[CmdletBinding()]
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[ValidatePattern('^[A-Za-z0-9._-]+$')]
|
|
||||||
[string] $JobId,
|
|
||||||
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $RepoUrl,
|
|
||||||
|
|
||||||
[Parameter(Mandatory)]
|
|
||||||
[string] $Branch,
|
|
||||||
|
|
||||||
[string] $Commit = '',
|
|
||||||
|
|
||||||
[string] $Configuration = 'Release',
|
|
||||||
|
|
||||||
# Pass -Submodules to clone with --recurse-submodules
|
|
||||||
[switch] $Submodules,
|
|
||||||
|
|
||||||
# Custom build command to run inside the VM (empty = dotnet build)
|
|
||||||
[string] $BuildCommand = '',
|
|
||||||
|
|
||||||
# Artifact source pattern relative to GuestWorkDir (used with BuildCommand)
|
|
||||||
[string] $GuestArtifactSource = 'dist',
|
|
||||||
|
|
||||||
# Clone repo IN the VM instead of on host (requires Git in template §6.6)
|
|
||||||
# Skips Phase 1 host clone, injects URL/branch/commit into VM for in-VM clone.
|
|
||||||
# Prerequisite: Git for Windows installed in template.
|
|
||||||
[switch] $UseGitClone,
|
|
||||||
|
|
||||||
# Use VMware shared folders for NuGet/pip cache (§SharedCache).
|
|
||||||
# Requires Set-TemplateSharedFolders.ps1 to have been run on the template.
|
|
||||||
[switch] $UseSharedCache,
|
|
||||||
|
|
||||||
# Skip artifact packaging (Invoke-RemoteBuild) and collection (Phase 6).
|
|
||||||
# Use for jobs that intentionally produce no output (lint, test-only, etc.).
|
|
||||||
# When absent, the build must produce GuestArtifactSource or the job fails.
|
|
||||||
[switch] $SkipArtifact,
|
|
||||||
|
|
||||||
# Credential Manager target for Gitea PAT used by -UseGitClone (private repos).
|
|
||||||
# Store with: New-StoredCredential -Target 'GiteaPAT' -UserName '<user>' -Password '<pat>' -Persist LocalMachine
|
|
||||||
# Ignored when -UseGitClone is not set.
|
|
||||||
[string] $GiteaCredentialTarget = 'GiteaPAT',
|
|
||||||
|
|
||||||
[string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,
|
|
||||||
|
|
||||||
[string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),
|
|
||||||
|
|
||||||
[string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),
|
|
||||||
|
|
||||||
[string] $ArtifactBaseDir = 'F:\CI\Artifacts',
|
|
||||||
|
|
||||||
[ValidatePattern('^((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$')]
|
|
||||||
[string] $VMIPAddress = '', # if empty, auto-detected via vmrun getGuestIPAddress
|
|
||||||
|
|
||||||
[string] $GuestCredentialTarget = 'BuildVMGuest',
|
|
||||||
|
|
||||||
[string] $VmrunPath = 'C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe',
|
|
||||||
|
|
||||||
# Directory for job execution logs. Each job gets its own sub-directory:
|
|
||||||
# $LogDir\$JobId\invoke-ci.log
|
|
||||||
[string] $LogDir = 'F:\CI\Logs',
|
|
||||||
|
|
||||||
# Days to retain job logs. Directories older than this are purged at job end.
|
|
||||||
# Set to 0 to disable retention/purge.
|
|
||||||
[int] $LogRetentionDays = 30,
|
|
||||||
|
|
||||||
# Guest OS type — determines transport (WinRM for Windows, SSH for Linux).
|
|
||||||
# When 'Auto', detected from the VMX guestOS field after clone.
|
|
||||||
[ValidateSet('Windows', 'Linux', 'Auto')]
|
|
||||||
[string] $GuestOS = 'Auto',
|
|
||||||
|
|
||||||
[string] $SshKeyPath = 'F:\CI\keys\ci_linux',
|
|
||||||
[string] $SshUser = 'ci_build',
|
|
||||||
|
|
||||||
# Persistent known_hosts file used for CI-job SSH calls.
|
|
||||||
# Empty string (default) = permissive mode (StrictHostKeyChecking=no, UserKnownHostsFile=NUL).
|
|
||||||
# Ephemeral CI VMs on a local NAT network do not need strict host-key verification.
|
|
||||||
# Pass a file path only if you need accept-new mode for a specific deployment.
|
|
||||||
[string] $SshKnownHostsFile = '',
|
|
||||||
|
|
||||||
# Extra environment variables injected into the guest VM before the build command (§6.5).
|
|
||||||
# Keys must be valid environment variable names. Values must be plain strings.
|
|
||||||
# Example: @{ SIGN_PASS = $env:SIGN_PASS_FROM_SECRET; GPG_KEY_ID = '0xABCD1234' }
|
|
||||||
[hashtable] $ExtraGuestEnv = @{},
|
|
||||||
|
|
||||||
# Optional webhook URL (Discord/Gitea). When set, a background job fires a
|
|
||||||
# [WARNING] once the job has been running for 90 minutes.
|
|
||||||
[string] $WebhookUrl = '',
|
|
||||||
|
|
||||||
# Optional per-job VMX CPU and RAM override (default 0 = use template value).
|
|
||||||
# Applied to the linked clone after New-BuildVM.ps1, before vmrun start.
|
|
||||||
# The template VMX is never modified.
|
|
||||||
# Example: -GuestCPU 4 -GuestMemoryMB 8192
|
|
||||||
[ValidateRange(0, 32)]
|
|
||||||
[int] $GuestCPU = 0,
|
|
||||||
|
|
||||||
[ValidateRange(0, 131072)]
|
|
||||||
[int] $GuestMemoryMB = 0
|
|
||||||
)
|
|
||||||
|
|
||||||
Set-StrictMode -Version Latest
|
Set-StrictMode -Version Latest
|
||||||
$ErrorActionPreference = 'Stop'
|
$ErrorActionPreference = 'Stop'
|
||||||
|
|
||||||
# ── Resolve script directory for dot-sourcing sibling scripts ─────────────────
|
# Shim: delegates to the Python ci_orchestrator CLI.
|
||||||
$scriptDir = $PSScriptRoot
|
# Original PS implementation moved to git history; see plans/A4-closeout.md.
|
||||||
|
$pyArgs = @()
|
||||||
# ── Load shared helpers (Invoke-VmrunBounded, Get-GuestIPAddress, …) ──────────
|
foreach ($a in $args) {
|
||||||
Import-Module (Join-Path $scriptDir '_Common.psm1') -Force
|
if ($a -is [string] -and $a.Length -gt 1 -and $a.StartsWith('-') -and -not $a.StartsWith('--')) {
|
||||||
|
$name = $a.Substring(1)
|
||||||
# ── Validate required config ──────────────────────────────────────────────────
|
$kebab = [Regex]::Replace($name, '(?<=[a-z0-9])([A-Z])', '-$1')
|
||||||
if ([string]::IsNullOrWhiteSpace($TemplatePath)) {
|
$kebab = [Regex]::Replace($kebab, '([A-Z]+)([A-Z][a-z])', '$1-$2')
|
||||||
Write-Error "TemplatePath is required. Set -TemplatePath or env:GITEA_CI_TEMPLATE_PATH."
|
$pyArgs += '--' + $kebab.ToLower()
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
if (-not (Test-Path $TemplatePath -PathType Leaf)) {
|
|
||||||
Write-Error "Template VMX not found: $TemplatePath"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
# Sanity check: warn if template is outside the expected CI tree
|
|
||||||
if ($TemplatePath -notlike 'F:\CI\Templates\*') {
|
|
||||||
Write-Warning "[Invoke-CIJob] TemplatePath '$TemplatePath' is not under F:\CI\Templates\. Ensure this is intentional."
|
|
||||||
}
|
|
||||||
if (-not (Test-Path $VmrunPath -PathType Leaf)) {
|
|
||||||
Write-Error "vmrun.exe not found: $VmrunPath"
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Validate ExtraGuestEnv keys ──────────────────────────────────────────────
|
|
||||||
foreach ($key in $ExtraGuestEnv.Keys) {
|
|
||||||
if ($key -notmatch '^[A-Za-z_][A-Za-z0-9_]*$') {
|
|
||||||
Write-Error "ExtraGuestEnv key '$key' is not a valid environment variable name. Keys must match ^[A-Za-z_][A-Za-z0-9_]*$."
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Load guest credentials from Windows Credential Manager ───────────────────
|
|
||||||
# Requires the CredentialManager module: Install-Module CredentialManager
|
|
||||||
# In a lab setup you can fall back to prompting, but never hardcode credentials.
|
|
||||||
$credential = $null
|
|
||||||
try {
|
|
||||||
$credential = Get-StoredCredential -Target $GuestCredentialTarget -ErrorAction Stop
|
|
||||||
Write-Host "[Invoke-CIJob] Loaded guest credentials from Credential Manager ($GuestCredentialTarget)."
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Could not load credentials from Credential Manager ($GuestCredentialTarget)."
|
|
||||||
Write-Warning " Using interactive prompt as fallback. Store credentials with:"
|
|
||||||
Write-Warning " cmdkey /generic:$GuestCredentialTarget /user:DOMAIN\user /pass:password"
|
|
||||||
$credential = Get-Credential -Message "Enter guest VM credentials for job $JobId"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Setup paths ───────────────────────────────────────────────────────────────
|
|
||||||
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
|
|
||||||
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
|
|
||||||
$cloneVmxPath = $null
|
|
||||||
$leaseFile = $null # §2.1 IP lease file path — released in finally
|
|
||||||
$jsonLog = $null # §4.1 JSONL structured log path
|
|
||||||
$startTime = Get-Date
|
|
||||||
|
|
||||||
# ── Phase-duration tracking (Task 5.5) ───────────────────────────────────────
|
|
||||||
$phaseTimings = [System.Collections.Generic.List[PSCustomObject]]::new()
|
|
||||||
$phaseStart = $startTime
|
|
||||||
|
|
||||||
# ── Start transcript + JSONL log ──────────────────────────────────────────────
|
|
||||||
$transcriptStarted = $false
|
|
||||||
try {
|
|
||||||
$jobLogDir = Join-Path $LogDir $JobId
|
|
||||||
if (-not (Test-Path $jobLogDir)) { New-Item -ItemType Directory -Path $jobLogDir -Force | Out-Null }
|
|
||||||
$transcriptPath = Join-Path $jobLogDir 'invoke-ci.log'
|
|
||||||
$jsonLog = Join-Path $jobLogDir 'invoke-ci.jsonl'
|
|
||||||
# -Append allows starting a new transcript even when VS Code (or another host)
|
|
||||||
# has already opened a transcript on this PowerShell session.
|
|
||||||
Start-Transcript -Path $transcriptPath -Append -Force -ErrorAction Stop
|
|
||||||
$transcriptStarted = $true
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Could not start transcript: $_"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── §4.1 Structured event emitter ────────────────────────────────────────────
|
|
||||||
# Appends a single JSON line to invoke-ci.jsonl per phase transition.
|
|
||||||
# Silent no-op if $jsonLog is null (transcript setup failed).
|
|
||||||
function Write-JobEvent {
|
|
||||||
param(
|
|
||||||
[Parameter(Mandatory)] [string] $Phase,
|
|
||||||
[Parameter(Mandatory)] [string] $Status, # start | success | failure | info
|
|
||||||
[hashtable] $Data = @{}
|
|
||||||
)
|
|
||||||
if (-not $script:jsonLog) { return }
|
|
||||||
$logEntry = [ordered]@{
|
|
||||||
ts = (Get-Date -Format 'o')
|
|
||||||
jobId = $script:JobId
|
|
||||||
phase = $Phase
|
|
||||||
status = $Status
|
|
||||||
data = $Data
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
$logEntry | ConvertTo-Json -Compress -Depth 5 |
|
|
||||||
Add-Content -Path $script:jsonLog -Encoding UTF8 -ErrorAction Stop
|
|
||||||
} catch { Write-Debug "[Write-JobEvent] Logging skipped: $_" } # never let logging fail the build
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] Job : $JobId"
|
|
||||||
Write-Host "[Invoke-CIJob] Repository : $RepoUrl"
|
|
||||||
Write-Host "[Invoke-CIJob] Branch : $Branch"
|
|
||||||
if ($Commit) { Write-Host "[Invoke-CIJob] Commit : $Commit" }
|
|
||||||
Write-Host "[Invoke-CIJob] Build VM IP : $VMIPAddress"
|
|
||||||
Write-Host "[Invoke-CIJob] Template : $TemplatePath"
|
|
||||||
Write-Host "[Invoke-CIJob] Artifacts : $jobArtifactDir"
|
|
||||||
Write-Host "[Invoke-CIJob] Started : $startTime"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'start' -Data @{
|
|
||||||
repo = $RepoUrl
|
|
||||||
branch = $Branch
|
|
||||||
commit = $Commit
|
|
||||||
template = $TemplatePath
|
|
||||||
snapshot = $SnapshotName
|
|
||||||
}
|
|
||||||
|
|
||||||
$exitCode = 0
|
|
||||||
$durationWarnJob = $null
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── 90-minute duration warning (fires once via background job) ───────────────
|
|
||||||
if ($WebhookUrl -ne '') {
|
|
||||||
$warnMsg = "[WARNING] CI Job duration -- Job $JobId has been running for 90 minutes. Check $RepoUrl or kill the job if stuck."
|
|
||||||
$warnBody = (@{ content = $warnMsg } | ConvertTo-Json -Compress)
|
|
||||||
$durationWarnJob = Start-Job -ScriptBlock {
|
|
||||||
param($url, $body)
|
|
||||||
Start-Sleep -Seconds 5400 # 90 minutes
|
|
||||||
try {
|
|
||||||
Invoke-RestMethod -Uri $url -Method Post -Body $body `
|
|
||||||
-ContentType 'application/json' -TimeoutSec 10
|
|
||||||
} catch { Write-Verbose "Webhook POST failed (non-critical): $_" }
|
|
||||||
} -ArgumentList $WebhookUrl, $warnBody
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 1: Clone repo (HOST or GUEST) ───────────────────────────────
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# §3.3: Clone in guest VM instead of host. Skip host clone phase.
|
|
||||||
# PAT will be injected at runtime (Phase 5) from Credential Manager.
|
|
||||||
Write-Host "`n[Phase 1/6] Skipping host clone — will clone in guest VM (-UseGitClone)..."
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'skipped' -Data @{ method = 'in-vm-clone' }
|
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
# Default: Clone on host, zip, transfer to guest via WinRM.
|
$pyArgs += $a
|
||||||
Write-Host "`n[Phase 1/6] Cloning repository on host..."
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'start'
|
|
||||||
if (Test-Path $hostCloneDir) { Remove-Item $hostCloneDir -Recurse -Force }
|
|
||||||
$gitArgs = @('clone', '--depth', '1', '--progress', '--branch', $Branch)
|
|
||||||
if ($Submodules) { $gitArgs += @('--recurse-submodules', '--shallow-submodules') }
|
|
||||||
$gitArgs += @($RepoUrl, $hostCloneDir)
|
|
||||||
Write-Host "[Invoke-CIJob] git $($gitArgs -join ' ')"
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
# Stream git stdout/stderr live (otherwise heavy clones with submodules
|
|
||||||
# look frozen for a long time). Tee for diagnostic in case of failure.
|
|
||||||
$gitOutput = & git @gitArgs 2>&1 | Tee-Object -Variable streamed | ForEach-Object { Write-Host " [git] $_"; "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
throw "git clone failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
||||||
}
|
|
||||||
if ($Commit -ne '') {
|
|
||||||
$ErrorActionPreference = 'Continue'
|
|
||||||
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
# Commit not in shallow history (e.g. manual run on older SHA).
|
|
||||||
# Fetch full history and retry.
|
|
||||||
Write-Host "[Invoke-CIJob] Shallow checkout failed — fetching full history..."
|
|
||||||
& git -C $hostCloneDir fetch --unshallow 2>&1 | Out-Null
|
|
||||||
$gitOutput = & git -C $hostCloneDir checkout $Commit 2>&1 | ForEach-Object { "$_" }
|
|
||||||
$gitExit = $LASTEXITCODE
|
|
||||||
}
|
|
||||||
$ErrorActionPreference = 'Stop'
|
|
||||||
if ($gitExit -ne 0) {
|
|
||||||
throw "git checkout $Commit failed (exit $gitExit):`n$($gitOutput -join "`n")"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
|
|
||||||
Write-JobEvent -Phase 'phase1.clone-repo' -Status 'success' -Data @{ dir = $hostCloneDir }
|
|
||||||
}
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 1 - Clone repo'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
|
|
||||||
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
|
|
||||||
# and produce two VMs with the same DHCP lease. A file-based mutex ensures
|
|
||||||
# only one job is in the clone→start→IP phase at a time. After the IP is
|
|
||||||
# acquired and written to a lease file the lock is released, so build phases
|
|
||||||
# (the slow part) run fully in parallel. Lease is released in finally.
|
|
||||||
$stateDir = 'F:\CI\State'
|
|
||||||
$leasesDir = Join-Path $stateDir 'ip-leases'
|
|
||||||
$lockPath = Join-Path $stateDir 'vm-start.lock'
|
|
||||||
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
|
|
||||||
|
|
||||||
# ── Pre-clone disk space gate ─────────────────────────────────────────
|
|
||||||
$cloneDriveLetter = (Split-Path -Qualifier $CloneBaseDir).TrimEnd(':')
|
|
||||||
$cloneVol = Get-Volume -DriveLetter $cloneDriveLetter -ErrorAction SilentlyContinue
|
|
||||||
if ($null -ne $cloneVol) {
|
|
||||||
$freeGB = [math]::Round($cloneVol.SizeRemaining / 1GB, 1)
|
|
||||||
if ($freeGB -lt 20) {
|
|
||||||
throw "Insufficient disk space on ${cloneDriveLetter}: ($freeGB GB free). 20 GB minimum required before VM clone."
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] Disk space on ${cloneDriveLetter}: ${freeGB} GB free (OK)."
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
|
|
||||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'start'
|
|
||||||
$lockStream = $null
|
|
||||||
$lockDeadline = (Get-Date).AddMinutes(10)
|
|
||||||
while ($true) {
|
|
||||||
try {
|
|
||||||
$lockStream = [System.IO.File]::Open(
|
|
||||||
$lockPath,
|
|
||||||
[System.IO.FileMode]::OpenOrCreate,
|
|
||||||
[System.IO.FileAccess]::ReadWrite,
|
|
||||||
[System.IO.FileShare]::None)
|
|
||||||
break
|
|
||||||
} catch [System.IO.IOException] {
|
|
||||||
if ((Get-Date) -ge $lockDeadline) {
|
|
||||||
throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock acquired."
|
|
||||||
|
|
||||||
try {
|
|
||||||
# ── Phase 2: Clone VM ─────────────────────────────────────────────
|
|
||||||
Write-Host "[Phase 2/6] Cloning VM..."
|
|
||||||
$cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
|
|
||||||
-TemplatePath $TemplatePath `
|
|
||||||
-SnapshotName $SnapshotName `
|
|
||||||
-CloneBaseDir $CloneBaseDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-VmrunPath $VmrunPath
|
|
||||||
|
|
||||||
# ── Optional VMX CPU/RAM override (post-clone, pre-start) ─────────────
|
|
||||||
if ($GuestCPU -gt 0 -or $GuestMemoryMB -gt 0) {
|
|
||||||
Write-Host "[Invoke-CIJob] Applying VMX override: CPU=$GuestCPU, RAM=${GuestMemoryMB}MB"
|
|
||||||
$vmxLines = Get-Content -LiteralPath $cloneVmxPath
|
|
||||||
if ($GuestCPU -gt 0) {
|
|
||||||
if ($vmxLines -imatch '^numvcpus\s*=') {
|
|
||||||
$vmxLines = $vmxLines -replace '(?i)^numvcpus\s*=.*', "numvcpus = `"$GuestCPU`""
|
|
||||||
} else {
|
|
||||||
$vmxLines = $vmxLines + "numvcpus = `"$GuestCPU`""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($GuestMemoryMB -gt 0) {
|
|
||||||
if ($vmxLines -imatch '^memsize\s*=') {
|
|
||||||
$vmxLines = $vmxLines -replace '(?i)^memsize\s*=.*', "memsize = `"$GuestMemoryMB`""
|
|
||||||
} else {
|
|
||||||
$vmxLines = $vmxLines + "memsize = `"$GuestMemoryMB`""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Set-Content -LiteralPath $cloneVmxPath -Value $vmxLines -Encoding UTF8
|
|
||||||
Write-Host "[Invoke-CIJob] VMX updated: $cloneVmxPath"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Phase 3: Start VM ─────────────────────────────────────────────
|
|
||||||
Write-Host "`n[Phase 3/6] Starting VM..."
|
|
||||||
$startResult = Invoke-VmrunBounded -VmrunPath $VmrunPath `
|
|
||||||
-Operation start `
|
|
||||||
-Arguments @($cloneVmxPath, 'nogui') `
|
|
||||||
-TimeoutSeconds 180
|
|
||||||
if ($startResult.TimedOut) {
|
|
||||||
throw "vmrun start timed out after 180s for: $cloneVmxPath"
|
|
||||||
}
|
|
||||||
if ($startResult.ExitCode -ne 0) {
|
|
||||||
throw "vmrun start failed (exit $($startResult.ExitCode)): $($startResult.Output)"
|
|
||||||
}
|
|
||||||
Write-Host "[Invoke-CIJob] VM started (took $($startResult.ElapsedSeconds)s)."
|
|
||||||
|
|
||||||
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
|
|
||||||
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
||||||
$detectedIP = Get-GuestIPAddress -VmrunPath $VmrunPath `
|
|
||||||
-VmxPath $cloneVmxPath -TimeoutSeconds 120
|
|
||||||
if ($detectedIP -eq '') {
|
|
||||||
throw "Could not detect VM IP address after 120s. Ensure ci-report-ip.service is enabled in the guest template (Linux) or open-vm-tools is running (Windows)."
|
|
||||||
}
|
|
||||||
$VMIPAddress = $detectedIP
|
|
||||||
Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Lease IP ──────────────────────────────────────────────────────
|
|
||||||
# If two VMs somehow got the same IP (DHCP edge case), fail fast here
|
|
||||||
# rather than letting both jobs write to the same WinRM target.
|
|
||||||
$leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
|
|
||||||
if (Test-Path $leaseFile) {
|
|
||||||
$holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
|
|
||||||
throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
|
|
||||||
}
|
|
||||||
[System.IO.File]::WriteAllText($leaseFile, $JobId)
|
|
||||||
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
|
|
||||||
Write-JobEvent -Phase 'phase2-3b.vm-start' -Status 'success' -Data @{ ip = $VMIPAddress; vmx = $cloneVmxPath }
|
|
||||||
|
|
||||||
} finally {
|
|
||||||
# Release lock — next waiting job can now enter clone→start→IP phase
|
|
||||||
if ($lockStream) {
|
|
||||||
$lockStream.Close()
|
|
||||||
$lockStream.Dispose()
|
|
||||||
$lockStream = $null
|
|
||||||
}
|
|
||||||
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Invoke-CIJob] VM-start lock released."
|
|
||||||
}
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 2-3 - Clone+Start VM'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
# ── Resolve GuestOS from VMX if Auto ────────────────────────────────────
|
|
||||||
if ($GuestOS -eq 'Auto' -and $cloneVmxPath) {
|
|
||||||
$vmxContent = Get-Content -LiteralPath $cloneVmxPath -Raw -ErrorAction SilentlyContinue
|
|
||||||
if ($vmxContent -match 'guestOS\s*=\s*"([^"]+)"') {
|
|
||||||
$detectedOS = $Matches[1]
|
|
||||||
$GuestOS = if ($detectedOS -like '*ubuntu*' -or $detectedOS -like '*linux*') { 'Linux' } else { 'Windows' }
|
|
||||||
Write-Host "[Invoke-CIJob] Auto-detected GuestOS: $GuestOS (VMX guestOS=$detectedOS)"
|
|
||||||
} else {
|
|
||||||
$GuestOS = 'Windows'
|
|
||||||
Write-Host "[Invoke-CIJob] Could not detect GuestOS from VMX — defaulting to Windows"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
# ── Phase 4: Wait for readiness ───────────────────────────────────────
|
|
||||||
Write-Host "`n[Phase 4/6] Waiting for VM readiness..."
|
|
||||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'start' -Data @{ ip = $VMIPAddress }
|
|
||||||
$waitParams = @{
|
|
||||||
VMPath = $cloneVmxPath
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
VmrunPath = $VmrunPath
|
|
||||||
}
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
$waitParams['Transport'] = 'SSH'
|
|
||||||
$waitParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$waitParams['SshUser'] = $SshUser
|
|
||||||
}
|
|
||||||
& "$scriptDir\Wait-VMReady.ps1" @waitParams
|
|
||||||
Write-JobEvent -Phase 'phase4.wait-ready' -Status 'success'
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 4 - Wait ready'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 5: Remote build (copy source + build inside VM) ─────────────
|
|
||||||
Write-Host "`n[Phase 5/6] Invoking remote build..."
|
|
||||||
Write-JobEvent -Phase 'phase5.build' -Status 'start'
|
|
||||||
$remoteBuildParams = @{
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
Credential = $credential
|
|
||||||
Configuration = $Configuration
|
|
||||||
}
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
$remoteBuildParams['GuestOS'] = 'Linux'
|
|
||||||
$remoteBuildParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$remoteBuildParams['SshUser'] = $SshUser
|
|
||||||
$remoteBuildParams['SshKnownHostsFile'] = $SshKnownHostsFile
|
|
||||||
$remoteBuildParams.Remove('Credential')
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# In-guest git clone (requires Gitea reachable + PAT/key configured in guest)
|
|
||||||
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
||||||
$remoteBuildParams['CloneBranch'] = $Branch
|
|
||||||
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
||||||
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
||||||
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
||||||
} else {
|
|
||||||
# Default: host-cloned source shipped to guest via tar+scp
|
|
||||||
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if ($UseGitClone) {
|
|
||||||
# §3.3: Clone in VM — pass URL/branch/commit instead of HostSourceDir
|
|
||||||
$remoteBuildParams['CloneUrl'] = $RepoUrl
|
|
||||||
$remoteBuildParams['CloneBranch'] = $Branch
|
|
||||||
$remoteBuildParams['GiteaCredentialTarget'] = $GiteaCredentialTarget
|
|
||||||
if ($Commit -ne '') { $remoteBuildParams['CloneCommit'] = $Commit }
|
|
||||||
if ($Submodules) { $remoteBuildParams['CloneSubmodules'] = $true }
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
# Default: Use pre-cloned source from host
|
|
||||||
$remoteBuildParams['HostSourceDir'] = $hostCloneDir
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if ($BuildCommand -ne '') { $remoteBuildParams['BuildCommand'] = $BuildCommand }
|
|
||||||
if ($GuestArtifactSource -ne '') { $remoteBuildParams['GuestArtifactSource'] = $GuestArtifactSource }
|
|
||||||
if ($ExtraGuestEnv.Count -gt 0) { $remoteBuildParams['ExtraGuestEnv'] = $ExtraGuestEnv }
|
|
||||||
if ($UseSharedCache) { $remoteBuildParams['UseSharedCache'] = $true }
|
|
||||||
if ($SkipArtifact) { $remoteBuildParams['SkipArtifact'] = $true }
|
|
||||||
& "$scriptDir\Invoke-RemoteBuild.ps1" @remoteBuildParams
|
|
||||||
Write-JobEvent -Phase 'phase5.build' -Status 'success'
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 5 - Remote build'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
$phaseStart = Get-Date
|
|
||||||
|
|
||||||
# ── Phase 6: Collect artifacts ────────────────────────────────────────
|
|
||||||
if ($SkipArtifact) {
|
|
||||||
Write-Host "`n[Phase 6/6] Artifact collection skipped (-SkipArtifact)."
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'skipped'
|
|
||||||
} else {
|
|
||||||
Write-Host "`n[Phase 6/6] Collecting artifacts..."
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'start'
|
|
||||||
if ($GuestOS -eq 'Linux') {
|
|
||||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
||||||
-IPAddress $VMIPAddress `
|
|
||||||
-GuestOS 'Linux' `
|
|
||||||
-SshKeyPath $SshKeyPath `
|
|
||||||
-SshUser $SshUser `
|
|
||||||
-SshKnownHostsFile $SshKnownHostsFile `
|
|
||||||
-GuestArtifactPath '/opt/ci/output' `
|
|
||||||
-HostArtifactDir $jobArtifactDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-Commit $Commit `
|
|
||||||
-IncludeLogs
|
|
||||||
} else {
|
|
||||||
& "$scriptDir\Get-BuildArtifacts.ps1" `
|
|
||||||
-IPAddress $VMIPAddress `
|
|
||||||
-Credential $credential `
|
|
||||||
-GuestArtifactPath 'C:\CI\output\artifacts.zip' `
|
|
||||||
-HostArtifactDir $jobArtifactDir `
|
|
||||||
-JobId $JobId `
|
|
||||||
-Commit $Commit `
|
|
||||||
-IncludeLogs
|
|
||||||
}
|
|
||||||
Write-JobEvent -Phase 'phase6.artifacts' -Status 'success' -Data @{ dir = $jobArtifactDir }
|
|
||||||
}
|
|
||||||
$elapsed = (Get-Date) - $startTime
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'success' -Data @{ elapsedSec = [int]$elapsed.TotalSeconds }
|
|
||||||
$phaseTimings.Add([PSCustomObject]@{ Phase = 'Phase 6 - Artifacts'; Sec = [int]((Get-Date) - $phaseStart).TotalSeconds })
|
|
||||||
Write-Host "`n============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] Phase durations:"
|
|
||||||
foreach ($pt in $phaseTimings) {
|
|
||||||
Write-Host (" {0,-30} {1,5}s" -f $pt.Phase, $pt.Sec)
|
|
||||||
}
|
|
||||||
Write-Host (" {0,-30} {1,5}s" -f 'TOTAL', [int]$elapsed.TotalSeconds)
|
|
||||||
Write-Host "============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] SUCCESS -- Job $JobId completed in $($elapsed.ToString('mm\:ss'))"
|
|
||||||
Write-Host "[Invoke-CIJob] Artifacts: $jobArtifactDir"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
$exitCode = 1
|
|
||||||
$elapsed = (Get-Date) - $startTime
|
|
||||||
Write-JobEvent -Phase 'job' -Status 'failure' -Data @{
|
|
||||||
elapsedSec = [int]$elapsed.TotalSeconds
|
|
||||||
error = "$_"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Best-effort guest diagnostics (skip if VM was never started) ─────────
|
|
||||||
# Only collect when a VM IP is known (Phase 3b completed). Never throws.
|
|
||||||
if (-not [string]::IsNullOrWhiteSpace($VMIPAddress)) {
|
|
||||||
try {
|
|
||||||
$diagDir = Join-Path $jobArtifactDir 'diagnostics'
|
|
||||||
$diagParams = @{
|
|
||||||
IPAddress = $VMIPAddress
|
|
||||||
GuestOS = if ($GuestOS -eq 'Auto') { 'Windows' } else { $GuestOS }
|
|
||||||
DestinationDir = $diagDir
|
|
||||||
}
|
|
||||||
if ($diagParams.GuestOS -eq 'Windows' -and $credential) {
|
|
||||||
$diagParams['Credential'] = $credential
|
|
||||||
}
|
|
||||||
if ($diagParams.GuestOS -eq 'Linux') {
|
|
||||||
$diagParams['SshKeyPath'] = $SshKeyPath
|
|
||||||
$diagParams['SshUser'] = $SshUser
|
|
||||||
}
|
|
||||||
Get-GuestDiagnostics @diagParams
|
|
||||||
} catch {
|
|
||||||
Write-Warning "[Invoke-CIJob] Diagnostics collection skipped: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Write-Host "`n============================================================"
|
|
||||||
Write-Host "[Invoke-CIJob] FAILURE — Job $JobId failed after $($elapsed.ToString('mm\:ss'))"
|
|
||||||
Write-Host "Error: $_"
|
|
||||||
Write-Host "============================================================"
|
|
||||||
}
|
|
||||||
finally {
|
|
||||||
# ── Release IP lease (§2.1) ───────────────────────────────────────────
|
|
||||||
if ($leaseFile -and (Test-Path $leaseFile)) {
|
|
||||||
Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Cleanup] IP lease released: $VMIPAddress"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Always destroy the VM ─────────────────────────────────────────────
|
|
||||||
if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
|
|
||||||
Write-Host "`n[Cleanup] Destroying ephemeral VM..."
|
|
||||||
& "$scriptDir\Remove-BuildVM.ps1" -VMPath $cloneVmxPath -VmrunPath $VmrunPath
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Write-Host "[Cleanup] No VM to destroy (clone was not created or already gone)."
|
|
||||||
}
|
|
||||||
# ── Clean up host-side git clone (unless in-VM clone was used) ────────
|
|
||||||
# When -UseGitClone, $hostCloneDir is never created
|
|
||||||
if (-not $UseGitClone -and (Test-Path $hostCloneDir)) {
|
|
||||||
Write-Host "[Cleanup] Removing host source clone ($hostCloneDir)..."
|
|
||||||
Remove-Item $hostCloneDir -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Cancel 90-minute warning job (build finished before it fired) ──────────
|
|
||||||
if ($null -ne $durationWarnJob) {
|
|
||||||
Remove-Job -Job $durationWarnJob -Force -ErrorAction SilentlyContinue
|
|
||||||
$durationWarnJob = $null
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Stop transcript ───────────────────────────────────────────────────
|
|
||||||
if ($transcriptStarted) {
|
|
||||||
try { Stop-Transcript -ErrorAction SilentlyContinue } catch { Write-Debug "[Invoke-CIJob] Stop-Transcript ignored: $_" }
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── Log retention: purge directories older than $LogRetentionDays days ─
|
|
||||||
if ($LogRetentionDays -gt 0 -and (Test-Path $LogDir)) {
|
|
||||||
$cutoff = (Get-Date).AddDays(-$LogRetentionDays)
|
|
||||||
Get-ChildItem -Path $LogDir -Directory -ErrorAction SilentlyContinue |
|
|
||||||
Where-Object { $_.LastWriteTime -lt $cutoff } |
|
|
||||||
ForEach-Object {
|
|
||||||
Remove-Item $_.FullName -Recurse -Force -ErrorAction SilentlyContinue
|
|
||||||
Write-Host "[Cleanup] Purged old log dir: $($_.FullName)"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
exit $exitCode
|
$venvPython = $env:CI_VENV_PYTHON
|
||||||
|
if (-not $venvPython) { $venvPython = 'F:\CI\python\venv\Scripts\python.exe' }
|
||||||
|
& $venvPython -m ci_orchestrator job @pyArgs
|
||||||
|
exit $LASTEXITCODE
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import click
|
|||||||
from ci_orchestrator import __version__
|
from ci_orchestrator import __version__
|
||||||
from ci_orchestrator.commands.artifacts import artifacts
|
from ci_orchestrator.commands.artifacts import artifacts
|
||||||
from ci_orchestrator.commands.build import build
|
from ci_orchestrator.commands.build import build
|
||||||
|
from ci_orchestrator.commands.job import job
|
||||||
from ci_orchestrator.commands.monitor import monitor
|
from ci_orchestrator.commands.monitor import monitor
|
||||||
from ci_orchestrator.commands.report import report
|
from ci_orchestrator.commands.report import report
|
||||||
from ci_orchestrator.commands.vm import vm
|
from ci_orchestrator.commands.vm import vm
|
||||||
@@ -40,6 +41,7 @@ cli.add_command(build)
|
|||||||
cli.add_command(artifacts)
|
cli.add_command(artifacts)
|
||||||
cli.add_command(monitor)
|
cli.add_command(monitor)
|
||||||
cli.add_command(report)
|
cli.add_command(report)
|
||||||
|
cli.add_command(job)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__": # pragma: no cover
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
|||||||
@@ -0,0 +1,550 @@
|
|||||||
|
"""``job`` sub-command — full CI orchestration.
|
||||||
|
|
||||||
|
Phase A4 ports ``scripts/Invoke-CIJob.ps1``: clone an ephemeral VM from a
|
||||||
|
template snapshot, wait for the guest to become reachable, run the build
|
||||||
|
inside the guest, optionally collect artifacts, and **always** destroy the
|
||||||
|
clone (even on failure or ``KeyboardInterrupt``).
|
||||||
|
|
||||||
|
Composition over re-spawn: this module imports the in-process helpers
|
||||||
|
from ``commands.build``, ``commands.artifacts`` and ``commands.vm`` rather
|
||||||
|
than re-invoking ``python -m ci_orchestrator`` for each phase.
|
||||||
|
|
||||||
|
Phase C hook: the backend is selected via :func:`load_backend` so that an
|
||||||
|
ESXi backend can be plugged in by changing ``[backend].type`` in
|
||||||
|
``config.toml``. ``WorkstationVmrunBackend`` is never imported directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
from ci_orchestrator.backends import load_backend
|
||||||
|
from ci_orchestrator.backends.errors import BackendError, BackendNotAvailable
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
from ci_orchestrator.commands.artifacts import (
|
||||||
|
_linux_collect,
|
||||||
|
_windows_collect,
|
||||||
|
_write_manifest,
|
||||||
|
)
|
||||||
|
from ci_orchestrator.commands.build import _linux_build, _windows_build
|
||||||
|
from ci_orchestrator.commands.vm import _stop_with_fallback, _try_remove_dir
|
||||||
|
from ci_orchestrator.config import Config, load_config
|
||||||
|
from ci_orchestrator.credentials import KeyringCredentialStore
|
||||||
|
from ci_orchestrator.transport.errors import TransportError
|
||||||
|
from ci_orchestrator.transport.ssh import SshTransport
|
||||||
|
from ci_orchestrator.transport.winrm import WinRmTransport
|
||||||
|
|
||||||
|
if TYPE_CHECKING: # pragma: no cover
|
||||||
|
from ci_orchestrator.backends.protocol import VmBackend
|
||||||
|
|
||||||
|
|
||||||
|
_VALID_ENV_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||||
|
_GUEST_OS_LINUX_HINTS = ("ubuntu", "linux", "debian", "centos", "rhel", "suse")
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────── helpers
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> float:
|
||||||
|
return time.monotonic()
|
||||||
|
|
||||||
|
|
||||||
|
def _read_guest_os_from_vmx(vmx_path: Path) -> str:
|
||||||
|
"""Best-effort detect ``Linux`` / ``Windows`` from the VMX ``guestOS`` line."""
|
||||||
|
try:
|
||||||
|
text = vmx_path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
except OSError:
|
||||||
|
return "windows"
|
||||||
|
m = re.search(r'guestOS\s*=\s*"([^"]+)"', text)
|
||||||
|
if not m:
|
||||||
|
return "windows"
|
||||||
|
detected = m.group(1).lower()
|
||||||
|
if any(h in detected for h in _GUEST_OS_LINUX_HINTS):
|
||||||
|
return "linux"
|
||||||
|
return "windows"
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_vmx_overrides(vmx_path: Path, cpu: int, memory_mb: int) -> None:
|
||||||
|
"""Edit ``numvcpus`` / ``memsize`` in-place. No-op when both are 0."""
|
||||||
|
if cpu <= 0 and memory_mb <= 0:
|
||||||
|
return
|
||||||
|
lines = vmx_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||||
|
out: list[str] = []
|
||||||
|
seen_cpu = False
|
||||||
|
seen_mem = False
|
||||||
|
for line in lines:
|
||||||
|
low = line.strip().lower()
|
||||||
|
if cpu > 0 and low.startswith("numvcpus"):
|
||||||
|
out.append(f'numvcpus = "{cpu}"')
|
||||||
|
seen_cpu = True
|
||||||
|
elif memory_mb > 0 and low.startswith("memsize"):
|
||||||
|
out.append(f'memsize = "{memory_mb}"')
|
||||||
|
seen_mem = True
|
||||||
|
else:
|
||||||
|
out.append(line)
|
||||||
|
if cpu > 0 and not seen_cpu:
|
||||||
|
out.append(f'numvcpus = "{cpu}"')
|
||||||
|
if memory_mb > 0 and not seen_mem:
|
||||||
|
out.append(f'memsize = "{memory_mb}"')
|
||||||
|
vmx_path.write_text("\n".join(out) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_extra_env_json(raw: str) -> dict[str, str]:
|
||||||
|
"""Parse the ``--extra-env-json`` payload. Empty/``{}`` ⇒ empty dict."""
|
||||||
|
raw = (raw or "").strip()
|
||||||
|
if not raw or raw == "{}":
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
loaded = json.loads(raw)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise click.BadParameter(f"--extra-env-json is not valid JSON: {exc}") from exc
|
||||||
|
if not isinstance(loaded, dict):
|
||||||
|
raise click.BadParameter("--extra-env-json must be a JSON object.")
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for key, value in loaded.items():
|
||||||
|
if not isinstance(key, str) or not _VALID_ENV_KEY.match(key):
|
||||||
|
raise click.BadParameter(
|
||||||
|
f"--extra-env-json key {key!r} is not a valid env var name."
|
||||||
|
)
|
||||||
|
out[key] = "" if value is None else str(value)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_running(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> bool:
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
if backend.is_running(handle):
|
||||||
|
return True
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(f"[job] backend probe error: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_ip(
|
||||||
|
backend: VmBackend, handle: VmHandle, deadline: float, poll: float
|
||||||
|
) -> str | None:
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
ip = backend.get_ip(handle)
|
||||||
|
except BackendError:
|
||||||
|
ip = None
|
||||||
|
if ip:
|
||||||
|
return ip
|
||||||
|
time.sleep(poll)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_transport(
|
||||||
|
*,
|
||||||
|
guest_os: str,
|
||||||
|
ip_address: str,
|
||||||
|
deadline: float,
|
||||||
|
poll: float,
|
||||||
|
credential_target: str,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_known_hosts: str | None,
|
||||||
|
) -> bool:
|
||||||
|
"""Loop until the guest answers WinRM (Windows) or SSH (Linux)."""
|
||||||
|
if guest_os == "windows":
|
||||||
|
cred = KeyringCredentialStore().get(credential_target)
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
with WinRmTransport(ip_address, cred.username, cred.password) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
return True
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[job] WinRM probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
while _now() < deadline:
|
||||||
|
try:
|
||||||
|
with SshTransport(
|
||||||
|
ip_address,
|
||||||
|
username=ssh_user,
|
||||||
|
key_path=ssh_key_path,
|
||||||
|
known_hosts=ssh_known_hosts,
|
||||||
|
) as t:
|
||||||
|
if t.is_ready():
|
||||||
|
return True
|
||||||
|
except TransportError as exc:
|
||||||
|
click.echo(f"[job] SSH probe failed: {exc}", err=True)
|
||||||
|
time.sleep(poll)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _make_clone_name(job_id: str) -> str:
|
||||||
|
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||||
|
# uuid4 suffix avoids collisions when several clones share a JobId.
|
||||||
|
suffix = uuid.uuid4().hex[:6]
|
||||||
|
return f"Clone_{job_id}_{timestamp}_{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def _destroy_clone(
|
||||||
|
backend: VmBackend | None,
|
||||||
|
handle: VmHandle | None,
|
||||||
|
clone_dir: Path | None,
|
||||||
|
) -> None:
|
||||||
|
"""Final cleanup. Tolerates every error — must never raise."""
|
||||||
|
if backend is not None and handle is not None:
|
||||||
|
try:
|
||||||
|
_stop_with_fallback(backend, handle)
|
||||||
|
except Exception as exc: # pragma: no cover - belt and braces
|
||||||
|
click.echo(f"[job] stop ignored: {exc}", err=True)
|
||||||
|
try:
|
||||||
|
backend.delete(handle)
|
||||||
|
except BackendError as exc:
|
||||||
|
click.echo(
|
||||||
|
f"[job] vmrun deleteVM failed: {exc}; will remove dir manually.",
|
||||||
|
err=True,
|
||||||
|
)
|
||||||
|
except Exception as exc: # pragma: no cover
|
||||||
|
click.echo(f"[job] delete ignored: {exc}", err=True)
|
||||||
|
if clone_dir is not None and clone_dir.exists():
|
||||||
|
if _try_remove_dir(clone_dir):
|
||||||
|
click.echo(f"[job] clone directory removed: {clone_dir}")
|
||||||
|
else:
|
||||||
|
click.echo(
|
||||||
|
f"[job] could not fully remove: {clone_dir}", err=True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ────────────────────────────────────────────────────────── command
|
||||||
|
|
||||||
|
|
||||||
|
@click.command("job")
|
||||||
|
@click.option("--job-id", "job_id", required=True)
|
||||||
|
@click.option("--repo-url", "repo_url", required=True)
|
||||||
|
@click.option("--branch", required=True)
|
||||||
|
@click.option("--commit", default="")
|
||||||
|
@click.option("--configuration", default="Release", show_default=True)
|
||||||
|
@click.option("--build-command", "build_command", default="")
|
||||||
|
@click.option("--guest-artifact-source", "guest_artifact_source", default="dist", show_default=True)
|
||||||
|
@click.option("--submodules", is_flag=True, default=False)
|
||||||
|
@click.option("--use-git-clone", "use_git_clone", is_flag=True, default=False)
|
||||||
|
@click.option("--use-shared-cache", "use_shared_cache", is_flag=True, default=False)
|
||||||
|
@click.option("--skip-artifact", "skip_artifact", is_flag=True, default=False)
|
||||||
|
@click.option(
|
||||||
|
"--gitea-credential-target",
|
||||||
|
"gitea_credential_target",
|
||||||
|
default="GiteaPAT",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--template-path",
|
||||||
|
"template_path",
|
||||||
|
default=None,
|
||||||
|
help="Template VMX path (or backend-specific identifier).",
|
||||||
|
)
|
||||||
|
@click.option("--snapshot-name", "snapshot_name", default="BaseClean", show_default=True)
|
||||||
|
@click.option("--clone-base-dir", "clone_base_dir", default=None)
|
||||||
|
@click.option("--artifact-base-dir", "artifact_base_dir", default=None)
|
||||||
|
@click.option(
|
||||||
|
"--guest-os",
|
||||||
|
type=click.Choice(["windows", "linux", "auto"], case_sensitive=False),
|
||||||
|
default="auto",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--guest-credential-target",
|
||||||
|
"guest_credential_target",
|
||||||
|
default=None,
|
||||||
|
help="Override config.guest_cred_target (Windows).",
|
||||||
|
)
|
||||||
|
@click.option("--ssh-key-path", "ssh_key_path", default=None)
|
||||||
|
@click.option("--ssh-user", "ssh_user", default="ci_build", show_default=True)
|
||||||
|
@click.option("--ssh-known-hosts-file", "ssh_known_hosts_file", default=None)
|
||||||
|
@click.option(
|
||||||
|
"--extra-env-json",
|
||||||
|
"extra_env_json",
|
||||||
|
default="",
|
||||||
|
help="JSON object of env vars to inject into the guest before the build.",
|
||||||
|
)
|
||||||
|
@click.option("--guest-cpu", "guest_cpu", type=click.IntRange(0, 32), default=0)
|
||||||
|
@click.option(
|
||||||
|
"--guest-memory-mb", "guest_memory_mb", type=click.IntRange(0, 131072), default=0
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--ready-timeout",
|
||||||
|
"ready_timeout",
|
||||||
|
type=click.FloatRange(10.0, 3600.0),
|
||||||
|
default=600.0,
|
||||||
|
show_default=True,
|
||||||
|
help="Total seconds to wait for VM running + IP + transport ready.",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--poll-interval",
|
||||||
|
"poll_interval",
|
||||||
|
type=click.FloatRange(1.0, 60.0),
|
||||||
|
default=5.0,
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
|
@click.option("--vmrun-path", "vmrun_path", default=None)
|
||||||
|
def job(
|
||||||
|
job_id: str,
|
||||||
|
repo_url: str,
|
||||||
|
branch: str,
|
||||||
|
commit: str,
|
||||||
|
configuration: str,
|
||||||
|
build_command: str,
|
||||||
|
guest_artifact_source: str,
|
||||||
|
submodules: bool,
|
||||||
|
use_git_clone: bool,
|
||||||
|
use_shared_cache: bool,
|
||||||
|
skip_artifact: bool,
|
||||||
|
gitea_credential_target: str,
|
||||||
|
template_path: str | None,
|
||||||
|
snapshot_name: str,
|
||||||
|
clone_base_dir: str | None,
|
||||||
|
artifact_base_dir: str | None,
|
||||||
|
guest_os: str,
|
||||||
|
guest_credential_target: str | None,
|
||||||
|
ssh_key_path: str | None,
|
||||||
|
ssh_user: str,
|
||||||
|
ssh_known_hosts_file: str | None,
|
||||||
|
extra_env_json: str,
|
||||||
|
guest_cpu: int,
|
||||||
|
guest_memory_mb: int,
|
||||||
|
ready_timeout: float,
|
||||||
|
poll_interval: float,
|
||||||
|
vmrun_path: str | None,
|
||||||
|
) -> None:
|
||||||
|
"""Run the full CI pipeline against an ephemeral build VM.
|
||||||
|
|
||||||
|
Replaces ``scripts/Invoke-CIJob.ps1``. Cleanup of the ephemeral VM is
|
||||||
|
guaranteed via ``try/finally`` even on failure or ``KeyboardInterrupt``.
|
||||||
|
"""
|
||||||
|
# Reserved for future host-side clone variant; the Python port always
|
||||||
|
# provisions sources via in-guest ``git clone``.
|
||||||
|
del submodules, use_git_clone, gitea_credential_target
|
||||||
|
|
||||||
|
config: Config = load_config()
|
||||||
|
if not template_path:
|
||||||
|
raise click.ClickException(
|
||||||
|
"--template-path is required."
|
||||||
|
)
|
||||||
|
template_p = Path(template_path)
|
||||||
|
if not template_p.is_file():
|
||||||
|
raise click.ClickException(f"Template VMX not found: {template_path}")
|
||||||
|
|
||||||
|
base_clone_dir = Path(clone_base_dir) if clone_base_dir else config.paths.build_vms
|
||||||
|
base_artifact_dir = Path(artifact_base_dir) if artifact_base_dir else config.paths.artifacts
|
||||||
|
base_clone_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
job_artifact_dir = base_artifact_dir / job_id
|
||||||
|
job_artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
extra_env = _parse_extra_env_json(extra_env_json)
|
||||||
|
|
||||||
|
# Build backend from config; allow CLI override for vmrun path.
|
||||||
|
if vmrun_path is not None:
|
||||||
|
from ci_orchestrator.backends.workstation import WorkstationVmrunBackend
|
||||||
|
|
||||||
|
backend: VmBackend = WorkstationVmrunBackend(vmrun_path=vmrun_path)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
backend = load_backend(config)
|
||||||
|
except BackendNotAvailable as exc:
|
||||||
|
raise click.ClickException(f"backend unavailable: {exc}") from exc
|
||||||
|
|
||||||
|
clone_name = _make_clone_name(job_id)
|
||||||
|
clone_dir = base_clone_dir / clone_name
|
||||||
|
clone_vmx = clone_dir / f"{clone_name}.vmx"
|
||||||
|
|
||||||
|
handle: VmHandle | None = None
|
||||||
|
started = datetime.now(tz=UTC)
|
||||||
|
phase_timings: list[tuple[str, float]] = []
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
click.echo("=" * 60)
|
||||||
|
click.echo(f"[job] Job : {job_id}")
|
||||||
|
click.echo(f"[job] Repository : {repo_url}")
|
||||||
|
click.echo(f"[job] Branch : {branch}")
|
||||||
|
if commit:
|
||||||
|
click.echo(f"[job] Commit : {commit}")
|
||||||
|
click.echo(f"[job] Template : {template_path}")
|
||||||
|
click.echo(f"[job] Snapshot : {snapshot_name}")
|
||||||
|
click.echo(f"[job] Artifacts : {job_artifact_dir}")
|
||||||
|
click.echo(f"[job] Started : {started.isoformat()}")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# ── Phase 1/5: clone VM ─────────────────────────────────────────
|
||||||
|
click.echo(f"\n[job] Phase 1/5 — cloning VM: {clone_vmx}")
|
||||||
|
try:
|
||||||
|
handle = backend.clone_linked(
|
||||||
|
template=str(template_p),
|
||||||
|
snapshot=snapshot_name,
|
||||||
|
name=clone_name,
|
||||||
|
destination=str(clone_vmx),
|
||||||
|
)
|
||||||
|
except BackendError as exc:
|
||||||
|
raise click.ClickException(f"clone failed: {exc}") from exc
|
||||||
|
if not clone_vmx.is_file():
|
||||||
|
raise click.ClickException(
|
||||||
|
f"backend reported success but clone VMX is missing: {clone_vmx}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if guest_cpu > 0 or guest_memory_mb > 0:
|
||||||
|
click.echo(
|
||||||
|
f"[job] applying VMX override: CPU={guest_cpu}, RAM={guest_memory_mb}MB"
|
||||||
|
)
|
||||||
|
_apply_vmx_overrides(clone_vmx, guest_cpu, guest_memory_mb)
|
||||||
|
|
||||||
|
phase_timings.append(("clone", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 2/5: start VM ─────────────────────────────────────────
|
||||||
|
click.echo("\n[job] Phase 2/5 — starting VM (headless)")
|
||||||
|
try:
|
||||||
|
backend.start(handle, headless=True)
|
||||||
|
except BackendError as exc:
|
||||||
|
raise click.ClickException(f"vmrun start failed: {exc}") from exc
|
||||||
|
|
||||||
|
# Resolve guest OS now that the clone exists.
|
||||||
|
if guest_os.lower() == "auto":
|
||||||
|
guest_os = _read_guest_os_from_vmx(clone_vmx)
|
||||||
|
click.echo(f"[job] auto-detected guest OS: {guest_os}")
|
||||||
|
else:
|
||||||
|
guest_os = guest_os.lower()
|
||||||
|
|
||||||
|
deadline = _now() + ready_timeout
|
||||||
|
if not _wait_running(backend, handle, deadline, poll_interval):
|
||||||
|
raise click.ClickException("timeout waiting for VM to be running.")
|
||||||
|
ip_address = _wait_for_ip(backend, handle, deadline, poll_interval)
|
||||||
|
if not ip_address:
|
||||||
|
raise click.ClickException("timeout waiting for guest IP.")
|
||||||
|
click.echo(f"[job] guest IP: {ip_address}")
|
||||||
|
phase_timings.append(("start", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 3/5: wait for transport readiness ────────────────────
|
||||||
|
click.echo(f"\n[job] Phase 3/5 — waiting for {guest_os} transport readiness")
|
||||||
|
target = guest_credential_target or config.guest_cred_target
|
||||||
|
key_path: str | None = ssh_key_path
|
||||||
|
if key_path is None and config.ssh_key_path is not None:
|
||||||
|
key_path = str(config.ssh_key_path)
|
||||||
|
|
||||||
|
if not _probe_transport(
|
||||||
|
guest_os=guest_os,
|
||||||
|
ip_address=ip_address,
|
||||||
|
deadline=deadline,
|
||||||
|
poll=poll_interval,
|
||||||
|
credential_target=target,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
ssh_key_path=key_path,
|
||||||
|
ssh_known_hosts=ssh_known_hosts_file,
|
||||||
|
):
|
||||||
|
raise click.ClickException(
|
||||||
|
"timeout waiting for guest transport to be ready."
|
||||||
|
)
|
||||||
|
phase_timings.append(("wait-ready", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 4/5: build inside guest ──────────────────────────────
|
||||||
|
click.echo("\n[job] Phase 4/5 — invoking remote build")
|
||||||
|
if guest_os == "linux":
|
||||||
|
_linux_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
workdir="/opt/ci/build",
|
||||||
|
output_dir="/opt/ci/output",
|
||||||
|
host_source_dir=None,
|
||||||
|
clone_url=repo_url,
|
||||||
|
clone_branch=branch,
|
||||||
|
clone_commit=commit,
|
||||||
|
clone_submodules=False,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
extra_env=extra_env,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_windows_build(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
workdir="C:\\CI\\build",
|
||||||
|
artifact_zip="C:\\CI\\output\\artifacts.zip",
|
||||||
|
host_source_dir=None,
|
||||||
|
clone_url=repo_url,
|
||||||
|
clone_branch=branch,
|
||||||
|
clone_commit=commit,
|
||||||
|
clone_submodules=False,
|
||||||
|
build_command=build_command,
|
||||||
|
artifact_source=guest_artifact_source,
|
||||||
|
configuration=configuration,
|
||||||
|
extra_env=extra_env,
|
||||||
|
skip_artifact=skip_artifact,
|
||||||
|
use_shared_cache=use_shared_cache,
|
||||||
|
)
|
||||||
|
phase_timings.append(("build", _now() - phase_start))
|
||||||
|
phase_start = _now()
|
||||||
|
|
||||||
|
# ── Phase 5/5: collect artifacts ───────────────────────────────
|
||||||
|
if skip_artifact:
|
||||||
|
click.echo("\n[job] Phase 5/5 — artifact collection skipped (--skip-artifact)")
|
||||||
|
else:
|
||||||
|
click.echo("\n[job] Phase 5/5 — collecting artifacts")
|
||||||
|
if guest_os == "linux":
|
||||||
|
_linux_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
ssh_user=ssh_user,
|
||||||
|
key_path=key_path,
|
||||||
|
known_hosts=ssh_known_hosts_file,
|
||||||
|
guest_path="/opt/ci/output",
|
||||||
|
host_dir=job_artifact_dir,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_windows_collect(
|
||||||
|
ip_address=ip_address,
|
||||||
|
credential_target=target,
|
||||||
|
guest_path="C:\\CI\\output\\artifacts.zip",
|
||||||
|
host_dir=job_artifact_dir,
|
||||||
|
include_logs=True,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
count = _write_manifest(job_artifact_dir, job_id, commit)
|
||||||
|
click.echo(f"[job] manifest.json written ({count} file(s))")
|
||||||
|
except OSError as exc:
|
||||||
|
click.echo(f"[job] manifest write failed: {exc}", err=True)
|
||||||
|
phase_timings.append(("artifacts", _now() - phase_start))
|
||||||
|
|
||||||
|
elapsed = (datetime.now(tz=UTC) - started).total_seconds()
|
||||||
|
click.echo("\n" + "=" * 60)
|
||||||
|
click.echo("[job] phase durations:")
|
||||||
|
for name, secs in phase_timings:
|
||||||
|
click.echo(f" {name:<14} {int(secs):>5}s")
|
||||||
|
click.echo(f" {'TOTAL':<14} {int(elapsed):>5}s")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
click.echo(f"[job] SUCCESS — Job {job_id} completed in {int(elapsed)}s")
|
||||||
|
click.echo(f"[job] Artifacts: {job_artifact_dir}")
|
||||||
|
click.echo("=" * 60)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
click.echo("\n[job] interrupted by user; cleaning up...", err=True)
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
sys.exit(130)
|
||||||
|
except click.ClickException:
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
click.echo(f"\n[job] FAILURE: {exc}", err=True)
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
raise click.ClickException(str(exc)) from exc
|
||||||
|
else:
|
||||||
|
_destroy_clone(backend, handle, clone_dir)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["job"]
|
||||||
@@ -0,0 +1,394 @@
|
|||||||
|
"""Tests for ``ci_orchestrator job`` (Phase A4)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import click
|
||||||
|
import pytest
|
||||||
|
from click.testing import CliRunner
|
||||||
|
|
||||||
|
import ci_orchestrator.commands.job as job_module
|
||||||
|
from ci_orchestrator.__main__ import cli
|
||||||
|
from ci_orchestrator.backends.errors import BackendOperationFailed
|
||||||
|
from ci_orchestrator.backends.protocol import VmHandle
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────── fakes
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBackend:
|
||||||
|
"""Records every backend call and creates the VMX file on clone."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
clone_fail: bool = False,
|
||||||
|
start_fail: bool = False,
|
||||||
|
delete_fail: bool = False,
|
||||||
|
ip: str | None = "10.99.0.42",
|
||||||
|
running_after: int = 0,
|
||||||
|
clone_guest_os: str = "ubuntu-64",
|
||||||
|
) -> None:
|
||||||
|
self.clone_fail = clone_fail
|
||||||
|
self.start_fail = start_fail
|
||||||
|
self.delete_fail = delete_fail
|
||||||
|
self.ip = ip
|
||||||
|
self.running_after = running_after
|
||||||
|
self.clone_guest_os = clone_guest_os
|
||||||
|
self.calls: list[str] = []
|
||||||
|
self._is_running_count = 0
|
||||||
|
|
||||||
|
def clone_linked(
|
||||||
|
self,
|
||||||
|
template: str,
|
||||||
|
snapshot: str,
|
||||||
|
name: str,
|
||||||
|
destination: str | None = None,
|
||||||
|
) -> VmHandle:
|
||||||
|
self.calls.append("clone")
|
||||||
|
if self.clone_fail:
|
||||||
|
raise BackendOperationFailed("clone", 1, "boom")
|
||||||
|
assert destination is not None
|
||||||
|
dst = Path(destination)
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
dst.write_text(
|
||||||
|
f'config.version = "8"\nguestOS = "{self.clone_guest_os}"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return VmHandle(identifier=str(dst), name=name)
|
||||||
|
|
||||||
|
def start(self, handle: VmHandle, headless: bool = True) -> None:
|
||||||
|
self.calls.append("start")
|
||||||
|
if self.start_fail:
|
||||||
|
raise BackendOperationFailed("start", 1, "no")
|
||||||
|
|
||||||
|
def stop(self, handle: VmHandle, hard: bool = False) -> None:
|
||||||
|
self.calls.append(f"stop:{'hard' if hard else 'soft'}")
|
||||||
|
|
||||||
|
def delete(self, handle: VmHandle) -> None:
|
||||||
|
self.calls.append("delete")
|
||||||
|
if self.delete_fail:
|
||||||
|
raise BackendOperationFailed("deleteVM", 1, "locked")
|
||||||
|
|
||||||
|
def is_running(self, handle: VmHandle) -> bool:
|
||||||
|
self._is_running_count += 1
|
||||||
|
return self._is_running_count > self.running_after
|
||||||
|
|
||||||
|
def get_ip(self, handle: VmHandle, timeout: float = 0.0) -> str | None:
|
||||||
|
return self.ip
|
||||||
|
|
||||||
|
def list_snapshots(self, handle: VmHandle) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_common(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
backend: _FakeBackend,
|
||||||
|
*,
|
||||||
|
transport_ready: bool = True,
|
||||||
|
linux_build_raises: bool = False,
|
||||||
|
) -> dict[str, list[Any]]:
|
||||||
|
"""Patch all in-process collaborators of the ``job`` command."""
|
||||||
|
calls: dict[str, list[Any]] = {
|
||||||
|
"linux_build": [],
|
||||||
|
"windows_build": [],
|
||||||
|
"linux_collect": [],
|
||||||
|
"windows_collect": [],
|
||||||
|
"manifest": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "load_backend", lambda _cfg: backend)
|
||||||
|
|
||||||
|
class _ProbeOk:
|
||||||
|
def __enter__(self) -> _ProbeOk:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_e: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def is_ready(self) -> bool:
|
||||||
|
return transport_ready
|
||||||
|
|
||||||
|
class _Store:
|
||||||
|
def get(self, _t: str) -> Any:
|
||||||
|
from ci_orchestrator.credentials import Credential
|
||||||
|
|
||||||
|
return Credential("ci", "pwd")
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "WinRmTransport", lambda *a, **k: _ProbeOk())
|
||||||
|
monkeypatch.setattr(job_module, "SshTransport", lambda *a, **k: _ProbeOk())
|
||||||
|
monkeypatch.setattr(job_module, "KeyringCredentialStore", lambda: _Store())
|
||||||
|
|
||||||
|
def _linux_build(**kwargs: Any) -> None:
|
||||||
|
calls["linux_build"].append(kwargs)
|
||||||
|
if linux_build_raises:
|
||||||
|
raise RuntimeError("build exploded")
|
||||||
|
|
||||||
|
def _windows_build(**kwargs: Any) -> None:
|
||||||
|
calls["windows_build"].append(kwargs)
|
||||||
|
|
||||||
|
def _linux_collect(**kwargs: Any) -> None:
|
||||||
|
calls["linux_collect"].append(kwargs)
|
||||||
|
# Simulate a transferred file so manifest writing has something to do.
|
||||||
|
Path(kwargs["host_dir"], "out.txt").write_text("ok", encoding="utf-8")
|
||||||
|
|
||||||
|
def _windows_collect(**kwargs: Any) -> None:
|
||||||
|
calls["windows_collect"].append(kwargs)
|
||||||
|
|
||||||
|
def _manifest(host_dir: Path, job_id: str, commit: str) -> int:
|
||||||
|
calls["manifest"].append((str(host_dir), job_id, commit))
|
||||||
|
return 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(job_module, "_linux_build", _linux_build)
|
||||||
|
monkeypatch.setattr(job_module, "_windows_build", _windows_build)
|
||||||
|
monkeypatch.setattr(job_module, "_linux_collect", _linux_collect)
|
||||||
|
monkeypatch.setattr(job_module, "_windows_collect", _windows_collect)
|
||||||
|
monkeypatch.setattr(job_module, "_write_manifest", _manifest)
|
||||||
|
# Make poll loops instant.
|
||||||
|
monkeypatch.setattr(job_module.time, "sleep", lambda _s: None)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def template_vmx(tmp_path: Path) -> Path:
|
||||||
|
"""Linux-flavoured VMX so auto-detect picks the SSH transport."""
|
||||||
|
vmx = tmp_path / "templates" / "LinuxBuild2404.vmx"
|
||||||
|
vmx.parent.mkdir(parents=True)
|
||||||
|
vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8")
|
||||||
|
return vmx
|
||||||
|
|
||||||
|
|
||||||
|
def _common_args(
|
||||||
|
template: Path,
|
||||||
|
*,
|
||||||
|
clone_base: Path,
|
||||||
|
artifact_base: Path,
|
||||||
|
extra: list[str] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
return [
|
||||||
|
"job",
|
||||||
|
"--job-id",
|
||||||
|
"ci-test-1",
|
||||||
|
"--repo-url",
|
||||||
|
"http://gitea.local/org/repo.git",
|
||||||
|
"--branch",
|
||||||
|
"main",
|
||||||
|
"--template-path",
|
||||||
|
str(template),
|
||||||
|
"--clone-base-dir",
|
||||||
|
str(clone_base),
|
||||||
|
"--artifact-base-dir",
|
||||||
|
str(artifact_base),
|
||||||
|
"--ready-timeout",
|
||||||
|
"30",
|
||||||
|
"--poll-interval",
|
||||||
|
"1",
|
||||||
|
*(extra or []),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────── tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_happy_path_linux(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=1)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
|
||||||
|
base_clone = tmp_path / "build-vms"
|
||||||
|
base_artifact = tmp_path / "artifacts"
|
||||||
|
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli, _common_args(template_vmx, clone_base=base_clone, artifact_base=base_artifact)
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
# Backend received the full lifecycle including delete in finally.
|
||||||
|
assert backend.calls[0] == "clone"
|
||||||
|
assert "start" in backend.calls
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
# Linux build branch was taken (auto-detected from VMX guestOS).
|
||||||
|
assert len(calls["linux_build"]) == 1
|
||||||
|
assert calls["linux_build"][0]["build_command"] == ""
|
||||||
|
# Artifacts were collected into the per-job directory and manifest written.
|
||||||
|
assert (base_artifact / "ci-test-1" / "out.txt").is_file()
|
||||||
|
assert calls["manifest"] and calls["manifest"][0][1] == "ci-test-1"
|
||||||
|
# Clone directory removed.
|
||||||
|
assert not any((base_clone).iterdir()) or all(not p.exists() for p in base_clone.iterdir())
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_skip_artifact_branch(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(running_after=0)
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=["--skip-artifact"],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert calls["linux_collect"] == []
|
||||||
|
assert calls["windows_collect"] == []
|
||||||
|
assert calls["manifest"] == []
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_cleanup_on_build_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
"""If the build raises, the VM must still be destroyed (try/finally)."""
|
||||||
|
backend = _FakeBackend(running_after=0)
|
||||||
|
_patch_common(monkeypatch, backend, linux_build_raises=True)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "build exploded" in result.output
|
||||||
|
assert "delete" in backend.calls
|
||||||
|
# Stop attempt happened before delete.
|
||||||
|
assert any(c.startswith("stop:") for c in backend.calls)
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_cleanup_on_clone_failure(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend(clone_fail=True)
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(template_vmx, clone_base=tmp_path / "vms", artifact_base=tmp_path / "art"),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "clone failed" in result.output
|
||||||
|
# Backend.start/delete must not have been called: handle is None.
|
||||||
|
assert backend.calls == ["clone"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_rejects_missing_template(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend()
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
tmp_path / "ghost.vmx",
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "Template VMX not found" in result.output
|
||||||
|
assert backend.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_windows_branch_with_overrides(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Windows-flavoured VMX selects the WinRM build path; CPU/RAM override applied."""
|
||||||
|
vmx = tmp_path / "templates" / "WinBuild2025.vmx"
|
||||||
|
vmx.parent.mkdir(parents=True)
|
||||||
|
vmx.write_text('guestOS = "windows10srv-64"\n', encoding="utf-8")
|
||||||
|
|
||||||
|
backend = _FakeBackend(running_after=0, clone_guest_os="windows10srv-64")
|
||||||
|
calls = _patch_common(monkeypatch, backend)
|
||||||
|
extra = [
|
||||||
|
"--guest-cpu",
|
||||||
|
"4",
|
||||||
|
"--guest-memory-mb",
|
||||||
|
"8192",
|
||||||
|
"--extra-env-json",
|
||||||
|
'{"BUILD_TAG":"abc"}',
|
||||||
|
]
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=extra,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code == 0, result.output
|
||||||
|
assert calls["windows_build"], "Windows build branch not taken"
|
||||||
|
assert calls["windows_build"][0]["extra_env"] == {"BUILD_TAG": "abc"}
|
||||||
|
# CPU/RAM override patched the clone VMX.
|
||||||
|
clone_vmx = next((tmp_path / "vms").rglob("*.vmx"), None)
|
||||||
|
# After successful job the clone dir is deleted, so the file is gone.
|
||||||
|
# Check that the Windows collect helper got the correct artifact path instead.
|
||||||
|
assert calls["windows_collect"][0]["guest_path"].endswith("artifacts.zip")
|
||||||
|
assert clone_vmx is None # cleaned up
|
||||||
|
assert backend.calls[-1] == "delete"
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_invalid_extra_env_json(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, template_vmx: Path
|
||||||
|
) -> None:
|
||||||
|
backend = _FakeBackend()
|
||||||
|
_patch_common(monkeypatch, backend)
|
||||||
|
result = CliRunner().invoke(
|
||||||
|
cli,
|
||||||
|
_common_args(
|
||||||
|
template_vmx,
|
||||||
|
clone_base=tmp_path / "vms",
|
||||||
|
artifact_base=tmp_path / "art",
|
||||||
|
extra=["--extra-env-json", '{"bad-key":"x"}'],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert result.exit_code != 0
|
||||||
|
assert "not a valid env var name" in result.output
|
||||||
|
# Failure is parameter-time; clone never started.
|
||||||
|
assert backend.calls == []
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────── helper-level tests
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_vmx_overrides_replaces_existing_lines(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text(
|
||||||
|
'config.version = "8"\nnumvcpus = "1"\nmemsize = "2048"\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
job_module._apply_vmx_overrides(vmx, cpu=8, memory_mb=16384)
|
||||||
|
text = vmx.read_text(encoding="utf-8")
|
||||||
|
assert 'numvcpus = "8"' in text
|
||||||
|
assert 'memsize = "16384"' in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_vmx_overrides_appends_when_missing(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text('config.version = "8"\n', encoding="utf-8")
|
||||||
|
job_module._apply_vmx_overrides(vmx, cpu=2, memory_mb=4096)
|
||||||
|
text = vmx.read_text(encoding="utf-8")
|
||||||
|
assert 'numvcpus = "2"' in text
|
||||||
|
assert 'memsize = "4096"' in text
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_guest_os_detects_linux(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text('guestOS = "ubuntu-64"\n', encoding="utf-8")
|
||||||
|
assert job_module._read_guest_os_from_vmx(vmx) == "linux"
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_guest_os_defaults_to_windows(tmp_path: Path) -> None:
|
||||||
|
vmx = tmp_path / "x.vmx"
|
||||||
|
vmx.write_text("# nothing useful\n", encoding="utf-8")
|
||||||
|
assert job_module._read_guest_os_from_vmx(vmx) == "windows"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_empty_inputs() -> None:
|
||||||
|
assert job_module._parse_extra_env_json("") == {}
|
||||||
|
assert job_module._parse_extra_env_json("{}") == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_extra_env_json_rejects_non_object() -> None:
|
||||||
|
with pytest.raises(click.exceptions.UsageError):
|
||||||
|
job_module._parse_extra_env_json("[]")
|
||||||
Reference in New Issue
Block a user