Simone/local-ci-cd-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sprint2): §2.1/2.2/2.3/2.5 — IP lock/lease, scheduled tasks, retention, versioned snapshot

Browse Source 
§2.1 — IP race with capacity>1 (Invoke-CIJob.ps1)
  Phases 2-3b (clone+start+IP) now run inside a file-based exclusive mutex
  (F:\CI\State\vm-start.lock via FileShare.None). Only one job at a time is
  in the VM-start phase, preventing concurrent vmrun starts from racing for
  the same DHCP lease. After IP is confirmed unique it is written to an IP
  lease file (F:\CI\State\ip-leases\<IP>.lease). Lock is released immediately
  so build phases run fully in parallel. Lease released in finally block.
  Collision detection: if a lease file already exists for the detected IP,
  the job fails fast with a clear error rather than silently sharing a WinRM target.

§2.5 — Versioned snapshot (Invoke-CIJob.ps1 + runner/config.yaml)
  $SnapshotName now defaults to GITEA_CI_SNAPSHOT_NAME env var, falling back
  to 'BaseClean'. config.yaml documents the commented-out variable for snapshot
  refresh workflow (BaseClean_<yyyyMMdd> naming convention).

§2.3 — Artifact/log retention (scripts/Invoke-RetentionPolicy.ps1)
  Purges per-job subdirs under F:\CI\Artifacts and F:\CI\Logs older than
  RetentionDays (default 30). Switches to AggressiveRetentionDays (default 7)
  when F: free space drops below MinFreeGB (default 50 GB). Also removes stale
  IP lease files older than 12 hours (defensive cleanup for crashed jobs).
  SupportsShouldProcess (-WhatIf) for dry-run.

§2.2 — Scheduled task registration (scripts/Register-CIScheduledTasks.ps1)
  Registers two tasks under \CI\ task folder as SYSTEM/HighestPrivilege:
    CI-CleanupOrphans  — every 6h + AtStartup, -MaxAgeHours 4
    CI-RetentionPolicy — daily 03:00 (+30min random) + AtStartup (+15min random)
  Idempotent (-Force), SupportsShouldProcess (-WhatIf), validates script paths.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Simone
2026-05-10 18:36:38 +02:00
parent d4e619ddc2
commit bc894a8058
5 changed files with 353 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/Invoke-CIJob.ps1
+87 -25
View File
@@ -97,7 +97,7 @@ param(
[string] $TemplatePath = $env:GITEA_CI_TEMPLATE_PATH,
[string] $SnapshotName = 'BaseClean',
[string] $SnapshotName = $(if ($env:GITEA_CI_SNAPSHOT_NAME) { $env:GITEA_CI_SNAPSHOT_NAME } else { 'BaseClean' }),
[string] $CloneBaseDir = $(if ($env:GITEA_CI_CLONE_BASE_DIR) { $env:GITEA_CI_CLONE_BASE_DIR } else { 'F:\CI\BuildVMs' }),
@@ -158,6 +158,7 @@ catch {
$jobArtifactDir = Join-Path $ArtifactBaseDir $JobId
$hostCloneDir = Join-Path $env:TEMP "ci-src-$JobId"
$cloneVmxPath = $null
$leaseFile = $null # §2.1 IP lease file path — released in finally
$startTime = Get-Date
# ── Start transcript ──────────────────────────────────────────────────────────
@@ -224,32 +225,87 @@ try {
}
Write-Host "[Invoke-CIJob] Source cloned to: $hostCloneDir"
# ── Phase 2: Clone VM ─────────────────────────────────────────────────
Write-Host "`n[Phase 2/6] Cloning VM..."
$cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
-TemplatePath $TemplatePath `
-SnapshotName $SnapshotName `
-CloneBaseDir $CloneBaseDir `
-JobId $JobId `
-VmrunPath $VmrunPath
# ── Phase 2-3b: Clone + Start + IP (serialized via file-based mutex) ────
# §2.1: With capacity:4 and DHCP on VMnet8, concurrent vmrun starts can race
# and produce two VMs with the same DHCP lease. A file-based mutex ensures
# only one job is in the clone→start→IP phase at a time. After the IP is
# acquired and written to a lease file the lock is released, so build phases
# (the slow part) run fully in parallel. Lease is released in finally.
$stateDir = 'F:\CI\State'
$leasesDir = Join-Path $stateDir 'ip-leases'
$lockPath = Join-Path $stateDir 'vm-start.lock'
if (-not (Test-Path $leasesDir)) { New-Item -ItemType Directory -Path $leasesDir -Force | Out-Null }
# ── Phase 3: Start VM ─────────────────────────────────────────────────
Write-Host "`n[Phase 3/6] Starting VM..."
$startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
if ($LASTEXITCODE -ne 0) {
throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
}
Write-Host "[Invoke-CIJob] VM started."
# ── Phase 3b: Resolve VM IP if not provided ─────────────────────────
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
Write-Host "[Invoke-CIJob] Auto-detecting VM IP via vmrun getGuestIPAddress..."
$detectedIP = & $VmrunPath getGuestIPAddress $cloneVmxPath -wait 2>&1
if ($LASTEXITCODE -ne 0 -or $detectedIP -notmatch '^(\d{1,3}\.){3}\d{1,3}$') {
throw "Could not detect VM IP address: $detectedIP"
Write-Host "`n[Phase 2/6] Acquiring VM-start lock (prevents DHCP collision with capacity>1)..."
$lockStream = $null
$lockDeadline = (Get-Date).AddMinutes(10)
while ($true) {
try {
$lockStream = [System.IO.File]::Open(
$lockPath,
[System.IO.FileMode]::OpenOrCreate,
[System.IO.FileAccess]::ReadWrite,
[System.IO.FileShare]::None)
break
} catch [System.IO.IOException] {
if ((Get-Date) -ge $lockDeadline) {
throw "[Phase 2] VM-start lock timeout after 10 min — a concurrent job may be stuck."
}
Write-Host "[Invoke-CIJob] VM-start lock busy, retrying in 5s..."
Start-Sleep -Seconds 5
}
$VMIPAddress = $detectedIP.Trim()
Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
}
Write-Host "[Invoke-CIJob] VM-start lock acquired."
try {
# ── Phase 2: Clone VM ─────────────────────────────────────────────
Write-Host "[Phase 2/6] Cloning VM..."
$cloneVmxPath = & "$scriptDir\New-BuildVM.ps1" `
-TemplatePath $TemplatePath `
-SnapshotName $SnapshotName `
-CloneBaseDir $CloneBaseDir `
-JobId $JobId `
-VmrunPath $VmrunPath
# ── Phase 3: Start VM ─────────────────────────────────────────────
Write-Host "`n[Phase 3/6] Starting VM..."
$startOutput = & $VmrunPath -T ws start $cloneVmxPath nogui 2>&1
if ($LASTEXITCODE -ne 0) {
throw "vmrun start failed (exit $LASTEXITCODE): $startOutput"
}
Write-Host "[Invoke-CIJob] VM started."
# ── Phase 3b: Resolve VM IP ───────────────────────────────────────
if ([string]::IsNullOrWhiteSpace($VMIPAddress)) {
Write-Host "[Invoke-CIJob] Auto-detecting VM IP via vmrun getGuestIPAddress -wait..."
$detectedIP = & $VmrunPath getGuestIPAddress $cloneVmxPath -wait 2>&1
if ($LASTEXITCODE -ne 0 -or $detectedIP -notmatch '^(\d{1,3}\.){3}\d{1,3}$') {
throw "Could not detect VM IP address: $detectedIP"
}
$VMIPAddress = $detectedIP.Trim()
Write-Host "[Invoke-CIJob] Detected VM IP: $VMIPAddress"
}
# ── Lease IP ──────────────────────────────────────────────────────
# If two VMs somehow got the same IP (DHCP edge case), fail fast here
# rather than letting both jobs write to the same WinRM target.
$leaseFile = Join-Path $leasesDir "$($VMIPAddress.Replace('.', '-')).lease"
if (Test-Path $leaseFile) {
$holder = (Get-Content $leaseFile -Raw -ErrorAction SilentlyContinue).Trim()
throw "IP collision: $VMIPAddress already leased by job '$holder'. Retry or reduce runner capacity."
}
[System.IO.File]::WriteAllText($leaseFile, $JobId)
Write-Host "[Invoke-CIJob] IP $VMIPAddress leased for job $JobId."
} finally {
# Release lock — next waiting job can now enter clone→start→IP phase
if ($lockStream) {
$lockStream.Close()
$lockStream.Dispose()
$lockStream = $null
}
Remove-Item $lockPath -Force -ErrorAction SilentlyContinue
Write-Host "[Invoke-CIJob] VM-start lock released."
}
# ── Phase 4: Wait for readiness ───────────────────────────────────────
@@ -295,6 +351,12 @@ catch {
Write-Host "============================================================"
}
finally {
# ── Release IP lease (§2.1) ───────────────────────────────────────────
if ($leaseFile -and (Test-Path $leaseFile)) {
Remove-Item $leaseFile -Force -ErrorAction SilentlyContinue
Write-Host "[Cleanup] IP lease released: $VMIPAddress"
}
# ── Always destroy the VM ─────────────────────────────────────────────
if ($null -ne $cloneVmxPath -and (Test-Path $cloneVmxPath -PathType Leaf)) {
Write-Host "`n[Cleanup] Destroying ephemeral VM..."
scripts/Invoke-RetentionPolicy.ps1
+129
View File
@@ -0,0 +1,129 @@
#Requires -Version 5.1
<#
.SYNOPSIS
Purges old CI artifacts, logs, and stale IP leases based on retention policy.
.DESCRIPTION
Removes per-job subdirectories under ArtifactDir and LogDir whose last-write
time is older than RetentionDays. If the monitored drive is below MinFreeGB,
switches automatically to AggressiveRetentionDays to reclaim space faster.
Also removes stale IP lease files under F:\CI\State\ip-leases\ that are
older than 12 hours (defensive cleanup for jobs that crashed without a finally).
Designed to run as a Windows Scheduled Task (see Register-CIScheduledTasks.ps1).
Safe to run while CI jobs are active — only deletes directories whose
last-write time predates the cutoff.
.PARAMETER ArtifactDir
Directory containing per-job artifact subdirectories. Default: F:\CI\Artifacts
.PARAMETER LogDir
Directory containing per-job log subdirectories. Default: F:\CI\Logs
.PARAMETER RetentionDays
Normal retention threshold in days. Default: 30
.PARAMETER AggressiveRetentionDays
Retention threshold used when drive free space drops below MinFreeGB. Default: 7
.PARAMETER MinFreeGB
If drive free space is below this value, AggressiveRetentionDays is used instead
of RetentionDays and a warning is emitted. Default: 50
.PARAMETER DriveLetter
Single drive letter (without colon) to check for free space. Default: F
.EXAMPLE
# Dry run — lists what would be deleted
.\Invoke-RetentionPolicy.ps1 -WhatIf
# Live run
.\Invoke-RetentionPolicy.ps1
#>
[CmdletBinding(SupportsShouldProcess)]
param(
[string] $ArtifactDir = 'F:\CI\Artifacts',
[string] $LogDir = 'F:\CI\Logs',
[ValidateRange(1, 365)]
[int] $RetentionDays = 30,
[ValidateRange(1, 365)]
[int] $AggressiveRetentionDays = 7,
[ValidateRange(1, 1000)]
[int] $MinFreeGB = 50,
[ValidatePattern('^[A-Za-z]$')]
[string] $DriveLetter = 'F'
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Continue' # Don't abort on individual file errors
# ── Determine effective retention threshold ───────────────────────────────────
$drive = Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue
$freeGB = if ($drive) { [math]::Round($drive.Free / 1GB, 1) } else { [double]::MaxValue }
$aggressiveMode = $freeGB -lt $MinFreeGB
$effectiveDays = if ($aggressiveMode) { $AggressiveRetentionDays } else { $RetentionDays }
$cutoff = (Get-Date).AddDays(-$effectiveDays)
Write-Host "[RetentionPolicy] Drive ${DriveLetter}: free ${freeGB} GB"
if ($aggressiveMode) {
Write-Warning "[RetentionPolicy] Free space below ${MinFreeGB} GB — aggressive retention: ${AggressiveRetentionDays}d"
} else {
Write-Host "[RetentionPolicy] Normal retention: ${effectiveDays}d (cutoff: $($cutoff.ToString('yyyy-MM-dd')))"
}
# ── Helper: purge old subdirectories under a base dir ─────────────────────────
function Remove-OldJobDirs {
param([string]$BaseDir, [string]$Label)
if (-not (Test-Path $BaseDir)) {
Write-Host "[RetentionPolicy] $Label dir not found: $BaseDir — skipping."
return
}
$old = Get-ChildItem -Path $BaseDir -Directory -ErrorAction SilentlyContinue |
Where-Object { $_.LastWriteTime -lt $cutoff }
if (-not @($old).Count) {
Write-Host "[RetentionPolicy] $Label: nothing to purge (cutoff: $($cutoff.ToString('yyyy-MM-dd')))."
return
}
foreach ($dir in $old) {
$agedays = [int]((Get-Date) - $dir.LastWriteTime).TotalDays
if ($PSCmdlet.ShouldProcess($dir.FullName, "Remove $Label dir (age: ${agedays}d)")) {
Remove-Item $dir.FullName -Recurse -Force -ErrorAction SilentlyContinue
Write-Host "[RetentionPolicy] Purged $Label (${agedays}d): $($dir.Name)"
}
}
}
Remove-OldJobDirs -BaseDir $ArtifactDir -Label 'artifact'
Remove-OldJobDirs -BaseDir $LogDir -Label 'log'
# ── Stale IP lease cleanup (§2.1 defensive) ───────────────────────────────────
# Lease files are normally deleted by Invoke-CIJob.ps1 finally block.
# If the host crashed mid-job the lease file is never removed. Remove leases
# older than 12 hours (well beyond any plausible build duration).
$leasesDir = 'F:\CI\State\ip-leases'
$leaseCutoff = (Get-Date).AddHours(-12)
if (Test-Path $leasesDir) {
$staleLeases = Get-ChildItem -Path $leasesDir -File -Filter '*.lease' -ErrorAction SilentlyContinue |
Where-Object { $_.LastWriteTime -lt $leaseCutoff }
foreach ($lf in @($staleLeases)) {
if ($PSCmdlet.ShouldProcess($lf.FullName, 'Remove stale IP lease')) {
Remove-Item $lf.FullName -Force -ErrorAction SilentlyContinue
Write-Host "[RetentionPolicy] Removed stale IP lease: $($lf.Name)"
}
}
}
# ── Summary ───────────────────────────────────────────────────────────────────
$freeGBAfter = if ($drive) { [math]::Round((Get-PSDrive -Name $DriveLetter -ErrorAction SilentlyContinue).Free / 1GB, 1) } else { $freeGB }
$delta = [math]::Round($freeGBAfter - $freeGB, 1)
Write-Host "[RetentionPolicy] Done. Drive ${DriveLetter}: free ${freeGBAfter} GB (delta: +${delta} GB)"
scripts/Register-CIScheduledTasks.ps1
+126
View File
@@ -0,0 +1,126 @@
#Requires -Version 5.1
#Requires -RunAsAdministrator
<#
.SYNOPSIS
Registers Windows Scheduled Tasks for CI system maintenance.
.DESCRIPTION
Creates or updates two tasks under the \CI\ task folder:
CI-CleanupOrphans — Runs Cleanup-OrphanedBuildVMs.ps1 every 6 hours
and at host startup. Destroys stale build VMs
(those older than -MaxAgeHours, default 4) that
were not cleaned up after a crash or timeout.
CI-RetentionPolicy — Runs Invoke-RetentionPolicy.ps1 daily at 3:00 AM
and at host startup (with a 15-min random delay).
Purges old artifact and log directories per the
configured retention window.
Both tasks run as SYSTEM with highest privilege. Idempotent: safe to re-run
after updating the scripts — existing tasks are overwritten (-Force).
.PARAMETER ScriptRoot
Directory containing the CI scripts. Default: N:\Code\Workspace\Local-CI-CD-System\scripts
.PARAMETER MaxAgeHours
Passed to Cleanup-OrphanedBuildVMs.ps1. VMs older than this are treated as
orphaned. Must exceed the longest expected build duration. Default: 4
.EXAMPLE
# Register (or update) tasks — run from an elevated PowerShell session
.\Register-CIScheduledTasks.ps1
# Preview what would be registered
.\Register-CIScheduledTasks.ps1 -WhatIf
#>
[CmdletBinding(SupportsShouldProcess)]
param(
[string] $ScriptRoot = 'N:\Code\Workspace\Local-CI-CD-System\scripts',
[ValidateRange(1, 168)]
[int] $MaxAgeHours = 4
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$psExe = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
$taskPath = '\CI\'
$principal = New-ScheduledTaskPrincipal -UserId 'SYSTEM' -RunLevel Highest
# Ensure task folder exists (Register-ScheduledTask creates it, but be explicit)
$schedulerService = New-Object -ComObject 'Schedule.Service'
$schedulerService.Connect()
$rootFolder = $schedulerService.GetFolder('\')
try { $rootFolder.GetFolder('CI') | Out-Null }
catch { $rootFolder.CreateFolder('CI') | Out-Null }
[System.Runtime.InteropServices.Marshal]::ReleaseComObject($schedulerService) | Out-Null
# ── Task 1: CI-CleanupOrphans ─────────────────────────────────────────────────
$cleanupScript = Join-Path $ScriptRoot 'Cleanup-OrphanedBuildVMs.ps1'
if (-not (Test-Path $cleanupScript -PathType Leaf)) {
Write-Warning "Script not found — skipping CI-CleanupOrphans: $cleanupScript"
} else {
$cleanupAction = New-ScheduledTaskAction `
-Execute $psExe `
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$cleanupScript`" -MaxAgeHours $MaxAgeHours"
# Repeat every 6 hours indefinitely (Once trigger + RepetitionInterval)
$repeat6h = New-ScheduledTaskTrigger -Once -At '00:00' `
-RepetitionInterval (New-TimeSpan -Hours 6)
$atStartup = New-ScheduledTaskTrigger -AtStartup
$cleanupSettings = New-ScheduledTaskSettingsSet `
-ExecutionTimeLimit (New-TimeSpan -Hours 1) `
-MultipleInstances IgnoreNew `
-StartWhenAvailable
if ($PSCmdlet.ShouldProcess('CI-CleanupOrphans', 'Register/update scheduled task')) {
Register-ScheduledTask `
-TaskName 'CI-CleanupOrphans' `
-TaskPath $taskPath `
-Action $cleanupAction `
-Trigger @($repeat6h, $atStartup) `
-Principal $principal `
-Settings $cleanupSettings `
-Description "Destroys orphaned CI build VMs older than ${MaxAgeHours}h (Local CI/CD System)" `
-Force | Out-Null
Write-Host "[Register] CI-CleanupOrphans registered (every 6h + at startup, -MaxAgeHours $MaxAgeHours)." -ForegroundColor Green
}
}
# ── Task 2: CI-RetentionPolicy ────────────────────────────────────────────────
$retentionScript = Join-Path $ScriptRoot 'Invoke-RetentionPolicy.ps1'
if (-not (Test-Path $retentionScript -PathType Leaf)) {
Write-Warning "Script not found — skipping CI-RetentionPolicy: $retentionScript"
} else {
$retentionAction = New-ScheduledTaskAction `
-Execute $psExe `
-Argument "-NoProfile -NonInteractive -ExecutionPolicy Bypass -File `"$retentionScript`""
$daily3am = New-ScheduledTaskTrigger -Daily -At '03:00' `
-RandomDelay (New-TimeSpan -Minutes 30)
$atStartup2 = New-ScheduledTaskTrigger -AtStartup `
-RandomDelay (New-TimeSpan -Minutes 15)
$retentionSettings = New-ScheduledTaskSettingsSet `
-ExecutionTimeLimit (New-TimeSpan -Hours 1) `
-MultipleInstances IgnoreNew `
-StartWhenAvailable
if ($PSCmdlet.ShouldProcess('CI-RetentionPolicy', 'Register/update scheduled task')) {
Register-ScheduledTask `
-TaskName 'CI-RetentionPolicy' `
-TaskPath $taskPath `
-Action $retentionAction `
-Trigger @($daily3am, $atStartup2) `
-Principal $principal `
-Settings $retentionSettings `
-Description 'Purges old CI artifacts, logs, and stale IP leases (Local CI/CD System)' `
-Force | Out-Null
Write-Host "[Register] CI-RetentionPolicy registered (daily 03:00 + at startup)." -ForegroundColor Green
}
}
Write-Host "`n[Register] Done. Verify with: Get-ScheduledTask -TaskPath '\CI\' | Format-Table TaskName, State" -ForegroundColor Cyan