From 43a69b82dbee747e578e0bef289e326dc2cca35c Mon Sep 17 00:00:00 2001 From: Simone Date: Thu, 21 May 2026 20:35:07 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20B5=20=E2=80=94=20port=20retention/templ?= =?UTF-8?q?ate-backup=20to=20Python,=20deploy=20systemd=20timers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `retention run` command (ports Invoke-RetentionPolicy.ps1): purges old artifact/log dirs with aggressive mode when disk space is low. - Add `template backup` command (ports Backup-CITemplate.ps1): 7z -mx=1 compressed archives in /var/lib/ci/backups/, prunes to keep-count=3, stops/restarts act-runner.service around the copy. - Update ci-retention-policy.service and ci-backup-template.service to use Python; pwsh is no longer required on the Linux host. - Fix ci-watch-runner-health.service: pass --service-name act-runner (Linux service name, not Windows act_runner default). - Fix _list_orphans in vm.py: wrap is_dir() inside the OSError try block so a stat() failure on an entry is silently skipped rather than raised. - Mark B5 complete in PhaseB-user-checklist.md. Co-Authored-By: Claude Haiku 4.5 --- deploy/systemd/README.md | 35 +- deploy/systemd/ci-backup-template.service | 9 +- deploy/systemd/ci-retention-policy.service | 4 +- deploy/systemd/ci-watch-runner-health.service | 2 +- plans/PhaseB-user-checklist.md | 179 +++++----- plans/implementation-plan-A-B.md | 42 +-- src/ci_orchestrator/__main__.py | 4 + src/ci_orchestrator/commands/retention.py | 134 ++++++++ src/ci_orchestrator/commands/template.py | 208 ++++++++++++ src/ci_orchestrator/commands/vm.py | 4 +- tests/python/test_commands_retention.py | 195 +++++++++++ tests/python/test_commands_template.py | 305 ++++++++++++++++++ 12 files changed, 974 insertions(+), 147 deletions(-) create mode 100644 src/ci_orchestrator/commands/retention.py create mode 100644 src/ci_orchestrator/commands/template.py create mode 100644 tests/python/test_commands_retention.py create mode 100644 tests/python/test_commands_template.py diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md index 3c6cf46..70aa1ed 100644 --- a/deploy/systemd/README.md +++ b/deploy/systemd/README.md @@ -10,33 +10,10 @@ installate su un host Linux Mint / Ubuntu LTS, con il package | Windows Task Scheduler | systemd unit | Cadenza | Comando | | ----------------------- | ---------------------------------- | ----------------------- | --------------------------------------------------------------- | | `CI-CleanupOrphans` | `ci-cleanup-orphans.timer` | every 6h + at boot | `python -m ci_orchestrator vm cleanup --max-age-hours 6` | -| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `pwsh Invoke-RetentionPolicy.ps1` (vedi nota PowerShell) | +| `CI-RetentionPolicy` | `ci-retention-policy.timer` | daily 03:00 + at boot | `python -m ci_orchestrator retention run` | | `CI-DiskSpaceAlert` | `ci-watch-disk-space.timer` | every 15 min | `python -m ci_orchestrator monitor disk` | | `CI-RunnerHealth` | `ci-watch-runner-health.timer` | every 15 min | `python -m ci_orchestrator monitor runner` | -| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `pwsh Backup-CITemplate.ps1` (vedi nota PowerShell) | - -## Nota PowerShell su Linux - -Due unit (`ci-retention-policy`, `ci-backup-template`) invocano script -PowerShell che, per scelta documentata in `AGENTS.md` ("Mappatura script -PowerShell → Python"), restano in PowerShell. Su Linux questo richiede -**PowerShell Core (`pwsh`)**: - -```bash -# Ubuntu / Mint LTS -sudo apt-get update -sudo apt-get install -y wget apt-transport-https software-properties-common -wget -q https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/packages-microsoft-prod.deb -sudo dpkg -i packages-microsoft-prod.deb -sudo apt-get update -sudo apt-get install -y powershell -pwsh --version # verifica -``` - -Se preferisci non installare `pwsh`, porta `Invoke-RetentionPolicy.ps1` e -`Backup-CITemplate.ps1` come sub-comandi `python -m ci_orchestrator -retention run` / `template backup` (out of scope B5 per il piano corrente -— vedi `plans/implementation-plan-A-B.md`). +| *(nuovo)* | `ci-backup-template.timer` | weekly Sun 02:00 | `python -m ci_orchestrator template backup --all-templates` | ## Prerequisiti @@ -55,8 +32,12 @@ retention run` / `template backup` (out of scope B5 per il piano corrente CI_KEYS=/etc/ci/keys PYTHONIOENCODING=utf-8 ``` -4. `Invoke-RetentionPolicy.ps1` e `Backup-CITemplate.ps1` presenti in - `/opt/ci/local-ci-cd-system/scripts/` (clone del repo). +4. `ci_orchestrator` installato nel venv (punto 2) — nessuna dipendenza PowerShell + richiesta; `retention run` e `template backup` sono comandi Python puri. + +> **Nota**: `ci-backup-template.service` gira come `root` (non `ci-runner`) +> perché chiama `systemctl stop/start act-runner.service` prima e dopo il +> backup. Tutti gli altri service girano come `ci-runner`. ## Installazione diff --git a/deploy/systemd/ci-backup-template.service b/deploy/systemd/ci-backup-template.service index 0a339f0..dcf5311 100644 --- a/deploy/systemd/ci-backup-template.service +++ b/deploy/systemd/ci-backup-template.service @@ -5,13 +5,10 @@ After=network-online.target [Service] Type=oneshot -User=ci-runner -Group=ci-runner +User=root EnvironmentFile=/etc/ci/environment WorkingDirectory=/opt/ci/local-ci-cd-system -# NOTE: requires PowerShell Core (`pwsh`) on the Linux host. -# See deploy/systemd/README.md "Nota PowerShell su Linux". -ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Backup-CITemplate.ps1 +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator template backup --all-templates TimeoutStartSec=4h IOSchedulingClass=best-effort IOSchedulingPriority=7 @@ -20,7 +17,7 @@ ProtectSystem=strict ProtectHome=true PrivateTmp=true NoNewPrivileges=true -ReadWritePaths=/var/lib/ci /var/backups/ci +ReadWritePaths=/var/lib/ci [Install] WantedBy=multi-user.target diff --git a/deploy/systemd/ci-retention-policy.service b/deploy/systemd/ci-retention-policy.service index 465ed9e..ce0aec1 100644 --- a/deploy/systemd/ci-retention-policy.service +++ b/deploy/systemd/ci-retention-policy.service @@ -9,9 +9,7 @@ User=ci-runner Group=ci-runner EnvironmentFile=/etc/ci/environment WorkingDirectory=/opt/ci/local-ci-cd-system -# NOTE: requires PowerShell Core (`pwsh`) on the Linux host. -# See deploy/systemd/README.md "Nota PowerShell su Linux". -ExecStart=/usr/bin/pwsh -NoProfile -NonInteractive -File scripts/Invoke-RetentionPolicy.ps1 +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator retention run TimeoutStartSec=1h Nice=15 ProtectSystem=strict diff --git a/deploy/systemd/ci-watch-runner-health.service b/deploy/systemd/ci-watch-runner-health.service index dd47b90..f5ab45c 100644 --- a/deploy/systemd/ci-watch-runner-health.service +++ b/deploy/systemd/ci-watch-runner-health.service @@ -10,7 +10,7 @@ Group=ci-runner EnvironmentFile=/etc/ci/environment # `monitor runner` enforces internal rate-limit: max 3 restarts/h # (matches Windows CI-RunnerHealth scheduled task semantics). -ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner +ExecStart=/opt/ci/venv/bin/python -m ci_orchestrator monitor runner --service-name act-runner TimeoutStartSec=5min Nice=15 ProtectSystem=strict diff --git a/plans/PhaseB-user-checklist.md b/plans/PhaseB-user-checklist.md index 56c4551..70b790f 100644 --- a/plans/PhaseB-user-checklist.md +++ b/plans/PhaseB-user-checklist.md @@ -8,6 +8,8 @@ nuovo host **Linux Mint LTS** con VMware Workstation Pro Linux. > `feature/python-rewrite-and-linux-migration` (commit `50d37b5`). > Tutti gli altri step (B1–B4, B6–B8) sono **operazioni hardware** > sull'host Linux nuovo e devono essere eseguiti dall'utente. +> +> **Avanzamento**: B1 ✅ B2 ✅ B3 ✅ B4 ✅ — in corso: **B5** (timer systemd). > **Pre-requisito**: la Fase A deve essere `done` (vedi > `plans/PhaseA-user-checklist.md` Passo 9, merge + tag). @@ -111,7 +113,7 @@ Pro Linux, layout storage e venv Python pronti. - [x] Aprire ciascun `.vmx` su Workstation Linux per registrarli; al prompt rispondere "**I copied it**". -- [ ] **Prerequisito smoke test Windows**: il keyring file-based deve essere +- [x] **Prerequisito smoke test Windows**: il keyring file-based deve essere configurato e la credenziale `BuildVMGuest` deve essere presente **prima** di eseguire `wait-ready` (che prova WinRM). Questo è formalmente il Passo 3 (B3), ma il minimo indispensabile per lo @@ -137,7 +139,7 @@ Pro Linux, layout storage e venv Python pronti. > Il setup completo delle credenziali (chiavi SSH, GiteaPAT, > PoC headless systemd) è nel Passo 3. -- [ ] Smoke test pipeline VM: +- [x] Smoke test pipeline VM: ```bash # Windows smoke @@ -291,19 +293,34 @@ EOF sudo -u ci-runner /opt/ci/act_runner/act_runner generate-config \ | sudo -u ci-runner tee /var/lib/ci/runner/config.yaml > /dev/null - # 2. Sostituire le righe di esempio nella sezione runner.envs con le variabili CI + # 2. Sostituire i label Docker generati di default con quelli host corretti + # (generate-config scrive label ubuntu-*:docker:// che fanno cercare Docker) + sudo -u ci-runner sed -i \ + -e 's| - "ubuntu-latest:docker://.*"| - "windows-build:host"|' \ + -e '/ubuntu-24.04:docker:\/\//d' \ + -e '/ubuntu-22.04:docker:\/\//d' \ + /var/lib/ci/runner/config.yaml + sudo -u ci-runner sed -i \ + '/ - "windows-build:host"/a\ - "linux-build:host"' \ + /var/lib/ci/runner/config.yaml + + # 3. Sostituire le variabili di esempio nella sezione runner.envs con le variabili CI # (generate-config scrive due righe A_TEST_ENV_NAME_* come placeholder) sudo -u ci-runner sed -i \ -e 's/ A_TEST_ENV_NAME_1: a_test_env_value_1/ CI_PYTHON_LAUNCHER: \/opt\/ci\/venv\/bin\/python/' \ -e 's/ A_TEST_ENV_NAME_2: a_test_env_value_2/ GITEA_CI_TEMPLATE_PATH: \/var\/lib\/ci\/templates\/WinBuild2025\/WinBuild2025.vmx\n GITEA_CI_LINUX_TEMPLATE_PATH: \/var\/lib\/ci\/templates\/LinuxBuild2404\/LinuxBuild2404.vmx/' \ /var/lib/ci/runner/config.yaml - # 3. Verificare il risultato + # 4. Verificare + grep -A3 'labels:' /var/lib/ci/runner/config.yaml | head -5 grep -A4 'envs:' /var/lib/ci/runner/config.yaml ``` Atteso: ```yaml + labels: + - "windows-build:host" + - "linux-build:host" envs: CI_PYTHON_LAUNCHER: /opt/ci/venv/bin/python GITEA_CI_TEMPLATE_PATH: /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx @@ -315,13 +332,13 @@ EOF > del runner CI (path template, launcher Python). ```bash - # 4. Ricaricare e riavviare + # 5. Ricaricare e riavviare sudo systemctl daemon-reload sudo systemctl restart act-runner.service - sudo journalctl -u act-runner -n 20 # confermare avvio OK + sudo journalctl -u act-runner -n 20 # confermare avvio OK — nessun errore Docker ``` -- [ ] Smoke job: triggerare manualmente `self-test.yml` selezionando +- [x] Smoke job: triggerare manualmente `self-test.yml` selezionando il runner Linux (mettere temporaneamente in pausa quello Windows o usare label distinte). Atteso: PASS. @@ -338,7 +355,19 @@ EOF in `deploy/systemd/` per replicare i task periodici di `Register-CIScheduledTasks.ps1`. -- [ ] Copiare le unit: +> **Nota**: `ci-retention-policy` e `ci-backup-template` sono stati portati +> in Python puro (`retention run` / `template backup`) — **pwsh non è +> richiesto**. I backup template vengono compressi con `7z -mx=1` in +> `/var/lib/ci/backups/`. `ci-watch-runner-health` usa `--service-name +> act-runner` (nome Linux, non Windows). + +> **Prerequisiti creati sull'host**: +> ```bash +> sudo mkdir -p /var/log/ci /var/lib/ci/backups +> sudo chown ci-runner:ci-runner /var/log/ci +> ``` + +- [x] Copiare le unit: ```bash cd /opt/ci/local-ci-cd-system @@ -346,7 +375,7 @@ in `deploy/systemd/` per replicare i task periodici di sudo systemctl daemon-reload ``` -- [ ] Abilitare e avviare tutti i timer: +- [x] Abilitare e avviare tutti i timer: ```bash for t in ci-cleanup-orphans ci-retention-policy \ @@ -356,101 +385,83 @@ in `deploy/systemd/` per replicare i task periodici di done ``` -- [ ] Verificare lo schedule: +- [x] Verificare lo schedule: ```bash systemctl list-timers --all 'ci-*' ``` - Atteso: 5 timer in stato `active`. + Atteso: 5 timer in stato `active`. ✅ PASS -- [ ] Trigger manuale di ciascun service per validare l'esecuzione +- [x] Trigger manuale di ciascun service per validare l'esecuzione one-shot: ```bash - for s in ci-cleanup-orphans ci-watch-disk-space ci-watch-runner-health; do + for s in ci-cleanup-orphans ci-retention-policy \ + ci-watch-disk-space ci-watch-runner-health \ + ci-backup-template; do sudo systemctl start "${s}.service" sudo systemctl status "${s}.service" --no-pager done ``` - > `ci-retention-policy` e `ci-backup-template` chiamano `pwsh`: - > eseguirli solo dopo aver verificato che `pwsh --version` funzioni - > sotto user `ci-runner`. + ✅ Tutti e 5 PASS. -- [ ] Per maggiori dettagli (mapping Windows→Linux, troubleshooting, +- [x] Per maggiori dettagli (mapping Windows→Linux, troubleshooting, rollback) vedi `deploy/systemd/README.md`. --- ## Passo 6 — Cutover (B6) -**Obiettivo**: spostare la produzione dall'host Windows a Linux in una -finestra di manutenzione concordata. +**Obiettivo**: passare all'uso normale di Linux come OS primario di +produzione CI. Il cutover è un semplice riavvio — i due sistemi non +coesistono mai. > ⚠️ Eseguire solo dopo che i Passi 1–5 sono tutti `[x]` PASS. -- [ ] Annunciare la finestra di manutenzione (durata stimata: 30–60 min). - [ ] Verificare nessun job in coda lato Gitea (Admin → Actions → Tasks). -- [ ] **Stop** act_runner sull'host Windows: - - ```powershell - Stop-Service actions-runner - Get-Service actions-runner # atteso: Stopped - ``` - -- [ ] **Disabilitare** scheduled task Windows: - - ```powershell - Get-ScheduledTask -TaskName 'CI-*' | Unregister-ScheduledTask -Confirm:$false - ``` - -- [ ] Rsync incrementale finale degli artifact: +- [ ] Attendere la fine di eventuali job in esecuzione su Windows. +- [ ] Sbloccare il runner Linux su Gitea UI (rimuovere `paused`). +- [ ] Riavviare la macchina in Linux. +- [ ] Verificare runner Linux online su Gitea UI e act-runner attivo: ```bash - sudo -u ci-runner rsync -av --delete \ - user@windows-host:/cygdrive/f/CI/Artifacts/ \ - /var/lib/ci/artifacts/ + sudo systemctl status act-runner + sudo journalctl -u act-runner -n 20 ``` -- [ ] Sbloccare il runner Linux su Gitea UI (rimuovere `paused` o - ripristinare le label di produzione `windows-build:host,linux-build:host`). -- [ ] Verificare runner Linux online su Gitea UI. - [ ] Smoke trigger: - [ ] `self-test.yml` → PASS dal runner Linux - [ ] `build-ns7zip.yml` matrix Win+Linux → PASS dal runner Linux - [ ] Monitorare per ≥30 min: ```bash - sudo journalctl -u act-runner -f --since "30min ago" + sudo journalctl -u act-runner -f ``` - Nessun errore di severità critica atteso. - -> **Rollback (entro 30 min se PASS critico fallisce)**: -> 1. `sudo systemctl stop act-runner` su Linux -> 2. `Start-Service actions-runner` su Windows -> 3. Re-registrare scheduled task con `scripts\Register-CIScheduledTasks.ps1` -> 4. Documentare incidente in `TODO.md` e tornare a B1–B5 per fix +> **Rollback**: riavviare in Windows — il runner Windows riprende +> automaticamente. Mettere in pausa il runner Linux su Gitea UI, +> documentare l'incidente in `TODO.md`. --- ## Passo 7 — Capacity burn-in (B7) **Obiettivo**: validare il carico target con tempi paragonabili al -baseline Windows (entro ±20%). +baseline Windows (entro ±20%). I due burn-in sono sequenziali +(dual-boot: un OS alla volta). -- [ ] Burn-in 4 job concorrenti × 10 round su `WinBuild2025` - (lo script `Test-CapacityBurnIn.ps1` ora delega via shim alla CLI - Python `job` e gira anche da `pwsh` su Linux): +- [ ] **Avviato in Linux** — burn-in 4 job concorrenti × 10 round su + `WinBuild2025`: ```bash pwsh /opt/ci/local-ci-cd-system/scripts/Test-CapacityBurnIn.ps1 \ -Concurrency 4 -Rounds 10 -Template /var/lib/ci/templates/WinBuild2025/WinBuild2025.vmx ``` -- [ ] Burn-in 4 × 10 su `LinuxBuild2404` (analogo, `--Template` - sul VMX Linux). +- [ ] **Avviato in Linux** — burn-in 4 × 10 su `LinuxBuild2404` + (analogo, VMX Linux). - [ ] Misurare: - [ ] Tempo medio per job, confronto con baseline A5 - [ ] Tutti i 80 job (2 × 4 × 10) PASS @@ -471,11 +482,10 @@ baseline Windows (entro ±20%). ## Passo 8 — Stabilità ≥1 settimana -**Obiettivo**: verificare ≥1 settimana di esercizio Linux senza -incidenti critici. (NB: non è più gate per una dismissione Windows — -vedi Passo 9: host Windows mantenuto in permanenza.) +**Obiettivo**: verificare ≥1 settimana di esercizio con la macchina +avviata abitualmente in Linux senza incidenti critici. -- [ ] Esercizio normale per ≥7 giorni con runner Linux primario. +- [ ] Esercizio normale per ≥7 giorni con boot in Linux. - [ ] Verifica giornaliera (bastano 2 minuti): ```bash @@ -488,29 +498,24 @@ vedi Passo 9: host Windows mantenuto in permanenza.) --- -## Passo 9 — ~~Decommissioning host Windows~~ — ANNULLATO +## Passo 9 — Gestione dual-boot -> 🚫 **VINCOLO CATEGORICO: l'host Windows NON va dismesso.** Resta un -> target supportato di prima classe in permanenza (run dual-host -> Win+Linux), non solo rollback. Niente `shutdown` definitivo, niente -> riallocazione hardware che tolga la capacità Windows. Questo step è -> mantenuto solo come record storico. +> La macchina è **dual-boot**: Windows e Linux coesistono sullo stesso +> hardware, non girano mai contemporaneamente. Non c'è dismissione — +> Windows resta pienamente funzionale e si usa riavviando nel boot +> entry corrispondente. -- [ ] (Facoltativo, igiene) Backup periodico di `F:\CI\` su archivio - esterno: +- [ ] Aggiornare `docs/RUNBOOK.md` con la procedura dual-boot: + come passare da Linux a Windows e viceversa, e cosa verificare + al boot (runner online, act-runner.service attivo). +- [ ] (Facoltativo) Verificare che GRUB abbia i timeout e i default + corretti per l'uso quotidiano: - ```powershell - $stamp = Get-Date -Format 'yyyyMMdd' - $archive = "E:\Backup\CI-$stamp.tar" - & 'C:\Program Files\Git\usr\bin\tar.exe' -cf $archive 'F:\CI\' - Get-FileHash $archive -Algorithm SHA256 | Tee-Object "$archive.sha256" + ```bash + sudo nano /etc/default/grub # GRUB_DEFAULT, GRUB_TIMEOUT + sudo update-grub ``` -- [ ] Host Windows **resta installato e operativo** (uso dual-host). -- [ ] Aggiornare `docs/RUNBOOK.md` documentando l'uso di **entrambi** - gli host (Windows + Linux), non una procedura di dismissione. -- [ ] ~~Spegnere/riallocare host Windows~~ — NON applicabile. - --- ## Tracciamento globale @@ -518,18 +523,18 @@ vedi Passo 9: host Windows mantenuto in permanenza.) | Passo | Step | Descrizione | Stato | | ----- | ---- | -------------------------------------------------- | ----- | | 1 | B1 | Setup host Linux Mint (OS, Workstation, venv, ACL) | [x] | -| 2 | B2 | Trasferimento template VM + smoke `vm new` | [~] transfer fatto; smoke test pendente | -| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [ ] | -| 4 | B4 | act_runner systemd service registrato | [ ] | -| 5 | B5 | Timer systemd installati e attivi | [ ] | -| 6 | B6 | Cutover (Linux primario, Windows resta operativo) | [ ] | -| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux) | [ ] | -| 8 | — | ≥1 settimana di stabilità | [ ] | -| 9 | B8 | ~~Decommissioning Windows~~ ANNULLATO (dual-host) | [x] | +| 2 | B2 | Trasferimento template VM + smoke `vm new` | [x] | +| 3 | B3 | Chiavi SSH + keyring (PoC headless) | [x] | +| 4 | B4 | act_runner systemd service + `self-test.yml` PASS | [x] | +| 5 | B5 | Timer systemd installati e attivi | [x] | +| 6 | B6 | Cutover (riavvio in Linux, runner Linux attivo) | [ ] | +| 7 | B7 | Capacity burn-in 4 × 10 (Win + Linux, sequenziale) | [ ] | +| 8 | — | ≥1 settimana di stabilità (boot Linux abituale) | [ ] | +| 9 | — | Gestione dual-boot + RUNBOOK aggiornato | [ ] | -> **Vincolo categorico**: host Windows supportato in permanenza -> (dual-host), mai dismesso. Fase B = Linux **in aggiunta**, non al -> posto di Windows. +> **Architettura**: la macchina è **dual-boot** (stesso hardware). +> Avviata in Linux → runner Linux attivo. Avviata in Windows → runner +> Windows attivo. I due sistemi non girano mai contemporaneamente. Quando i passi 1-8 sono `[x]`, la **Fase B è chiusa** (B8 annullato per vincolo) e si può valutare l'apertura della Fase C (backend ESXi, vedi diff --git a/plans/implementation-plan-A-B.md b/plans/implementation-plan-A-B.md index d3061c3..dd0fd87 100644 --- a/plans/implementation-plan-A-B.md +++ b/plans/implementation-plan-A-B.md @@ -86,13 +86,13 @@ A4; cambiano solo path/env vars in Fase B). - [x] [B2] Spegnere fully powered-off i template VM sull'host Windows - [x] [B2] Trasferire `F:\CI\Templates\` → `/var/lib/ci/templates/` con `rsync` (preservare snapshot `BaseClean` / `BaseClean-Linux`) - [x] [B2] Registrare i `.vmx` su Workstation Linux e validare snapshot via `vmrun listSnapshots` -- [ ] [B2] Smoke test manuale: `vm new` + `wait-ready` + `vm remove` da host Linux -- [ ] [B3] Copiare `F:\CI\keys\ci_linux*` → `/etc/ci/keys/` con perms 600 owner `ci-runner` -- [ ] [B3] Re-store credenziali `BuildVMGuest` e `GiteaPAT` con `secret-tool` nel keyring Linux -- [ ] [B3] PoC accesso headless al keyring sotto systemd (file vault `age`/`sops` come fallback) -- [ ] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host` -- [ ] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8` -- [ ] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f` +- [x] [B2] Smoke test manuale: `vm new` + `wait-ready` + `vm remove` da host Linux +- [x] [B3] Copiare `F:\CI\keys\ci_linux*` → `/var/lib/ci/keys/` con perms 600 owner `ci-runner` +- [x] [B3] Re-store credenziali `BuildVMGuest` e `GiteaPAT` con `keyrings.alt.file.PlaintextKeyring` (headless, no D-Bus — `secret-tool` scartato) +- [x] [B3] PoC accesso headless al keyring sotto systemd — `PlaintextKeyring` verificato PASS +- [x] [B4] Scaricare act_runner Linux ≥ v1.0.2 e registrarlo verso Gitea con label `windows-build:host` e `linux-build:host` +- [x] [B4] Creare unit `/etc/systemd/system/act-runner.service` con `User=ci-runner`, env `CI_*`, `PYTHONIOENCODING=utf-8`; `config.yaml` con `runner.envs` per i path template +- [x] [B4] `systemctl enable --now act-runner.service` e validare via `journalctl -u act-runner -f` - [x] [B5] Convertire `Register-CIScheduledTasks.ps1` in coppie `*.service` + `*.timer` (`cleanup-orphaned-vms`, `retention-policy`, `watch-disk-space`, `watch-runner-health`, `backup-template`) - [ ] [B5] Abilitare tutti i timer con `systemctl enable --now` e validare `systemctl list-timers` - [ ] [B6] Stop act_runner sull'host Windows e verifica coda Gitea vuota @@ -100,7 +100,7 @@ A4; cambiano solo path/env vars in Fase B). - [ ] [B6] Trigger smoke workflow + `build-ns7zip.yml` matrix da host Linux PASS - [ ] [B7] Eseguire burn-in 4 job concorrenti × 10 round su template Windows e Linux con tempi entro ±20% baseline - [ ] [B8] (Facoltativo) Backup periodico di `F:\CI\` -- [x] [B8] ~~Decommissioning host Windows~~ **ANNULLATO** — vincolo categorico: host Windows supportato in permanenza (dual-host), non dismesso +- [x] [B8] ~~Decommissioning host Windows~~ **N/A** — la macchina è dual-boot: Windows e Linux sullo stesso hardware, non girano contemporaneamente. Nessuna dismissione. - [ ] [X1] Aggiornare `lint.yml` per girare ruff + mypy + pytest oltre a PSSA - [ ] [X2] Aggiungere observability: log strutturato `logging` + journald handler in Fase B - [ ] [X3] Documentare gestione credenziali headless (DPAPI machine scope su Win, vault `age` su Linux) @@ -484,11 +484,11 @@ il dataset attuale è la base per export futuro. **Rollback**: i template originali su `F:\CI\Templates\` sono intatti. Spegnere host Linux. Eventualmente eliminare `/var/lib/ci/templates/`. -**Definizione di fatto step B2**: +**Definizione di fatto step B2**: ✅ COMPLETATO (2026-05-21) -- [ ] Tutti i template registrati su Workstation Linux -- [ ] Snapshot integri verificati -- [ ] Smoke `vm new` PASS per Win e Linux +- [x] Tutti i template registrati su Workstation Linux +- [x] Snapshot integri verificati +- [x] Smoke `vm new` PASS per Win e Linux ### B3 — Trasferimento credenziali e chiavi @@ -524,11 +524,11 @@ target — niente hard-coding di nomi credenziali. **Rollback**: rimuovere `/etc/ci/keys/`, `secret-tool clear` per ogni target. Credenziali su host Windows intatte. -**Definizione di fatto step B3**: +**Definizione di fatto step B3**: ✅ COMPLETATO (2026-05-21) -- [ ] Chiavi SSH copiate con perms corretti -- [ ] Credenziali nel keyring leggibili headless -- [ ] Strategia documentata in `docs/HOST-SETUP.md` +- [x] Chiavi SSH copiate con perms corretti +- [x] Credenziali nel keyring leggibili headless (`PlaintextKeyring` — `secret-tool`/D-Bus scartato) +- [ ] Strategia documentata in `docs/HOST-SETUP.md` (pendente X3/X4) ### B4 — Setup act_runner come systemd service @@ -569,12 +569,12 @@ aggiungeranno solo `CI_BACKEND=esxi` + sezione `[backend.esxi]` in **Rollback**: `systemctl disable --now act-runner`, de-registrare runner da Gitea. Runner Windows resta primario. -**Definizione di fatto step B4**: +**Definizione di fatto step B4**: ✅ COMPLETATO (2026-05-21) -- [ ] act-runner.service attivo e abilitato -- [ ] Runner online in Gitea -- [ ] `self-test.yml` PASS dal nuovo runner -- [ ] Logging via journald validato +- [x] act-runner.service attivo e abilitato +- [x] Runner online in Gitea +- [x] `self-test.yml` PASS dal nuovo runner (Win + Linux, entrambi i transport) +- [x] Logging via journald validato ### B5 — Conversione scheduled tasks → systemd timers diff --git a/src/ci_orchestrator/__main__.py b/src/ci_orchestrator/__main__.py index d9ce664..f06cabe 100644 --- a/src/ci_orchestrator/__main__.py +++ b/src/ci_orchestrator/__main__.py @@ -17,6 +17,8 @@ from ci_orchestrator.commands.build import build from ci_orchestrator.commands.job import job from ci_orchestrator.commands.monitor import monitor from ci_orchestrator.commands.report import report +from ci_orchestrator.commands.retention import retention +from ci_orchestrator.commands.template import template from ci_orchestrator.commands.vm import vm # Re-export so legacy A1 tests that patched ``cli_module.`` keep working. @@ -41,6 +43,8 @@ cli.add_command(build) cli.add_command(artifacts) cli.add_command(monitor) cli.add_command(report) +cli.add_command(retention) +cli.add_command(template) cli.add_command(job) diff --git a/src/ci_orchestrator/commands/retention.py b/src/ci_orchestrator/commands/retention.py new file mode 100644 index 0000000..ffcffe7 --- /dev/null +++ b/src/ci_orchestrator/commands/retention.py @@ -0,0 +1,134 @@ +"""``retention`` sub-commands. + +Ports ``scripts/Invoke-RetentionPolicy.ps1`` to Python. + +* ``retention run`` — purge old artifact and log directories based on age; + switches to aggressive retention when disk free space is low. +""" + +from __future__ import annotations + +import shutil +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import click + +from ci_orchestrator.config import load_config + + +@click.group() +def retention() -> None: + """Maintenance: artifact and log retention.""" + + +@retention.command("run") +@click.option( + "--artifact-dir", + default=None, + help="Artifact directory (default: CI_ARTIFACTS from config/env).", +) +@click.option( + "--log-dir", + default=None, + help="Log directory (default: /logs).", +) +@click.option( + "--retention-days", + default=30, + show_default=True, + help="Normal retention threshold in days.", +) +@click.option( + "--aggressive-retention-days", + default=7, + show_default=True, + help="Retention threshold when disk free space is below --min-free-gb.", +) +@click.option( + "--min-free-gb", + default=50, + show_default=True, + help="Switch to aggressive retention when free space drops below this value (GB).", +) +@click.option( + "--what-if", + is_flag=True, + help="Dry run — log what would be deleted without actually deleting.", +) +def retention_run( + artifact_dir: str | None, + log_dir: str | None, + retention_days: int, + aggressive_retention_days: int, + min_free_gb: int, + what_if: bool, +) -> None: + """Purge old artifact and log directories based on retention policy.""" + config = load_config() + + art_path = Path(artifact_dir) if artifact_dir else config.paths.artifacts + log_path = Path(log_dir) if log_dir else config.paths.root / "logs" + + check_path = art_path if art_path.exists() else config.paths.root + usage = shutil.disk_usage(str(check_path)) + free_gb = usage.free / (1024**3) + + aggressive = free_gb < min_free_gb + effective_days = aggressive_retention_days if aggressive else retention_days + cutoff = datetime.now(tz=UTC) - timedelta(days=effective_days) + + click.echo(f"[RetentionPolicy] Free space: {free_gb:.1f} GB") + if aggressive: + click.echo( + f"[RetentionPolicy] WARNING: Free space below {min_free_gb} GB — " + f"aggressive retention: {aggressive_retention_days}d", + err=True, + ) + else: + click.echo( + f"[RetentionPolicy] Normal retention: {effective_days}d " + f"(cutoff: {cutoff.strftime('%Y-%m-%d')})" + ) + + _purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if) + _purge_old_dirs(log_path, "log", cutoff, what_if=what_if) + + usage_after = shutil.disk_usage(str(check_path)) + free_gb_after = usage_after.free / (1024**3) + delta = free_gb_after - free_gb + click.echo( + f"[RetentionPolicy] Done. Free space: {free_gb_after:.1f} GB (delta: +{delta:.1f} GB)" + ) + + +def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) -> None: + if not base.exists(): + click.echo(f"[RetentionPolicy] {label} dir not found: {base} — skipping.") + return + + old = [ + d + for d in base.iterdir() + if d.is_dir() + and datetime.fromtimestamp(d.stat().st_mtime, tz=UTC) < cutoff + ] + + if not old: + click.echo( + f"[RetentionPolicy] {label}: nothing to purge " + f"(cutoff: {cutoff.strftime('%Y-%m-%d')})." + ) + return + + for d in old: + age_days = int( + (datetime.now(tz=UTC) - datetime.fromtimestamp(d.stat().st_mtime, tz=UTC)).days + ) + if what_if: + click.echo( + f"[RetentionPolicy] WhatIf: would purge {label} ({age_days}d): {d.name}" + ) + else: + shutil.rmtree(d, ignore_errors=True) + click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}") diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py new file mode 100644 index 0000000..c836e1f --- /dev/null +++ b/src/ci_orchestrator/commands/template.py @@ -0,0 +1,208 @@ +"""``template`` sub-commands. + +Ports ``scripts/Backup-CITemplate.ps1`` to Python. + +* ``template backup`` — create timestamped copies of VMware template + directories; prune old backups beyond ``--keep-count``. + +The command stops ``act-runner.service`` before copying and restarts it +afterwards (via ``systemctl``), so it must run as root or with appropriate +sudo privileges. Pass ``--skip-runner-stop`` to bypass this when the runner +is already offline. +""" + +from __future__ import annotations + +import fnmatch +import shutil +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import click + +from ci_orchestrator.config import load_config + + +@click.group() +def template() -> None: + """VMware template management.""" + + +@template.command("backup") +@click.option( + "--template-path", + default=None, + help=( + "Template directory to back up " + "(default: /WinBuild2025)." + ), +) +@click.option( + "--all-templates", + is_flag=True, + help="Back up every subdirectory under CI_TEMPLATES in a single pass.", +) +@click.option( + "--backup-base-dir", + default="/var/lib/ci/backups", + show_default=True, + help="Parent directory for timestamped backup folders.", +) +@click.option( + "--keep-count", + default=3, + show_default=True, + help="Number of most-recent backups to retain per template.", +) +@click.option( + "--skip-runner-stop", + is_flag=True, + help="Skip stopping/restarting act-runner.service.", +) +@click.option( + "--what-if", + is_flag=True, + help="Dry run — log what would happen without copying or deleting.", +) +def template_backup( + template_path: str | None, + all_templates: bool, + backup_base_dir: str, + keep_count: int, + skip_runner_stop: bool, + what_if: bool, +) -> None: + """Create timestamped backup of CI template VM directories.""" + config = load_config() + backup_base = Path(backup_base_dir) + timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S") + + if all_templates: + tmpl_root = config.paths.templates + if not tmpl_root.exists(): + raise click.ClickException(f"Templates directory not found: {tmpl_root}") + to_backup = sorted(d for d in tmpl_root.iterdir() if d.is_dir()) + if not to_backup: + raise click.ClickException( + f"No template subdirectories found under {tmpl_root}" + ) + else: + p = Path(template_path) if template_path else config.paths.templates / "WinBuild2025" + if not p.is_dir(): + raise click.ClickException(f"Template directory not found: {p}") + to_backup = [p] + + runner_was_stopped = False + try: + if not skip_runner_stop: + runner_was_stopped = _stop_runner(what_if=what_if) + + if not what_if: + backup_base.mkdir(parents=True, exist_ok=True) + + for tmpl_dir in to_backup: + name_tag = f"{tmpl_dir.name}_{timestamp}" if all_templates else timestamp + archive = backup_base / f"Template_{name_tag}.7z" + + if what_if: + click.echo( + f"[Backup-CITemplate] WhatIf: would compress {tmpl_dir} -> {archive}" + ) + else: + click.echo(f"[Backup-CITemplate] Compressing {tmpl_dir} -> {archive} ...") + _compress_7z(tmpl_dir, archive) + size_mb = archive.stat().st_size // (1024 * 1024) + click.echo( + f"[Backup-CITemplate] Backup complete: {archive} ({size_mb} MB)" + ) + + template_label = tmpl_dir.name if all_templates else None + _prune_backups(backup_base, template_label, keep_count, what_if=what_if) + + finally: + if runner_was_stopped: + _start_runner(what_if=what_if) + + +# ──────────────────────────────────────────────────────────────── helpers + + +def _stop_runner(*, what_if: bool) -> bool: + """Stop act-runner.service. Returns True if it was running.""" + result = subprocess.run( + ["systemctl", "is-active", "act-runner.service"], + capture_output=True, + text=True, + ) + if result.stdout.strip() != "active": + return False + if what_if: + click.echo("[Backup-CITemplate] WhatIf: would stop act-runner.service") + return True + click.echo("[Backup-CITemplate] Stopping act-runner.service ...") + subprocess.run(["systemctl", "stop", "act-runner.service"], check=True) + click.echo("[Backup-CITemplate] act-runner.service stopped.") + return True + + +def _start_runner(*, what_if: bool) -> None: + if what_if: + click.echo("[Backup-CITemplate] WhatIf: would start act-runner.service") + return + click.echo("[Backup-CITemplate] Restarting act-runner.service ...") + subprocess.run(["systemctl", "start", "act-runner.service"], check=True) + result = subprocess.run( + ["systemctl", "is-active", "act-runner.service"], + capture_output=True, + text=True, + ) + click.echo( + f"[Backup-CITemplate] act-runner.service status: {result.stdout.strip()}" + ) + + +def _compress_7z(source: Path, archive: Path) -> None: + """Create a 7z archive of ``source`` at ``archive`` using compression level 1.""" + subprocess.run( + ["7z", "a", "-mx=1", "-mmt=on", str(archive), str(source)], + check=True, + stdout=subprocess.DEVNULL, + ) + + +def _prune_backups( + backup_base: Path, + template_name: str | None, + keep_count: int, + *, + what_if: bool, +) -> None: + if template_name: + pattern = f"Template_{template_name}_????????_??????.7z" + else: + pattern = "Template_????????_??????.7z" + + existing = sorted( + [ + f + for f in backup_base.iterdir() + if f.is_file() and fnmatch.fnmatch(f.name, pattern) + ], + key=lambda f: f.stat().st_mtime, + reverse=True, + ) + + to_remove = existing[keep_count:] + for old in to_remove: + if what_if: + click.echo( + f"[Backup-CITemplate] WhatIf: would prune old backup: {old.name}" + ) + else: + click.echo(f"[Backup-CITemplate] Pruning old backup: {old.name}") + old.unlink() + + if to_remove: + label = template_name or "template" + click.echo(f"[Backup-CITemplate] Retained {keep_count} backup(s) for {label}.") diff --git a/src/ci_orchestrator/commands/vm.py b/src/ci_orchestrator/commands/vm.py index 8d60f56..e1911dc 100644 --- a/src/ci_orchestrator/commands/vm.py +++ b/src/ci_orchestrator/commands/vm.py @@ -194,9 +194,9 @@ def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]: cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours) out: list[Path] = [] for entry in clone_base.iterdir(): - if not entry.is_dir(): - continue try: + if not entry.is_dir(): + continue mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC) except OSError: continue diff --git a/tests/python/test_commands_retention.py b/tests/python/test_commands_retention.py new file mode 100644 index 0000000..8793f92 --- /dev/null +++ b/tests/python/test_commands_retention.py @@ -0,0 +1,195 @@ +"""Tests for ``ci_orchestrator retention run``.""" + +from __future__ import annotations + +import shutil +import time +from pathlib import Path +from typing import Any + +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.retention as ret +from ci_orchestrator.__main__ import cli + + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _fake_usage(free_gb: float, total_gb: float = 500.0) -> shutil._ntuple_diskusage: + gb = 1024**3 + return shutil._ntuple_diskusage( + int(total_gb * gb), + int((total_gb - free_gb) * gb), + int(free_gb * gb), + ) + + +def _make_old_dir(base: Path, name: str, age_days: int = 40) -> Path: + d = base / name + d.mkdir(parents=True) + old_mtime = time.time() - age_days * 86400 + import os + + os.utime(d, (old_mtime, old_mtime)) + return d + + +def _make_recent_dir(base: Path, name: str) -> Path: + d = base / name + d.mkdir(parents=True) + return d + + +# ── retention run — basic purge ─────────────────────────────────────────────── + + +def test_purge_old_artifact_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + art = tmp_path / "artifacts" + art.mkdir() + old = _make_old_dir(art, "job-001") + recent = _make_recent_dir(art, "job-002") + + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + ["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")], + ) + assert result.exit_code == 0, result.output + assert not old.exists(), "old dir should have been deleted" + assert recent.exists(), "recent dir should survive" + assert "Purged artifact" in result.output + + +def test_nothing_to_purge_message(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + art = tmp_path / "artifacts" + art.mkdir() + _make_recent_dir(art, "job-001") + + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + ["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")], + ) + assert result.exit_code == 0, result.output + assert "nothing to purge" in result.output + + +def test_missing_artifact_dir_skipped(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + [ + "retention", + "run", + "--artifact-dir", + str(tmp_path / "nonexistent"), + "--log-dir", + str(tmp_path / "logs"), + ], + ) + assert result.exit_code == 0, result.output + assert "not found" in result.output + + +# ── aggressive mode ─────────────────────────────────────────────────────────── + + +def test_aggressive_mode_triggered(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + art = tmp_path / "artifacts" + art.mkdir() + _make_old_dir(art, "job-001", age_days=10) # 10 days old, > aggressive threshold (7d) + + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=5)) + + result = CliRunner().invoke( + cli, + [ + "retention", + "run", + "--artifact-dir", + str(art), + "--log-dir", + str(tmp_path / "logs"), + "--min-free-gb", + "50", + "--aggressive-retention-days", + "7", + ], + ) + assert result.exit_code == 0, result.output + assert "aggressive retention" in result.output + assert "WARNING" in result.output + + +def test_aggressive_mode_not_triggered_when_space_ok( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + art = tmp_path / "artifacts" + art.mkdir() + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + ["retention", "run", "--artifact-dir", str(art), "--log-dir", str(tmp_path / "logs")], + ) + assert result.exit_code == 0, result.output + assert "Normal retention" in result.output + assert "WARNING" not in result.output + + +# ── --what-if ──────────────────────────────────────────────────────────────── + + +def test_what_if_does_not_delete(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + art = tmp_path / "artifacts" + art.mkdir() + old = _make_old_dir(art, "job-001") + + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + [ + "retention", + "run", + "--artifact-dir", + str(art), + "--log-dir", + str(tmp_path / "logs"), + "--what-if", + ], + ) + assert result.exit_code == 0, result.output + assert old.exists(), "--what-if must not delete anything" + assert "WhatIf" in result.output + + +# ── log dir purge ───────────────────────────────────────────────────────────── + + +def test_purge_old_log_dirs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + log = tmp_path / "logs" + log.mkdir() + old = _make_old_dir(log, "run-001") + + monkeypatch.setattr(ret.shutil, "disk_usage", lambda _p: _fake_usage(free_gb=200)) + + result = CliRunner().invoke( + cli, + [ + "retention", + "run", + "--artifact-dir", + str(tmp_path / "artifacts"), + "--log-dir", + str(log), + ], + ) + assert result.exit_code == 0, result.output + assert not old.exists() + assert "Purged log" in result.output diff --git a/tests/python/test_commands_template.py b/tests/python/test_commands_template.py new file mode 100644 index 0000000..5f69f65 --- /dev/null +++ b/tests/python/test_commands_template.py @@ -0,0 +1,305 @@ +"""Tests for ``ci_orchestrator template backup``.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import pytest +from click.testing import CliRunner + +import ci_orchestrator.commands.template as tmpl_mod +from ci_orchestrator.__main__ import cli + + +# ── fixtures ────────────────────────────────────────────────────────────────── + + +@pytest.fixture() +def template_dir(tmp_path: Path) -> Path: + """A fake template directory with a couple of files.""" + t = tmp_path / "templates" / "WinBuild2025" + t.mkdir(parents=True) + (t / "WinBuild2025.vmx").write_text("vmx content") + (t / "WinBuild2025.vmdk").write_bytes(b"\x00" * 1024) + return t + + +@pytest.fixture() +def backup_dir(tmp_path: Path) -> Path: + d = tmp_path / "backups" + d.mkdir() + return d + + +def _fake_compress(source: Path, archive: Path) -> None: + """Simulate 7z by writing a small placeholder file.""" + archive.write_bytes(b"fake-7z-archive") + + +@pytest.fixture() +def _no_systemctl(monkeypatch: pytest.MonkeyPatch) -> None: + """Prevent real systemctl and 7z calls in tests.""" + monkeypatch.setattr(tmpl_mod, "_stop_runner", lambda **_kw: False) + monkeypatch.setattr(tmpl_mod, "_start_runner", lambda **_kw: None) + monkeypatch.setattr(tmpl_mod, "_compress_7z", _fake_compress) + + +# ── basic backup ───────────────────────────────────────────────────────────── + + +def test_backup_creates_timestamped_archive( + template_dir: Path, backup_dir: Path, _no_systemctl: None +) -> None: + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(template_dir), + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code == 0, result.output + archives = list(backup_dir.iterdir()) + assert len(archives) == 1 + assert archives[0].name.startswith("Template_") + assert archives[0].suffix == ".7z" + + +def test_backup_missing_template_raises( + backup_dir: Path, tmp_path: Path, _no_systemctl: None +) -> None: + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(tmp_path / "nonexistent"), + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code != 0 + assert "not found" in result.output + + +# ── --all-templates ─────────────────────────────────────────────────────────── + + +def test_all_templates_backs_up_every_subdir( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + tmpl_root = tmp_path / "templates" + for name in ("WinBuild2025", "WinBuild2022", "LinuxBuild2404"): + d = tmpl_root / name + d.mkdir(parents=True) + (d / f"{name}.vmx").write_text("vmx") + + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--all-templates", + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code == 0, result.output + names = {f.name for f in backup_dir.iterdir() if f.suffix == ".7z"} + assert any("WinBuild2025" in n for n in names) + assert any("WinBuild2022" in n for n in names) + assert any("LinuxBuild2404" in n for n in names) + + +def test_all_templates_empty_dir_raises( + tmp_path: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch, _no_systemctl: None +) -> None: + tmpl_root = tmp_path / "templates" + tmpl_root.mkdir() + monkeypatch.setenv("CI_TEMPLATES", str(tmpl_root)) + + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--all-templates", + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + ], + ) + assert result.exit_code != 0 + assert "No template" in result.output + + +# ── pruning ─────────────────────────────────────────────────────────────────── + + +def test_prune_keeps_only_keep_count( + template_dir: Path, backup_dir: Path, _no_systemctl: None +) -> None: + for i in range(3): + old = backup_dir / f"Template_2025010{i}_120000.7z" + old.write_bytes(b"old-archive") + mtime = time.time() - (3 - i) * 3600 + os.utime(old, (mtime, mtime)) + + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(template_dir), + "--backup-base-dir", + str(backup_dir), + "--keep-count", + "2", + "--skip-runner-stop", + ], + ) + assert result.exit_code == 0, result.output + remaining = [f for f in backup_dir.iterdir() if f.suffix == ".7z"] + assert len(remaining) == 2, f"Expected 2 archives, got {len(remaining)}: {[f.name for f in remaining]}" + assert "Pruning" in result.output + + +# ── --what-if ──────────────────────────────────────────────────────────────── + + +def test_what_if_no_copy(template_dir: Path, backup_dir: Path, _no_systemctl: None) -> None: + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(template_dir), + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + "--what-if", + ], + ) + assert result.exit_code == 0, result.output + assert not list(backup_dir.iterdir()), "--what-if must not create anything" + assert "WhatIf" in result.output + + +# ── runner stop/start ───────────────────────────────────────────────────────── + + +def test_stop_runner_called_when_active( + template_dir: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stopped: list[bool] = [] + started: list[bool] = [] + + monkeypatch.setattr(tmpl_mod, "_stop_runner", lambda **kw: stopped.append(True) or True) + monkeypatch.setattr(tmpl_mod, "_start_runner", lambda **kw: started.append(True)) + monkeypatch.setattr(tmpl_mod, "_compress_7z", _fake_compress) + + result = CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(template_dir), + "--backup-base-dir", + str(backup_dir), + ], + ) + assert result.exit_code == 0, result.output + assert stopped, "_stop_runner should have been called" + assert started, "_start_runner should have been called" + + +def test_skip_runner_stop_skips_systemctl( + template_dir: Path, backup_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stopped: list[bool] = [] + monkeypatch.setattr(tmpl_mod, "_stop_runner", lambda **kw: stopped.append(True) or True) + monkeypatch.setattr(tmpl_mod, "_compress_7z", _fake_compress) + + CliRunner().invoke( + cli, + [ + "template", + "backup", + "--template-path", + str(template_dir), + "--backup-base-dir", + str(backup_dir), + "--skip-runner-stop", + ], + ) + assert not stopped, "--skip-runner-stop should prevent _stop_runner from being called" + + +# ── unit helpers: _stop_runner / _start_runner ─────────────────────────────── + + +def test_stop_runner_noop_when_inactive(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + tmpl_mod.subprocess, + "run", + lambda cmd, **_kw: type("R", (), {"stdout": "inactive\n", "returncode": 0})(), + ) + assert tmpl_mod._stop_runner(what_if=False) is False + + +def test_stop_runner_returns_true_when_active(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[list[str]] = [] + + def _fake_run(cmd: list[str], **_kw: object) -> object: + calls.append(cmd) + return type("R", (), {"stdout": "active\n", "returncode": 0})() + + monkeypatch.setattr(tmpl_mod.subprocess, "run", _fake_run) + result = tmpl_mod._stop_runner(what_if=False) + assert result is True + assert any("stop" in c for c in calls) + + +def test_stop_runner_what_if_does_not_call_stop(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[list[str]] = [] + + def _fake_run(cmd: list[str], **_kw: object) -> object: + calls.append(cmd) + return type("R", (), {"stdout": "active\n", "returncode": 0})() + + monkeypatch.setattr(tmpl_mod.subprocess, "run", _fake_run) + result = tmpl_mod._stop_runner(what_if=True) + assert result is True + assert not any("stop" in c for c in calls), "what_if=True must not call systemctl stop" + + +# ── _compress_7z args ───────────────────────────────────────────────────────── + + +def test_compress_7z_uses_mx1(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls: list[list[str]] = [] + + def _fake_run(cmd: list[str], **_kw: object) -> object: + calls.append(cmd) + return type("R", (), {"returncode": 0})() + + monkeypatch.setattr(tmpl_mod.subprocess, "run", _fake_run) + tmpl_mod._compress_7z(tmp_path / "src", tmp_path / "out.7z") + assert calls, "subprocess.run should have been called" + cmd = calls[0] + assert cmd[0] == "7z" + assert "-mx=1" in cmd