feat: B5 — port retention/template-backup to Python, deploy systemd timers
- Add `retention run` command (ports Invoke-RetentionPolicy.ps1): purges old artifact/log dirs with aggressive mode when disk space is low. - Add `template backup` command (ports Backup-CITemplate.ps1): 7z -mx=1 compressed archives in /var/lib/ci/backups/, prunes to keep-count=3, stops/restarts act-runner.service around the copy. - Update ci-retention-policy.service and ci-backup-template.service to use Python; pwsh is no longer required on the Linux host. - Fix ci-watch-runner-health.service: pass --service-name act-runner (Linux service name, not Windows act_runner default). - Fix _list_orphans in vm.py: wrap is_dir() inside the OSError try block so a stat() failure on an entry is silently skipped rather than raised. - Mark B5 complete in PhaseB-user-checklist.md. Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,134 @@
|
||||
"""``retention`` sub-commands.
|
||||
|
||||
Ports ``scripts/Invoke-RetentionPolicy.ps1`` to Python.
|
||||
|
||||
* ``retention run`` — purge old artifact and log directories based on age;
|
||||
switches to aggressive retention when disk free space is low.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
|
||||
@click.group()
|
||||
def retention() -> None:
|
||||
"""Maintenance: artifact and log retention."""
|
||||
|
||||
|
||||
@retention.command("run")
|
||||
@click.option(
|
||||
"--artifact-dir",
|
||||
default=None,
|
||||
help="Artifact directory (default: CI_ARTIFACTS from config/env).",
|
||||
)
|
||||
@click.option(
|
||||
"--log-dir",
|
||||
default=None,
|
||||
help="Log directory (default: <CI_ROOT>/logs).",
|
||||
)
|
||||
@click.option(
|
||||
"--retention-days",
|
||||
default=30,
|
||||
show_default=True,
|
||||
help="Normal retention threshold in days.",
|
||||
)
|
||||
@click.option(
|
||||
"--aggressive-retention-days",
|
||||
default=7,
|
||||
show_default=True,
|
||||
help="Retention threshold when disk free space is below --min-free-gb.",
|
||||
)
|
||||
@click.option(
|
||||
"--min-free-gb",
|
||||
default=50,
|
||||
show_default=True,
|
||||
help="Switch to aggressive retention when free space drops below this value (GB).",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
is_flag=True,
|
||||
help="Dry run — log what would be deleted without actually deleting.",
|
||||
)
|
||||
def retention_run(
|
||||
artifact_dir: str | None,
|
||||
log_dir: str | None,
|
||||
retention_days: int,
|
||||
aggressive_retention_days: int,
|
||||
min_free_gb: int,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Purge old artifact and log directories based on retention policy."""
|
||||
config = load_config()
|
||||
|
||||
art_path = Path(artifact_dir) if artifact_dir else config.paths.artifacts
|
||||
log_path = Path(log_dir) if log_dir else config.paths.root / "logs"
|
||||
|
||||
check_path = art_path if art_path.exists() else config.paths.root
|
||||
usage = shutil.disk_usage(str(check_path))
|
||||
free_gb = usage.free / (1024**3)
|
||||
|
||||
aggressive = free_gb < min_free_gb
|
||||
effective_days = aggressive_retention_days if aggressive else retention_days
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(days=effective_days)
|
||||
|
||||
click.echo(f"[RetentionPolicy] Free space: {free_gb:.1f} GB")
|
||||
if aggressive:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] WARNING: Free space below {min_free_gb} GB — "
|
||||
f"aggressive retention: {aggressive_retention_days}d",
|
||||
err=True,
|
||||
)
|
||||
else:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] Normal retention: {effective_days}d "
|
||||
f"(cutoff: {cutoff.strftime('%Y-%m-%d')})"
|
||||
)
|
||||
|
||||
_purge_old_dirs(art_path, "artifact", cutoff, what_if=what_if)
|
||||
_purge_old_dirs(log_path, "log", cutoff, what_if=what_if)
|
||||
|
||||
usage_after = shutil.disk_usage(str(check_path))
|
||||
free_gb_after = usage_after.free / (1024**3)
|
||||
delta = free_gb_after - free_gb
|
||||
click.echo(
|
||||
f"[RetentionPolicy] Done. Free space: {free_gb_after:.1f} GB (delta: +{delta:.1f} GB)"
|
||||
)
|
||||
|
||||
|
||||
def _purge_old_dirs(base: Path, label: str, cutoff: datetime, *, what_if: bool) -> None:
|
||||
if not base.exists():
|
||||
click.echo(f"[RetentionPolicy] {label} dir not found: {base} — skipping.")
|
||||
return
|
||||
|
||||
old = [
|
||||
d
|
||||
for d in base.iterdir()
|
||||
if d.is_dir()
|
||||
and datetime.fromtimestamp(d.stat().st_mtime, tz=UTC) < cutoff
|
||||
]
|
||||
|
||||
if not old:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] {label}: nothing to purge "
|
||||
f"(cutoff: {cutoff.strftime('%Y-%m-%d')})."
|
||||
)
|
||||
return
|
||||
|
||||
for d in old:
|
||||
age_days = int(
|
||||
(datetime.now(tz=UTC) - datetime.fromtimestamp(d.stat().st_mtime, tz=UTC)).days
|
||||
)
|
||||
if what_if:
|
||||
click.echo(
|
||||
f"[RetentionPolicy] WhatIf: would purge {label} ({age_days}d): {d.name}"
|
||||
)
|
||||
else:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
click.echo(f"[RetentionPolicy] Purged {label} ({age_days}d): {d.name}")
|
||||
@@ -0,0 +1,208 @@
|
||||
"""``template`` sub-commands.
|
||||
|
||||
Ports ``scripts/Backup-CITemplate.ps1`` to Python.
|
||||
|
||||
* ``template backup`` — create timestamped copies of VMware template
|
||||
directories; prune old backups beyond ``--keep-count``.
|
||||
|
||||
The command stops ``act-runner.service`` before copying and restarts it
|
||||
afterwards (via ``systemctl``), so it must run as root or with appropriate
|
||||
sudo privileges. Pass ``--skip-runner-stop`` to bypass this when the runner
|
||||
is already offline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import shutil
|
||||
import subprocess
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import click
|
||||
|
||||
from ci_orchestrator.config import load_config
|
||||
|
||||
|
||||
@click.group()
|
||||
def template() -> None:
|
||||
"""VMware template management."""
|
||||
|
||||
|
||||
@template.command("backup")
|
||||
@click.option(
|
||||
"--template-path",
|
||||
default=None,
|
||||
help=(
|
||||
"Template directory to back up "
|
||||
"(default: <CI_TEMPLATES>/WinBuild2025)."
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--all-templates",
|
||||
is_flag=True,
|
||||
help="Back up every subdirectory under CI_TEMPLATES in a single pass.",
|
||||
)
|
||||
@click.option(
|
||||
"--backup-base-dir",
|
||||
default="/var/lib/ci/backups",
|
||||
show_default=True,
|
||||
help="Parent directory for timestamped backup folders.",
|
||||
)
|
||||
@click.option(
|
||||
"--keep-count",
|
||||
default=3,
|
||||
show_default=True,
|
||||
help="Number of most-recent backups to retain per template.",
|
||||
)
|
||||
@click.option(
|
||||
"--skip-runner-stop",
|
||||
is_flag=True,
|
||||
help="Skip stopping/restarting act-runner.service.",
|
||||
)
|
||||
@click.option(
|
||||
"--what-if",
|
||||
is_flag=True,
|
||||
help="Dry run — log what would happen without copying or deleting.",
|
||||
)
|
||||
def template_backup(
|
||||
template_path: str | None,
|
||||
all_templates: bool,
|
||||
backup_base_dir: str,
|
||||
keep_count: int,
|
||||
skip_runner_stop: bool,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
"""Create timestamped backup of CI template VM directories."""
|
||||
config = load_config()
|
||||
backup_base = Path(backup_base_dir)
|
||||
timestamp = datetime.now(tz=UTC).strftime("%Y%m%d_%H%M%S")
|
||||
|
||||
if all_templates:
|
||||
tmpl_root = config.paths.templates
|
||||
if not tmpl_root.exists():
|
||||
raise click.ClickException(f"Templates directory not found: {tmpl_root}")
|
||||
to_backup = sorted(d for d in tmpl_root.iterdir() if d.is_dir())
|
||||
if not to_backup:
|
||||
raise click.ClickException(
|
||||
f"No template subdirectories found under {tmpl_root}"
|
||||
)
|
||||
else:
|
||||
p = Path(template_path) if template_path else config.paths.templates / "WinBuild2025"
|
||||
if not p.is_dir():
|
||||
raise click.ClickException(f"Template directory not found: {p}")
|
||||
to_backup = [p]
|
||||
|
||||
runner_was_stopped = False
|
||||
try:
|
||||
if not skip_runner_stop:
|
||||
runner_was_stopped = _stop_runner(what_if=what_if)
|
||||
|
||||
if not what_if:
|
||||
backup_base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for tmpl_dir in to_backup:
|
||||
name_tag = f"{tmpl_dir.name}_{timestamp}" if all_templates else timestamp
|
||||
archive = backup_base / f"Template_{name_tag}.7z"
|
||||
|
||||
if what_if:
|
||||
click.echo(
|
||||
f"[Backup-CITemplate] WhatIf: would compress {tmpl_dir} -> {archive}"
|
||||
)
|
||||
else:
|
||||
click.echo(f"[Backup-CITemplate] Compressing {tmpl_dir} -> {archive} ...")
|
||||
_compress_7z(tmpl_dir, archive)
|
||||
size_mb = archive.stat().st_size // (1024 * 1024)
|
||||
click.echo(
|
||||
f"[Backup-CITemplate] Backup complete: {archive} ({size_mb} MB)"
|
||||
)
|
||||
|
||||
template_label = tmpl_dir.name if all_templates else None
|
||||
_prune_backups(backup_base, template_label, keep_count, what_if=what_if)
|
||||
|
||||
finally:
|
||||
if runner_was_stopped:
|
||||
_start_runner(what_if=what_if)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────── helpers
|
||||
|
||||
|
||||
def _stop_runner(*, what_if: bool) -> bool:
|
||||
"""Stop act-runner.service. Returns True if it was running."""
|
||||
result = subprocess.run(
|
||||
["systemctl", "is-active", "act-runner.service"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.stdout.strip() != "active":
|
||||
return False
|
||||
if what_if:
|
||||
click.echo("[Backup-CITemplate] WhatIf: would stop act-runner.service")
|
||||
return True
|
||||
click.echo("[Backup-CITemplate] Stopping act-runner.service ...")
|
||||
subprocess.run(["systemctl", "stop", "act-runner.service"], check=True)
|
||||
click.echo("[Backup-CITemplate] act-runner.service stopped.")
|
||||
return True
|
||||
|
||||
|
||||
def _start_runner(*, what_if: bool) -> None:
|
||||
if what_if:
|
||||
click.echo("[Backup-CITemplate] WhatIf: would start act-runner.service")
|
||||
return
|
||||
click.echo("[Backup-CITemplate] Restarting act-runner.service ...")
|
||||
subprocess.run(["systemctl", "start", "act-runner.service"], check=True)
|
||||
result = subprocess.run(
|
||||
["systemctl", "is-active", "act-runner.service"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
click.echo(
|
||||
f"[Backup-CITemplate] act-runner.service status: {result.stdout.strip()}"
|
||||
)
|
||||
|
||||
|
||||
def _compress_7z(source: Path, archive: Path) -> None:
|
||||
"""Create a 7z archive of ``source`` at ``archive`` using compression level 1."""
|
||||
subprocess.run(
|
||||
["7z", "a", "-mx=1", "-mmt=on", str(archive), str(source)],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
|
||||
def _prune_backups(
|
||||
backup_base: Path,
|
||||
template_name: str | None,
|
||||
keep_count: int,
|
||||
*,
|
||||
what_if: bool,
|
||||
) -> None:
|
||||
if template_name:
|
||||
pattern = f"Template_{template_name}_????????_??????.7z"
|
||||
else:
|
||||
pattern = "Template_????????_??????.7z"
|
||||
|
||||
existing = sorted(
|
||||
[
|
||||
f
|
||||
for f in backup_base.iterdir()
|
||||
if f.is_file() and fnmatch.fnmatch(f.name, pattern)
|
||||
],
|
||||
key=lambda f: f.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
to_remove = existing[keep_count:]
|
||||
for old in to_remove:
|
||||
if what_if:
|
||||
click.echo(
|
||||
f"[Backup-CITemplate] WhatIf: would prune old backup: {old.name}"
|
||||
)
|
||||
else:
|
||||
click.echo(f"[Backup-CITemplate] Pruning old backup: {old.name}")
|
||||
old.unlink()
|
||||
|
||||
if to_remove:
|
||||
label = template_name or "template"
|
||||
click.echo(f"[Backup-CITemplate] Retained {keep_count} backup(s) for {label}.")
|
||||
@@ -194,9 +194,9 @@ def _list_orphans(clone_base: Path, max_age_hours: int) -> list[Path]:
|
||||
cutoff = datetime.now(tz=UTC) - timedelta(hours=max_age_hours)
|
||||
out: list[Path] = []
|
||||
for entry in clone_base.iterdir():
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
try:
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
mtime = datetime.fromtimestamp(entry.stat().st_mtime, tz=UTC)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user