From 6104811e7eb2ff099b2a1c676602e6a8c4308208 Mon Sep 17 00:00:00 2001
From: Simone <simone.chifari@gmail.com>
Date: Sun, 31 May 2026 17:45:32 +0200
Subject: [PATCH] fix: eliminate Linux build clock skew from RTC/NTP race

Linux clones could boot with the RTC interpreted as local time (guest
runs hours ahead of UTC). NTP then steps the clock backward at runtime,
landing after source mtimes were stamped, so make reported "Clock skew
detected" and future-dated files.

- build run: enable NTP and block until NTPSynchronized before stamping
  the sources, so the backward correction happens pre-touch. Drop
  `hwclock -s`, which read the local-time RTC and pushed the clock further
  ahead.
- template prepare-linux: set-local-rtc 0 so clones interpret the RTC as
  UTC and boot at the correct time; NTP then only nudges, never steps 2h.

Linux/SSH path only; WinRM branch untouched.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/ci_orchestrator/commands/build.py    | 26 ++++++++++++++++++++++++
 src/ci_orchestrator/commands/template.py | 21 +++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/src/ci_orchestrator/commands/build.py b/src/ci_orchestrator/commands/build.py
index a917515..fd4319d 100644
--- a/src/ci_orchestrator/commands/build.py
+++ b/src/ci_orchestrator/commands/build.py
@@ -151,6 +151,24 @@ def _linux_build(
         key_path=key_path,
         known_hosts=known_hosts,
     ) as t:
+        # Clock-skew guard. Ephemeral clones can boot with a wrong clock
+        # (e.g. RTC kept in local time → guest reads it as UTC and runs hours
+        # ahead). Enable NTP and BLOCK until the clock is actually synced, so
+        # the (possibly backward) correction step happens HERE — before the
+        # sources are stamped below. If the step landed *after* the touch,
+        # make would again see source mtimes in the future and warn about
+        # clock skew. Do NOT run `hwclock -s`: on a clone whose RTC is in
+        # local time it pushes the clock further ahead, the very skew we are
+        # removing. Best-effort (check=False); the touch after checkout is the
+        # backstop when NTP is unreachable and the clock simply stays put.
+        t.run(
+            "sudo timedatectl set-ntp true 2>/dev/null || true; "
+            'for _ in $(seq 1 60); do '
+            '[ "$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" '
+            "= yes ] && break; sleep 1; done",
+            check=False,
+        )
+
         t.run(f"rm -rf {_sh_quote(workdir)} && mkdir -p {_sh_quote(workdir)}")
 
         if host_source_dir:
@@ -190,6 +208,14 @@ def _linux_build(
                 "Linux build requires either --host-source-dir or --clone-url."
             )
 
+        # Normalise source mtimes to the (now-synced) guest clock so make
+        # never sees a file dated in the future and skips the skew warning
+        # plus the spurious rebuild it triggers.
+        t.run(
+            f"find {_sh_quote(workdir)} -exec touch -d {_sh_quote('now')} {{}} +",
+            check=False,
+        )
+
         env_prefix = "".join(
             f"export {k}={_sh_quote(v)}; " for k, v in extra_env.items()
         )
diff --git a/src/ci_orchestrator/commands/template.py b/src/ci_orchestrator/commands/template.py
index a4cfa93..5db4496 100644
--- a/src/ci_orchestrator/commands/template.py
+++ b/src/ci_orchestrator/commands/template.py
@@ -1736,6 +1736,27 @@ def template_prepare_linux(
             )
         click.echo("[prepare-linux] machine-id cleared — clones get unique DHCP ID.")
 
+        # ── Step 7c: Enable NTP time sync (clock-skew guard for clones) ──────
+        # Bake persistent NTP into the snapshot so every clone boots and
+        # corrects its clock automatically. Without this, a clone whose RTC
+        # drifted behind the host produces source mtimes "in the future" and
+        # make emits "Clock skew detected". This is the clean fix; the build
+        # path also force-syncs + touches sources as a runtime safety net.
+        click.echo("[prepare-linux] Step 7c — Enabling NTP time sync ...")
+        # set-local-rtc 0 forces the RTC to be interpreted as UTC. Without it
+        # a clone can boot hours ahead (RTC in local time read as UTC); NTP
+        # then steps the clock *backward* at runtime, which is what produces
+        # the "Clock skew detected" / future-mtime warnings in make.
+        rc_ntp, _, err_ntp = _pl_run(
+            client,
+            "sudo timedatectl set-local-rtc 0 2>/dev/null; "
+            "sudo systemctl enable --now systemd-timesyncd 2>/dev/null; "
+            "sudo timedatectl set-ntp true",
+        )
+        if rc_ntp != 0:
+            raise click.ClickException(f"enabling NTP failed (exit {rc_ntp}):\n{err_ntp}")
+        click.echo("[prepare-linux] NTP time sync enabled.")
+
         # ── Step 8: Graceful shutdown ─────────────────────────────────────────
         if do_snapshot:
             click.echo("[prepare-linux] Step 8 — Sending graceful shutdown ...")