From 19186981a5b1d3b882fcff2c8e482e5c1c17bbf9 Mon Sep 17 00:00:00 2001 From: Simone Date: Sun, 19 Jul 2026 19:47:16 +0200 Subject: [PATCH] fix(host): bring up VMware host stack at boot via native systemd units Build VMs stopped starting after a reboot into a new kernel: vmmon/vmnet were left unloaded and vmnet1/vmnet8 (DHCP/NAT) never came up, so vmware-vmx died (status=237/KEYRING) and clones got no IP. Root cause: the LSB->systemd sysv-generated vmware.service intermittently fails to generate at early boot ("systemd-sysv-generator: Cannot find unit vmware.service"), and the existing module-rebuild guard only builds the modules -- it never loads them or starts vmnet networking. Add a native vmware-host.service that runs `/etc/init.d/vmware start` (loads vmmon/vmci + brings up vmnet), ordered after the rebuild guard, and mask the flaky sysv vmware.service so the native unit is the sole owner. Provisioned by scripts/install-vmware-host-services.sh and wired into setup-host-linux.sh (new step 8/9). VMware ships no native unit and no kernel postinst hook, so this native wrapper around VMware's own tools is the reliable boot path. Verified: linked clone boots and gets a vmnet8 DHCP IP. --- deploy/systemd/README.md | 47 ++++++++++ deploy/systemd/vmware-host.service | 21 +++++ deploy/systemd/vmware-modules-rebuild.service | 17 ++++ deploy/systemd/vmware-modules-rebuild.sh | 28 ++++++ docs/RUNBOOK-PhaseC.md | 3 +- scripts/install-vmware-host-services.sh | 90 +++++++++++++++++++ setup-host-linux.sh | 42 ++++++--- 7 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 deploy/systemd/vmware-host.service create mode 100644 deploy/systemd/vmware-modules-rebuild.service create mode 100755 deploy/systemd/vmware-modules-rebuild.sh create mode 100755 scripts/install-vmware-host-services.sh diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md index 70aa1ed..ea4567a 100644 --- a/deploy/systemd/README.md +++ b/deploy/systemd/README.md @@ -84,3 +84,50 @@ sudo systemctl daemon-reload A questo punto il fallback è ri-attivare i task su Windows con `scripts/Register-CIScheduledTasks.ps1`. + +--- + +## Servizi VMware host (prerequisito avvio VM) + +Distinti dai timer periodici sopra: due unit **native** che portano su lo stack +host di VMware Workstation a ogni boot. Servono perché VMware su Linux **non** +fornisce unit systemd native, **non** registra `vmmon`/`vmnet` con DKMS e **non** +installa hook di post-install del kernel. + +| unit | ruolo | +| ---- | ----- | +| `vmware-modules-rebuild.service` | ricostruisce `vmmon`/`vmnet` via `vmware-modconfig` se il `.ko` per il kernel in esecuzione manca (guard *build-if-missing*). Ordinata `Before=vmware-host.service`. Script: `/usr/local/sbin/vmware-modules-rebuild.sh`. | +| `vmware-host.service` | esegue `/etc/init.d/vmware start` → carica `vmmon`/`vmci` e avvia la rete `vmnet1`/`vmnet8` (DHCP/NAT). Ordinata `After=vmware-modules-rebuild.service`. | + +Il sysv `vmware.service` (LSB) viene **mascherato**: si è dimostrato inaffidabile +da generare al boot (`systemd-sysv-generator: Cannot find unit vmware.service`), +lasciando `vmmon` non caricato e la rete VM giù → le build VM non partono. +`vmware-host.service` ne diventa l'unico owner. + +### Installazione + +```bash +sudo ./scripts/install-vmware-host-services.sh # installa + enable + start (idempotente) +``` + +Installato anche automaticamente da `setup-host-linux.sh` (Step 8/9). + +### Verifica + +```bash +systemctl is-enabled vmware-host.service vmware-modules-rebuild.service +lsmod | grep -E 'vmmon|vmnet' +ip -o link show | grep vmnet # attese: vmnet1, vmnet8 +``` + +### Recovery (stack giù, VM non partono) + +```bash +sudo systemctl start vmware-host.service +``` + +### Rollback + +```bash +sudo ./scripts/install-vmware-host-services.sh --uninstall +``` diff --git a/deploy/systemd/vmware-host.service b/deploy/systemd/vmware-host.service new file mode 100644 index 0000000..66bdcd2 --- /dev/null +++ b/deploy/systemd/vmware-host.service @@ -0,0 +1,21 @@ +[Unit] +Description=Start VMware Workstation host stack (vmmon/vmci + vmnet networking) +Documentation=man:vmware(1) +# Native replacement for the flaky LSB->systemd sysv-generated vmware.service, +# which intermittently fails to generate at early boot ("Cannot find unit +# vmware.service"), leaving vmmon unloaded and vmnet DHCP/NAT down so VMs +# cannot start. The rebuild guard ensures the modules are built first; this +# unit loads them and brings up vmnet1/vmnet8 via VMware's own init script. +Wants=vmware-modules-rebuild.service +After=vmware-modules-rebuild.service network-pre.target local-fs.target +ConditionPathExists=/etc/init.d/vmware + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/etc/init.d/vmware start +ExecStop=/etc/init.d/vmware stop +TimeoutStartSec=120 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/vmware-modules-rebuild.service b/deploy/systemd/vmware-modules-rebuild.service new file mode 100644 index 0000000..93a86d8 --- /dev/null +++ b/deploy/systemd/vmware-modules-rebuild.service @@ -0,0 +1,17 @@ +[Unit] +Description=Rebuild VMware host modules (vmmon/vmnet) for running kernel if missing +Documentation=man:vmware-modconfig(1) +# VMware modules are not DKMS-managed; ensure they exist before vmware-host starts. +After=local-fs.target +Before=vmware-host.service +ConditionPathExists=/usr/bin/vmware-modconfig + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/vmware-modules-rebuild.sh +# Building modules can take a little while on a cold kernel. +TimeoutStartSec=600 + +[Install] +WantedBy=multi-user.target diff --git a/deploy/systemd/vmware-modules-rebuild.sh b/deploy/systemd/vmware-modules-rebuild.sh new file mode 100755 index 0000000..b00a72a --- /dev/null +++ b/deploy/systemd/vmware-modules-rebuild.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Rebuild VMware host modules (vmmon/vmnet) for the running kernel if absent. +# VMware Workstation does not register these with DKMS, so a kernel upgrade +# leaves them unbuilt until the GUI or vmware-modconfig is run. This runs at +# boot, before vmware-host.service, and self-heals using VMware's own builder. +set -eu + +KVER="$(uname -r)" + +if /sbin/modinfo vmmon >/dev/null 2>&1 && /sbin/modinfo vmnet >/dev/null 2>&1; then + # Module objects already present for the running kernel — nothing to do. + exit 0 +fi + +logger -t vmware-modules-rebuild "vmmon/vmnet missing for kernel ${KVER} — rebuilding via vmware-modconfig" + +if ! command -v vmware-modconfig >/dev/null 2>&1; then + logger -t vmware-modules-rebuild "ERROR: vmware-modconfig not found" + exit 1 +fi + +# vmware-modconfig builds against the running kernel's headers. +if /usr/bin/vmware-modconfig --console --install-all; then + logger -t vmware-modules-rebuild "rebuild complete for kernel ${KVER}" +else + logger -t vmware-modules-rebuild "ERROR: vmware-modconfig failed (headers for ${KVER} installed?)" + exit 1 +fi diff --git a/docs/RUNBOOK-PhaseC.md b/docs/RUNBOOK-PhaseC.md index e53b152..93ea27a 100644 --- a/docs/RUNBOOK-PhaseC.md +++ b/docs/RUNBOOK-PhaseC.md @@ -557,5 +557,6 @@ original `.ps1` scripts remain in `scripts/archived/`. | `bench run` round FAIL: orphan | Clone was not destroyed | `ci vm cleanup`; check vmrun and `/var/lib/ci/build-vms` | | WinRM jobs drop at concurrency 4 | Known instability (TODO §3.6) | Use `--concurrency 3` | | IP-acquire timeout (Linux) | vmnet8 DHCP wedged | `sudo systemctl restart vmware-networks` | -| `vmrun` "operation was canceled" | vmmon/vmnet unbuilt after a kernel bump | `sudo vmware-modconfig --console --install-all` (or reboot; the systemd guard rebuilds) | +| `vmrun` "operation was canceled" | vmmon/vmnet unbuilt after a kernel bump | `sudo vmware-modconfig --console --install-all` (or reboot; `vmware-modules-rebuild.service` rebuilds) | +| VMs won't start after a reboot — no `/dev/vmnet*`, `vmware-vmx` dies `status=237/KEYRING` | Host stack didn't come up at boot: `vmmon` unloaded + vmnet DHCP/NAT down (the flaky sysv `vmware.service` failed to generate — `systemd-sysv-generator: Cannot find unit vmware.service`) | `sudo systemctl start vmware-host.service`, then confirm it's enabled. Provisioned by `scripts/install-vmware-host-services.sh` — see [deploy/systemd](../deploy/systemd/README.md#servizi-vmware-host-prerequisito-avvio-vm) | | `ci: command not found` | Alias not loaded | `source ~/.bashrc`, or re-run [§0.2](#02-install-the-ci-alias) | diff --git a/scripts/install-vmware-host-services.sh b/scripts/install-vmware-host-services.sh new file mode 100755 index 0000000..7b24e0e --- /dev/null +++ b/scripts/install-vmware-host-services.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# install-vmware-host-services.sh — install the native systemd units that bring +# up the VMware Workstation host stack at boot on the Linux CI host. +# +# Why this exists: VMware Workstation on Linux ships NO native systemd unit, +# does NOT register vmmon/vmnet with DKMS, and installs NO kernel postinst hook. +# Two failure modes follow, both of which stop build VMs from starting: +# 1. After a kernel upgrade the vmmon/vmnet .ko are unbuilt until +# vmware-modconfig runs (vmrun start => "Error: The operation was +# canceled"). +# 2. The LSB-generated sysv `vmware.service` has proven unreliable to generate +# at early boot ("systemd-sysv-generator: Cannot find unit vmware.service"), +# so vmmon stays unloaded and vmnet1/vmnet8 (DHCP/NAT) never come up. +# +# This installs two native systemd units built on VMware's own tools: +# - vmware-modules-rebuild.service : rebuilds vmmon/vmnet via vmware-modconfig +# if the .ko for the running kernel is missing (build-if-missing guard). +# - vmware-host.service : runs `/etc/init.d/vmware start` to load +# the modules and bring up vmnet networking; ordered After the guard. +# The flaky sysv `vmware.service` is masked so vmware-host.service is the sole +# owner of the host stack. +# +# Usage: +# sudo ./scripts/install-vmware-host-services.sh # install + enable + start +# sudo ./scripts/install-vmware-host-services.sh --uninstall +# +# Idempotent: re-running overwrites the units in place and re-enables them. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(dirname "$SCRIPT_DIR")" +SRC="$REPO_DIR/deploy/systemd" +ACTION="install" + +log() { printf '[install-vmware-host-services] %s\n' "$*"; } +warn() { printf '[install-vmware-host-services] WARN: %s\n' "$*" >&2; } +die() { printf '[install-vmware-host-services] ERROR: %s\n' "$*" >&2; exit 1; } + +while [ $# -gt 0 ]; do + case "$1" in + --uninstall) ACTION="uninstall"; shift ;; + -h|--help) sed -n '2,29p' "$0"; exit 0 ;; + *) die "unknown argument: $1 (use --uninstall)" ;; + esac +done + +[ "$(id -u)" -eq 0 ] || die "run as root: sudo bash $0" + +if [ "$ACTION" = "uninstall" ]; then + systemctl disable --now vmware-host.service 2>/dev/null || true + systemctl disable --now vmware-modules-rebuild.service 2>/dev/null || true + rm -f /etc/systemd/system/vmware-host.service \ + /etc/systemd/system/vmware-modules-rebuild.service \ + /usr/local/sbin/vmware-modules-rebuild.sh + systemctl unmask vmware.service 2>/dev/null || true + systemctl daemon-reload + log "uninstalled (units removed, vmware.service unmasked)" + exit 0 +fi + +[ -d "$SRC" ] || die "source dir not found: $SRC (clone the repo first)" +command -v vmware-modconfig >/dev/null 2>&1 || warn "vmware-modconfig not found — is VMware Workstation installed?" +[ -x /etc/init.d/vmware ] || warn "/etc/init.d/vmware not found — is VMware Workstation installed?" + +# 1. rebuild guard script + units +install -m 0755 -o root -g root "$SRC/vmware-modules-rebuild.sh" /usr/local/sbin/vmware-modules-rebuild.sh +install -m 0644 -o root -g root "$SRC/vmware-modules-rebuild.service" /etc/systemd/system/vmware-modules-rebuild.service +install -m 0644 -o root -g root "$SRC/vmware-host.service" /etc/systemd/system/vmware-host.service +log "installed rebuild script + 2 systemd units" + +# 2. mask the flaky sysv vmware.service so the native unit is the sole owner +if [ "$(systemctl is-enabled vmware.service 2>/dev/null || true)" = "masked" ]; then + log "vmware.service already masked" +else + systemctl mask vmware.service && log "masked sysv vmware.service" || warn "could not mask vmware.service" +fi + +# 3. reload + enable + start (start is idempotent — init script start is re-runnable) +systemctl daemon-reload +systemctl enable vmware-modules-rebuild.service +systemctl enable vmware-host.service +log "enabled vmware-modules-rebuild.service + vmware-host.service" + +if systemctl start vmware-host.service; then + log "vmware-host.service started" +else + warn "vmware-host.service start failed — check: journalctl -u vmware-host.service" +fi + +log "done — verify: lsmod | grep -E 'vmmon|vmnet'; ip -o link show | grep vmnet" diff --git a/setup-host-linux.sh b/setup-host-linux.sh index ffe13c4..324ff94 100755 --- a/setup-host-linux.sh +++ b/setup-host-linux.sh @@ -14,7 +14,10 @@ # 5. (Opzionale) Clona il repo e installa il package nel venv # 6. (Opt-in, --with-pwsh) Installa PowerShell Core # 7. Installa l'alias shell `ci` a livello di sistema (/etc/profile.d/) -# 8. Autorizza ci-runner sulla sessione X del desktop user (xhost persistente) +# 8. Installa i servizi VMware host (unit systemd native): caricano vmmon/vmnet +# e avviano la rete vmnet (DHCP/NAT) a ogni boot, e ricostruiscono i moduli +# dopo un upgrade del kernel. Sostituisce il flaky sysv vmware.service. +# 9. Autorizza ci-runner sulla sessione X del desktop user (xhost persistente) # così `ci vmware` può aprire la GUI di VMware Workstation. # L'host Linux NON ha più bisogno di pwsh per il normale funzionamento # (orchestratore Python). Installalo solo se ti serve per script legacy. @@ -106,7 +109,7 @@ echo " Utente : $CI_USER" echo "" # ── Step 1: utente di servizio ──────────────────────────────────────────────── -info "[1/8] Utente di servizio $CI_USER" +info "[1/9] Utente di servizio $CI_USER" if id "$CI_USER" &>/dev/null; then warn "Utente $CI_USER esiste già — skip" else @@ -115,7 +118,7 @@ else fi # ── Step 2: mount partizione CI ─────────────────────────────────────────────── -info "[2/8] Mount $CI_DISK → $CI_ROOT" +info "[2/9] Mount $CI_DISK → $CI_ROOT" if mountpoint -q "$CI_ROOT" 2>/dev/null; then warn "$CI_ROOT già montato — skip riorganizzazione e fstab" @@ -173,7 +176,7 @@ else fi # ── Step 3: layout directory e permessi ─────────────────────────────────────── -info "[3/8] Layout directory e permessi" +info "[3/9] Layout directory e permessi" # Installa acl se mancante if ! command -v setfacl &>/dev/null; then @@ -210,7 +213,7 @@ echo " Layout OK — ACL default su build-vms applicata" # ── Step 4: Python 3.11+ e venv ────────────────────────────────────────────── if [[ "$SKIP_PYTHON" == false ]]; then - info "[4/8] Python 3.11+ e venv produzione" + info "[4/9] Python 3.11+ e venv produzione" # Cerca la prima versione di Python >= 3.11 già installata PYTHON_BIN="" @@ -241,12 +244,12 @@ if [[ "$SKIP_PYTHON" == false ]]; then echo " venv creato in $OPT_CI/venv ($($PYTHON_BIN --version))" fi else - info "[4/8] Python/venv — SKIPPED" + info "[4/9] Python/venv — SKIPPED" fi # ── Step 5: Clone repo e pip install ───────────────────────────────────────── if [[ "$SKIP_CLONE" == false ]]; then - info "[5/8] Clone repo e pip install" + info "[5/9] Clone repo e pip install" REPO_DIR="$OPT_CI/local-ci-cd-system" if [[ -d "$REPO_DIR/.git" ]]; then warn "Repo già presente in $REPO_DIR — skip clone (eseguire git pull manualmente)" @@ -260,12 +263,12 @@ if [[ "$SKIP_CLONE" == false ]]; then || die "ci_orchestrator non importabile" echo " Verifica OK: ${verify_out%%$'\n'*}" else - info "[5/8] Clone repo — SKIPPED" + info "[5/9] Clone repo — SKIPPED" fi # ── Step 6: PowerShell Core (opt-in via --with-pwsh) ────────────────────────── if [[ "$WITH_PWSH" == true ]]; then - info "[6/8] PowerShell Core" + info "[6/9] PowerShell Core" if command -v pwsh &>/dev/null; then warn "pwsh già installato: $(pwsh --version) — skip" else @@ -287,11 +290,11 @@ if [[ "$WITH_PWSH" == true ]]; then echo " Installato: $(pwsh --version)" fi else - info "[6/8] PowerShell Core — SKIPPED (opt-in con --with-pwsh)" + info "[6/9] PowerShell Core — SKIPPED (opt-in con --with-pwsh)" fi # ── Step 7: alias shell `ci` (system-wide) ──────────────────────────────────── -info "[7/8] Alias shell 'ci'" +info "[7/9] Alias shell 'ci'" ALIAS_SCRIPT="$OPT_CI/local-ci-cd-system/scripts/install-ci-alias.sh" if [[ -x "$ALIAS_SCRIPT" ]]; then # Installa /etc/profile.d/ci-alias.sh per tutti gli utenti login. @@ -303,8 +306,21 @@ else warn "install-ci-alias.sh non trovato in $ALIAS_SCRIPT — skip (clona il repo, Step 5)" fi -# ── Step 8: autorizzazione X per la GUI di VMware (xhost persistente) ────────── -info "[8/8] Accesso GUI VMware (xhost ci-runner)" +# ── Step 8: servizi VMware host (moduli + rete al boot) ─────────────────────── +info "[8/9] Servizi VMware host (vmmon/vmnet + rete al boot)" +VMSVC_SCRIPT="$OPT_CI/local-ci-cd-system/scripts/install-vmware-host-services.sh" +if [[ -x "$VMSVC_SCRIPT" ]]; then + # Installa le unit systemd native che caricano vmmon/vmnet e avviano la rete + # vmnet (DHCP/NAT) a ogni boot, e ricostruiscono i moduli dopo un upgrade del + # kernel. Sostituisce il flaky sysv vmware.service. Vedi deploy/systemd/. + bash "$VMSVC_SCRIPT" + echo " Servizi VMware host installati — verifica: ip -o link show | grep vmnet" +else + warn "install-vmware-host-services.sh non trovato in $VMSVC_SCRIPT — skip (clona il repo, Step 5)" +fi + +# ── Step 9: autorizzazione X per la GUI di VMware (xhost persistente) ────────── +info "[9/9] Accesso GUI VMware (xhost ci-runner)" GUI_SCRIPT="$OPT_CI/local-ci-cd-system/scripts/install-vmware-gui-access.sh" if [[ "$SKIP_GUI_ACCESS" == true ]]; then info " SKIPPED (--skip-gui-access)"