agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
2026-07-12 07:48:49 +02:00
parent bec4bac076
commit bc4eda926b
18 changed files with 1249 additions and 104 deletions
@@ -0,0 +1,60 @@
+package main
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+// F10/rc255 (CAMPAIGN-3): a guest-hook phase body that PANICS must never crash the process — the hook
+// must return cleanly so the guest start proceeds (a nonzero exit blocks the start). runHookPhase
+// recovers the panic and returns.
+func TestRunHookPhase_PanicRecovered(t *testing.T) {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		runHookPhase("9201", "pre-start", 5*time.Second, func(context.Context) {
+			panic("simulated heal panic (e.g. a future Heal bug)")
+		})
+	}()
+	select {
+	case <-done:
+		// returned cleanly — the guest start would proceed
+	case <-time.After(3 * time.Second):
+		t.Fatal("runHookPhase did not return after a panicking body (would have crashed the hook)")
+	}
+}
+
+// A phase body that overruns the timeout must be abandoned — the hook returns rather than hanging the
+// PVE start task. (The body's context is cancelled; the hook does not wait for the body to notice.)
+func TestRunHookPhase_TimeoutReturns(t *testing.T) {
+	bodyCtxCancelled := make(chan struct{}, 1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		runHookPhase("9201", "post-start", 20*time.Millisecond, func(ctx context.Context) {
+			<-ctx.Done() // simulate a body that respects cancellation eventually
+			bodyCtxCancelled <- struct{}{}
+		})
+	}()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("runHookPhase did not return after the timeout (would hang the guest start)")
+	}
+	select {
+	case <-bodyCtxCancelled:
+		// the body's context was cancelled at the deadline — the intended signal
+	case <-time.After(time.Second):
+		t.Fatal("the phase body's context was not cancelled at the timeout")
+	}
+}
+
+// A body that errors (returns normally, no panic) is fine — the hook returns cleanly.
+func TestRunHookPhase_NormalBodyReturns(t *testing.T) {
+	ran := false
+	runHookPhase("9201", "pre-start", time.Second, func(context.Context) { ran = true })
+	if !ran {
+		t.Fatal("the phase body must run")
+	}
+}
@@ -44,6 +44,7 @@ import (
 	"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/selfheal"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
 	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
@@ -67,18 +68,46 @@ func runGuestHook(args []string) {
 	vmid, phase := args[0], args[1]
 	switch phase {
 	case guesthook.PhasePreStart:
-		created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
-		if len(created) > 0 {
-			fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
-		}
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
-		}
+		// F10/rc255 (CAMPAIGN-3): a pre-start hook that panics or hangs would BLOCK the guest start (the
+		// campaign saw a guest bricked while a unit sat start-limited). Every phase runs recover-wrapped
+		// under a hard timeout and this function ALWAYS returns cleanly (main then exits 0).
+		runHookPhase(vmid, "pre-start", 30*time.Second, func(context.Context) {
+			created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf")
+			if len(created) > 0 {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created)
+			}
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err)
+			}
+		})
 	case guesthook.PhasePostStart:
-		// Bounded: a hook must be fast; a wedged systemd call must not hold the start task.
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-		postStartNetworkReassertFn(ctx, vmid)
+		runHookPhase(vmid, "post-start", 30*time.Second, func(ctx context.Context) {
+			postStartNetworkReassertFn(ctx, vmid)
+		})
+	}
+}
+
+// runHookPhase runs one guest-hook phase body under a hard timeout with a panic recover, so NO phase
+// can ever fail the guest start (CAMPAIGN-3 F10/rc255). A panic is logged to the PVE task log (stderr)
+// and swallowed; a body that overruns the timeout is abandoned (its context is cancelled) while the
+// hook returns. This is the Go belt; the wrapper script's `|| true; exit 0` is the shell belt.
+func runHookPhase(vmid, phase string, timeout time.Duration, body func(context.Context)) {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	done := make(chan struct{})
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s PANIC recovered (boot continues): %v\n", vmid, phase, r)
+			}
+			close(done)
+		}()
+		body(ctx)
+	}()
+	select {
+	case <-done:
+	case <-ctx.Done():
+		fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s timed out after %s (boot continues)\n", vmid, phase, timeout)
 	}
 }

@@ -795,6 +824,16 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 		// bind that a re-provision dropped — before serving, so the drive is back in the guest config
 		// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
 		localSrv.ReassertGuestBinds(ctx)
+		// F12 (CAMPAIGN-3, CRITICAL): before re-arming, migrate any already-installed network-storage
+		// units to the current template — the units installed before 0.85 carry the network-online
+		// ordering cycle that makes each host boot a coin flip (networking lost on one boot, the automount
+		// on the next). A general template-drift reconcile (content-hash compare, batched daemon-reload),
+		// idempotent. Runs BEFORE the reassert sweep so the re-armed triggers are the fixed units.
+		if mountReasserter != nil {
+			if n := mountReasserter.MigrateNetworkUnits(ctx); n > 0 {
+				logger.Info("network-storage units migrated to the current template (F12 boot-ordering fix)", "migrated", n)
+			}
+		}
 		// RCA fix 1 (AUDIT-nas-cwa-rca-2026-07-11): re-arm idle NAS automount triggers ONCE at startup —
 		// covers the host-boot ordering where guests autostarted before the agent (their fresh namespaces
 		// missed the triggers). Deliberately NOT in the 20 s ticker: an idle trigger is healthy and must
@@ -822,6 +861,45 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 				}
 			}
 		}()
+		// CAMPAIGN-3 Part 6: node self-heal watchdog (appliance-gated). One heal — host networking
+		// (F12-class defense in depth: any boot leaving networking down is detected + remedied with the
+		// exact `systemctl start networking.service` the morning recovery ran by hand). On a byo host the
+		// check WARNs but the remedy is structurally unreachable (the Manager gates before any exec).
+		{
+			shMode := proxmox.RunnerMode(cfg.Privileged.Mode)
+			if shMode == "" {
+				shMode = proxmox.RunnerSudo
+			}
+			shRunner := &proxmox.ExecRunner{Mode: shMode, SudoPath: cfg.Privileged.SudoPath}
+			systemctlBin := cfg.Privileged.Systemctl
+			if systemctlBin == "" {
+				systemctlBin = "/usr/bin/systemctl"
+			}
+			shMgr := &selfheal.Manager{
+				Appliance: cfg.IsAppliance(),
+				Interval:  60 * time.Second,
+				Heals: []selfheal.Heal{&selfheal.NetworkingHeal{
+					IsActive: selfheal.SystemctlIsActive("networking.service"),
+					HasRoute: selfheal.HasDefaultRoute,
+					Start: func(ctx context.Context) error {
+						_, stderr, err := shRunner.Run(ctx, systemctlBin, "start", "networking.service")
+						if err != nil {
+							return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(stderr)))
+						}
+						return nil
+					},
+					Sleep:  selfheal.RealSleep,
+					Logger: logger,
+				}},
+				Logger: logger,
+			}
+			mode := "byo"
+			if cfg.IsAppliance() {
+				mode = "appliance"
+			}
+			logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60)
+			go shMgr.Watch(ctx)
+		}
 		// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
 		localSrv.RecoverFormatJob(ctx)
 		// F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete