From bc4eda926b97d7c84c235ecb5f66951444e5e45c Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 12 Jul 2026 07:48:49 +0200 Subject: [PATCH] agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf --- .../guesthook_bulletproof_test.go | 60 ++++++ cmd/felhom-agent/main.go | 100 ++++++++- configs/felhom-agent.sudoers | 23 +- internal/capability/manifest.go | 7 + internal/config/config.go | 18 ++ internal/config/config_test.go | 38 ++++ internal/guesthook/install.go | 7 +- internal/guesthook/netreassert.go | 4 +- internal/guesthook/netreassert_test.go | 4 +- internal/localapi/netreassert.go | 4 +- internal/selfheal/selfheal.go | 161 ++++++++++++++ internal/selfheal/selfheal_test.go | 134 ++++++++++++ internal/storage/hostops.go | 46 ++-- internal/storage/netmigrate_test.go | 170 +++++++++++++++ internal/storage/netmount.go | 200 +++++++++++++++++- internal/storage/netreassert.go | 123 ++++++++--- internal/storage/netreassert_test.go | 158 +++++++++++--- internal/storage/netresidue_test.go | 96 +++++++++ 18 files changed, 1249 insertions(+), 104 deletions(-) create mode 100644 cmd/felhom-agent/guesthook_bulletproof_test.go create mode 100644 internal/selfheal/selfheal.go create mode 100644 internal/selfheal/selfheal_test.go create mode 100644 internal/storage/netmigrate_test.go create mode 100644 internal/storage/netresidue_test.go diff --git a/cmd/felhom-agent/guesthook_bulletproof_test.go b/cmd/felhom-agent/guesthook_bulletproof_test.go new file mode 100644 index 0000000..f27aae6 --- /dev/null +++ b/cmd/felhom-agent/guesthook_bulletproof_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "testing" + "time" +) + +// F10/rc255 (CAMPAIGN-3): a guest-hook phase body that PANICS must never crash the process — the hook +// must return cleanly so the guest start proceeds (a nonzero exit blocks the start). runHookPhase +// recovers the panic and returns. +func TestRunHookPhase_PanicRecovered(t *testing.T) { + done := make(chan struct{}) + go func() { + defer close(done) + runHookPhase("9201", "pre-start", 5*time.Second, func(context.Context) { + panic("simulated heal panic (e.g. a future Heal bug)") + }) + }() + select { + case <-done: + // returned cleanly — the guest start would proceed + case <-time.After(3 * time.Second): + t.Fatal("runHookPhase did not return after a panicking body (would have crashed the hook)") + } +} + +// A phase body that overruns the timeout must be abandoned — the hook returns rather than hanging the +// PVE start task. (The body's context is cancelled; the hook does not wait for the body to notice.) +func TestRunHookPhase_TimeoutReturns(t *testing.T) { + bodyCtxCancelled := make(chan struct{}, 1) + done := make(chan struct{}) + go func() { + defer close(done) + runHookPhase("9201", "post-start", 20*time.Millisecond, func(ctx context.Context) { + <-ctx.Done() // simulate a body that respects cancellation eventually + bodyCtxCancelled <- struct{}{} + }) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runHookPhase did not return after the timeout (would hang the guest start)") + } + select { + case <-bodyCtxCancelled: + // the body's context was cancelled at the deadline — the intended signal + case <-time.After(time.Second): + t.Fatal("the phase body's context was not cancelled at the timeout") + } +} + +// A body that errors (returns normally, no panic) is fine — the hook returns cleanly. +func TestRunHookPhase_NormalBodyReturns(t *testing.T) { + ran := false + runHookPhase("9201", "pre-start", time.Second, func(context.Context) { ran = true }) + if !ran { + t.Fatal("the phase body must run") + } +} diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index b70ea2f..5a4d45a 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -44,6 +44,7 @@ import ( "gitea.dooplex.hu/admin/felhom-agent/internal/provision" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" + "gitea.dooplex.hu/admin/felhom-agent/internal/selfheal" "gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate" "gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs" "gitea.dooplex.hu/admin/felhom-agent/internal/storage" @@ -67,18 +68,46 @@ func runGuestHook(args []string) { vmid, phase := args[0], args[1] switch phase { case guesthook.PhasePreStart: - created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf") - if len(created) > 0 { - fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created) - } - if err != nil { - fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err) - } + // F10/rc255 (CAMPAIGN-3): a pre-start hook that panics or hangs would BLOCK the guest start (the + // campaign saw a guest bricked while a unit sat start-limited). Every phase runs recover-wrapped + // under a hard timeout and this function ALWAYS returns cleanly (main then exits 0). + runHookPhase(vmid, "pre-start", 30*time.Second, func(context.Context) { + created, err := guesthook.Heal("/etc/pve/lxc/" + vmid + ".conf") + if len(created) > 0 { + fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s pre-start — created %d placeholder(s) for absent drive(s): %v\n", vmid, len(created), created) + } + if err != nil { + fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s heal error (boot continues): %v\n", vmid, err) + } + }) case guesthook.PhasePostStart: - // Bounded: a hook must be fast; a wedged systemd call must not hold the start task. - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - postStartNetworkReassertFn(ctx, vmid) + runHookPhase(vmid, "post-start", 30*time.Second, func(ctx context.Context) { + postStartNetworkReassertFn(ctx, vmid) + }) + } +} + +// runHookPhase runs one guest-hook phase body under a hard timeout with a panic recover, so NO phase +// can ever fail the guest start (CAMPAIGN-3 F10/rc255). A panic is logged to the PVE task log (stderr) +// and swallowed; a body that overruns the timeout is abandoned (its context is cancelled) while the +// hook returns. This is the Go belt; the wrapper script's `|| true; exit 0` is the shell belt. +func runHookPhase(vmid, phase string, timeout time.Duration, body func(context.Context)) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + done := make(chan struct{}) + go func() { + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s PANIC recovered (boot continues): %v\n", vmid, phase, r) + } + close(done) + }() + body(ctx) + }() + select { + case <-done: + case <-ctx.Done(): + fmt.Fprintf(os.Stderr, "felhom-agent guest-hook: vmid %s %s timed out after %s (boot continues)\n", vmid, phase, timeout) } } @@ -795,6 +824,16 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int // bind that a re-provision dropped — before serving, so the drive is back in the guest config // (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped. localSrv.ReassertGuestBinds(ctx) + // F12 (CAMPAIGN-3, CRITICAL): before re-arming, migrate any already-installed network-storage + // units to the current template — the units installed before 0.85 carry the network-online + // ordering cycle that makes each host boot a coin flip (networking lost on one boot, the automount + // on the next). A general template-drift reconcile (content-hash compare, batched daemon-reload), + // idempotent. Runs BEFORE the reassert sweep so the re-armed triggers are the fixed units. + if mountReasserter != nil { + if n := mountReasserter.MigrateNetworkUnits(ctx); n > 0 { + logger.Info("network-storage units migrated to the current template (F12 boot-ordering fix)", "migrated", n) + } + } // RCA fix 1 (AUDIT-nas-cwa-rca-2026-07-11): re-arm idle NAS automount triggers ONCE at startup — // covers the host-boot ordering where guests autostarted before the agent (their fresh namespaces // missed the triggers). Deliberately NOT in the 20 s ticker: an idle trigger is healthy and must @@ -822,6 +861,45 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int } } }() + // CAMPAIGN-3 Part 6: node self-heal watchdog (appliance-gated). One heal — host networking + // (F12-class defense in depth: any boot leaving networking down is detected + remedied with the + // exact `systemctl start networking.service` the morning recovery ran by hand). On a byo host the + // check WARNs but the remedy is structurally unreachable (the Manager gates before any exec). + { + shMode := proxmox.RunnerMode(cfg.Privileged.Mode) + if shMode == "" { + shMode = proxmox.RunnerSudo + } + shRunner := &proxmox.ExecRunner{Mode: shMode, SudoPath: cfg.Privileged.SudoPath} + systemctlBin := cfg.Privileged.Systemctl + if systemctlBin == "" { + systemctlBin = "/usr/bin/systemctl" + } + shMgr := &selfheal.Manager{ + Appliance: cfg.IsAppliance(), + Interval: 60 * time.Second, + Heals: []selfheal.Heal{&selfheal.NetworkingHeal{ + IsActive: selfheal.SystemctlIsActive("networking.service"), + HasRoute: selfheal.HasDefaultRoute, + Start: func(ctx context.Context) error { + _, stderr, err := shRunner.Run(ctx, systemctlBin, "start", "networking.service") + if err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(stderr))) + } + return nil + }, + Sleep: selfheal.RealSleep, + Logger: logger, + }}, + Logger: logger, + } + mode := "byo" + if cfg.IsAppliance() { + mode = "appliance" + } + logger.Info("selfheal: node watchdog starting", "mode", mode, "interval_s", 60) + go shMgr.Watch(ctx) + } // F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved). localSrv.RecoverFormatJob(ctx) // F2-b: recover any guest left with a stale vzdump lock by a reboot-during-backup (unlock → delete diff --git a/configs/felhom-agent.sudoers b/configs/felhom-agent.sudoers index d3d8f73..aa3692c 100644 --- a/configs/felhom-agent.sudoers +++ b/configs/felhom-agent.sudoers @@ -143,11 +143,23 @@ Cmnd_Alias FELHOM_STALELOCK = \ # `.automount` variants + the unit-file removal. The unit FILE name is the systemd-escaped mountpoint, # which always begins `mnt-felhom` (the mountpoint is /mnt/felhom-drives/), so the rm glob is scoped # to felhom mount units only. mkdir of the mountpoint reuses FELHOM_INTERMEDIARY's /mnt/felhom-drives/*. +# CAMPAIGN-3 additions (loud, per the no-widening rule): +# - `systemctl reset-failed -- mnt-felhom*`: F10 (CRITICAL) — a NAS automount that hit +# mount-start-limit-hit during an outage was re-armable by NO platform path; the reassert now +# reset-failed's the stuck unit before `enable --now` (which the start-limit otherwise refuses), +# and RemoveNetworkMount clears failed-state residue (F2). Scoped to felhom mount units (the unit +# name is the systemd-escaped mountpoint, always beginning `mnt-felhom`). reset-failed only clears +# a unit's failed latch — it cannot start/stop/alter anything. +# - `rmdir /mnt/felhom-drives/*`: F1 — remove the now-empty mountpoint dir a removed share leaves +# behind (the campaign accumulated 10 stub-shaped leftovers). rmdir ONLY (never rm -rf): it refuses +# a non-empty dir, so unexpected data is preserved, not destroyed — a fail-safe grant. Cmnd_Alias FELHOM_NETMOUNT = \ /usr/bin/install -o root -g root -m 0644 -- /var/lib/felhom-agent/units/* /etc/systemd/system/*.automount, \ /usr/bin/systemctl enable --now -- *.automount, \ /usr/bin/systemctl disable -- *.automount, \ /usr/bin/systemctl stop -- *.automount, \ + /usr/bin/systemctl reset-failed -- mnt-felhom*, \ + /usr/bin/rmdir /mnt/felhom-drives/*, \ /usr/bin/rm -f /etc/systemd/system/mnt-felhom* # Offsite WG tunnel (S3, doc 06 §3.3). The agent manages wg-quick@wg-felhom as an agent-managed @@ -222,4 +234,13 @@ Cmnd_Alias FELHOM_OOB = \ /usr/sbin/nft add element inet felhom_oob operator_ips *, \ /usr/sbin/nft add element inet felhom_oob ssh_port * -felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR +# Node self-heal (CAMPAIGN-3 Part 6, F12-class defense in depth). The ONE fixed unit the appliance +# watchdog may (re)start when a boot leaves networking down — the exact command the morning recovery +# ran by hand after the F12 host loss. FIXED unit, no glob: this grant alone cannot harm — starting +# networking.service is precisely what the boot should have done. The remedy is ALSO code-gated on +# deployment_mode="appliance" (the Manager refuses to invoke it on a byo host); the sudoers grant is +# the coarse floor, the mode gate is the fine one. +Cmnd_Alias FELHOM_SELFHEAL = \ + /usr/bin/systemctl start networking.service + +felhom-agent ALL=(root) NOPASSWD: FELHOM_MOUNT, FELHOM_DISK, FELHOM_PROVISION, FELHOM_FORMAT, FELHOM_DNSMASQ, FELHOM_GUESTHOOK, FELHOM_INTERMEDIARY, FELHOM_CONTROLLERSWAP, FELHOM_STALELOCK, FELHOM_NETMOUNT, FELHOM_WG, FELHOM_SELFUPDATE, FELHOM_SSHD, FELHOM_OOB, FELHOM_PBSDR, FELHOM_SELFHEAL diff --git a/internal/capability/manifest.go b/internal/capability/manifest.go index 42d9cef..ee27ea8 100644 --- a/internal/capability/manifest.go +++ b/internal/capability/manifest.go @@ -73,6 +73,13 @@ var manifest = []Capability{ {"mount-unit-disable", "mount unit disable", "/usr/bin/systemctl", []string{"disable", "--", "felhom-x.mount"}, false}, {"mount-unit-stop", "mount unit stop", "/usr/bin/systemctl", []string{"stop", "--", "felhom-x.mount"}, false}, + // ---- Network storage re-arm + cleanup (CAMPAIGN-3 F10/F1) ---- + {"netmount-reset-failed", "NAS automount re-arm after start-limit (F10)", "/usr/bin/systemctl", []string{"reset-failed", "--", "mnt-felhom\\x2ddrives-media.automount"}, false}, + {"netmount-rmdir", "removed-share mountpoint cleanup (F1)", "/usr/bin/rmdir", []string{"/mnt/felhom-drives/media"}, false}, + + // ---- Node self-heal (CAMPAIGN-3 F12-class, appliance-gated in code) ---- + {"selfheal-networking-start", "appliance networking recovery at boot (F12 defense in depth)", "/usr/bin/systemctl", []string{"start", "networking.service"}, false}, + // ---- Provisioning back-half ---- {"provision-chown", "bootstrap mount guest-root chown", "/usr/bin/chown", []string{"-R", "100000:100000", "/var/lib/felhom-agent/guests/9201"}, false}, {"provision-config-mount", "bootstrap config bind mount", "/usr/sbin/pct", []string{"set", "9201", "-mp0", "/var/lib/felhom-agent/guests/9201"}, false}, diff --git a/internal/config/config.go b/internal/config/config.go index 634f52d..80f9b53 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -36,11 +36,26 @@ type Config struct { SelfUpdate SelfUpdateConfig `json:"selfupdate"` LogLevel string `json:"log_level"` // debug|info|warn|error (default info) + // DeploymentMode gates host-service self-heal (CAMPAIGN-3 Part 6). "appliance" = a Felhom-managed + // node the agent may remediate (e.g. start networking at boot — F12-class defense in depth). Any + // other value, including absent/unknown, is treated as "byo" (a customer's own host): the self-heal + // CHECK still runs and WARNs, but the REMEDY is structurally unreachable. Fail-safe to byo — never + // touch a host we do not own. Distinct from Privileged.Mode (sudo vs direct exec) — do NOT overload. + DeploymentMode string `json:"deployment_mode,omitempty"` + // SourcePath is the file this config was loaded from ("" = all-env). Set by Load, never // serialized — the pbsdr bridge's escrow.pbs_storage_id seed writes back to it. SourcePath string `json:"-"` } +// DeploymentModeAppliance is the ONLY value that unlocks host-service self-heal. Everything else, +// including "" and any typo, is byo (fail-safe — a host we do not own is never remediated). +const DeploymentModeAppliance = "appliance" + +// IsAppliance reports whether this node is a Felhom-managed appliance (self-heal remedies allowed). +// Fail-safe: absent/unknown → false (byo). +func (c *Config) IsAppliance() bool { return c.DeploymentMode == DeploymentModeAppliance } + // OOBConfig configures the dedicated felhom-sshd OOB access instance + belt (TASK H1). **Enabled // DEFAULTS TO FALSE** — a rollout to a box without explicit oob.enabled=true is a no-op (no port // claim, no config render, no belt mutation, no oob report stanza) until the operator endpoint + @@ -575,6 +590,9 @@ func applyEnv(cfg *Config) { if v := os.Getenv("FELHOM_AGENT_LOG_LEVEL"); v != "" { cfg.LogLevel = v } + if v := os.Getenv("FELHOM_AGENT_DEPLOYMENT_MODE"); v != "" { + cfg.DeploymentMode = v + } // hub if v := os.Getenv("FELHOM_AGENT_HUB_URL"); v != "" { cfg.Hub.URL = v diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7d21a73..4e5792b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -133,3 +133,41 @@ func TestLoadFileThenEnvOverride(t *testing.T) { t.Errorf("default endpoint lost: %q", cfg.Proxmox.Endpoint) } } + +// CAMPAIGN-3 Part 6: deployment_mode gates node self-heal, and it is FAIL-SAFE to byo — absent or any +// unknown value is byo, ONLY the exact "appliance" unlocks the remedy. +func TestIsAppliance_FailSafeToByo(t *testing.T) { + cases := []struct { + mode string + want bool + }{ + {"appliance", true}, + {"byo", false}, + {"", false}, // absent field → byo (fail-safe) + {"Appliance", false}, // case-sensitive — a typo must not unlock the remedy + {"garbage", false}, + } + for _, c := range cases { + cfg := &Config{DeploymentMode: c.mode} + if got := cfg.IsAppliance(); got != c.want { + t.Errorf("IsAppliance(mode=%q) = %t, want %t", c.mode, got, c.want) + } + } +} + +// The env overlay can set deployment_mode (FELHOM_AGENT_DEPLOYMENT_MODE). +func TestDeploymentModeEnvOverlay(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "agent.json") + if err := os.WriteFile(path, []byte(`{"proxmox":{"node":"n","token":"u@pve!t=s"},"deployment_mode":"byo"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("FELHOM_AGENT_DEPLOYMENT_MODE", "appliance") + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cfg.IsAppliance() { + t.Errorf("env overlay did not set deployment_mode: %q", cfg.DeploymentMode) + } +} diff --git a/internal/guesthook/install.go b/internal/guesthook/install.go index 7ca2c61..c4b46a2 100644 --- a/internal/guesthook/install.go +++ b/internal/guesthook/install.go @@ -28,9 +28,14 @@ var SnippetPath = filepath.Join(SnippetDir, SnippetName) // snippetBody is the tiny wrapper PVE execs as `