// Package selfheal is a minimal check/remedy registry for node-level self-healing, gated hard on // appliance deployment mode (CAMPAIGN-3 Part 6). // // Only ONE heal ships now: host networking recovery — F12-class defense in depth. The F12 fix (the // automount template no longer creates the boot ordering cycle) is the CURE for that specific instance; // this watchdog is the belt for the CLASS: any boot that leaves networking down for ANY reason (a future // ordering bug, a flaky NIC bring-up, a botched netplan) is detected and — on a Felhom-managed appliance // only — remedied with the exact command the morning recovery ran by hand (`systemctl start // networking.service`). The registry shape is the deliverable so future appliance heals slot in behind // the SAME gate; resist growing a zoo of heals. // // THE GATE (never touch a host we do not own): a BYO host runs the CHECK and WARNs, but the Manager // refuses to call any Remediate before the appliance test — the remedy is structurally unreachable on // byo (unit-tested: byo + unhealthy → zero privileged invocations). package selfheal import ( "context" "fmt" "log/slog" "os/exec" "strings" "time" ) // Heal is one node-level check + remedy. Healthy reports the current state (and a human detail for the // unhealthy log); Remediate attempts recovery and returns a terminal error if it gives up. type Heal interface { Name() string Healthy(ctx context.Context) (ok bool, detail string) Remediate(ctx context.Context) error } // Manager runs the registered heals at boot and on a periodic watchdog. The appliance gate lives HERE, // before any Remediate — a byo node never reaches a remedy. type Manager struct { Appliance bool Interval time.Duration // watchdog cadence (≈60s); <=0 disables the periodic loop Heals []Heal Logger *slog.Logger } // RunOnce evaluates every heal once (the boot pass). For each unhealthy heal it WARNs (both facts in // the detail); on an appliance it then remediates, on byo it stops at the WARN — the remedy exec is // never reached off-appliance. func (m *Manager) RunOnce(ctx context.Context) { for _, h := range m.Heals { ok, detail := h.Healthy(ctx) if ok { m.Logger.Debug("selfheal: node check healthy", "heal", h.Name()) continue } m.Logger.Warn("selfheal: node check FAILED", "heal", h.Name(), "detail", detail, "mode", m.mode()) if !m.Appliance { // GATE: byo hosts are the customer's to manage — never remediate. (Red-proof: byo + // unhealthy must record ZERO privileged invocations; the remedy is unreachable here.) m.Logger.Warn("selfheal: byo host — remedy skipped (not ours to touch)", "heal", h.Name()) continue } if err := h.Remediate(ctx); err != nil { m.Logger.Error("selfheal: remedy GAVE UP — host needs attention", "heal", h.Name(), "err", err) } } } // Watch runs RunOnce once immediately (the boot pass) then on every Interval tick until ctx is done. func (m *Manager) Watch(ctx context.Context) { m.RunOnce(ctx) if m.Interval <= 0 { return } t := time.NewTicker(m.Interval) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: m.RunOnce(ctx) } } } func (m *Manager) mode() string { if m.Appliance { return "appliance" } return "byo" } // NetworkingHeal is the one shipped heal: host networking (F12-class). Healthy ⇔ networking.service is // active AND a default route exists. All three probes/actions are seams (unit-tested with fakes); Start // is the ONLY privileged one (`systemctl start networking.service`, appliance-gated by the Manager). type NetworkingHeal struct { IsActive func(ctx context.Context) bool // `systemctl is-active networking.service` (unprivileged) HasRoute func(ctx context.Context) bool // a default route exists (unprivileged) Start func(ctx context.Context) error // PRIVILEGED `systemctl start networking.service` Sleep func(ctx context.Context, d time.Duration) Logger *slog.Logger } func (n *NetworkingHeal) Name() string { return "networking" } // Healthy reports both facts so the WARN detail names exactly what is down. func (n *NetworkingHeal) Healthy(ctx context.Context) (bool, string) { active := n.IsActive(ctx) route := n.HasRoute(ctx) if active && route { return true, "" } return false, fmt.Sprintf("networking.service active=%t, default route present=%t", active, route) } // networkingBackoffs is the wait after each start attempt before re-checking — 10s/30s/60s gives a slow // NIC/DHCP time to come up. Package var so tests can shrink it. var networkingBackoffs = []time.Duration{10 * time.Second, 30 * time.Second, 60 * time.Second} // Remediate runs up to 3 `systemctl start networking.service` attempts, each followed by its backoff and // a health re-check. ERROR per failed attempt; INFO on recovery; a terminal give-up error if all fail. func (n *NetworkingHeal) Remediate(ctx context.Context) error { for attempt := 1; attempt <= len(networkingBackoffs); attempt++ { if err := n.Start(ctx); err != nil { n.Logger.Error("node self-heal: networking start attempt failed", "attempt", attempt, "err", err) } n.Sleep(ctx, networkingBackoffs[attempt-1]) if ok, _ := n.Healthy(ctx); ok { n.Logger.Info("node self-heal: networking recovered", "attempt", attempt) return nil } n.Logger.Error("node self-heal: networking still down after start attempt", "attempt", attempt) } return fmt.Errorf("networking still down after %d attempts (host needs physical/console attention)", len(networkingBackoffs)) } // --- production seams ------------------------------------------------------------------------------ // SystemctlIsActive is the default IsActive: `systemctl is-active ` prints "active" when up. // Unprivileged (unit state is world-readable) — NOT routed through sudo. func SystemctlIsActive(unit string) func(context.Context) bool { return func(ctx context.Context) bool { out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "--", unit).Output() return strings.TrimSpace(string(out)) == "active" } } // HasDefaultRoute is the default HasRoute: `ip route show default` is non-empty when a default route // exists. Unprivileged read. func HasDefaultRoute(ctx context.Context) bool { out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output() return err == nil && strings.TrimSpace(string(out)) != "" } // RealSleep is the default Sleep: a context-cancellable time.Sleep. func RealSleep(ctx context.Context, d time.Duration) { t := time.NewTimer(d) defer t.Stop() select { case <-ctx.Done(): case <-t.C: } }