agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/selfheal/selfheal.go

@@ -0,0 +1,161 @@
+// Package selfheal is a minimal check/remedy registry for node-level self-healing, gated hard on
+// appliance deployment mode (CAMPAIGN-3 Part 6).
+//
+// Only ONE heal ships now: host networking recovery — F12-class defense in depth. The F12 fix (the
+// automount template no longer creates the boot ordering cycle) is the CURE for that specific instance;
+// this watchdog is the belt for the CLASS: any boot that leaves networking down for ANY reason (a future
+// ordering bug, a flaky NIC bring-up, a botched netplan) is detected and — on a Felhom-managed appliance
+// only — remedied with the exact command the morning recovery ran by hand (`systemctl start
+// networking.service`). The registry shape is the deliverable so future appliance heals slot in behind
+// the SAME gate; resist growing a zoo of heals.
+//
+// THE GATE (never touch a host we do not own): a BYO host runs the CHECK and WARNs, but the Manager
+// refuses to call any Remediate before the appliance test — the remedy is structurally unreachable on
+// byo (unit-tested: byo + unhealthy → zero privileged invocations).
+package selfheal
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// Heal is one node-level check + remedy. Healthy reports the current state (and a human detail for the
+// unhealthy log); Remediate attempts recovery and returns a terminal error if it gives up.
+type Heal interface {
+	Name() string
+	Healthy(ctx context.Context) (ok bool, detail string)
+	Remediate(ctx context.Context) error
+}
+
+// Manager runs the registered heals at boot and on a periodic watchdog. The appliance gate lives HERE,
+// before any Remediate — a byo node never reaches a remedy.
+type Manager struct {
+	Appliance bool
+	Interval  time.Duration // watchdog cadence (≈60s); <=0 disables the periodic loop
+	Heals     []Heal
+	Logger    *slog.Logger
+}
+
+// RunOnce evaluates every heal once (the boot pass). For each unhealthy heal it WARNs (both facts in
+// the detail); on an appliance it then remediates, on byo it stops at the WARN — the remedy exec is
+// never reached off-appliance.
+func (m *Manager) RunOnce(ctx context.Context) {
+	for _, h := range m.Heals {
+		ok, detail := h.Healthy(ctx)
+		if ok {
+			m.Logger.Debug("selfheal: node check healthy", "heal", h.Name())
+			continue
+		}
+		m.Logger.Warn("selfheal: node check FAILED", "heal", h.Name(), "detail", detail, "mode", m.mode())
+		if !m.Appliance {
+			// GATE: byo hosts are the customer's to manage — never remediate. (Red-proof: byo +
+			// unhealthy must record ZERO privileged invocations; the remedy is unreachable here.)
+			m.Logger.Warn("selfheal: byo host — remedy skipped (not ours to touch)", "heal", h.Name())
+			continue
+		}
+		if err := h.Remediate(ctx); err != nil {
+			m.Logger.Error("selfheal: remedy GAVE UP — host needs attention", "heal", h.Name(), "err", err)
+		}
+	}
+}
+
+// Watch runs RunOnce once immediately (the boot pass) then on every Interval tick until ctx is done.
+func (m *Manager) Watch(ctx context.Context) {
+	m.RunOnce(ctx)
+	if m.Interval <= 0 {
+		return
+	}
+	t := time.NewTicker(m.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			m.RunOnce(ctx)
+		}
+	}
+}
+
+func (m *Manager) mode() string {
+	if m.Appliance {
+		return "appliance"
+	}
+	return "byo"
+}
+
+// NetworkingHeal is the one shipped heal: host networking (F12-class). Healthy ⇔ networking.service is
+// active AND a default route exists. All three probes/actions are seams (unit-tested with fakes); Start
+// is the ONLY privileged one (`systemctl start networking.service`, appliance-gated by the Manager).
+type NetworkingHeal struct {
+	IsActive func(ctx context.Context) bool  // `systemctl is-active networking.service` (unprivileged)
+	HasRoute func(ctx context.Context) bool  // a default route exists (unprivileged)
+	Start    func(ctx context.Context) error // PRIVILEGED `systemctl start networking.service`
+	Sleep    func(ctx context.Context, d time.Duration)
+	Logger   *slog.Logger
+}
+
+func (n *NetworkingHeal) Name() string { return "networking" }
+
+// Healthy reports both facts so the WARN detail names exactly what is down.
+func (n *NetworkingHeal) Healthy(ctx context.Context) (bool, string) {
+	active := n.IsActive(ctx)
+	route := n.HasRoute(ctx)
+	if active && route {
+		return true, ""
+	}
+	return false, fmt.Sprintf("networking.service active=%t, default route present=%t", active, route)
+}
+
+// networkingBackoffs is the wait after each start attempt before re-checking — 10s/30s/60s gives a slow
+// NIC/DHCP time to come up. Package var so tests can shrink it.
+var networkingBackoffs = []time.Duration{10 * time.Second, 30 * time.Second, 60 * time.Second}
+
+// Remediate runs up to 3 `systemctl start networking.service` attempts, each followed by its backoff and
+// a health re-check. ERROR per failed attempt; INFO on recovery; a terminal give-up error if all fail.
+func (n *NetworkingHeal) Remediate(ctx context.Context) error {
+	for attempt := 1; attempt <= len(networkingBackoffs); attempt++ {
+		if err := n.Start(ctx); err != nil {
+			n.Logger.Error("node self-heal: networking start attempt failed", "attempt", attempt, "err", err)
+		}
+		n.Sleep(ctx, networkingBackoffs[attempt-1])
+		if ok, _ := n.Healthy(ctx); ok {
+			n.Logger.Info("node self-heal: networking recovered", "attempt", attempt)
+			return nil
+		}
+		n.Logger.Error("node self-heal: networking still down after start attempt", "attempt", attempt)
+	}
+	return fmt.Errorf("networking still down after %d attempts (host needs physical/console attention)", len(networkingBackoffs))
+}
+
+// --- production seams ------------------------------------------------------------------------------
+
+// SystemctlIsActive is the default IsActive: `systemctl is-active <unit>` prints "active" when up.
+// Unprivileged (unit state is world-readable) — NOT routed through sudo.
+func SystemctlIsActive(unit string) func(context.Context) bool {
+	return func(ctx context.Context) bool {
+		out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "--", unit).Output()
+		return strings.TrimSpace(string(out)) == "active"
+	}
+}
+
+// HasDefaultRoute is the default HasRoute: `ip route show default` is non-empty when a default route
+// exists. Unprivileged read.
+func HasDefaultRoute(ctx context.Context) bool {
+	out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output()
+	return err == nil && strings.TrimSpace(string(out)) != ""
+}
+
+// RealSleep is the default Sleep: a context-cancellable time.Sleep.
+func RealSleep(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}