agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf
2026-07-12 07:48:49 +02:00
parent bec4bac076
commit bc4eda926b
18 changed files with 1249 additions and 104 deletions
@@ -0,0 +1,161 @@
+// Package selfheal is a minimal check/remedy registry for node-level self-healing, gated hard on
+// appliance deployment mode (CAMPAIGN-3 Part 6).
+//
+// Only ONE heal ships now: host networking recovery — F12-class defense in depth. The F12 fix (the
+// automount template no longer creates the boot ordering cycle) is the CURE for that specific instance;
+// this watchdog is the belt for the CLASS: any boot that leaves networking down for ANY reason (a future
+// ordering bug, a flaky NIC bring-up, a botched netplan) is detected and — on a Felhom-managed appliance
+// only — remedied with the exact command the morning recovery ran by hand (`systemctl start
+// networking.service`). The registry shape is the deliverable so future appliance heals slot in behind
+// the SAME gate; resist growing a zoo of heals.
+//
+// THE GATE (never touch a host we do not own): a BYO host runs the CHECK and WARNs, but the Manager
+// refuses to call any Remediate before the appliance test — the remedy is structurally unreachable on
+// byo (unit-tested: byo + unhealthy → zero privileged invocations).
+package selfheal
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+// Heal is one node-level check + remedy. Healthy reports the current state (and a human detail for the
+// unhealthy log); Remediate attempts recovery and returns a terminal error if it gives up.
+type Heal interface {
+	Name() string
+	Healthy(ctx context.Context) (ok bool, detail string)
+	Remediate(ctx context.Context) error
+}
+
+// Manager runs the registered heals at boot and on a periodic watchdog. The appliance gate lives HERE,
+// before any Remediate — a byo node never reaches a remedy.
+type Manager struct {
+	Appliance bool
+	Interval  time.Duration // watchdog cadence (≈60s); <=0 disables the periodic loop
+	Heals     []Heal
+	Logger    *slog.Logger
+}
+
+// RunOnce evaluates every heal once (the boot pass). For each unhealthy heal it WARNs (both facts in
+// the detail); on an appliance it then remediates, on byo it stops at the WARN — the remedy exec is
+// never reached off-appliance.
+func (m *Manager) RunOnce(ctx context.Context) {
+	for _, h := range m.Heals {
+		ok, detail := h.Healthy(ctx)
+		if ok {
+			m.Logger.Debug("selfheal: node check healthy", "heal", h.Name())
+			continue
+		}
+		m.Logger.Warn("selfheal: node check FAILED", "heal", h.Name(), "detail", detail, "mode", m.mode())
+		if !m.Appliance {
+			// GATE: byo hosts are the customer's to manage — never remediate. (Red-proof: byo +
+			// unhealthy must record ZERO privileged invocations; the remedy is unreachable here.)
+			m.Logger.Warn("selfheal: byo host — remedy skipped (not ours to touch)", "heal", h.Name())
+			continue
+		}
+		if err := h.Remediate(ctx); err != nil {
+			m.Logger.Error("selfheal: remedy GAVE UP — host needs attention", "heal", h.Name(), "err", err)
+		}
+	}
+}
+
+// Watch runs RunOnce once immediately (the boot pass) then on every Interval tick until ctx is done.
+func (m *Manager) Watch(ctx context.Context) {
+	m.RunOnce(ctx)
+	if m.Interval <= 0 {
+		return
+	}
+	t := time.NewTicker(m.Interval)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			m.RunOnce(ctx)
+		}
+	}
+}
+
+func (m *Manager) mode() string {
+	if m.Appliance {
+		return "appliance"
+	}
+	return "byo"
+}
+
+// NetworkingHeal is the one shipped heal: host networking (F12-class). Healthy ⇔ networking.service is
+// active AND a default route exists. All three probes/actions are seams (unit-tested with fakes); Start
+// is the ONLY privileged one (`systemctl start networking.service`, appliance-gated by the Manager).
+type NetworkingHeal struct {
+	IsActive func(ctx context.Context) bool  // `systemctl is-active networking.service` (unprivileged)
+	HasRoute func(ctx context.Context) bool  // a default route exists (unprivileged)
+	Start    func(ctx context.Context) error // PRIVILEGED `systemctl start networking.service`
+	Sleep    func(ctx context.Context, d time.Duration)
+	Logger   *slog.Logger
+}
+
+func (n *NetworkingHeal) Name() string { return "networking" }
+
+// Healthy reports both facts so the WARN detail names exactly what is down.
+func (n *NetworkingHeal) Healthy(ctx context.Context) (bool, string) {
+	active := n.IsActive(ctx)
+	route := n.HasRoute(ctx)
+	if active && route {
+		return true, ""
+	}
+	return false, fmt.Sprintf("networking.service active=%t, default route present=%t", active, route)
+}
+
+// networkingBackoffs is the wait after each start attempt before re-checking — 10s/30s/60s gives a slow
+// NIC/DHCP time to come up. Package var so tests can shrink it.
+var networkingBackoffs = []time.Duration{10 * time.Second, 30 * time.Second, 60 * time.Second}
+
+// Remediate runs up to 3 `systemctl start networking.service` attempts, each followed by its backoff and
+// a health re-check. ERROR per failed attempt; INFO on recovery; a terminal give-up error if all fail.
+func (n *NetworkingHeal) Remediate(ctx context.Context) error {
+	for attempt := 1; attempt <= len(networkingBackoffs); attempt++ {
+		if err := n.Start(ctx); err != nil {
+			n.Logger.Error("node self-heal: networking start attempt failed", "attempt", attempt, "err", err)
+		}
+		n.Sleep(ctx, networkingBackoffs[attempt-1])
+		if ok, _ := n.Healthy(ctx); ok {
+			n.Logger.Info("node self-heal: networking recovered", "attempt", attempt)
+			return nil
+		}
+		n.Logger.Error("node self-heal: networking still down after start attempt", "attempt", attempt)
+	}
+	return fmt.Errorf("networking still down after %d attempts (host needs physical/console attention)", len(networkingBackoffs))
+}
+
+// --- production seams ------------------------------------------------------------------------------
+
+// SystemctlIsActive is the default IsActive: `systemctl is-active <unit>` prints "active" when up.
+// Unprivileged (unit state is world-readable) — NOT routed through sudo.
+func SystemctlIsActive(unit string) func(context.Context) bool {
+	return func(ctx context.Context) bool {
+		out, _ := exec.CommandContext(ctx, "systemctl", "is-active", "--", unit).Output()
+		return strings.TrimSpace(string(out)) == "active"
+	}
+}
+
+// HasDefaultRoute is the default HasRoute: `ip route show default` is non-empty when a default route
+// exists. Unprivileged read.
+func HasDefaultRoute(ctx context.Context) bool {
+	out, err := exec.CommandContext(ctx, "ip", "route", "show", "default").Output()
+	return err == nil && strings.TrimSpace(string(out)) != ""
+}
+
+// RealSleep is the default Sleep: a context-cancellable time.Sleep.
+func RealSleep(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
@@ -0,0 +1,134 @@
+package selfheal
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+)
+
+func quietLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError + 1}))
+}
+
+// fakeHeal is a controllable Heal: healthy toggles, and it counts remediation invocations.
+type fakeHeal struct {
+	healthy    bool
+	remediated int
+}
+
+func (f *fakeHeal) Name() string { return "fake" }
+func (f *fakeHeal) Healthy(context.Context) (bool, string) {
+	return f.healthy, "fake detail"
+}
+func (f *fakeHeal) Remediate(context.Context) error {
+	f.remediated++
+	f.healthy = true
+	return nil
+}
+
+// THE BYO GATE (red-proof): a byo Manager must NEVER reach a remedy, even with an unhealthy heal.
+func TestManager_ByoNeverRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: false, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 0 {
+		t.Fatalf("byo host must NEVER remediate, got %d invocations", h.remediated)
+	}
+}
+
+// An appliance Manager remediates an unhealthy heal.
+func TestManager_ApplianceRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: true, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 1 {
+		t.Fatalf("appliance must remediate an unhealthy heal once, got %d", h.remediated)
+	}
+}
+
+// A healthy heal is never remediated (either mode).
+func TestManager_HealthyNoRemediate(t *testing.T) {
+	for _, appliance := range []bool{true, false} {
+		h := &fakeHeal{healthy: true}
+		m := &Manager{Appliance: appliance, Heals: []Heal{h}, Logger: quietLogger()}
+		m.RunOnce(context.Background())
+		if h.remediated != 0 {
+			t.Fatalf("appliance=%t: a healthy heal must not be remediated, got %d", appliance, h.remediated)
+		}
+	}
+}
+
+// THE BYO GATE at the networking layer: force the networking check unhealthy under byo and assert the
+// privileged Start is invoked ZERO times (the gate is before any exec).
+func TestNetworkingHeal_ByoZeroPrivilegedInvocations(t *testing.T) {
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false },
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	m := &Manager{Appliance: false, Heals: []Heal{n}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if starts != 0 {
+		t.Fatalf("byo networking heal must invoke `systemctl start` ZERO times, got %d", starts)
+	}
+}
+
+// The remedy state machine: recovers on the 2nd attempt (start brings it up) → returns nil, one INFO.
+func TestNetworkingHeal_RecoversMidAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	active := false
+	attempts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return active },
+		HasRoute: func(context.Context) bool { return true },
+		Start: func(context.Context) error {
+			attempts++
+			if attempts >= 2 {
+				active = true // the 2nd start brings networking up
+			}
+			return nil
+		},
+		Sleep:  func(context.Context, time.Duration) {},
+		Logger: quietLogger(),
+	}
+	if err := n.Remediate(context.Background()); err != nil {
+		t.Fatalf("remedy should recover by attempt 2, got err %v", err)
+	}
+	if attempts != 2 {
+		t.Fatalf("recovery should have taken 2 start attempts, got %d", attempts)
+	}
+}
+
+// The remedy gives up after all attempts if networking never comes up — a terminal error, logged as
+// give-up by the Manager.
+func TestNetworkingHeal_GivesUpAfterAllAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false }, // never recovers
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	err := n.Remediate(context.Background())
+	if err == nil {
+		t.Fatal("remedy must return a terminal error when networking never recovers")
+	}
+	if starts != len(networkingBackoffs) {
+		t.Fatalf("remedy must try exactly %d times, got %d", len(networkingBackoffs), starts)
+	}
+}
+
+// withFastBackoffs swaps the real 10/30/60s backoffs for near-zero waits (the Sleep seam is faked
+// anyway, but keep the count/shape). Returns a restore func.
+func withFastBackoffs() func() {
+	orig := networkingBackoffs
+	networkingBackoffs = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	return func() { networkingBackoffs = orig }
+}