agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/selfheal/selfheal_test.go

@@ -0,0 +1,134 @@
+package selfheal
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"testing"
+	"time"
+)
+
+func quietLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError + 1}))
+}
+
+// fakeHeal is a controllable Heal: healthy toggles, and it counts remediation invocations.
+type fakeHeal struct {
+	healthy    bool
+	remediated int
+}
+
+func (f *fakeHeal) Name() string { return "fake" }
+func (f *fakeHeal) Healthy(context.Context) (bool, string) {
+	return f.healthy, "fake detail"
+}
+func (f *fakeHeal) Remediate(context.Context) error {
+	f.remediated++
+	f.healthy = true
+	return nil
+}
+
+// THE BYO GATE (red-proof): a byo Manager must NEVER reach a remedy, even with an unhealthy heal.
+func TestManager_ByoNeverRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: false, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 0 {
+		t.Fatalf("byo host must NEVER remediate, got %d invocations", h.remediated)
+	}
+}
+
+// An appliance Manager remediates an unhealthy heal.
+func TestManager_ApplianceRemediates(t *testing.T) {
+	h := &fakeHeal{healthy: false}
+	m := &Manager{Appliance: true, Heals: []Heal{h}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if h.remediated != 1 {
+		t.Fatalf("appliance must remediate an unhealthy heal once, got %d", h.remediated)
+	}
+}
+
+// A healthy heal is never remediated (either mode).
+func TestManager_HealthyNoRemediate(t *testing.T) {
+	for _, appliance := range []bool{true, false} {
+		h := &fakeHeal{healthy: true}
+		m := &Manager{Appliance: appliance, Heals: []Heal{h}, Logger: quietLogger()}
+		m.RunOnce(context.Background())
+		if h.remediated != 0 {
+			t.Fatalf("appliance=%t: a healthy heal must not be remediated, got %d", appliance, h.remediated)
+		}
+	}
+}
+
+// THE BYO GATE at the networking layer: force the networking check unhealthy under byo and assert the
+// privileged Start is invoked ZERO times (the gate is before any exec).
+func TestNetworkingHeal_ByoZeroPrivilegedInvocations(t *testing.T) {
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false },
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	m := &Manager{Appliance: false, Heals: []Heal{n}, Logger: quietLogger()}
+	m.RunOnce(context.Background())
+	if starts != 0 {
+		t.Fatalf("byo networking heal must invoke `systemctl start` ZERO times, got %d", starts)
+	}
+}
+
+// The remedy state machine: recovers on the 2nd attempt (start brings it up) → returns nil, one INFO.
+func TestNetworkingHeal_RecoversMidAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	active := false
+	attempts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return active },
+		HasRoute: func(context.Context) bool { return true },
+		Start: func(context.Context) error {
+			attempts++
+			if attempts >= 2 {
+				active = true // the 2nd start brings networking up
+			}
+			return nil
+		},
+		Sleep:  func(context.Context, time.Duration) {},
+		Logger: quietLogger(),
+	}
+	if err := n.Remediate(context.Background()); err != nil {
+		t.Fatalf("remedy should recover by attempt 2, got err %v", err)
+	}
+	if attempts != 2 {
+		t.Fatalf("recovery should have taken 2 start attempts, got %d", attempts)
+	}
+}
+
+// The remedy gives up after all attempts if networking never comes up — a terminal error, logged as
+// give-up by the Manager.
+func TestNetworkingHeal_GivesUpAfterAllAttempts(t *testing.T) {
+	defer withFastBackoffs()()
+	starts := 0
+	n := &NetworkingHeal{
+		IsActive: func(context.Context) bool { return false }, // never recovers
+		HasRoute: func(context.Context) bool { return false },
+		Start:    func(context.Context) error { starts++; return nil },
+		Sleep:    func(context.Context, time.Duration) {},
+		Logger:   quietLogger(),
+	}
+	err := n.Remediate(context.Background())
+	if err == nil {
+		t.Fatal("remedy must return a terminal error when networking never recovers")
+	}
+	if starts != len(networkingBackoffs) {
+		t.Fatalf("remedy must try exactly %d times, got %d", len(networkingBackoffs), starts)
+	}
+}
+
+// withFastBackoffs swaps the real 10/30/60s backoffs for near-zero waits (the Sleep seam is faked
+// anyway, but keep the count/shape). Returns a restore func.
+func withFastBackoffs() func() {
+	orig := networkingBackoffs
+	networkingBackoffs = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	return func() { networkingBackoffs = orig }
+}