agent v0.27.0: slice 10 P3 — self-heal watchdog reconcile + 4-state intent model

IntentStore (durable-id-keyed: new/enrolled/ejected/decommissioned, OnAbsent
replug rule). Watchdog re-mounts only enrolled drives (out-of-band unmount heals;
ejected/decommissioned/new left alone) + exp-backoff flapping guard (alert@4,
cap@8). guest-attach records enrolled; eject records ejected. Non-hollow tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

internal/storage/watchdog_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net"
+	"path/filepath"
 	"sync"
 	"testing"
 	"time"
@@ -358,3 +359,124 @@ func TestHostLiveness_NetworkDial(t *testing.T) {
 		t.Error("network target without endpoint must not be flagged down")
 	}
 }
+
+// ---- P3 self-heal reconcile (intent-gated) -----------------------------------------------
+
+// newReconcileWD builds a watchdog with one mount-backed, durable-id'd target ("usb"), a fake
+// remounter, an intent store, and the OnAbsent hook wired to it. Synchronous dispatch + manual clock.
+func newReconcileWD(intent *IntentStore) (*Watchdog, *mapLiveness, *fakeRemounter, *time.Time) {
+	known := &staticKnown{targets: []KnownTarget{{
+		Name: "usb", DurableID: "uuid:U", MountBacked: true,
+		MountPath: "/mnt/usb", BackingDevice: "/dev/sdb1", UUID: "U",
+	}}}
+	live := &mapLiveness{present: map[string]bool{}, device: map[string]bool{}}
+	rem := &fakeRemounter{}
+	clock := time.Unix(1_700_000_000, 0).UTC()
+	var reader IntentReader
+	if intent != nil {
+		reader = intent
+	}
+	w := NewWatchdog(WatchdogOptions{
+		Targets: known, Liveness: live, Remounter: rem, Intent: reader,
+		OnAbsent: func(id string) {
+			if intent != nil {
+				_ = intent.OnAbsent(id)
+			}
+		},
+		Interval: time.Second, Debounce: 30 * time.Second, Logger: quietLogger(),
+	})
+	w.now = func() time.Time { return clock }
+	w.spawn = func(f func()) { f() }
+	return w, live, rem, &clock
+}
+
+// colleague's out-of-band unmount (intent=enrolled, device present, not mounted) → reconciled.
+func TestWatchdog_Reconcile_EnrolledColleagueUnmount(t *testing.T) {
+	intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
+	_ = intent.SetEnrolled("uuid:U")
+	w, live, rem, _ := newReconcileWD(intent)
+	ctx := context.Background()
+	live.set("usb", true)
+	w.tick(ctx) // baseline present
+	live.set("usb", false)
+	live.setDevice("usb", true) // unmounted, device still there (out-of-band unmount)
+	w.tick(ctx)
+	if rem.count() != 1 {
+		t.Fatalf("enrolled drive out-of-band-unmounted should be reconciled: remounts=%d", rem.count())
+	}
+}
+
+// ejected / new / decommissioned → NEVER reconciled (only enrolled is).
+func TestWatchdog_Reconcile_RespectsIntent(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		setup func(*IntentStore)
+	}{
+		{"ejected", func(s *IntentStore) { _ = s.SetEjected("uuid:U") }},
+		{"new", func(s *IntentStore) {}}, // no record
+		{"decommissioned", func(s *IntentStore) { _ = s.SetDecommissioned("uuid:U") }},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
+			tc.setup(intent)
+			w, live, rem, _ := newReconcileWD(intent)
+			ctx := context.Background()
+			live.set("usb", true)
+			w.tick(ctx)
+			live.set("usb", false)
+			live.setDevice("usb", true)
+			w.tick(ctx)
+			if rem.count() != 0 {
+				t.Fatalf("%s drive must NOT be reconciled: remounts=%d", tc.name, rem.count())
+			}
+		})
+	}
+}
+
+// ejected → physically absent → (OnAbsent clears to enrolled) → replug → reconciled (replug rule).
+func TestWatchdog_Reconcile_EjectedAbsentReplugAutoMounts(t *testing.T) {
+	intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
+	_ = intent.SetEjected("uuid:U")
+	w, live, rem, _ := newReconcileWD(intent)
+	ctx := context.Background()
+	live.set("usb", true)
+	w.tick(ctx) // baseline present
+	// physically pull: not present AND device gone
+	live.set("usb", false)
+	live.setDevice("usb", false)
+	w.tick(ctx) // present→absent, device gone → OnAbsent clears ejected→enrolled
+	if intent.Get("uuid:U") != IntentEnrolled {
+		t.Fatalf("ejected→absent should clear to enrolled, got %q", intent.Get("uuid:U"))
+	}
+	// replug: device back, not yet mounted
+	live.setDevice("usb", true)
+	w.tick(ctx)
+	if rem.count() != 1 {
+		t.Fatalf("replugged drive (now enrolled) should auto-mount: remounts=%d", rem.count())
+	}
+}
+
+// flapping: a re-mount that never sticks backs off and STOPS after MaxRetries (no infinite loop).
+func TestWatchdog_Reconcile_FlappingBacksOffAndCaps(t *testing.T) {
+	intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
+	_ = intent.SetEnrolled("uuid:U")
+	w, live, rem, clock := newReconcileWD(intent)
+	ctx := context.Background()
+	live.set("usb", true)
+	w.tick(ctx) // baseline present
+	// Drive is stuck: device present but the re-mount never makes it present.
+	live.set("usb", false)
+	live.setDevice("usb", true)
+	for i := 0; i < 20; i++ {
+		w.tick(ctx)
+		*clock = clock.Add(2 * time.Hour) // always exceed the (growing) backoff window
+	}
+	if rem.count() != flappingMaxRetries {
+		t.Fatalf("flapping re-mount should cap at %d, got %d (infinite loop?)", flappingMaxRetries, rem.count())
+	}
+	// further ticks add no more re-mounts (stays capped)
+	w.tick(ctx)
+	if rem.count() != flappingMaxRetries {
+		t.Fatalf("capped flapping must not resume: got %d", rem.count())
+	}
+}