agent v0.27.0: slice 10 P3 — self-heal watchdog reconcile + 4-state intent model

IntentStore (durable-id-keyed: new/enrolled/ejected/decommissioned, OnAbsent replug rule). Watchdog re-mounts only enrolled drives (out-of-band unmount heals; ejected/decommissioned/new left alone) + exp-backoff flapping guard (alert@4, cap@8). guest-attach records enrolled; eject records ejected. Non-hollow tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-12 17:49:25 +02:00
parent bc4f2b9168
commit 237b85f420
8 changed files with 590 additions and 35 deletions
@@ -63,6 +63,22 @@ type Transition struct {
 	To   string
 }

+// IntentReader gates self-heal by the drive's INTENT (slice 10 P3): the watchdog reconciles
+// (re-mounts) ONLY a drive whose intent is `enrolled`. Satisfied by *IntentStore. When nil, the
+// watchdog is ungated (legacy observe+remount-any behaviour) — production always wires it.
+type IntentReader interface {
+	Get(durableID string) DriveIntent
+}
+
+// Flapping guard (3C): a drive whose re-mount keeps not sticking gets exponential backoff, an alert
+// after AlertThreshold consecutive failed cycles, and stops being retried after MaxRetries — instead
+// of looping forever. Reset when the drive goes present (healed) or absent (gone, not flapping).
+const (
+	flappingAlertThreshold = 4 // consecutive failed re-mount cycles before raising the alert
+	flappingMaxRetries     = 8 // after this many, stop retrying (alert stands until present/absent)
+	flappingBackoffCap     = 5 // backoff window = debounce * 2^min(fails, cap)
+)
+
 // Watchdog is the third daemon goroutine (alongside the hub loop + reconcile engine). It
 // fast-polls the known target set, detects attached↔disconnected transitions, and triggers
 // an immediate, debounced out-of-band host-report so the hub learns of a drop in seconds.
@@ -80,13 +96,19 @@ type Watchdog struct {
 	now       func() time.Time
 	spawn     func(func()) // spawn a background task (overridable in tests; default `go f()`)

-	mu          sync.Mutex
-	last        map[string]bool // name -> last observed present (only for seen targets)
-	lastFire    time.Time
-	fired       bool                 // lastFire is valid
-	pending     bool                 // a transition is awaiting the debounce window
-	lastRemount map[string]time.Time // name -> last re-mount dispatch (rate-limit)
-	lastUUID    map[string]string    // name -> last fs-UUID observed while ATTACHED (re-mount key)
+	intent   IntentReader      // P3: gate re-mount by drive intent (nil = ungated legacy)
+	onAbsent func(durableID string) // P3: called when a known target goes ABSENT (clears `ejected`)
+
+	mu             sync.Mutex
+	last           map[string]bool      // name -> last observed present (only for seen targets)
+	lastFire       time.Time
+	fired          bool                 // lastFire is valid
+	pending        bool                 // a transition is awaiting the debounce window
+	lastRemount    map[string]time.Time // name -> last re-mount dispatch (rate-limit / backoff base)
+	lastUUID       map[string]string    // name -> last fs-UUID observed while ATTACHED (re-mount key)
+	remountFails   map[string]int       // name -> consecutive failed re-mount cycles (flapping guard)
+	remountPending map[string]bool      // name -> a re-mount was dispatched, awaiting present-confirm
+	flapAlerted    map[string]bool      // name -> the "keeps dropping" alert has fired (once)
 }

 // WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
@@ -99,6 +121,11 @@ type WatchdogOptions struct {
 	Interval  time.Duration
 	Debounce  time.Duration
 	Logger    *slog.Logger
+	// Intent gates self-heal to `enrolled` drives (slice 10 P3). OnAbsent is called (durable-id) when a
+	// known target goes physically absent, so the intent store can clear an `ejected` flag (replug
+	// rule). Both optional — nil Intent = ungated legacy remount; nil OnAbsent = no intent clearing.
+	Intent   IntentReader
+	OnAbsent func(durableID string)
 }

 // NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
@@ -121,21 +148,49 @@ func NewWatchdog(opts WatchdogOptions) *Watchdog {
 		trigger = func() {}
 	}
 	return &Watchdog{
-		targets:     opts.Targets,
-		liveness:    opts.Liveness,
-		remounter:   opts.Remounter,
-		interval:    interval,
-		debounce:    debounce,
-		trigger:     trigger,
-		logger:      logger,
-		now:         func() time.Time { return time.Now().UTC() },
-		spawn:       func(f func()) { go f() },
-		last:        map[string]bool{},
-		lastRemount: map[string]time.Time{},
-		lastUUID:    map[string]string{},
+		targets:        opts.Targets,
+		liveness:       opts.Liveness,
+		remounter:      opts.Remounter,
+		intent:         opts.Intent,
+		onAbsent:       opts.OnAbsent,
+		interval:       interval,
+		debounce:       debounce,
+		trigger:        trigger,
+		logger:         logger,
+		now:            func() time.Time { return time.Now().UTC() },
+		spawn:          func(f func()) { go f() },
+		last:           map[string]bool{},
+		lastRemount:    map[string]time.Time{},
+		lastUUID:       map[string]string{},
+		remountFails:   map[string]int{},
+		remountPending: map[string]bool{},
+		flapAlerted:    map[string]bool{},
 	}
 }

+// reconcileAllowed reports whether self-heal may re-mount this target: only an `enrolled` drive is
+// auto-mounted (3C: never auto-adopt a new/unknown drive; respect an intentional eject; never touch a
+// decommissioned drive). An ungated watchdog (no intent reader) allows it (legacy).
+func (w *Watchdog) reconcileAllowed(t KnownTarget) bool {
+	if w.intent == nil {
+		return true
+	}
+	return w.intent.Get(t.DurableID) == IntentEnrolled
+}
+
+// backoffWindow returns the re-mount backoff for a target with `fails` consecutive failed cycles:
+// debounce * 2^min(fails, cap). fails==0 → the plain debounce.
+func (w *Watchdog) backoffWindow(fails int) time.Duration {
+	if fails > flappingBackoffCap {
+		fails = flappingBackoffCap
+	}
+	win := w.debounce
+	for i := 0; i < fails; i++ {
+		win *= 2
+	}
+	return win
+}
+
 // Run fast-polls until ctx is cancelled. The first tick establishes the baseline (no
 // trigger); subsequent ticks detect transitions. Returns nil on ctx cancellation.
 func (w *Watchdog) Run(ctx context.Context) error {
@@ -207,24 +262,52 @@ func (w *Watchdog) tick(ctx context.Context) {
 	var transitions []Transition
 	var remounts []KnownTarget
 	current := make(map[string]bool, len(probes))
+	var absentNow []KnownTarget  // P3: targets that just went physically ABSENT (clear `ejected`)
+	var flapAlerts []KnownTarget // P3: "drive keeps dropping" alerts to surface
 	for _, p := range probes {
 		current[p.t.Name] = p.present
 		if prev, seen := w.last[p.t.Name]; seen && prev != p.present {
 			transitions = append(transitions, Transition{Name: p.t.Name, From: stateStr(prev), To: stateStr(p.present)})
-		}
-		// Re-mount candidate: a mount-backed target that is NOT mounted but whose backing
-		// device is physically present (a disconnected→device-back state). Rate-limited per
-		// target to the debounce window so a persistent mount failure can't storm HostOps.
-		if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent {
-			if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.debounce {
-				w.lastRemount[p.t.Name] = now
-				remounts = append(remounts, p.t)
+			// Physically ABSENT (device gone, not merely unmounted): a mount-backed target that went
+			// !present AND whose device is no longer present. The intent store clears `ejected` here so
+			// a replug auto-mounts (the replug rule); flapping state resets (it's gone, not flapping).
+			if prev && !p.present && p.t.MountBacked && !p.devicePresent {
+				absentNow = append(absentNow, p.t)
+				w.resetFlap(p.t.Name)
 			}
 		}
-		// Once a target is present again, clear its re-mount rate-limit so a future cycle
-		// re-mounts promptly.
+		// Re-mount candidate (self-heal): an ENROLLED, mount-backed target that is NOT mounted but
+		// whose backing device IS physically present (out-of-band unmount → device-still-there). Gated
+		// by intent (never auto-adopt a new/ejected/decommissioned drive) + an exponential-backoff
+		// flapping guard (3C): a re-mount that doesn't stick (still !present next cycle) counts as a
+		// failure → grow the backoff; alert after AlertThreshold; stop after MaxRetries.
+		if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent && w.reconcileAllowed(p.t) {
+			fails := w.remountFails[p.t.Name]
+			// Only act once a full backoff window has elapsed since the last dispatch (or the first
+			// time). Gating the WHOLE evaluation on the window means a re-mount that's merely slow
+			// (the dispatch is async, ticks are fast) isn't miscounted as a failure — a failure is
+			// "still not present a full window after we dispatched".
+			if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.backoffWindow(fails) {
+				if w.remountPending[p.t.Name] { // prior dispatch didn't take within its window → failed
+					fails++
+					w.remountFails[p.t.Name] = fails
+					w.remountPending[p.t.Name] = false
+				}
+				if fails >= flappingAlertThreshold && !w.flapAlerted[p.t.Name] {
+					w.flapAlerted[p.t.Name] = true
+					flapAlerts = append(flapAlerts, p.t)
+				}
+				if fails < flappingMaxRetries {
+					w.lastRemount[p.t.Name] = now
+					w.remountPending[p.t.Name] = true
+					remounts = append(remounts, p.t)
+				}
+			}
+		}
+		// Healed (present again): reset re-mount rate-limit + flapping state so a future drop re-mounts
+		// promptly and the alert can re-fire.
 		if p.present {
-			delete(w.lastRemount, p.t.Name)
+			w.resetFlap(p.t.Name)
 		}
 	}
 	w.last = current // targets no longer known drop out
@@ -254,10 +337,33 @@ func (w *Watchdog) tick(ctx context.Context) {
 	}
 	for _, t := range remounts {
 		t := t
-		w.logger.Info("storage: watchdog dispatching benign re-mount (device returned)",
-			"target", t.Name, "where", t.MountPath)
+		w.logger.Info("storage: watchdog self-heal — re-mounting enrolled drive (device returned, no eject intent)",
+			"target", t.Name, "where", t.MountPath, "durable_id", t.DurableID)
 		w.spawn(func() { w.remounter.Remount(ctx, t) })
 	}
+	// P3: a drive that went physically absent — clear an `ejected` intent so a replug auto-mounts.
+	for _, t := range absentNow {
+		if w.onAbsent != nil && t.DurableID != "" {
+			w.logger.Info("storage: watchdog — known drive went absent; clearing any eject intent (replug will auto-mount)",
+				"target", t.Name, "durable_id", t.DurableID)
+			w.onAbsent(t.DurableID)
+		}
+	}
+	// P3 flapping guard: a drive whose re-mount keeps not sticking — alert instead of looping silently.
+	for _, t := range flapAlerts {
+		w.logger.Warn("storage: watchdog ALERT — drive keeps dropping (re-mount not sticking); backing off",
+			"target", t.Name, "where", t.MountPath, "durable_id", t.DurableID, "failed_cycles", flappingAlertThreshold)
+		w.trigger() // surface out-of-band so the hub/operator sees the unhealthy drive
+	}
+}
+
+// resetFlap clears all per-target re-mount/backoff/alert state (target healed or went absent). The
+// caller holds w.mu.
+func (w *Watchdog) resetFlap(name string) {
+	delete(w.lastRemount, name)
+	delete(w.remountFails, name)
+	delete(w.remountPending, name)
+	delete(w.flapAlerted, name)
 }

 func stateStr(present bool) string {