agent v0.27.0: slice 10 P3 — self-heal watchdog reconcile + 4-state intent model
IntentStore (durable-id-keyed: new/enrolled/ejected/decommissioned, OnAbsent replug rule). Watchdog re-mounts only enrolled drives (out-of-band unmount heals; ejected/decommissioned/new left alone) + exp-backoff flapping guard (alert@4, cap@8). guest-attach records enrolled; eject records ejected. Non-hollow tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,131 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Drive INTENT model (slice 10 P3 self-heal). The agent persists, per external user-data drive
|
||||
// (keyed by DURABLE-ID — UUID/WWN, never sdX/path), the operator/customer INTENT — which is distinct
|
||||
// from "is it currently mounted". The self-heal reconciler acts ONLY on this intent, so an
|
||||
// out-of-band unmount (a colleague's Proxmox action) — which records NO intent — gets healed, while an
|
||||
// official eject is respected.
|
||||
//
|
||||
// Four states (3A):
|
||||
// - new : durable-id NOT in the store. Never auto-mounted (the user must enroll it).
|
||||
// - enrolled : desired = mounted + bound into the guest. Drift (present && unmounted) with
|
||||
// this intent → reconcile.
|
||||
// - ejected : an intentional, temporary unmount, set ONLY via the official eject endpoint.
|
||||
// Present && unmounted → left alone. CLEARED to `enrolled` when the drive goes
|
||||
// physically ABSENT, so a replug auto-mounts (the replug rule, for free).
|
||||
// - decommissioned : permanent. Never auto-mounted; SURVIVES absent/present; cleared only by an
|
||||
// explicit re-commission.
|
||||
//
|
||||
// The elegant invariant: intent is recorded ONLY through the official enroll/eject/decommission
|
||||
// paths. "Is it unmounted" ≠ "did someone officially eject it."
|
||||
type DriveIntent string
|
||||
|
||||
const (
|
||||
IntentNew DriveIntent = "" // not in the store
|
||||
IntentEnrolled DriveIntent = "enrolled" // desired mounted → reconcile drift
|
||||
IntentEjected DriveIntent = "ejected" // intentional unmount → leave alone
|
||||
IntentDecommissioned DriveIntent = "decommissioned" // permanent → never auto-mount
|
||||
)
|
||||
|
||||
// IntentStore is the durable, durable-id-keyed intent map. Thread-safe; atomic file writes
|
||||
// (tmp+rename), 0600. Shared between the local-API (records enroll/eject) and the self-heal
|
||||
// reconciler (reads to gate remounts; clears `ejected` on absent).
|
||||
type IntentStore struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
m map[string]DriveIntent // durable-id -> intent
|
||||
}
|
||||
|
||||
// OpenIntentStore loads (or initializes) the store at path. A missing file is an empty store; a
|
||||
// corrupt file is an error (fail loud — the reconciler's gate must not silently lose intent).
|
||||
func OpenIntentStore(path string) (*IntentStore, error) {
|
||||
s := &IntentStore{path: path, m: map[string]DriveIntent{}}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return s, nil
|
||||
}
|
||||
return nil, fmt.Errorf("intent store: read %s: %w", path, err)
|
||||
}
|
||||
if len(data) > 0 {
|
||||
if err := json.Unmarshal(data, &s.m); err != nil {
|
||||
return nil, fmt.Errorf("intent store: parse %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Get returns the intent for a durable-id (IntentNew if absent). An empty durable-id is always
|
||||
// IntentNew — the reconciler must never act on a drive whose identity it can't pin.
|
||||
func (s *IntentStore) Get(durableID string) DriveIntent {
|
||||
if durableID == "" {
|
||||
return IntentNew
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.m[durableID]
|
||||
}
|
||||
|
||||
// SetEnrolled records a drive as enrolled (on successful enroll/guest-attach). Idempotent.
|
||||
func (s *IntentStore) SetEnrolled(durableID string) error { return s.set(durableID, IntentEnrolled) }
|
||||
|
||||
// SetEjected records an intentional eject (official eject endpoint only).
|
||||
func (s *IntentStore) SetEjected(durableID string) error { return s.set(durableID, IntentEjected) }
|
||||
|
||||
// SetDecommissioned records a permanent decommission (operator path).
|
||||
func (s *IntentStore) SetDecommissioned(durableID string) error {
|
||||
return s.set(durableID, IntentDecommissioned)
|
||||
}
|
||||
|
||||
// OnAbsent transitions a drive that has gone physically ABSENT: an `ejected` drive becomes `enrolled`
|
||||
// again (so a replug auto-mounts — the replug rule); `decommissioned` and `enrolled` are unchanged;
|
||||
// `new` stays new. This is the ONLY place ejected→enrolled happens.
|
||||
func (s *IntentStore) OnAbsent(durableID string) error {
|
||||
if durableID == "" {
|
||||
return nil
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.m[durableID] == IntentEjected {
|
||||
s.m[durableID] = IntentEnrolled
|
||||
return s.saveLocked()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *IntentStore) set(durableID string, intent DriveIntent) error {
|
||||
if durableID == "" {
|
||||
return fmt.Errorf("intent store: refusing to record intent for an empty durable-id")
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.m[durableID] == intent {
|
||||
return nil // idempotent — no write
|
||||
}
|
||||
s.m[durableID] = intent
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *IntentStore) saveLocked() error {
|
||||
data, err := json.MarshalIndent(s.m, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.path)
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIntentStore_StatesAndPersistence(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "intent.json")
|
||||
s, err := OpenIntentStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
// new: absent → IntentNew
|
||||
if s.Get("uuid:A") != IntentNew {
|
||||
t.Fatalf("absent should be new, got %q", s.Get("uuid:A"))
|
||||
}
|
||||
if err := s.SetEnrolled("uuid:A"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SetEjected("uuid:B"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SetDecommissioned("uuid:C"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// reload → states persisted
|
||||
s2, err := OpenIntentStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
if s2.Get("uuid:A") != IntentEnrolled || s2.Get("uuid:B") != IntentEjected || s2.Get("uuid:C") != IntentDecommissioned {
|
||||
t.Fatalf("states not persisted: A=%q B=%q C=%q", s2.Get("uuid:A"), s2.Get("uuid:B"), s2.Get("uuid:C"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntentStore_OnAbsent_ReplyRule(t *testing.T) {
|
||||
s, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
_ = s.SetEjected("uuid:E")
|
||||
_ = s.SetDecommissioned("uuid:D")
|
||||
_ = s.SetEnrolled("uuid:N")
|
||||
|
||||
// ejected → absent → enrolled (so a replug auto-mounts)
|
||||
if err := s.OnAbsent("uuid:E"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Get("uuid:E") != IntentEnrolled {
|
||||
t.Fatalf("ejected→absent should clear to enrolled, got %q", s.Get("uuid:E"))
|
||||
}
|
||||
// decommissioned survives absent
|
||||
_ = s.OnAbsent("uuid:D")
|
||||
if s.Get("uuid:D") != IntentDecommissioned {
|
||||
t.Fatalf("decommissioned must survive absent, got %q", s.Get("uuid:D"))
|
||||
}
|
||||
// enrolled unchanged by absent
|
||||
_ = s.OnAbsent("uuid:N")
|
||||
if s.Get("uuid:N") != IntentEnrolled {
|
||||
t.Fatalf("enrolled should stay enrolled, got %q", s.Get("uuid:N"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntentStore_EmptyDurableIDRefused(t *testing.T) {
|
||||
s, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
if err := s.SetEnrolled(""); err == nil {
|
||||
t.Fatal("recording intent for an empty durable-id must be refused")
|
||||
}
|
||||
if s.Get("") != IntentNew {
|
||||
t.Fatal("empty durable-id must read as new")
|
||||
}
|
||||
}
|
||||
|
||||
// ReconcilePermitted is the gate the reconciler uses: only an `enrolled` drive (intent-none) is
|
||||
// auto-mounted; new/ejected/decommissioned are not. This documents the decision table.
|
||||
func TestIntentStore_ReconcileGate(t *testing.T) {
|
||||
s, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
_ = s.SetEnrolled("uuid:enr")
|
||||
_ = s.SetEjected("uuid:ej")
|
||||
_ = s.SetDecommissioned("uuid:dec")
|
||||
cases := map[string]bool{
|
||||
"uuid:enr": true, // enrolled → reconcile
|
||||
"uuid:ej": false, // ejected → leave alone
|
||||
"uuid:dec": false, // decommissioned → never
|
||||
"uuid:new": false, // new → never auto-mount
|
||||
}
|
||||
for id, want := range cases {
|
||||
got := s.Get(id) == IntentEnrolled
|
||||
if got != want {
|
||||
t.Errorf("reconcile-permitted(%s): got %v want %v (intent %q)", id, got, want, s.Get(id))
|
||||
}
|
||||
}
|
||||
}
|
||||
+138
-32
@@ -63,6 +63,22 @@ type Transition struct {
|
||||
To string
|
||||
}
|
||||
|
||||
// IntentReader gates self-heal by the drive's INTENT (slice 10 P3): the watchdog reconciles
|
||||
// (re-mounts) ONLY a drive whose intent is `enrolled`. Satisfied by *IntentStore. When nil, the
|
||||
// watchdog is ungated (legacy observe+remount-any behaviour) — production always wires it.
|
||||
type IntentReader interface {
|
||||
Get(durableID string) DriveIntent
|
||||
}
|
||||
|
||||
// Flapping guard (3C): a drive whose re-mount keeps not sticking gets exponential backoff, an alert
|
||||
// after AlertThreshold consecutive failed cycles, and stops being retried after MaxRetries — instead
|
||||
// of looping forever. Reset when the drive goes present (healed) or absent (gone, not flapping).
|
||||
const (
|
||||
flappingAlertThreshold = 4 // consecutive failed re-mount cycles before raising the alert
|
||||
flappingMaxRetries = 8 // after this many, stop retrying (alert stands until present/absent)
|
||||
flappingBackoffCap = 5 // backoff window = debounce * 2^min(fails, cap)
|
||||
)
|
||||
|
||||
// Watchdog is the third daemon goroutine (alongside the hub loop + reconcile engine). It
|
||||
// fast-polls the known target set, detects attached↔disconnected transitions, and triggers
|
||||
// an immediate, debounced out-of-band host-report so the hub learns of a drop in seconds.
|
||||
@@ -80,13 +96,19 @@ type Watchdog struct {
|
||||
now func() time.Time
|
||||
spawn func(func()) // spawn a background task (overridable in tests; default `go f()`)
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
lastRemount map[string]time.Time // name -> last re-mount dispatch (rate-limit)
|
||||
lastUUID map[string]string // name -> last fs-UUID observed while ATTACHED (re-mount key)
|
||||
intent IntentReader // P3: gate re-mount by drive intent (nil = ungated legacy)
|
||||
onAbsent func(durableID string) // P3: called when a known target goes ABSENT (clears `ejected`)
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]bool // name -> last observed present (only for seen targets)
|
||||
lastFire time.Time
|
||||
fired bool // lastFire is valid
|
||||
pending bool // a transition is awaiting the debounce window
|
||||
lastRemount map[string]time.Time // name -> last re-mount dispatch (rate-limit / backoff base)
|
||||
lastUUID map[string]string // name -> last fs-UUID observed while ATTACHED (re-mount key)
|
||||
remountFails map[string]int // name -> consecutive failed re-mount cycles (flapping guard)
|
||||
remountPending map[string]bool // name -> a re-mount was dispatched, awaiting present-confirm
|
||||
flapAlerted map[string]bool // name -> the "keeps dropping" alert has fired (once)
|
||||
}
|
||||
|
||||
// WatchdogOptions configures a Watchdog. Targets, Liveness and Trigger are required; the
|
||||
@@ -99,6 +121,11 @@ type WatchdogOptions struct {
|
||||
Interval time.Duration
|
||||
Debounce time.Duration
|
||||
Logger *slog.Logger
|
||||
// Intent gates self-heal to `enrolled` drives (slice 10 P3). OnAbsent is called (durable-id) when a
|
||||
// known target goes physically absent, so the intent store can clear an `ejected` flag (replug
|
||||
// rule). Both optional — nil Intent = ungated legacy remount; nil OnAbsent = no intent clearing.
|
||||
Intent IntentReader
|
||||
OnAbsent func(durableID string)
|
||||
}
|
||||
|
||||
// NewWatchdog builds a Watchdog. A nil Trigger is tolerated (the watchdog still tracks
|
||||
@@ -121,21 +148,49 @@ func NewWatchdog(opts WatchdogOptions) *Watchdog {
|
||||
trigger = func() {}
|
||||
}
|
||||
return &Watchdog{
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
remounter: opts.Remounter,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
spawn: func(f func()) { go f() },
|
||||
last: map[string]bool{},
|
||||
lastRemount: map[string]time.Time{},
|
||||
lastUUID: map[string]string{},
|
||||
targets: opts.Targets,
|
||||
liveness: opts.Liveness,
|
||||
remounter: opts.Remounter,
|
||||
intent: opts.Intent,
|
||||
onAbsent: opts.OnAbsent,
|
||||
interval: interval,
|
||||
debounce: debounce,
|
||||
trigger: trigger,
|
||||
logger: logger,
|
||||
now: func() time.Time { return time.Now().UTC() },
|
||||
spawn: func(f func()) { go f() },
|
||||
last: map[string]bool{},
|
||||
lastRemount: map[string]time.Time{},
|
||||
lastUUID: map[string]string{},
|
||||
remountFails: map[string]int{},
|
||||
remountPending: map[string]bool{},
|
||||
flapAlerted: map[string]bool{},
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileAllowed reports whether self-heal may re-mount this target: only an `enrolled` drive is
|
||||
// auto-mounted (3C: never auto-adopt a new/unknown drive; respect an intentional eject; never touch a
|
||||
// decommissioned drive). An ungated watchdog (no intent reader) allows it (legacy).
|
||||
func (w *Watchdog) reconcileAllowed(t KnownTarget) bool {
|
||||
if w.intent == nil {
|
||||
return true
|
||||
}
|
||||
return w.intent.Get(t.DurableID) == IntentEnrolled
|
||||
}
|
||||
|
||||
// backoffWindow returns the re-mount backoff for a target with `fails` consecutive failed cycles:
|
||||
// debounce * 2^min(fails, cap). fails==0 → the plain debounce.
|
||||
func (w *Watchdog) backoffWindow(fails int) time.Duration {
|
||||
if fails > flappingBackoffCap {
|
||||
fails = flappingBackoffCap
|
||||
}
|
||||
win := w.debounce
|
||||
for i := 0; i < fails; i++ {
|
||||
win *= 2
|
||||
}
|
||||
return win
|
||||
}
|
||||
|
||||
// Run fast-polls until ctx is cancelled. The first tick establishes the baseline (no
|
||||
// trigger); subsequent ticks detect transitions. Returns nil on ctx cancellation.
|
||||
func (w *Watchdog) Run(ctx context.Context) error {
|
||||
@@ -207,24 +262,52 @@ func (w *Watchdog) tick(ctx context.Context) {
|
||||
var transitions []Transition
|
||||
var remounts []KnownTarget
|
||||
current := make(map[string]bool, len(probes))
|
||||
var absentNow []KnownTarget // P3: targets that just went physically ABSENT (clear `ejected`)
|
||||
var flapAlerts []KnownTarget // P3: "drive keeps dropping" alerts to surface
|
||||
for _, p := range probes {
|
||||
current[p.t.Name] = p.present
|
||||
if prev, seen := w.last[p.t.Name]; seen && prev != p.present {
|
||||
transitions = append(transitions, Transition{Name: p.t.Name, From: stateStr(prev), To: stateStr(p.present)})
|
||||
}
|
||||
// Re-mount candidate: a mount-backed target that is NOT mounted but whose backing
|
||||
// device is physically present (a disconnected→device-back state). Rate-limited per
|
||||
// target to the debounce window so a persistent mount failure can't storm HostOps.
|
||||
if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent {
|
||||
if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.debounce {
|
||||
w.lastRemount[p.t.Name] = now
|
||||
remounts = append(remounts, p.t)
|
||||
// Physically ABSENT (device gone, not merely unmounted): a mount-backed target that went
|
||||
// !present AND whose device is no longer present. The intent store clears `ejected` here so
|
||||
// a replug auto-mounts (the replug rule); flapping state resets (it's gone, not flapping).
|
||||
if prev && !p.present && p.t.MountBacked && !p.devicePresent {
|
||||
absentNow = append(absentNow, p.t)
|
||||
w.resetFlap(p.t.Name)
|
||||
}
|
||||
}
|
||||
// Once a target is present again, clear its re-mount rate-limit so a future cycle
|
||||
// re-mounts promptly.
|
||||
// Re-mount candidate (self-heal): an ENROLLED, mount-backed target that is NOT mounted but
|
||||
// whose backing device IS physically present (out-of-band unmount → device-still-there). Gated
|
||||
// by intent (never auto-adopt a new/ejected/decommissioned drive) + an exponential-backoff
|
||||
// flapping guard (3C): a re-mount that doesn't stick (still !present next cycle) counts as a
|
||||
// failure → grow the backoff; alert after AlertThreshold; stop after MaxRetries.
|
||||
if w.remounter != nil && p.t.MountBacked && !p.present && p.devicePresent && w.reconcileAllowed(p.t) {
|
||||
fails := w.remountFails[p.t.Name]
|
||||
// Only act once a full backoff window has elapsed since the last dispatch (or the first
|
||||
// time). Gating the WHOLE evaluation on the window means a re-mount that's merely slow
|
||||
// (the dispatch is async, ticks are fast) isn't miscounted as a failure — a failure is
|
||||
// "still not present a full window after we dispatched".
|
||||
if last, ok := w.lastRemount[p.t.Name]; !ok || now.Sub(last) >= w.backoffWindow(fails) {
|
||||
if w.remountPending[p.t.Name] { // prior dispatch didn't take within its window → failed
|
||||
fails++
|
||||
w.remountFails[p.t.Name] = fails
|
||||
w.remountPending[p.t.Name] = false
|
||||
}
|
||||
if fails >= flappingAlertThreshold && !w.flapAlerted[p.t.Name] {
|
||||
w.flapAlerted[p.t.Name] = true
|
||||
flapAlerts = append(flapAlerts, p.t)
|
||||
}
|
||||
if fails < flappingMaxRetries {
|
||||
w.lastRemount[p.t.Name] = now
|
||||
w.remountPending[p.t.Name] = true
|
||||
remounts = append(remounts, p.t)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Healed (present again): reset re-mount rate-limit + flapping state so a future drop re-mounts
|
||||
// promptly and the alert can re-fire.
|
||||
if p.present {
|
||||
delete(w.lastRemount, p.t.Name)
|
||||
w.resetFlap(p.t.Name)
|
||||
}
|
||||
}
|
||||
w.last = current // targets no longer known drop out
|
||||
@@ -254,10 +337,33 @@ func (w *Watchdog) tick(ctx context.Context) {
|
||||
}
|
||||
for _, t := range remounts {
|
||||
t := t
|
||||
w.logger.Info("storage: watchdog dispatching benign re-mount (device returned)",
|
||||
"target", t.Name, "where", t.MountPath)
|
||||
w.logger.Info("storage: watchdog self-heal — re-mounting enrolled drive (device returned, no eject intent)",
|
||||
"target", t.Name, "where", t.MountPath, "durable_id", t.DurableID)
|
||||
w.spawn(func() { w.remounter.Remount(ctx, t) })
|
||||
}
|
||||
// P3: a drive that went physically absent — clear an `ejected` intent so a replug auto-mounts.
|
||||
for _, t := range absentNow {
|
||||
if w.onAbsent != nil && t.DurableID != "" {
|
||||
w.logger.Info("storage: watchdog — known drive went absent; clearing any eject intent (replug will auto-mount)",
|
||||
"target", t.Name, "durable_id", t.DurableID)
|
||||
w.onAbsent(t.DurableID)
|
||||
}
|
||||
}
|
||||
// P3 flapping guard: a drive whose re-mount keeps not sticking — alert instead of looping silently.
|
||||
for _, t := range flapAlerts {
|
||||
w.logger.Warn("storage: watchdog ALERT — drive keeps dropping (re-mount not sticking); backing off",
|
||||
"target", t.Name, "where", t.MountPath, "durable_id", t.DurableID, "failed_cycles", flappingAlertThreshold)
|
||||
w.trigger() // surface out-of-band so the hub/operator sees the unhealthy drive
|
||||
}
|
||||
}
|
||||
|
||||
// resetFlap clears all per-target re-mount/backoff/alert state (target healed or went absent). The
|
||||
// caller holds w.mu.
|
||||
func (w *Watchdog) resetFlap(name string) {
|
||||
delete(w.lastRemount, name)
|
||||
delete(w.remountFails, name)
|
||||
delete(w.remountPending, name)
|
||||
delete(w.flapAlerted, name)
|
||||
}
|
||||
|
||||
func stateStr(present bool) string {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -358,3 +359,124 @@ func TestHostLiveness_NetworkDial(t *testing.T) {
|
||||
t.Error("network target without endpoint must not be flagged down")
|
||||
}
|
||||
}
|
||||
|
||||
// ---- P3 self-heal reconcile (intent-gated) -----------------------------------------------
|
||||
|
||||
// newReconcileWD builds a watchdog with one mount-backed, durable-id'd target ("usb"), a fake
|
||||
// remounter, an intent store, and the OnAbsent hook wired to it. Synchronous dispatch + manual clock.
|
||||
func newReconcileWD(intent *IntentStore) (*Watchdog, *mapLiveness, *fakeRemounter, *time.Time) {
|
||||
known := &staticKnown{targets: []KnownTarget{{
|
||||
Name: "usb", DurableID: "uuid:U", MountBacked: true,
|
||||
MountPath: "/mnt/usb", BackingDevice: "/dev/sdb1", UUID: "U",
|
||||
}}}
|
||||
live := &mapLiveness{present: map[string]bool{}, device: map[string]bool{}}
|
||||
rem := &fakeRemounter{}
|
||||
clock := time.Unix(1_700_000_000, 0).UTC()
|
||||
var reader IntentReader
|
||||
if intent != nil {
|
||||
reader = intent
|
||||
}
|
||||
w := NewWatchdog(WatchdogOptions{
|
||||
Targets: known, Liveness: live, Remounter: rem, Intent: reader,
|
||||
OnAbsent: func(id string) {
|
||||
if intent != nil {
|
||||
_ = intent.OnAbsent(id)
|
||||
}
|
||||
},
|
||||
Interval: time.Second, Debounce: 30 * time.Second, Logger: quietLogger(),
|
||||
})
|
||||
w.now = func() time.Time { return clock }
|
||||
w.spawn = func(f func()) { f() }
|
||||
return w, live, rem, &clock
|
||||
}
|
||||
|
||||
// colleague's out-of-band unmount (intent=enrolled, device present, not mounted) → reconciled.
|
||||
func TestWatchdog_Reconcile_EnrolledColleagueUnmount(t *testing.T) {
|
||||
intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
_ = intent.SetEnrolled("uuid:U")
|
||||
w, live, rem, _ := newReconcileWD(intent)
|
||||
ctx := context.Background()
|
||||
live.set("usb", true)
|
||||
w.tick(ctx) // baseline present
|
||||
live.set("usb", false)
|
||||
live.setDevice("usb", true) // unmounted, device still there (out-of-band unmount)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 1 {
|
||||
t.Fatalf("enrolled drive out-of-band-unmounted should be reconciled: remounts=%d", rem.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ejected / new / decommissioned → NEVER reconciled (only enrolled is).
|
||||
func TestWatchdog_Reconcile_RespectsIntent(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
setup func(*IntentStore)
|
||||
}{
|
||||
{"ejected", func(s *IntentStore) { _ = s.SetEjected("uuid:U") }},
|
||||
{"new", func(s *IntentStore) {}}, // no record
|
||||
{"decommissioned", func(s *IntentStore) { _ = s.SetDecommissioned("uuid:U") }},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
tc.setup(intent)
|
||||
w, live, rem, _ := newReconcileWD(intent)
|
||||
ctx := context.Background()
|
||||
live.set("usb", true)
|
||||
w.tick(ctx)
|
||||
live.set("usb", false)
|
||||
live.setDevice("usb", true)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 0 {
|
||||
t.Fatalf("%s drive must NOT be reconciled: remounts=%d", tc.name, rem.count())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ejected → physically absent → (OnAbsent clears to enrolled) → replug → reconciled (replug rule).
|
||||
func TestWatchdog_Reconcile_EjectedAbsentReplugAutoMounts(t *testing.T) {
|
||||
intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
_ = intent.SetEjected("uuid:U")
|
||||
w, live, rem, _ := newReconcileWD(intent)
|
||||
ctx := context.Background()
|
||||
live.set("usb", true)
|
||||
w.tick(ctx) // baseline present
|
||||
// physically pull: not present AND device gone
|
||||
live.set("usb", false)
|
||||
live.setDevice("usb", false)
|
||||
w.tick(ctx) // present→absent, device gone → OnAbsent clears ejected→enrolled
|
||||
if intent.Get("uuid:U") != IntentEnrolled {
|
||||
t.Fatalf("ejected→absent should clear to enrolled, got %q", intent.Get("uuid:U"))
|
||||
}
|
||||
// replug: device back, not yet mounted
|
||||
live.setDevice("usb", true)
|
||||
w.tick(ctx)
|
||||
if rem.count() != 1 {
|
||||
t.Fatalf("replugged drive (now enrolled) should auto-mount: remounts=%d", rem.count())
|
||||
}
|
||||
}
|
||||
|
||||
// flapping: a re-mount that never sticks backs off and STOPS after MaxRetries (no infinite loop).
|
||||
func TestWatchdog_Reconcile_FlappingBacksOffAndCaps(t *testing.T) {
|
||||
intent, _ := OpenIntentStore(filepath.Join(t.TempDir(), "i.json"))
|
||||
_ = intent.SetEnrolled("uuid:U")
|
||||
w, live, rem, clock := newReconcileWD(intent)
|
||||
ctx := context.Background()
|
||||
live.set("usb", true)
|
||||
w.tick(ctx) // baseline present
|
||||
// Drive is stuck: device present but the re-mount never makes it present.
|
||||
live.set("usb", false)
|
||||
live.setDevice("usb", true)
|
||||
for i := 0; i < 20; i++ {
|
||||
w.tick(ctx)
|
||||
*clock = clock.Add(2 * time.Hour) // always exceed the (growing) backoff window
|
||||
}
|
||||
if rem.count() != flappingMaxRetries {
|
||||
t.Fatalf("flapping re-mount should cap at %d, got %d (infinite loop?)", flappingMaxRetries, rem.count())
|
||||
}
|
||||
// further ticks add no more re-mounts (stays capped)
|
||||
w.tick(ctx)
|
||||
if rem.count() != flappingMaxRetries {
|
||||
t.Fatalf("capped flapping must not resume: got %d", rem.count())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user