v0.114.0 — R-113: drive presence means the DEVICE, not the bind
BoundUnderParent, the one field the controller's drive-absent gate keys on,
reported only "is this path a mount target in the guest's mountinfo". The
drive's raw mount at /mnt/<name> is a systemd mount unit bound to its device and
dies with it, but the agent's own bind of <raw>/felhom-data under the shared
parent is an ordinary bind: nothing ties it to the device, so its mountinfo
entry OUTLIVES the device as a stale shell. Presence read that survivor as true,
planDriveGates never produced a Stop action, and nothing fired on any channel --
not backup_target_absent, not the generic storage_disconnected. Measured live in
E-2d: detached at 10:58:37Z, silent for 4.5 minutes while the agent itself
logged "enrolled drive absent by UUID" every 20s (felhom.eu
audits/E2D-fresh-vm-2026-07-29.md §5.2).
The fix: BoundUnderParent becomes a CONJUNCTION -- bound under the parent AND
the drive's raw host mount still mounted (devicePresent, new deviceCheck seam
defaulting to isHostMountpoint). Applied at BOTH /disks construction sites. The
union path matters more, not less: it hardcodes State:"attached", so the
raw-mount check is the only device truth that row carries, and it is exactly the
shape E-2d detached.
Why a conjunction and not a replacement: half 2 alone would regress boot
ordering, where the raw drive mounts early and the bind lands ~18s later; the
gate depends on that window reading ABSENT. The conjunction leaves that
byte-identical and closes only the case the gate could never see.
Unknown is never absent: devicePresent("") returns TRUE. A false absent stops a
working customer's apps -- the failure mode of this fix, not of the bug.
Controller UNCHANGED, no MinAgent bump. BoundUnderParent has exactly one
functional consumer (planDriveGates, intermediary.go:226); every other mention
in both repos is a comment or a test, and boot convergence deliberately moved
off it to pollLiveBinds/driveBindLive. The alternative -- a new DevicePresent
bool the controller ANDs in -- was rejected as dangerous: a bool absent from an
older agent's JSON decodes to false, so every drive on a pre-0.114.0 agent would
have read ABSENT and stopped its apps.
Tests +6 in internal/localapi (208 -> 214): groups A-D plus a wire-contract test
asserting the ENCODED bound_under_parent, since that is what crosses to the
controller. Four red-proofs run and reverted (drop the conjunction on each path;
invert unknown; drop the bind half); disks.go verified byte-identical after.
NOT LIVE-VALIDATED. No drive was pulled. Leg awaiting Session C: device loss ->
gate Stop -> SetDisconnected -> backup_target_absent on the wire.
This commit is contained in:
@@ -0,0 +1,186 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// R-113 — BoundUnderParent must mean THE DEVICE IS THERE, not "a mount entry with this name exists".
|
||||
//
|
||||
// THE BUG THESE PIN. The drive's raw mount at /mnt/<name> is a systemd mount unit bound to its device
|
||||
// and dies with it. The agent's own bind of <raw>/felhom-data under the shared parent is an ordinary
|
||||
// bind — nothing ties it to the device — so it OUTLIVES the device as a stale shell. Before v0.114.0
|
||||
// BoundUnderParent was half 1 only, so a pulled drive kept reporting present, the controller's
|
||||
// drive-absent gate never produced a Stop action, and NOTHING fired on any channel: not
|
||||
// backup_target_absent, not the generic storage_disconnected. Measured live in E-2d with the device
|
||||
// detached — /mnt/mentes2 NOT mounted while /mnt/felhom-drives/mentes2 still read /dev/sdb[/felhom-data]
|
||||
// (felhom.eu audits/E2D-fresh-vm-2026-07-29.md §5.2).
|
||||
//
|
||||
// RED-PROOF. Drop `&& s.devicePresent(...)` from either construction site in disks.go and
|
||||
// TestDisks_DevicePresence_ObservePath_DeviceLossReadsAbsent / _UnionPath_... fail with
|
||||
// "reports present — the bind outlived the device (R-113)".
|
||||
//
|
||||
// These drive the REAL production path: NewServer → GET /disks through srv.Handler() → the JSON the
|
||||
// controller actually parses. The two lowest-level mount reads are injected (a unit test cannot create
|
||||
// real mounts), but nothing above them is faked, and the wire test below asserts the encoded field.
|
||||
|
||||
// presenceServer builds a /disks server over one Observe target and/or one registry drive, with the
|
||||
// bind and device checks independently controllable — the two conditions whose CONJUNCTION is the fix.
|
||||
func presenceServer(t *testing.T, obs []hub.StorageTarget, known []storage.KnownTarget, bound, device bool) *Server {
|
||||
t.Helper()
|
||||
opts := Options{
|
||||
ListenAddr: "127.0.0.1:0",
|
||||
Guests: &fakeGuestsCfg{},
|
||||
Backups: &fakeBackups{},
|
||||
Store: &fakeStore{},
|
||||
Storage: fakeStorage{targets: obs},
|
||||
Tokens: staticTokens{"A": 8200},
|
||||
Disks: &fakeDiskOps{probe: storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}},
|
||||
DiskGate: &fakeGate{},
|
||||
HostReader: sysOnSDA(),
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
}
|
||||
if known != nil {
|
||||
opts.DriveTargets = fakeKnownTargets{drives: known}
|
||||
}
|
||||
srv, err := NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
srv.boundCheck = func(string) bool { return bound }
|
||||
srv.deviceCheck = func(string) bool { return device }
|
||||
srv.resolveStorageDevice = func(string) (string, error) { return "/dev/sdb1", nil }
|
||||
return srv
|
||||
}
|
||||
|
||||
func diskByMount(t *testing.T, srv *Server, mount string) DiskInfo {
|
||||
t.Helper()
|
||||
for _, di := range decodeDisks(t, do(t, srv.Handler(), "GET", "/disks", "A", "").Body.Bytes()) {
|
||||
if di.MountPath == mount {
|
||||
return di
|
||||
}
|
||||
}
|
||||
t.Fatalf("no disk reported for mount %q", mount)
|
||||
return DiskInfo{}
|
||||
}
|
||||
|
||||
var obsUSB = []hub.StorageTarget{
|
||||
{Name: "usb", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb1", MountPath: "/mnt/felhom-usb", State: hub.StorageStateAttached},
|
||||
}
|
||||
|
||||
var knownUSB = []storage.KnownTarget{
|
||||
{Name: "mentes2", Type: hub.StorageTypeUSB, MountPath: "/mnt/mentes2", DurableID: "uuid:9303", UUID: "9303"},
|
||||
}
|
||||
|
||||
// ── Group A — device loss is seen ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestDisks_DevicePresence_ObservePath_DeviceLossReadsAbsent(t *testing.T) {
|
||||
// The exact E-2d shape: the bind survives (bound=true), the device is gone (device=false).
|
||||
di := diskByMount(t, presenceServer(t, obsUSB, nil, true, false), "/mnt/felhom-usb")
|
||||
if di.BoundUnderParent {
|
||||
t.Error("BoundUnderParent reports present — the bind outlived the device (R-113). " +
|
||||
"The controller's gate would emit no Stop action, so no alarm can fire.")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDisks_DevicePresence_UnionPath_DeviceLossReadsAbsent(t *testing.T) {
|
||||
// The union path matters MORE: a registry drive with no PVE dir-storage hardcodes State:"attached",
|
||||
// so the raw-mount check is the only device truth the row carries. This is what E-2d detached.
|
||||
di := diskByMount(t, presenceServer(t, nil, knownUSB, true, false), "/mnt/mentes2")
|
||||
if di.BoundUnderParent {
|
||||
t.Error("union-path drive reports present — the bind outlived the device (R-113)")
|
||||
}
|
||||
if di.State != hub.StorageStateAttached {
|
||||
t.Logf("note: union-path State is %q", di.State) // hardcoded; see the OBSERVATION in the report
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group B — the healthy drive, and the return ─────────────────────────────────────────────────
|
||||
|
||||
func TestDisks_DevicePresence_HealthyReadsPresent(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
obs []hub.StorageTarget
|
||||
known []storage.KnownTarget
|
||||
mount string
|
||||
}{
|
||||
{"observe", obsUSB, nil, "/mnt/felhom-usb"},
|
||||
{"union", nil, knownUSB, "/mnt/mentes2"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
di := diskByMount(t, presenceServer(t, c.obs, c.known, true, true), c.mount)
|
||||
if !di.BoundUnderParent {
|
||||
t.Error("a bound drive whose device is present must read PRESENT — " +
|
||||
"a false absent stops a working customer's apps (Scenario C's failure mode)")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group C — the over-correction guard: boot ordering must not regress ─────────────────────────
|
||||
|
||||
func TestDisks_DevicePresence_BootWindowStillReadsAbsent(t *testing.T) {
|
||||
// Boot ordering: the raw drive mounts EARLY (device=true), the agent binds under the parent ~18s
|
||||
// LATER (bound=false). Presence must stay FALSE in that window — unchanged from before R-113 — so
|
||||
// apps stay stopped until the bind is live and the gate's Return branch recreates them.
|
||||
di := diskByMount(t, presenceServer(t, obsUSB, nil, false, true), "/mnt/felhom-usb")
|
||||
if di.BoundUnderParent {
|
||||
t.Error("boot window reports present before the bind landed — this regresses the reboot " +
|
||||
"convergence the controller's gate comment at intermediary.go:220-224 depends on")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Group D — unknown must never mean absent ────────────────────────────────────────────────────
|
||||
|
||||
func TestDisks_DevicePresence_UnknownIsNotAbsent(t *testing.T) {
|
||||
// devicePresent has nothing to ask about when there is no raw mount path. It must answer TRUE.
|
||||
// Absence of a signal is not evidence of absence of a device — and the cost of getting this
|
||||
// backwards is stopping a healthy customer's apps.
|
||||
srv := presenceServer(t, nil, nil, true, false)
|
||||
srv.deviceCheck = nil // exercise the real devicePresent, not the injected fake
|
||||
if !srv.devicePresent("") {
|
||||
t.Error("devicePresent(\"\") = false — an unanswerable question was reported as ABSENT")
|
||||
}
|
||||
}
|
||||
|
||||
// ── The wire contract — what the controller actually parses ─────────────────────────────────────
|
||||
|
||||
// TestDisks_DevicePresence_WireFieldIsFalseOnDeviceLoss travels construction → HTTP handler → JSON
|
||||
// encoding and asserts the ENCODED field, because that is what crosses to the controller. A struct-level
|
||||
// assertion would not catch the field being dropped from the wire (e.g. an omitempty regression), and
|
||||
// `bound_under_parent` is the single field the controller's drive-absent gate keys on.
|
||||
func TestDisks_DevicePresence_WireFieldIsFalseOnDeviceLoss(t *testing.T) {
|
||||
body := do(t, presenceServer(t, nil, knownUSB, true, false).Handler(), "GET", "/disks", "A", "").Body.Bytes()
|
||||
if !strings.Contains(string(body), `"bound_under_parent"`) {
|
||||
t.Fatalf("the wire has no bound_under_parent field at all — the controller's gate reads nothing: %s", body)
|
||||
}
|
||||
var wire struct {
|
||||
Data struct {
|
||||
Disks []map[string]any `json:"disks"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &wire); err != nil {
|
||||
t.Fatalf("decode /disks: %v", err)
|
||||
}
|
||||
var seen bool
|
||||
for _, d := range wire.Data.Disks {
|
||||
if d["mount_path"] != "/mnt/mentes2" {
|
||||
continue
|
||||
}
|
||||
seen = true
|
||||
if v, ok := d["bound_under_parent"].(bool); !ok || v {
|
||||
t.Errorf("wire bound_under_parent = %v (want false) — the device is gone", d["bound_under_parent"])
|
||||
}
|
||||
}
|
||||
if !seen {
|
||||
t.Fatalf("the drive never reached the wire: %s", body)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user