package main import ( "context" "errors" "go/ast" "io" "log/slog" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/capability" ) // R-185 — a tier the box cannot READ must say so. // // THE OBSERVATION (demo-felhom, 2026-08-03, reproduced at the start of this session): root lists // three archives on `felhom-backup`; the agent's own token gets `{"data":[]}` from the same // endpoint; and `local`, which has the grant, lists through that same token. The token is the // variable, not the storage. // // The defect is NOT the missing grant — that is one command. It is that an empty content listing is // what a FORBIDDEN tier and a NEWBORN tier both return, so the box could not tell them apart and // said nothing. These tests pin the distinction. // permAnswer is the shape /access/permissions really returns, taken from the live measurement: // an UNGRANTED path answers with the privileges inherited from the box-wide grant — NOT empty, and // NOT a 403. var ( permGranted = map[string]int{"Datastore.Allocate": 1, "Datastore.AllocateSpace": 1} permUngranted = map[string]int{"Sys.Audit": 1, "SDN.Use": 1, "Datastore.Audit": 1} ) // probeWith calls the PRODUCTION decision with a permissions answer. **Naming the seam:** everything // below is true up to `storeGrantVerdict`; that the live call feeds it the real API answer is what // Part 0's measurement established and what the live run on the box demonstrates. An earlier draft // of this file re-implemented the branch here — it passed, and would have kept passing while // production diverged, which is the hollow shape this project keeps catching in its own tests. func probeWith(privs map[string]int, targetID string, critical bool) capability.Status { return storeGrantVerdict(targetID, critical, privs, nil) } // ── SCENARIO A — a forbidden storage is REPORTED, not passed over ──────────────────────────── // // COMPANION RED-PROOF (observed 2026-08-03): delete the store-grant probes from `probeAll` in // main.go — i.e. restore `append(capProber.Probe(ctx), poolReadStatus(ctx, px))` — and // TestMainWiresTheStoreGrantProbe fails with "main.go never calls storeGrantStatuses". That is // today's behaviour on the live box: complete silence about a tier it cannot read. func TestStoreGrant_ForbiddenStorageIsDegradedAndNamed(t *testing.T) { s := probeWith(permUngranted, "felhom-backup", true) if s.Status != capability.StatusDegraded { t.Fatalf("a storage the agent may not read must be DEGRADED, not %q — silence is the defect", s.Status) } if !s.Critical { t.Fatal("it must be CRITICAL: the hub alerts only on critical, so a non-critical entry is the same silence with extra steps") } if !strings.Contains(s.Reason, "felhom-backup") { t.Fatalf("the reason must NAME the storage — 'a grant is missing' costs a diagnosis at 07:00; got %q", s.Reason) } if !strings.Contains(s.Reason, "FelhomAgentStore") { t.Fatalf("the reason must name the ROLE to grant, so the fix is in the alert; got %q", s.Reason) } } // THE TRAP THE LIVE MEASUREMENT CAUGHT, pinned so it cannot be re-introduced: the ungranted answer // is not empty and not a 403 — it carries the INHERITED box-wide privileges. A probe that asked // "did the path come back?" or "does it have Datastore.Audit?" would report the blinded storage // healthy. func TestStoreGrant_InheritedPrivilegesAreNotAGrant(t *testing.T) { if len(permUngranted) == 0 { t.Fatal("fixture wrong: the ungranted answer is NOT empty — that is the whole trap") } if permUngranted["Datastore.Audit"] != 1 { t.Fatal("fixture wrong: the ungranted path DOES carry Datastore.Audit, inherited box-wide") } if s := probeWith(permUngranted, "felhom-backup", true); s.Status != capability.StatusDegraded { t.Fatalf("checking for the wrong privilege reports a blinded storage healthy; got %q", s.Status) } // ...and the privilege actually checked is the one whose absence was measured to blind listing. if storeGrantRequiredPriv != "Datastore.AllocateSpace" { t.Fatalf("the probed privilege changed to %q — re-measure before trusting it", storeGrantRequiredPriv) } } // ── SCENARIO B — a newborn tier is still silent ────────────────────────────────────────────── // // A storage the agent IS allowed to read but which simply holds no archives yet is HEALTHY. The // probe must not look at content at all, or every freshly provisioned box alarms and the signal dies. // // COMPANION RED-PROOF (observed): make the probe degrade on an empty content listing instead of on // the permission — a granted-but-empty storage then reports degraded, i.e. every newborn box alarms. func TestStoreGrant_GrantedButEmptyIsHealthy(t *testing.T) { s := probeWith(permGranted, "felhom-pbs", true) if s.Status != capability.StatusOK { t.Fatalf("a readable tier is healthy whether or not it holds archives yet; got %q (%s)", s.Status, s.Reason) } if s.Reason != "" { t.Fatalf("a healthy probe carries no reason; got %q", s.Reason) } } // ── SCENARIO C — the two states are distinguishable at a glance ────────────────────────────── func TestStoreGrant_ForbiddenAndNewbornAreDistinguishable(t *testing.T) { forbidden := probeWith(permUngranted, "felhom-backup", true) newborn := probeWith(permGranted, "felhom-pbs", true) if forbidden.Status == newborn.Status { t.Fatalf("the two states must differ — today both read as 'no settled archive yet'; got %q for both", forbidden.Status) } if forbidden.Name == newborn.Name { t.Fatalf("each tier needs its own capability id, or one tier's fault hides another's; got %q twice", forbidden.Name) } } // §8.3, weighed once and pinned: a box with NO dedicated target ("local" — host-install's own // DEGRADED fallback) must not turn an ordinary configuration into an operator page. It is still // probed and still reported; only the paging differs. func TestStoreGrant_TheFallbackTargetIsNotCritical(t *testing.T) { if storeGrantCritical("local") { t.Fatal("a box whose backup target is the 'local' fallback must not page the operator about " + "an ordinary, documented configuration") } for _, dedicated := range []string{"felhom-backup", "felhom-pbs", "some-nvme"} { if !storeGrantCritical(dedicated) { t.Fatalf("a DEDICATED target that cannot be read is user-facing and must be critical; %q was not", dedicated) } } // The fallback is still reported — silence for it would be the original defect, scoped smaller. if s := probeWith(permUngranted, "local", storeGrantCritical("local")); s.Status != capability.StatusDegraded { t.Fatalf("the fallback target must still report degraded when unreadable; got %q", s.Status) } } // A probe that cannot ask must never answer "ok" — unknown reported as healthy is worse than no // probe, because it looks like coverage. func TestStoreGrant_UnreachablePVEIsDegradedNotOK(t *testing.T) { s := storeGrantStatus(context.Background(), nil, "felhom-backup", true, nil) if s.Status != capability.StatusDegraded { t.Fatalf("an unaskable probe must be DEGRADED, never ok; got %q", s.Status) } if s.Reason == "" { t.Fatal("it must say why it could not ask") } } // ── SCENARIO H — the seam ──────────────────────────────────────────────────────────────────── // // This project's "built but never wired" count reached six last week. The fix for a SILENCE must not // itself be silent. AST, not grep: a commented-out call still contains the string. func TestMainWiresTheStoreGrantProbe(t *testing.T) { f := parseMainForWiring(t) var wired bool ast.Inspect(f, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) if !ok { return true } if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" { wired = true } return true }) if !wired { t.Error("main.go never calls storeGrantStatuses — the probe would exist and report to nobody, " + "which is precisely the silence R-185 is about") } } // ── R-190 — the grant repairs itself, and the repair is VISIBLE ────────────────────────────── // // R-190 is a storage grant that demonstrably worked at 04:44 on 2026-08-03 and was gone by 09:24, // with a host reinstall, logged `pveum` activity and cluster-log entries all ruled out. The cause is // open; the resilience is not conditional on it. // // The half that matters is the RECORD. R-190's own words: the probe sees the state, nothing sees the // transition. A self-repair that leaves only "ok" behind destroys the only evidence a loss happened, // so a recurring loss becomes undetectable forever — strictly worse than the fault it fixes. // fakeRepairRunner records wrapper invocations and can be made to fail. type fakeRepairRunner struct { calls [][]string fail bool } func (f *fakeRepairRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) { f.calls = append(f.calls, append([]string{name}, args...)) if f.fail { return nil, []byte("pveum: refused"), errors.New("exit status 2") } return nil, nil, nil } func newRepairer(f *fakeRepairRunner) *storeGrantRepairer { return &storeGrantRepairer{run: f.Run, log: slog.New(slog.NewTextHandler(io.Discard, nil))} } // ── SCENARIO F — the repair is BOUNDED ─────────────────────────────────────────────────────── // // COMPANION RED-PROOF (observed 2026-08-04): make mayAttempt always return true (drop the // storeGrantRepairMinInterval check) → // // --- FAIL: TestGrantRepair_IsBounded // storegrant_test.go: a repair must not run on every cycle; 5 cycles produced 5 attempt(s) // // which is a re-grant every report cycle, forever, against a fault an ACL cannot fix. Restored. func TestGrantRepair_IsBounded(t *testing.T) { f := &fakeRepairRunner{} r := newRepairer(f) // Jittered, so the series never lands exactly on the interval boundary — a perfectly regular // series is how a threshold test passes its own mutation, which has happened here before. base := time.Date(2026, 8, 4, 9, 17, 43, 0, time.UTC) offsets := []time.Duration{0, 13*time.Minute + 7*time.Second, 27*time.Minute + 51*time.Second, 41*time.Minute + 19*time.Second, 55*time.Minute + 3*time.Second} attempts := 0 for _, off := range offsets { if r.mayAttempt("felhom-backup", base.Add(off)) { attempts++ } } if attempts != 1 { t.Fatalf("a repair must not run on every cycle; %d cycles produced %d attempt(s) within %s", len(offsets), attempts, storeGrantRepairMinInterval) } // ...and once the interval has genuinely passed, it may try again — a bound is not a ban. if !r.mayAttempt("felhom-backup", base.Add(storeGrantRepairMinInterval+2*time.Minute+11*time.Second)) { t.Fatal("after the interval a repair must be allowed again — otherwise one failure disables the repair forever") } // A DIFFERENT tier is not throttled by this one's attempt. if !r.mayAttempt("felhom-pbs", base.Add(time.Minute)) { t.Fatal("the bound must be per tier — one tier's attempt must not suppress another's") } } // A nil repairer (or one with no runner) never attempts, and never panics. func TestGrantRepair_NilIsSafe(t *testing.T) { var r *storeGrantRepairer if r.mayAttempt("felhom-backup", time.Now()) { t.Fatal("a nil repairer must never claim an attempt") } if (&storeGrantRepairer{}).mayAttempt("felhom-backup", time.Now()) { t.Fatal("a repairer with no runner must never claim an attempt") } } // The repair calls the EXISTING wrapper verb, with the storage id — no new privileged surface. func TestGrantRepair_CallsTheExistingWrapperVerb(t *testing.T) { f := &fakeRepairRunner{} r := newRepairer(f) if err := r.repair(context.Background(), "felhom-backup"); err != nil { t.Fatalf("repair should succeed with a healthy runner: %v", err) } if len(f.calls) != 1 { t.Fatalf("exactly one wrapper invocation expected; got %d", len(f.calls)) } got := f.calls[0] want := []string{"/usr/local/sbin/felhom-backup-target-apply", "grant", "felhom-backup"} if len(got) != len(want) { t.Fatalf("wrapper argv = %v, want %v", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("wrapper argv = %v, want %v — the sudoers vector is `grant *`; anything else is a policy change", got, want) } } } // A repair that FAILS must surface the failure, not swallow it (Scenario E's precondition). func TestGrantRepair_FailureIsReturned(t *testing.T) { f := &fakeRepairRunner{fail: true} if err := newRepairer(f).repair(context.Background(), "felhom-backup"); err == nil { t.Fatal("a failed wrapper run must return its error — a repair that cannot run must never read as done") } } // ── SCENARIO D (the half that matters) — the REPAIR MUST BE VISIBLE ────────────────────────── // // A repair that leaves only "ok" behind is worse than the fault: the tier works, and the fact that a // permission vanished is gone with it. R-190 exists because nothing saw the transition. // // The channel is the hub's EXISTING ok→degraded→ok edge (§8.5) — nothing new was built. That only // works if the agent deliberately reports ONE degraded cycle after repairing, and if the explanation // rides the field the hub actually puts in the operator's e-mail. The hub's message is built from the // capability NAME and FEATURE (`internal/monitor/host_capability.go` emitTransition) — **not** from // Reason — so the Feature must carry it. // // COMPANION RED-PROOF (observed 2026-08-04): after a successful repair, report ok instead — // // s.Status = capability.StatusOK; s.Feature unchanged // // → --- FAIL: TestGrantRepair_ARepairedGrantIsReportedAsATransition // // storegrant_test.go: a self-repair must still report DEGRADED for one cycle so the hub raises // its edge; got "ok" — the loss would be invisible // // i.e. exactly the silence R-190 is about. Restored. func TestGrantRepair_ARepairedGrantIsReportedAsATransition(t *testing.T) { // THE PRODUCTION verdict, not a copy of it. An earlier draft of this test built the Status // itself and asserted its own construction — it would have passed while production reported ok, // which is precisely the silence being guarded against. if pre := probeWith(permUngranted, "felhom-backup", true); pre.Status != capability.StatusDegraded { t.Fatalf("precondition: a missing grant is degraded; got %q", pre.Status) } s := storeGrantRepairedVerdict("felhom-backup", true) if s.Status != capability.StatusDegraded { t.Fatalf("a self-repair must still report DEGRADED for one cycle so the hub raises its edge; "+ "got %q — the loss would be invisible", s.Status) } // The hub e-mails the FEATURE text. If the explanation is not there, the operator is told a // capability was degraded and never learns it repaired itself or that anything vanished. for _, want := range []string{"MISSING", "RESTORED", "felhom-backup", "R-190"} { if !strings.Contains(s.Feature, want) { t.Fatalf("the Feature text is what the hub puts in the operator's e-mail; it must contain %q. Got: %s", want, s.Feature) } } if !s.Critical { t.Fatal("the transition must be CRITICAL or the hub does not alert on it at all") } } // ── SCENARIO H — the seam ──────────────────────────────────────────────────────────────────── // // The wrapper's `grant` verb is itself a "built but never wired" example: it exists, is // sudoers-permitted for any id, and had only ever been called at storage CREATION. The repair must // not become the seventh instance. AST, not grep — a commented-out call still contains the string. func TestMainWiresTheGrantRepair(t *testing.T) { f := parseMainForWiring(t) var built, passed bool ast.Inspect(f, func(n ast.Node) bool { switch node := n.(type) { case *ast.CompositeLit: if id, ok := node.Type.(*ast.Ident); ok && id.Name == "storeGrantRepairer" { built = true } case *ast.CallExpr: if id, ok := node.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" && len(node.Args) == 4 { if a, ok := node.Args[3].(*ast.Ident); ok && a.Name == "grantRepairer" { passed = true } } } return true }) if !built { t.Error("main.go never constructs a storeGrantRepairer — nothing would ever repair a lost grant") } if !passed { t.Error("storeGrantStatuses is not passed the repairer — the probe would detect the loss and " + "leave it, which is v0.123.0's behaviour and not R-190's mitigation") } } // The transition must survive a probe that is NOT the one feeding the hub. // // MEASURED LIVE 2026-08-04, and this test exists because the first implementation failed it in // production while every unit test passed: `probeAll` is called independently by the self-check LOG // and by the collector building a host-report. The repairing call was the log's; the report three // seconds later found the grant present and reported `ok`. The agent's journal had the record and the // hub had nothing — the exact silence R-190 is about, re-created inside its own mitigation. // // COMPANION RED-PROOF (observed): delete the `recentlyRepaired` branch from the healthy path → // // --- FAIL: TestGrantRepair_TransitionSurvivesALaterProbe // storegrant_test.go: a probe AFTER the repair must still report the transition; got "ok" — // the host-report would carry ok and the operator would never learn the grant vanished // // Restored. func TestGrantRepair_TransitionSurvivesALaterProbe(t *testing.T) { r := newRepairer(&fakeRepairRunner{}) // Jittered, never landing on the window boundary. repairedAt := time.Date(2026, 8, 4, 9, 39, 34, 0, time.UTC) r.noteRepaired("felhom-backup", repairedAt) // The DECISION a later probe makes — the production function, not the helper it calls. An // earlier draft asserted `recentlyRepaired` directly and its red-proof PASSED, because removing // the latch's USE left the helper untouched. healthy := probeWith(permGranted, "felhom-backup", true) if healthy.Status != capability.StatusOK { t.Fatalf("precondition: a granted tier is ok; got %q", healthy.Status) } got := storeGrantHealthyVerdict("felhom-backup", true, healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(3*time.Second))) if got.Status != capability.StatusDegraded { t.Fatalf("a probe AFTER the repair must still report the transition; got %q — the host-report "+ "would carry ok and the operator would never learn the grant vanished", got.Status) } if !strings.Contains(got.Feature, "RESTORED") { t.Fatalf("the later probe must carry the explanation into the hub's e-mail; got: %s", got.Feature) } // Outside the window it reports plain ok again. late := storeGrantHealthyVerdict("felhom-backup", true, healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute))) if late.Status != capability.StatusOK { t.Fatalf("outside the window a healthy tier reports ok; got %q — a permanent degraded state "+ "would be its own false alarm", late.Status) } if !r.recentlyRepaired("felhom-backup", repairedAt.Add(14*time.Minute+37*time.Second)) { t.Fatal("the latch must outlast the 900s hub report interval, or the record never reaches the hub") } // ...and it clears on its own rather than latching a box degraded forever. if r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute+7*time.Second)) { t.Fatal("the latch must clear — a permanent degraded state would be its own false alarm") } // It is per tier. if r.recentlyRepaired("felhom-pbs", repairedAt.Add(time.Second)) { t.Fatal("one tier's repair must not latch another tier's status") } // The window MUST exceed the report interval — the property, asserted rather than assumed. if storeGrantRepairReportWindow <= 15*time.Minute { t.Fatalf("the report window (%s) must exceed the 900s hub report interval, or a transition can "+ "be missed entirely", storeGrantRepairReportWindow) } }