felhom-agent/cmd/felhom-agent/storegrant_test.go

package main

import (
	"context"
	"errors"
	"go/ast"
	"io"
	"log/slog"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
)

// R-185 — a tier the box cannot READ must say so.
//
// THE OBSERVATION (demo-felhom, 2026-08-03, reproduced at the start of this session): root lists
// three archives on `felhom-backup`; the agent's own token gets `{"data":[]}` from the same
// endpoint; and `local`, which has the grant, lists through that same token. The token is the
// variable, not the storage.
//
// The defect is NOT the missing grant — that is one command. It is that an empty content listing is
// what a FORBIDDEN tier and a NEWBORN tier both return, so the box could not tell them apart and
// said nothing. These tests pin the distinction.

// permAnswer is the shape /access/permissions really returns, taken from the live measurement:
// an UNGRANTED path answers with the privileges inherited from the box-wide grant — NOT empty, and
// NOT a 403.
var (
	permGranted   = map[string]int{"Datastore.Allocate": 1, "Datastore.AllocateSpace": 1}
	permUngranted = map[string]int{"Sys.Audit": 1, "SDN.Use": 1, "Datastore.Audit": 1}
)

// probeWith calls the PRODUCTION decision with a permissions answer. **Naming the seam:** everything
// below is true up to `storeGrantVerdict`; that the live call feeds it the real API answer is what
// Part 0's measurement established and what the live run on the box demonstrates. An earlier draft
// of this file re-implemented the branch here — it passed, and would have kept passing while
// production diverged, which is the hollow shape this project keeps catching in its own tests.
func probeWith(privs map[string]int, targetID string, critical bool) capability.Status {
	return storeGrantVerdict(targetID, critical, privs, nil)
}

// ── SCENARIO A — a forbidden storage is REPORTED, not passed over ────────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-03): delete the store-grant probes from `probeAll` in
// main.go — i.e. restore `append(capProber.Probe(ctx), poolReadStatus(ctx, px))` — and
// TestMainWiresTheStoreGrantProbe fails with "main.go never calls storeGrantStatuses". That is
// today's behaviour on the live box: complete silence about a tier it cannot read.
func TestStoreGrant_ForbiddenStorageIsDegradedAndNamed(t *testing.T) {
	s := probeWith(permUngranted, "felhom-backup", true)

	if s.Status != capability.StatusDegraded {
		t.Fatalf("a storage the agent may not read must be DEGRADED, not %q — silence is the defect", s.Status)
	}
	if !s.Critical {
		t.Fatal("it must be CRITICAL: the hub alerts only on critical, so a non-critical entry is the same silence with extra steps")
	}
	if !strings.Contains(s.Reason, "felhom-backup") {
		t.Fatalf("the reason must NAME the storage — 'a grant is missing' costs a diagnosis at 07:00; got %q", s.Reason)
	}
	if !strings.Contains(s.Reason, "FelhomAgentStore") {
		t.Fatalf("the reason must name the ROLE to grant, so the fix is in the alert; got %q", s.Reason)
	}
}

// THE TRAP THE LIVE MEASUREMENT CAUGHT, pinned so it cannot be re-introduced: the ungranted answer
// is not empty and not a 403 — it carries the INHERITED box-wide privileges. A probe that asked
// "did the path come back?" or "does it have Datastore.Audit?" would report the blinded storage
// healthy.
func TestStoreGrant_InheritedPrivilegesAreNotAGrant(t *testing.T) {
	if len(permUngranted) == 0 {
		t.Fatal("fixture wrong: the ungranted answer is NOT empty — that is the whole trap")
	}
	if permUngranted["Datastore.Audit"] != 1 {
		t.Fatal("fixture wrong: the ungranted path DOES carry Datastore.Audit, inherited box-wide")
	}
	if s := probeWith(permUngranted, "felhom-backup", true); s.Status != capability.StatusDegraded {
		t.Fatalf("checking for the wrong privilege reports a blinded storage healthy; got %q", s.Status)
	}
	// ...and the privilege actually checked is the one whose absence was measured to blind listing.
	if storeGrantRequiredPriv != "Datastore.AllocateSpace" {
		t.Fatalf("the probed privilege changed to %q — re-measure before trusting it", storeGrantRequiredPriv)
	}
}

// ── SCENARIO B — a newborn tier is still silent ──────────────────────────────────────────────
//
// A storage the agent IS allowed to read but which simply holds no archives yet is HEALTHY. The
// probe must not look at content at all, or every freshly provisioned box alarms and the signal dies.
//
// COMPANION RED-PROOF (observed): make the probe degrade on an empty content listing instead of on
// the permission — a granted-but-empty storage then reports degraded, i.e. every newborn box alarms.
func TestStoreGrant_GrantedButEmptyIsHealthy(t *testing.T) {
	s := probeWith(permGranted, "felhom-pbs", true)
	if s.Status != capability.StatusOK {
		t.Fatalf("a readable tier is healthy whether or not it holds archives yet; got %q (%s)", s.Status, s.Reason)
	}
	if s.Reason != "" {
		t.Fatalf("a healthy probe carries no reason; got %q", s.Reason)
	}
}

// ── SCENARIO C — the two states are distinguishable at a glance ──────────────────────────────
func TestStoreGrant_ForbiddenAndNewbornAreDistinguishable(t *testing.T) {
	forbidden := probeWith(permUngranted, "felhom-backup", true)
	newborn := probeWith(permGranted, "felhom-pbs", true)

	if forbidden.Status == newborn.Status {
		t.Fatalf("the two states must differ — today both read as 'no settled archive yet'; got %q for both", forbidden.Status)
	}
	if forbidden.Name == newborn.Name {
		t.Fatalf("each tier needs its own capability id, or one tier's fault hides another's; got %q twice", forbidden.Name)
	}
}

// §8.3, weighed once and pinned: a box with NO dedicated target ("local" — host-install's own
// DEGRADED fallback) must not turn an ordinary configuration into an operator page. It is still
// probed and still reported; only the paging differs.
func TestStoreGrant_TheFallbackTargetIsNotCritical(t *testing.T) {
	if storeGrantCritical("local") {
		t.Fatal("a box whose backup target is the 'local' fallback must not page the operator about " +
			"an ordinary, documented configuration")
	}
	for _, dedicated := range []string{"felhom-backup", "felhom-pbs", "some-nvme"} {
		if !storeGrantCritical(dedicated) {
			t.Fatalf("a DEDICATED target that cannot be read is user-facing and must be critical; %q was not", dedicated)
		}
	}
	// The fallback is still reported — silence for it would be the original defect, scoped smaller.
	if s := probeWith(permUngranted, "local", storeGrantCritical("local")); s.Status != capability.StatusDegraded {
		t.Fatalf("the fallback target must still report degraded when unreadable; got %q", s.Status)
	}
}

// A probe that cannot ask must never answer "ok" — unknown reported as healthy is worse than no
// probe, because it looks like coverage.
func TestStoreGrant_UnreachablePVEIsDegradedNotOK(t *testing.T) {
	s := storeGrantStatus(context.Background(), nil, "felhom-backup", true, nil)
	if s.Status != capability.StatusDegraded {
		t.Fatalf("an unaskable probe must be DEGRADED, never ok; got %q", s.Status)
	}
	if s.Reason == "" {
		t.Fatal("it must say why it could not ask")
	}
}

// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
//
// This project's "built but never wired" count reached six last week. The fix for a SILENCE must not
// itself be silent. AST, not grep: a commented-out call still contains the string.
func TestMainWiresTheStoreGrantProbe(t *testing.T) {
	f := parseMainForWiring(t)

	var wired bool
	ast.Inspect(f, func(n ast.Node) bool {
		call, ok := n.(*ast.CallExpr)
		if !ok {
			return true
		}
		if id, ok := call.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" {
			wired = true
		}
		return true
	})
	if !wired {
		t.Error("main.go never calls storeGrantStatuses — the probe would exist and report to nobody, " +
			"which is precisely the silence R-185 is about")
	}
}

// ── R-190 — the grant repairs itself, and the repair is VISIBLE ──────────────────────────────
//
// R-190 is a storage grant that demonstrably worked at 04:44 on 2026-08-03 and was gone by 09:24,
// with a host reinstall, logged `pveum` activity and cluster-log entries all ruled out. The cause is
// open; the resilience is not conditional on it.
//
// The half that matters is the RECORD. R-190's own words: the probe sees the state, nothing sees the
// transition. A self-repair that leaves only "ok" behind destroys the only evidence a loss happened,
// so a recurring loss becomes undetectable forever — strictly worse than the fault it fixes.

// fakeRepairRunner records wrapper invocations and can be made to fail.
type fakeRepairRunner struct {
	calls [][]string
	fail  bool
}

func (f *fakeRepairRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
	f.calls = append(f.calls, append([]string{name}, args...))
	if f.fail {
		return nil, []byte("pveum: refused"), errors.New("exit status 2")
	}
	return nil, nil, nil
}

func newRepairer(f *fakeRepairRunner) *storeGrantRepairer {
	return &storeGrantRepairer{run: f.Run, log: slog.New(slog.NewTextHandler(io.Discard, nil))}
}

// ── SCENARIO F — the repair is BOUNDED ───────────────────────────────────────────────────────
//
// COMPANION RED-PROOF (observed 2026-08-04): make mayAttempt always return true (drop the
// storeGrantRepairMinInterval check) →
//
//	--- FAIL: TestGrantRepair_IsBounded
//	    storegrant_test.go: a repair must not run on every cycle; 5 cycles produced 5 attempt(s)
//
// which is a re-grant every report cycle, forever, against a fault an ACL cannot fix. Restored.
func TestGrantRepair_IsBounded(t *testing.T) {
	f := &fakeRepairRunner{}
	r := newRepairer(f)
	// Jittered, so the series never lands exactly on the interval boundary — a perfectly regular
	// series is how a threshold test passes its own mutation, which has happened here before.
	base := time.Date(2026, 8, 4, 9, 17, 43, 0, time.UTC)
	offsets := []time.Duration{0, 13*time.Minute + 7*time.Second, 27*time.Minute + 51*time.Second,
		41*time.Minute + 19*time.Second, 55*time.Minute + 3*time.Second}
	attempts := 0
	for _, off := range offsets {
		if r.mayAttempt("felhom-backup", base.Add(off)) {
			attempts++
		}
	}
	if attempts != 1 {
		t.Fatalf("a repair must not run on every cycle; %d cycles produced %d attempt(s) within %s",
			len(offsets), attempts, storeGrantRepairMinInterval)
	}
	// ...and once the interval has genuinely passed, it may try again — a bound is not a ban.
	if !r.mayAttempt("felhom-backup", base.Add(storeGrantRepairMinInterval+2*time.Minute+11*time.Second)) {
		t.Fatal("after the interval a repair must be allowed again — otherwise one failure disables the repair forever")
	}
	// A DIFFERENT tier is not throttled by this one's attempt.
	if !r.mayAttempt("felhom-pbs", base.Add(time.Minute)) {
		t.Fatal("the bound must be per tier — one tier's attempt must not suppress another's")
	}
}

// A nil repairer (or one with no runner) never attempts, and never panics.
func TestGrantRepair_NilIsSafe(t *testing.T) {
	var r *storeGrantRepairer
	if r.mayAttempt("felhom-backup", time.Now()) {
		t.Fatal("a nil repairer must never claim an attempt")
	}
	if (&storeGrantRepairer{}).mayAttempt("felhom-backup", time.Now()) {
		t.Fatal("a repairer with no runner must never claim an attempt")
	}
}

// The repair calls the EXISTING wrapper verb, with the storage id — no new privileged surface.
func TestGrantRepair_CallsTheExistingWrapperVerb(t *testing.T) {
	f := &fakeRepairRunner{}
	r := newRepairer(f)
	if err := r.repair(context.Background(), "felhom-backup"); err != nil {
		t.Fatalf("repair should succeed with a healthy runner: %v", err)
	}
	if len(f.calls) != 1 {
		t.Fatalf("exactly one wrapper invocation expected; got %d", len(f.calls))
	}
	got := f.calls[0]
	want := []string{"/usr/local/sbin/felhom-backup-target-apply", "grant", "felhom-backup"}
	if len(got) != len(want) {
		t.Fatalf("wrapper argv = %v, want %v", got, want)
	}
	for i := range want {
		if got[i] != want[i] {
			t.Fatalf("wrapper argv = %v, want %v — the sudoers vector is `grant *`; anything else is a policy change", got, want)
		}
	}
}

// A repair that FAILS must surface the failure, not swallow it (Scenario E's precondition).
func TestGrantRepair_FailureIsReturned(t *testing.T) {
	f := &fakeRepairRunner{fail: true}
	if err := newRepairer(f).repair(context.Background(), "felhom-backup"); err == nil {
		t.Fatal("a failed wrapper run must return its error — a repair that cannot run must never read as done")
	}
}

// ── SCENARIO D (the half that matters) — the REPAIR MUST BE VISIBLE ──────────────────────────
//
// A repair that leaves only "ok" behind is worse than the fault: the tier works, and the fact that a
// permission vanished is gone with it. R-190 exists because nothing saw the transition.
//
// The channel is the hub's EXISTING ok→degraded→ok edge (§8.5) — nothing new was built. That only
// works if the agent deliberately reports ONE degraded cycle after repairing, and if the explanation
// rides the field the hub actually puts in the operator's e-mail. The hub's message is built from the
// capability NAME and FEATURE (`internal/monitor/host_capability.go` emitTransition) — **not** from
// Reason — so the Feature must carry it.
//
// COMPANION RED-PROOF (observed 2026-08-04): after a successful repair, report ok instead —
//
//	s.Status = capability.StatusOK; s.Feature unchanged
//
// →	--- FAIL: TestGrantRepair_ARepairedGrantIsReportedAsATransition
//
//	storegrant_test.go: a self-repair must still report DEGRADED for one cycle so the hub raises
//	its edge; got "ok" — the loss would be invisible
//
// i.e. exactly the silence R-190 is about. Restored.
func TestGrantRepair_ARepairedGrantIsReportedAsATransition(t *testing.T) {
	// THE PRODUCTION verdict, not a copy of it. An earlier draft of this test built the Status
	// itself and asserted its own construction — it would have passed while production reported ok,
	// which is precisely the silence being guarded against.
	if pre := probeWith(permUngranted, "felhom-backup", true); pre.Status != capability.StatusDegraded {
		t.Fatalf("precondition: a missing grant is degraded; got %q", pre.Status)
	}
	s := storeGrantRepairedVerdict("felhom-backup", true)

	if s.Status != capability.StatusDegraded {
		t.Fatalf("a self-repair must still report DEGRADED for one cycle so the hub raises its edge; "+
			"got %q — the loss would be invisible", s.Status)
	}
	// The hub e-mails the FEATURE text. If the explanation is not there, the operator is told a
	// capability was degraded and never learns it repaired itself or that anything vanished.
	for _, want := range []string{"MISSING", "RESTORED", "felhom-backup", "R-190"} {
		if !strings.Contains(s.Feature, want) {
			t.Fatalf("the Feature text is what the hub puts in the operator's e-mail; it must contain %q. Got: %s", want, s.Feature)
		}
	}
	if !s.Critical {
		t.Fatal("the transition must be CRITICAL or the hub does not alert on it at all")
	}
}

// ── SCENARIO H — the seam ────────────────────────────────────────────────────────────────────
//
// The wrapper's `grant` verb is itself a "built but never wired" example: it exists, is
// sudoers-permitted for any id, and had only ever been called at storage CREATION. The repair must
// not become the seventh instance. AST, not grep — a commented-out call still contains the string.
func TestMainWiresTheGrantRepair(t *testing.T) {
	f := parseMainForWiring(t)

	var built, passed bool
	ast.Inspect(f, func(n ast.Node) bool {
		switch node := n.(type) {
		case *ast.CompositeLit:
			if id, ok := node.Type.(*ast.Ident); ok && id.Name == "storeGrantRepairer" {
				built = true
			}
		case *ast.CallExpr:
			if id, ok := node.Fun.(*ast.Ident); ok && id.Name == "storeGrantStatuses" && len(node.Args) == 4 {
				if a, ok := node.Args[3].(*ast.Ident); ok && a.Name == "grantRepairer" {
					passed = true
				}
			}
		}
		return true
	})
	if !built {
		t.Error("main.go never constructs a storeGrantRepairer — nothing would ever repair a lost grant")
	}
	if !passed {
		t.Error("storeGrantStatuses is not passed the repairer — the probe would detect the loss and " +
			"leave it, which is v0.123.0's behaviour and not R-190's mitigation")
	}
}

// The transition must survive a probe that is NOT the one feeding the hub.
//
// MEASURED LIVE 2026-08-04, and this test exists because the first implementation failed it in
// production while every unit test passed: `probeAll` is called independently by the self-check LOG
// and by the collector building a host-report. The repairing call was the log's; the report three
// seconds later found the grant present and reported `ok`. The agent's journal had the record and the
// hub had nothing — the exact silence R-190 is about, re-created inside its own mitigation.
//
// COMPANION RED-PROOF (observed): delete the `recentlyRepaired` branch from the healthy path →
//
//	--- FAIL: TestGrantRepair_TransitionSurvivesALaterProbe
//	    storegrant_test.go: a probe AFTER the repair must still report the transition; got "ok" —
//	    the host-report would carry ok and the operator would never learn the grant vanished
//
// Restored.
func TestGrantRepair_TransitionSurvivesALaterProbe(t *testing.T) {
	r := newRepairer(&fakeRepairRunner{})
	// Jittered, never landing on the window boundary.
	repairedAt := time.Date(2026, 8, 4, 9, 39, 34, 0, time.UTC)
	r.noteRepaired("felhom-backup", repairedAt)

	// The DECISION a later probe makes — the production function, not the helper it calls. An
	// earlier draft asserted `recentlyRepaired` directly and its red-proof PASSED, because removing
	// the latch's USE left the helper untouched.
	healthy := probeWith(permGranted, "felhom-backup", true)
	if healthy.Status != capability.StatusOK {
		t.Fatalf("precondition: a granted tier is ok; got %q", healthy.Status)
	}
	got := storeGrantHealthyVerdict("felhom-backup", true,
		healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(3*time.Second)))
	if got.Status != capability.StatusDegraded {
		t.Fatalf("a probe AFTER the repair must still report the transition; got %q — the host-report "+
			"would carry ok and the operator would never learn the grant vanished", got.Status)
	}
	if !strings.Contains(got.Feature, "RESTORED") {
		t.Fatalf("the later probe must carry the explanation into the hub's e-mail; got: %s", got.Feature)
	}
	// Outside the window it reports plain ok again.
	late := storeGrantHealthyVerdict("felhom-backup", true,
		healthy, r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute)))
	if late.Status != capability.StatusOK {
		t.Fatalf("outside the window a healthy tier reports ok; got %q — a permanent degraded state "+
			"would be its own false alarm", late.Status)
	}
	if !r.recentlyRepaired("felhom-backup", repairedAt.Add(14*time.Minute+37*time.Second)) {
		t.Fatal("the latch must outlast the 900s hub report interval, or the record never reaches the hub")
	}
	// ...and it clears on its own rather than latching a box degraded forever.
	if r.recentlyRepaired("felhom-backup", repairedAt.Add(storeGrantRepairReportWindow+time.Minute+7*time.Second)) {
		t.Fatal("the latch must clear — a permanent degraded state would be its own false alarm")
	}
	// It is per tier.
	if r.recentlyRepaired("felhom-pbs", repairedAt.Add(time.Second)) {
		t.Fatal("one tier's repair must not latch another tier's status")
	}
	// The window MUST exceed the report interval — the property, asserted rather than assumed.
	if storeGrantRepairReportWindow <= 15*time.Minute {
		t.Fatalf("the report window (%s) must exceed the 900s hub report interval, or a transition can "+
			"be missed entirely", storeGrantRepairReportWindow)
	}
}