felhom-agent/internal/pbsdr/manager_test.go

package pbsdr

// The apply-bridge laws, each pinned by a non-hollow test (fake exec recorder — no docker/pct/
// real /dev, the REUSE §4 doctrine): set-only re-apply, secret-on-stdin-never-argv,
// verify-pin-BEFORE-consume, non-destructive adoption (no consume, tenancy entry-owned),
// consumed-but-failed loud + recover-only-via-fresh-secret, marker idempotency, old-hub compat.

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"testing"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// recordedCall is one exec through the runner seam — argv AND the stdin bytes.
type recordedCall struct {
	Name  string
	Args  []string
	Stdin string
}

type fakeRunner struct {
	mu    sync.Mutex
	calls []recordedCall
	// failVerb → error for calls whose first arg matches (e.g. "create").
	failVerb string
	failErr  error
}

func (f *fakeRunner) Run(ctx context.Context, name string, args ...string) ([]byte, []byte, error) {
	return f.RunStdin(ctx, nil, name, args...)
}

func (f *fakeRunner) RunStdin(_ context.Context, stdin io.Reader, name string, args ...string) ([]byte, []byte, error) {
	var in []byte
	if stdin != nil {
		in, _ = io.ReadAll(stdin)
	}
	f.mu.Lock()
	f.calls = append(f.calls, recordedCall{Name: name, Args: args, Stdin: string(in)})
	f.mu.Unlock()
	if f.failVerb != "" && len(args) > 0 && args[0] == f.failVerb {
		return nil, []byte("boom-stderr"), f.failErr
	}
	return nil, nil, nil
}

func (f *fakeRunner) recorded() []recordedCall {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]recordedCall(nil), f.calls...)
}

type fakeStorage struct {
	entry  *proxmox.StorageEntryConfig
	found  bool
	active []bool // consumed per StorageActive call; last value repeats
	calls  int
	// entryErrs is consumed per StorageEntry call (nil = the normal (entry,found,nil) answer);
	// after the slice is exhausted every call answers normally. Lets a test model the R-22
	// pre-check 403 that self-grant must recover from without aborting.
	entryErrs  []error
	entryCalls int
}

func (f *fakeStorage) StorageEntry(context.Context, string) (*proxmox.StorageEntryConfig, bool, error) {
	i := f.entryCalls
	f.entryCalls++
	if i < len(f.entryErrs) && f.entryErrs[i] != nil {
		return nil, false, f.entryErrs[i]
	}
	return f.entry, f.found, nil
}

func (f *fakeStorage) StorageActive(context.Context, string) (bool, error) {
	i := f.calls
	f.calls++
	if i >= len(f.active) {
		if len(f.active) == 0 {
			return false, nil
		}
		return f.active[len(f.active)-1], nil
	}
	return f.active[i], nil
}

type fakeConsumer struct {
	secret string
	err    error
	calls  int
}

func (f *fakeConsumer) ConsumePBSToken(context.Context) (string, error) {
	f.calls++
	if f.err != nil {
		return "", f.err
	}
	return f.secret, nil
}

const testFP = "c6:07:28:3f:5b:7b:5a:41:90:28:d7:ca:4f:37:14:70:56:39:2e:2f:0b:71:e8:06:ca:60:4a:d5:56:5f:3c:fd"

func testBlock() *hub.WirePBSDR {
	return &hub.WirePBSDR{
		Enabled: true, StorageID: "felhom-pbs", PBSTunnelIP: "10.77.0.1",
		Datastore: "felhom-offsite", Namespace: "peti", TokenID: "felhom@pbs!peti",
		Fingerprint: testFP,
	}
}

// newTestManager: fakes everywhere; the fingerprint probe defaults to PASS (override per test);
// a real temp agent.json so the escrow seed is asserted end-to-end.
func newTestManager(t *testing.T, r *fakeRunner, st *fakeStorage, c *fakeConsumer) (*Manager, string) {
	t.Helper()
	dir := t.TempDir()
	cfgPath := filepath.Join(dir, "agent.json")
	if err := os.WriteFile(cfgPath, []byte(`{"log_level":"info","escrow":{"posture":"zero_knowledge"},"custom_unknown":{"keep":1}}`), 0o600); err != nil {
		t.Fatalf("seed config: %v", err)
	}
	m := NewManager(r, st, c, dir, "/etc/pve/priv/storage", cfgPath,
		slog.New(slog.NewTextHandler(io.Discard, nil)))
	m.probeFP = func(context.Context, string, string) error { return nil }
	return m, cfgPath
}

func TestFreshPath_SecretOnStdinNeverArgv(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{found: false, active: []bool{true}} // post-apply probe active
	c := &fakeConsumer{secret: "SUPER-SECRET"}
	m, cfgPath := newTestManager(t, r, st, c)

	m.Apply(context.Background(), true, testBlock())

	calls := r.recorded()
	if len(calls) != 2 || calls[0].Args[0] != "create" || calls[1].Args[0] != "grant" {
		t.Fatalf("calls = %+v, want [create, grant]", calls)
	}
	// THE STDIN LAW: the secret appears in NO argv, ONLY on the create call's stdin.
	// (Red-proof: pass it as an argument → this fails with the secret visible in Args.)
	for _, call := range calls {
		for _, a := range call.Args {
			if strings.Contains(a, "SUPER-SECRET") {
				t.Fatalf("secret leaked into argv: %v", call.Args)
			}
		}
	}
	if calls[0].Stdin != "SUPER-SECRET\n" {
		t.Errorf("create stdin = %q, want the secret + newline", calls[0].Stdin)
	}
	if calls[1].Stdin != "" {
		t.Errorf("grant received stdin %q", calls[1].Stdin)
	}
	// Non-secret coords ride argv, descriptor-exact.
	want := []string{"create", "felhom-pbs", "10.77.0.1", "felhom-offsite", "peti", "felhom@pbs!peti", testFP, "/etc/pve/priv/storage"}
	if fmt.Sprint(calls[0].Args) != fmt.Sprint(want) {
		t.Errorf("create argv = %v, want %v", calls[0].Args, want)
	}
	if c.calls != 1 {
		t.Errorf("consume calls = %d, want 1", c.calls)
	}
	// Converged: marker applied + escrow seeded + unknown config keys preserved.
	if s := m.Status(); s == nil || s.State != "applied" {
		t.Fatalf("status = %+v, want applied", s)
	}
	raw, _ := os.ReadFile(cfgPath)
	var doc map[string]json.RawMessage
	json.Unmarshal(raw, &doc)
	var esc map[string]string
	json.Unmarshal(doc["escrow"], &esc)
	if esc["pbs_storage_id"] != "felhom-pbs" {
		t.Errorf("escrow not seeded: %s", doc["escrow"])
	}
	if esc["posture"] != "zero_knowledge" {
		t.Errorf("existing escrow fields clobbered: %s", doc["escrow"])
	}
	if _, ok := doc["custom_unknown"]; !ok {
		t.Error("unknown config key dropped by the seed write")
	}
	if strings.Contains(string(raw), "SUPER-SECRET") {
		t.Error("secret leaked into agent.json")
	}
}

// TestSelfGrant_PreCheck403DoesNotAbortBeforeGrant pins R-22 (F4 from tests/VALIDATION-n100):
// on a non-default storage id whose ACL the token lacks, the token-auth pre-check GET /storage/<id>
// 403s. The fix must NOT abort — it must run the root-run `grant` (which creates that very ACL),
// re-read, and converge. RED-PROOF: the pre-fix code returns on the StorageEntry error before any
// runner call, so NO grant runs and the box never converges — this test then fails on
// "self-grant never ran". No secret may be consumed on this (adoption) path.
func TestSelfGrant_PreCheck403DoesNotAbortBeforeGrant(t *testing.T) {
	r := &fakeRunner{}
	forbidden := &proxmox.APIError{StatusCode: 403, Method: "GET", Path: "/storage/felhom-offsite",
		Body: "Permission check failed (/storage/felhom-offsite, Datastore.Audit)"}
	st := &fakeStorage{
		// First read 403s (no ACL); after the self-grant the entry reads healthy → adoption.
		entryErrs: []error{forbidden},
		entry: &proxmox.StorageEntryConfig{Storage: "felhom-offsite", Type: "pbs",
			Namespace: "peti", Fingerprint: testFP},
		found:  true,
		active: []bool{true},
	}
	c := &fakeConsumer{secret: "MUST-NOT-BURN"}
	m, _ := newTestManager(t, r, st, c)
	block := testBlock()
	block.StorageID = "felhom-offsite"
	block.Namespace = "peti"

	m.Apply(context.Background(), true, block)

	// A 403 on IsForbidden must be recognised as such (regression guard on the type assertion).
	if !forbidden.IsForbidden() {
		t.Fatal("precondition: crafted APIError is not IsForbidden")
	}
	calls := r.recorded()
	grants := 0
	for _, call := range calls {
		if len(call.Args) > 0 && call.Args[0] == "grant" {
			grants++
		}
	}
	if grants == 0 {
		t.Fatalf("self-grant never ran — the pre-check 403 aborted before the root grant (R-22 regression); calls=%+v", calls)
	}
	if c.calls != 0 {
		t.Fatalf("a secret was consumed on the self-grant/adoption path (%d calls) — the no-consume law", c.calls)
	}
	if s := m.Status(); s == nil || (s.State != "adopted" && s.State != "applied") {
		t.Fatalf("status = %+v, want converged (adopted/applied) after self-grant", s)
	}
	// The pre-check was re-read (not dropped): 2 StorageEntry calls — the 403, then the post-grant read.
	if st.entryCalls < 2 {
		t.Fatalf("StorageEntry called %d times — the post-grant re-read is missing", st.entryCalls)
	}
}

func TestVerifyPinBeforeConsume(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{found: false}
	c := &fakeConsumer{secret: "S"}
	m, _ := newTestManager(t, r, st, c)
	m.probeFP = func(context.Context, string, string) error { return errors.New("pin mismatch") }

	m.Apply(context.Background(), true, testBlock())

	// THE ORDERING LAW: a failing pre-consume verify means NOTHING consumed, NOTHING executed.
	// (Red-proof: reorder consume before the probe → calls=1 → FAIL.)
	if c.calls != 0 {
		t.Fatalf("consume calls = %d, want 0 — the secret was touched before the fingerprint verify", c.calls)
	}
	if len(r.recorded()) != 0 {
		t.Fatalf("runner calls = %+v, want none", r.recorded())
	}
	if s := m.Status(); s == nil || s.State != "verify_failed" {
		t.Fatalf("status = %+v, want verify_failed", s)
	}
}

func TestAdoption_HealthyEntryNoConsumeTenancyEntryOwned(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{
		found: true,
		entry: &proxmox.StorageEntryConfig{Storage: "felhom-offsite", Type: "pbs",
			Server: "10.77.0.1", Datastore: "felhom-offsite", Namespace: "demo-felhom-01",
			Username: "felhom@pbs!demo-felhom-01", Fingerprint: testFP},
		active: []bool{true},
	}
	c := &fakeConsumer{secret: "STAGED-BUT-MUST-STAY"}
	m, _ := newTestManager(t, r, st, c)

	block := testBlock()
	block.StorageID = "felhom-offsite"
	block.Namespace = "demo" // the hub-provisioned tenancy differs — the entry must WIN
	m.Apply(context.Background(), true, block)

	if c.calls != 0 {
		t.Fatalf("adoption consumed the staged secret (%d calls) — the no-consume law", c.calls)
	}
	calls := r.recorded()
	if len(calls) != 1 || calls[0].Args[0] != "grant" {
		t.Fatalf("adoption calls = %+v, want exactly [grant]", calls)
	}
	s := m.Status()
	if s == nil || s.State != "adopted" {
		t.Fatalf("status = %+v, want adopted", s)
	}
	if s.Namespace != "demo-felhom-01" {
		t.Errorf("reported namespace = %q, want the ENTRY's (demo-felhom-01) — tenancy is entry-owned", s.Namespace)
	}
	if !strings.Contains(s.Message, "entry wins") {
		t.Errorf("adoption note missing from message: %q", s.Message)
	}
}

// TestSetOnlyLaw pins the data-loss-class guard twice over: (1) the re-apply path over an
// existing entry uses `reconcile` (pvesm set) — never create, never any deletion verb; (2) the
// wrapper script itself contains no deletion path (grep gate over the shipped file).
// Red-proof: introduce a remove+re-add path → the recorded verbs change → FAIL.
func TestSetOnlyLaw(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{
		found: true,
		entry: &proxmox.StorageEntryConfig{Storage: "felhom-pbs", Type: "pbs", Namespace: "peti", Fingerprint: testFP},
		// unhealthy → recovery path (verify → consume → reconcile) → post-apply healthy
		active: []bool{false, true},
	}
	c := &fakeConsumer{secret: "FRESH"}
	m, _ := newTestManager(t, r, st, c)

	m.Apply(context.Background(), true, testBlock())

	calls := r.recorded()
	if len(calls) != 2 || calls[0].Args[0] != "reconcile" || calls[1].Args[0] != "grant" {
		t.Fatalf("re-apply calls = %+v, want [reconcile, grant] (set-only)", calls)
	}
	deletionish := regexp.MustCompile(`remove|delete|destroy`)
	for _, call := range calls {
		for _, a := range call.Args {
			if deletionish.MatchString(a) {
				t.Fatalf("deletion-class verb reached the wrapper: %v — K destruction path", call.Args)
			}
		}
	}
	if calls[0].Stdin != "FRESH\n" {
		t.Errorf("reconcile stdin = %q, want the fresh secret (re-issue recovery)", calls[0].Stdin)
	}

	// (2) the wrapper file: no deletion path, grep-assertable (the spike's set-only law).
	wrapper, err := os.ReadFile(filepath.Join("..", "..", "configs", "felhom-pbs-apply"))
	if err != nil {
		t.Fatalf("read wrapper: %v", err)
	}
	if regexp.MustCompile(`pvesm (remove|delete)`).Match(wrapper) {
		t.Fatal("configs/felhom-pbs-apply contains a pvesm deletion verb — the set-only law is dead")
	}
	if regexp.MustCompile(`rm\s+.*\.enc`).Match(wrapper) {
		t.Fatal("configs/felhom-pbs-apply deletes a .enc file — K destruction path")
	}
}

func TestConsumedButFailed_LoudAndRecoversOnlyViaFreshSecret(t *testing.T) {
	r := &fakeRunner{failVerb: "create", failErr: errors.New("pvesm add exploded")}
	st := &fakeStorage{found: false}
	c := &fakeConsumer{secret: "BURNED"}
	m, _ := newTestManager(t, r, st, c)
	block := testBlock()

	// 1. consume + create fails → LOUD persistent state.
	m.Apply(context.Background(), true, block)
	s := m.Status()
	if s == nil || s.State != "consumed_failed" || !s.ConsumedFailed {
		t.Fatalf("status = %+v, want consumed_failed", s)
	}
	if _, err := os.Stat(m.consumedFailedPath()); err != nil {
		t.Fatal("consumed-failed state file not written")
	}

	// 2. next ticks WITHOUT a fresh secret: no silent recovery, state stays loud.
	c.err = hub.ErrNoPBSSecret
	m.Apply(context.Background(), true, block)
	if s := m.Status(); s == nil || s.State != "consumed_failed" {
		t.Fatalf("status after no-secret retry = %+v, want consumed_failed (never quiet waiting)", s)
	}

	// 3. operator re-issue staged a FRESH secret → the bridge recovers on the next tick.
	c.err = nil
	c.secret = "FRESH-AFTER-REISSUE"
	r.failVerb = ""
	st.active = []bool{true}
	m.Apply(context.Background(), true, block)
	if s := m.Status(); s == nil || s.State != "applied" {
		t.Fatalf("status after re-issue = %+v, want applied", s)
	}
	if _, err := os.Stat(m.consumedFailedPath()); !os.IsNotExist(err) {
		t.Error("consumed-failed state not cleared after recovery")
	}
}

func TestMarkerIdempotency(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{found: false, active: []bool{true}}
	c := &fakeConsumer{secret: "S"}
	m, _ := newTestManager(t, r, st, c)
	block := testBlock()

	m.Apply(context.Background(), true, block)
	n := len(r.recorded())
	m.Apply(context.Background(), true, block) // same descriptor hash → pure no-op
	if len(r.recorded()) != n || c.calls != 1 {
		t.Fatalf("re-apply over an unchanged descriptor ran ops (calls %d→%d, consume %d)", n, len(r.recorded()), c.calls)
	}
	if s := m.Status(); s == nil || s.State != "applied" || s.AppliedAt == "" {
		t.Fatalf("status = %+v, want applied with applied_at", s)
	}
}

func TestOldHubAndDisabledCompat(t *testing.T) {
	r := &fakeRunner{}
	c := &fakeConsumer{secret: "S"}
	m, _ := newTestManager(t, r, &fakeStorage{}, c)

	m.Apply(context.Background(), false, nil) // no desired data yet
	m.Apply(context.Background(), true, nil)  // pre-slice-1 hub: no pbs_dr key
	if len(r.recorded()) != 0 || c.calls != 0 || m.Status() != nil {
		t.Fatalf("nil-block Apply had effects (runner %d, consume %d, status %+v)", len(r.recorded()), c.calls, m.Status())
	}

	m.Apply(context.Background(), true, &hub.WirePBSDR{Enabled: false, StorageID: "felhom-pbs"})
	if len(r.recorded()) != 0 || c.calls != 0 {
		t.Fatal("disabled descriptor ran ops (teardown is not this slice)")
	}
	if s := m.Status(); s == nil || s.State != "disabled" {
		t.Fatalf("status = %+v, want disabled", s)
	}
}

// DRConfigured (v0.86.0) — the capability prober's GatePBSDR answer. Walks the full lifecycle:
// unconfigured → enabled(applied) → restart(marker only) → disabled. Red-proof partner: make
// DRConfigured return status!=nil (ignore the "disabled" state) → the disabled case fails.
func TestDRConfigured_Lifecycle(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{found: false, active: []bool{true}}
	c := &fakeConsumer{secret: "S"}
	m, _ := newTestManager(t, r, st, c)

	// Fresh box, nothing fetched: not configured.
	if m.DRConfigured() {
		t.Fatal("fresh manager reports DR configured")
	}
	// Old hub / no descriptor: still not configured.
	m.Apply(context.Background(), true, nil)
	if m.DRConfigured() {
		t.Fatal("nil-block reports DR configured")
	}
	// Enabled descriptor applied: configured.
	m.Apply(context.Background(), true, testBlock())
	if s := m.Status(); s == nil || s.State != "applied" {
		t.Fatalf("precondition: status = %+v, want applied", s)
	}
	if !m.DRConfigured() {
		t.Fatal("applied box reports DR NOT configured")
	}
	// Agent restart (fresh manager over the same state dir): the persisted marker must answer
	// BEFORE the first desired-state fetch — an applied box never flaps to inactive at boot.
	m2 := NewManager(r, st, c, filepath.Dir(m.stateDir), "/etc/pve/priv/storage", "",
		slog.New(slog.NewTextHandler(io.Discard, nil)))
	if !m2.DRConfigured() {
		t.Fatal("restarted manager (marker on disk) reports DR NOT configured")
	}
	// Operator turns the tier OFF: descriptor disabled wins over the stale marker.
	m2.Apply(context.Background(), true, &hub.WirePBSDR{Enabled: false, StorageID: "felhom-pbs"})
	if m2.DRConfigured() {
		t.Fatal("disabled descriptor still reports DR configured")
	}
}

// TestWireFieldNames pins the cross-repo descriptor contract (hub/internal/web/pbsdr.go
// pbsDRDescriptor): the exact JSON the hub writes must land in WirePBSDR field-for-field.
func TestWireFieldNames(t *testing.T) {
	hubJSON := `{"desired_state":{"guests":[],"pbs_dr":{"enabled":true,"storage_id":"felhom-pbs",` +
		`"pbs_tunnel_ip":"10.77.0.1","datastore":"felhom-offsite","namespace":"peti",` +
		`"token_id":"felhom@pbs!peti","fingerprint":"aa:bb"}},"generation":7}`
	var resp hub.DesiredStateResponse
	if err := json.Unmarshal([]byte(hubJSON), &resp); err != nil {
		t.Fatalf("unmarshal: %v", err)
	}
	b := resp.DesiredState.PBSDR
	if b == nil || !b.Enabled || b.StorageID != "felhom-pbs" || b.PBSTunnelIP != "10.77.0.1" ||
		b.Datastore != "felhom-offsite" || b.Namespace != "peti" ||
		b.TokenID != "felhom@pbs!peti" || b.Fingerprint != "aa:bb" {
		t.Fatalf("wire mapping wrong: %+v", b)
	}
}

func TestEscrowSeed_NeverClobbersDifferentValue(t *testing.T) {
	r := &fakeRunner{}
	st := &fakeStorage{found: false, active: []bool{true}}
	c := &fakeConsumer{secret: "S"}
	m, cfgPath := newTestManager(t, r, st, c)
	os.WriteFile(cfgPath, []byte(`{"escrow":{"pbs_storage_id":"operator-set-id"}}`), 0o600)

	m.Apply(context.Background(), true, testBlock())

	raw, _ := os.ReadFile(cfgPath)
	var doc struct {
		Escrow struct {
			PBSStorageID string `json:"pbs_storage_id"`
		} `json:"escrow"`
	}
	json.Unmarshal(raw, &doc)
	if doc.Escrow.PBSStorageID != "operator-set-id" {
		t.Fatalf("an operator-set escrow.pbs_storage_id was clobbered: %s", raw)
	}
}