felhom.eu/hub/internal/pbsdrheal/reconciler_test.go

package pbsdrheal

// Non-hollow reconciler tests mapping 1:1 to the TASK's integration scenarios A–F. A REAL store
// (t.TempDir sqlite) supplies the work-set + reports; a FAKE Actions seam counts restage/reissue
// calls without any SSH/ep0. reconcileOnce is driven directly (same package) for determinism. Audit
// events are asserted against the real store. Each test is the durable guard for its §10 red-proof.

import (
	"context"
	"io"
	"log"
	"path/filepath"
	"sync"
	"testing"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

type fakeActions struct {
	mu            sync.Mutex
	restageHosts  []string
	reissueCusts  []string
	restageResult bool // what Restage returns (does a stored row exist?)
	restageErr    error
	reissueErr    error
}

func (f *fakeActions) Restage(hostID string) (bool, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.restageHosts = append(f.restageHosts, hostID)
	return f.restageResult, f.restageErr
}

func (f *fakeActions) Reissue(ctx context.Context, customerID string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.reissueCusts = append(f.reissueCusts, customerID)
	return f.reissueErr
}

func (f *fakeActions) restages() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.restageHosts) }
func (f *fakeActions) reissues() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.reissueCusts) }

func newHealStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatalf("store.New: %v", err)
	}
	t.Cleanup(func() { st.Close() })
	return st
}

// seedHost writes a host with a pbs_dr descriptor of the requested enable/provision shape.
func seedHost(t *testing.T, st *store.Store, hostID, customerID string, enabled, provisioned bool) {
	t.Helper()
	if err := st.UpsertHost(&store.Host{HostID: hostID, CustomerID: customerID, APIKey: "k-" + hostID}); err != nil {
		t.Fatalf("UpsertHost %s: %v", hostID, err)
	}
	ns := ""
	if provisioned {
		ns = customerID
	}
	desc := `{"pbs_dr":{"enabled":` + boolStr(enabled) + `,"namespace":"` + ns + `","storage_id":"felhom-pbs"}}`
	if _, err := st.SetHostDesired(hostID, []byte(desc)); err != nil {
		t.Fatalf("SetHostDesired %s: %v", hostID, err)
	}
}

func seedReport(t *testing.T, st *store.Store, hostID, customerID, state string) {
	t.Helper()
	if err := st.SaveHostReport(hostID, customerID, []byte(`{"pbs_dr":{"state":"`+state+`"}}`), store.HostReportDenorm{AgentVersion: "0.88.0"}); err != nil {
		t.Fatalf("SaveHostReport %s: %v", hostID, err)
	}
}

func boolStr(b bool) string {
	if b {
		return "true"
	}
	return "false"
}

func newRec(st *store.Store, act Actions) *Reconciler {
	r := NewReconciler(st, act, log.New(io.Discard, "", 0))
	r.debounceReports = 1 // fire on the first stuck report unless a test overrides
	return r
}

func genOf(t *testing.T, st *store.Store, hostID string) int64 {
	t.Helper()
	h, err := st.GetHost(hostID)
	if err != nil || h == nil {
		t.Fatalf("GetHost %s: (%v, %v)", hostID, h, err)
	}
	return h.DesiredGeneration
}

func eventTypes(t *testing.T, st *store.Store, customerID string) []string {
	t.Helper()
	evs, err := st.GetRecentEvents(customerID, 50)
	if err != nil {
		t.Fatalf("GetRecentEvents: %v", err)
	}
	var out []string
	for _, e := range evs {
		out = append(out, e.EventType)
	}
	return out
}

// A — the headline: a re-installed box (descriptor enabled+provisioned, stored CONSUMED secret,
// waiting_secret) is re-staged — NOT reissued, and NO generation bump.
func TestScenarioA_WaitingSecretRestages(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hA", "cA", true, true)
	seedReport(t, st, "hA", "cA", stateWaitingSecret)
	genBefore := genOf(t, st, "hA")
	fake := &fakeActions{restageResult: true} // a stored secret exists
	r := newRec(st, fake)

	r.reconcileOnce(context.Background())

	if fake.restages() != 1 || fake.reissues() != 0 {
		t.Fatalf("restages=%d reissues=%d, want 1/0 (re-stage, not re-issue)", fake.restages(), fake.reissues())
	}
	if fake.restageHosts[0] != "hA" {
		t.Errorf("restaged host = %s, want hA", fake.restageHosts[0])
	}
	if g := genOf(t, st, "hA"); g != genBefore {
		t.Errorf("generation bumped by a re-stage heal: %d -> %d", genBefore, g)
	}
	if got := eventTypes(t, st, "cA"); len(got) != 1 || got[0] != eventRestaged {
		t.Errorf("events = %v, want [%s]", got, eventRestaged)
	}
}

// B — no stored secret → escalate to Re-issue exactly once.
func TestScenarioB_NoStoredSecretReissues(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hB", "cB", true, true)
	seedReport(t, st, "hB", "cB", stateWaitingSecret)
	fake := &fakeActions{restageResult: false} // no row to re-stage
	r := newRec(st, fake)

	r.reconcileOnce(context.Background())

	if fake.restages() != 1 || fake.reissues() != 1 {
		t.Fatalf("restages=%d reissues=%d, want 1/1 (tried re-stage, then escalated)", fake.restages(), fake.reissues())
	}
	if fake.reissueCusts[0] != "cB" {
		t.Errorf("reissued customer = %s, want cB", fake.reissueCusts[0])
	}
	if got := eventTypes(t, st, "cB"); len(got) != 1 || got[0] != eventReissued {
		t.Errorf("events = %v, want [%s]", got, eventReissued)
	}
}

// C — a converged/healthy host is a pure no-op (idempotency; the gen-thrash guard).
func TestScenarioC_ConvergedHostNoOp(t *testing.T) {
	for _, state := range []string{"applied", "adopted", "disabled", "verify_failed"} {
		t.Run(state, func(t *testing.T) {
			st := newHealStore(t)
			seedHost(t, st, "hC", "cC", true, true)
			seedReport(t, st, "hC", "cC", state)
			genBefore := genOf(t, st, "hC")
			fake := &fakeActions{restageResult: true}
			r := newRec(st, fake)

			r.reconcileOnce(context.Background())

			if fake.restages() != 0 || fake.reissues() != 0 {
				t.Fatalf("state %s: restages=%d reissues=%d, want 0/0 (no-op)", state, fake.restages(), fake.reissues())
			}
			if g := genOf(t, st, "hC"); g != genBefore {
				t.Errorf("state %s: generation changed on a healthy host", state)
			}
			if got := eventTypes(t, st, "cC"); len(got) != 0 {
				t.Errorf("state %s: events = %v, want none", state, got)
			}
		})
	}
}

// D — debounce: a single waiting_secret report does NOT heal; a second DISTINCT one does.
func TestScenarioD_DebounceRequiresTwoDistinctReports(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hD", "cD", true, true)
	fake := &fakeActions{restageResult: true}
	r := NewReconciler(st, fake, log.New(io.Discard, "", 0)) // default debounceReports = 2

	seedReport(t, st, "hD", "cD", stateWaitingSecret) // report #1
	r.reconcileOnce(context.Background())
	if fake.restages() != 0 {
		t.Fatalf("healed on the FIRST waiting_secret report (restages=%d) — debounce broken", fake.restages())
	}
	// Re-observing the SAME report must also not advance the debounce.
	r.reconcileOnce(context.Background())
	if fake.restages() != 0 {
		t.Fatalf("healed on a RE-OBSERVED same report (restages=%d) — debounce must count distinct reports", fake.restages())
	}
	seedReport(t, st, "hD", "cD", stateWaitingSecret) // report #2 (distinct)
	r.reconcileOnce(context.Background())
	if fake.restages() != 1 {
		t.Fatalf("did not heal after 2 distinct waiting_secret reports (restages=%d)", fake.restages())
	}
}

// E — consumed_failed is escalated (Re-issue), never re-staged (a re-stage re-feeds a burned secret).
func TestScenarioE_ConsumedFailedEscalates(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hE", "cE", true, true)
	seedReport(t, st, "hE", "cE", stateConsumedFailed)
	fake := &fakeActions{restageResult: true} // even if a secret were re-stageable, must NOT re-stage
	r := newRec(st, fake)

	r.reconcileOnce(context.Background())

	if fake.restages() != 0 || fake.reissues() != 1 {
		t.Fatalf("restages=%d reissues=%d, want 0/1 (consumed_failed → re-issue only)", fake.restages(), fake.reissues())
	}
	if got := eventTypes(t, st, "cE"); len(got) != 1 || got[0] != eventConsumedFailed {
		t.Errorf("events = %v, want [%s] (distinct consumed_failed signal)", got, eventConsumedFailed)
	}
}

// F — DR-OFF (disabled descriptor) and enabled-but-unprovisioned hosts are never in the work set.
func TestScenarioF_OutOfScopeHostsUntouched(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hOff", "cOff", false, true)   // descriptor disabled
	seedReport(t, st, "hOff", "cOff", stateWaitingSecret)
	seedHost(t, st, "hUnprov", "cUnprov", true, false) // enabled but never provisioned
	seedReport(t, st, "hUnprov", "cUnprov", stateWaitingSecret)
	fake := &fakeActions{restageResult: true}
	r := newRec(st, fake)

	r.reconcileOnce(context.Background())

	if fake.restages() != 0 || fake.reissues() != 0 {
		t.Fatalf("out-of-scope hosts acted on: restages=%d reissues=%d, want 0/0", fake.restages(), fake.reissues())
	}
}

// RestrictToHost scopes the work set: a stuck host OTHER than the allowed one is untouched.
func TestRestrictToHost_ScopesWorkSet(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hAllowed", "cAllowed", true, true)
	seedReport(t, st, "hAllowed", "cAllowed", stateWaitingSecret)
	seedHost(t, st, "hOther", "cOther", true, true)
	seedReport(t, st, "hOther", "cOther", stateWaitingSecret)
	fake := &fakeActions{restageResult: true}
	r := newRec(st, fake)
	r.RestrictToHost("hAllowed")

	r.reconcileOnce(context.Background())

	if fake.restages() != 1 || fake.restageHosts[0] != "hAllowed" {
		t.Fatalf("restaged hosts = %v, want exactly [hAllowed] (scope restriction)", fake.restageHosts)
	}
}

// After a heal, the SAME stuck report must not re-heal on the next tick (only a fresh report does).
func TestNoReHealOnSameReport(t *testing.T) {
	st := newHealStore(t)
	seedHost(t, st, "hR", "cR", true, true)
	seedReport(t, st, "hR", "cR", stateWaitingSecret)
	fake := &fakeActions{restageResult: true}
	r := newRec(st, fake)

	r.reconcileOnce(context.Background())
	r.reconcileOnce(context.Background()) // same report, no new evidence
	if fake.restages() != 1 {
		t.Fatalf("re-healed the same stuck report: restages=%d, want 1", fake.restages())
	}
}