felhom.eu/hub/internal/offsiteheal/reconciler_test.go

package offsiteheal

// Non-hollow reconciler tests mapping 1:1 to the task's integration scenarios A–F. A REAL store
// (t.TempDir sqlite) supplies the work set, the customer configs and the reports; a FAKE Actions seam
// counts restage/reissue calls without touching a storage provider. ReconcileOnce is driven directly
// for determinism — the debounce is exercised by feeding DISTINCT REPORTS, never by sleeping, because
// the debounce counts fresh evidence rather than elapsed time.

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"path/filepath"
	"sync"
	"testing"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

type fakeActions struct {
	mu            sync.Mutex
	restaged      []string
	reissued      []string
	restageResult bool // does a stored secret row exist?
	restageErr    error
	reissueErr    error
}

func (f *fakeActions) Restage(customerID string) (bool, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.restaged = append(f.restaged, customerID)
	return f.restageResult, f.restageErr
}

func (f *fakeActions) Reissue(_ context.Context, customerID string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.reissued = append(f.reissued, customerID)
	return f.reissueErr
}

func (f *fakeActions) restages() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.restaged) }
func (f *fakeActions) reissues() int { f.mu.Lock(); defer f.mu.Unlock(); return len(f.reissued) }

func newHealStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatalf("store.New: %v", err)
	}
	t.Cleanup(func() { st.Close() })
	return st
}

// seedCustomer writes an ACTIVE customer whose offsite descriptor is enabled (or not).
func seedCustomer(t *testing.T, st *store.Store, customerID string, offsiteEnabled bool) {
	t.Helper()
	cfgJSON := `{"offsite":{"enabled":false}}`
	if offsiteEnabled {
		cfgJSON = `{"offsite":{"enabled":true,"type":"shared","host":"x.your-storagebox.de","user":"u","port":23,"repo_path":"/home/felhom-repo"}}`
	}
	if err := st.SaveCustomerConfig(&store.CustomerConfig{
		CustomerID: customerID, CustomerName: customerID, Domain: customerID + ".hu",
		APIKey: "k-" + customerID, ConfigJSON: cfgJSON,
	}); err != nil {
		t.Fatalf("SaveCustomerConfig %s: %v", customerID, err)
	}
}

// pushReport writes ONE controller report. state=="" → a healthy configured box (enabled offsite
// object); otherwise the declaration shape controller v0.199.0 emits. Each call is a DISTINCT report,
// which is the debounce's currency.
func pushReport(t *testing.T, st *store.Store, customerID, state string, seq int) {
	t.Helper()
	var off map[string]any
	if state == "" {
		off = map[string]any{"enabled": true, "escrow_state": "escrowed", "last_status": "ok"}
	} else {
		off = map[string]any{"enabled": false, "state": state}
	}
	b, err := json.Marshal(map[string]any{
		"controller_version": fmt.Sprintf("0.199.0-%d", seq), // makes each row distinguishable
		"offsite":            off,
	})
	if err != nil {
		t.Fatal(err)
	}
	if err := st.SaveReport(customerID, b); err != nil {
		t.Fatalf("SaveReport %s: %v", customerID, err)
	}
}

// pushReportNoOffsite writes a report with NO offsite object at all — the pre-v0.199.0 shape a
// stranded box used to send, and the shape a box that never had off-site backups still sends.
func pushReportNoOffsite(t *testing.T, st *store.Store, customerID string, seq int) {
	t.Helper()
	b, _ := json.Marshal(map[string]any{"controller_version": fmt.Sprintf("0.198.0-%d", seq)})
	if err := st.SaveReport(customerID, b); err != nil {
		t.Fatalf("SaveReport %s: %v", customerID, err)
	}
}

func newRec(st *store.Store, act Actions) *Reconciler {
	return NewReconciler(st, act, log.New(io.Discard, "", 0))
}

func countEvents(t *testing.T, st *store.Store, customerID, eventType string) int {
	t.Helper()
	ev, err := st.GetLatestEventByType(customerID, eventType)
	if err != nil {
		t.Fatalf("GetLatestEventByType: %v", err)
	}
	if ev == nil {
		return 0
	}
	return 1
}

// SCENARIO A + D — a declaring box is served, and the STORED secret is re-armed rather than a fresh
// credential minted.
//
// RED-PROOF (D): invert the order in heal() so Reissue is called before Restage — the assertion
// "no mint" fails, and the test names the cost: external churn for a credential the hub already has.
func TestScenarioAD_DeclaringBoxIsRestagedNotMinted(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true} // a stored secret exists
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	pushReport(t, st, "c1", StateNeedsCredential, 1)
	r.ReconcileOnce(context.Background())
	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("acted on a SINGLE declaration — the debounce did not hold (restages=%d reissues=%d)", act.restages(), act.reissues())
	}

	pushReport(t, st, "c1", StateNeedsCredential, 2) // a second DISTINCT report
	r.ReconcileOnce(context.Background())

	if act.restages() != 1 {
		t.Fatalf("a sustained declaration was not served: restages=%d", act.restages())
	}
	if act.reissues() != 0 {
		t.Fatalf("a fresh credential was minted although a stored one could be re-armed — external churn for nothing (reissues=%d)", act.reissues())
	}
	if countEvents(t, st, "c1", eventRestaged) != 1 {
		t.Error("no offsite_selfheal_restaged audit event — an automatic remediation must be visible")
	}
	if countEvents(t, st, "c1", eventReissued) != 0 {
		t.Error("a reissue event was recorded for a restage")
	}
}

// SCENARIO B — a box that never had off-site backups says nothing, so nothing happens. (The
// declaration itself is the controller's guard; here we assert the hub side is a no-op on silence.)
func TestScenarioB_SilentBoxIsNeverActedOn(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	for i := 1; i <= 4; i++ {
		pushReportNoOffsite(t, st, "c1", i)
		r.ReconcileOnce(context.Background())
	}
	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("a silent box was acted on: restages=%d reissues=%d", act.restages(), act.reissues())
	}
}

// SCENARIO C — a healthy box is a PURE no-op, repeatedly. No restage, no mint, no event.
func TestScenarioC_HealthyBoxIsAPureNoOp(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	for i := 1; i <= 5; i++ {
		pushReport(t, st, "c1", "", i) // healthy: enabled offsite object, no declaration
		r.ReconcileOnce(context.Background())
	}
	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("a healthy box was acted on: restages=%d reissues=%d", act.restages(), act.reissues())
	}
	if countEvents(t, st, "c1", eventRestaged)+countEvents(t, st, "c1", eventReissued) != 0 {
		t.Error("a healthy box produced a self-heal event")
	}
}

// SCENARIO E — nothing to re-arm escalates to a mint, ONCE, and does not repeat while the staged
// secret sits unconsumed (a fresh report must re-confirm before acting again).
func TestScenarioE_NothingToRestageEscalatesOnce(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: false} // NO stored secret
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	pushReport(t, st, "c1", StateNeedsCredential, 1)
	r.ReconcileOnce(context.Background())
	pushReport(t, st, "c1", StateNeedsCredential, 2)
	r.ReconcileOnce(context.Background())

	if act.reissues() != 1 {
		t.Fatalf("escalation did not happen exactly once: reissues=%d", act.reissues())
	}
	if countEvents(t, st, "c1", eventReissued) != 1 {
		t.Error("no offsite_selfheal_reissued audit event")
	}

	// Repeated ticks on the SAME report must not mint again — this is the "a mint on every tick"
	// failure the sibling's resetAfterHeal exists to prevent.
	for i := 0; i < 5; i++ {
		r.ReconcileOnce(context.Background())
	}
	if act.reissues() != 1 {
		t.Fatalf("minted repeatedly on an unchanged report: reissues=%d", act.reissues())
	}
}

// SCENARIO F — a BLIP is absorbed: one declaring report followed by a healthy one triggers nothing.
//
// RED-PROOF: set debounceReports to 1 (i.e. remove the debounce) — the blip acts, and a credential is
// churned by what was only a restart.
func TestScenarioF_BlipIsAbsorbedByTheDebounce(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	pushReport(t, st, "c1", StateNeedsCredential, 1) // the blip
	r.ReconcileOnce(context.Background())
	pushReport(t, st, "c1", "", 2) // healthy again
	r.ReconcileOnce(context.Background())

	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("a one-report blip triggered a credential action: restages=%d reissues=%d", act.restages(), act.reissues())
	}

	// And the streak must have been FORGOTTEN, not merely paused: a single later declaration must
	// still not act.
	pushReport(t, st, "c1", StateNeedsCredential, 3)
	r.ReconcileOnce(context.Background())
	if act.restages() != 0 {
		t.Fatal("the debounce streak survived a healthy report — a flapping box would be healed on every other cycle")
	}
}

// A DISABLED offsite descriptor is the operator's own choice and must never be re-credentialed,
// even if the box declares (e.g. an old declaration left in the newest report).
func TestDisabledDescriptorIsNeverHealed(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", false) // offsite disabled in the config

	for i := 1; i <= 4; i++ {
		pushReport(t, st, "c1", StateNeedsCredential, i)
		r.ReconcileOnce(context.Background())
	}
	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("a customer whose offsite is DISABLED was re-credentialed: restages=%d reissues=%d", act.restages(), act.reissues())
	}
}

// A blocked customer is never healed.
func TestBlockedCustomerIsNeverHealed(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)
	// SaveCustomerConfig always writes status 'active' (its INSERT does not carry the column), so the
	// block must go through the real setter — asserted below, because a test that silently failed to
	// block would pass for the wrong reason.
	if err := st.SetCustomerConfigStatus("c1", "blocked"); err != nil {
		t.Fatalf("SetCustomerConfigStatus: %v", err)
	}
	if !st.IsCustomerBlocked("c1") {
		t.Fatal("precondition: the customer is not actually blocked — this test would pass vacuously")
	}
	for i := 1; i <= 4; i++ {
		pushReport(t, st, "c1", StateNeedsCredential, i)
		r.ReconcileOnce(context.Background())
	}
	if act.restages() != 0 || act.reissues() != 0 {
		t.Fatalf("a BLOCKED customer was healed: restages=%d reissues=%d", act.restages(), act.reissues())
	}
}

// A restage ERROR must not escalate to a mint — the reconciler does nothing and retries. Acting on a
// failed read is the "never act on a partial view" rule.
func TestRestageErrorDoesNotEscalate(t *testing.T) {
	st := newHealStore(t)
	act := &fakeActions{restageResult: true, restageErr: fmt.Errorf("db is busy")}
	r := newRec(st, act)
	seedCustomer(t, st, "c1", true)

	pushReport(t, st, "c1", StateNeedsCredential, 1)
	r.ReconcileOnce(context.Background())
	pushReport(t, st, "c1", StateNeedsCredential, 2)
	r.ReconcileOnce(context.Background())

	if act.reissues() != 0 {
		t.Fatalf("a failed re-stage escalated to an external mint: reissues=%d", act.reissues())
	}
	if countEvents(t, st, "c1", eventRestaged) != 0 {
		t.Error("a failed re-stage recorded a success event")
	}
}