felhom.eu/hub/internal/monitor/offsite_test.go

package monitor

import (
	"fmt"
	"strings"
	"testing"
	"time"
)

// saveOffsiteReport records a customer report whose report_json carries the given offsite object
// (or none, when raw==""). The 1.1s sleep before a RE-save of the same customer makes received_at
// strictly increasing (second resolution) so GetCustomers' MAX(received_at) picks the new row
// deterministically.
var offsiteSaved = map[string]bool{}

func saveOffsiteReport(t *testing.T, st interface {
	SaveReport(string, []byte) error
}, customerID, raw string) {
	t.Helper()
	if offsiteSaved[customerID] {
		time.Sleep(1100 * time.Millisecond)
	}
	offsiteSaved[customerID] = true
	body := `{"customer_id":"` + customerID + `"`
	if raw != "" {
		body += `,"offsite":` + raw
	}
	body += `}`
	if err := st.SaveReport(customerID, []byte(body)); err != nil {
		t.Fatal(err)
	}
}

func offsiteJSON(enabled bool, escrow, lastRun, lastStatus string, sizeBytes int64, quotaGB int) string {
	return fmt.Sprintf(`{"enabled":%v,"escrow_state":%q,"last_run":%q,"last_status":%q,"snapshot_count":3,"repo_size_bytes":%d,"quota_gb":%d}`,
		enabled, escrow, lastRun, lastStatus, sizeBytes, quotaGB)
}

// Scenario D — staleness: enabled+escrowed with no run >48h (or never) alerts once (deduped across
// sweeps); a pending customer NEVER alerts (normal onboarding — the red-proofed filter); recovery re-arms.
func TestOffsite_StalenessAlert(t *testing.T) {
	st := newDiskStore(t)
	now := time.Now().UTC()
	old := now.Add(-72 * time.Hour).Format(time.RFC3339)
	fresh := now.Add(-1 * time.Hour).Format(time.RFC3339)

	// stale-c: escrowed, last run 72h ago → stale. pend-c: PENDING with the same old run → silent.
	saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50))
	saveOffsiteReport(t, st, "pend-c", offsiteJSON(true, "pending", old, "", 1<<30, 50))

	var types []string
	oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog())
	oc.Check()
	if got := count(types, "offsite_stale"); got != 1 {
		t.Fatalf("want exactly 1 offsite_stale (stale-c only; pending must NEVER alert), got %d (%v)", got, types)
	}
	if oc.GetStaleState("pend-c") != "ok" {
		t.Fatalf("a pending customer must not be stale, got %s", oc.GetStaleState("pend-c"))
	}
	// repeated sweep with the same data → NO re-page (dedupe)
	oc.Check()
	if got := count(types, "offsite_stale"); got != 1 {
		t.Fatalf("staleness must not re-page every sweep, got %d", got)
	}
	// recovery: a fresh run clears + re-arms; going stale again re-alerts
	saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", fresh, "ok", 1<<30, 50))
	oc.Check()
	if oc.GetStaleState("stale-c") != "ok" {
		t.Fatal("recovery must clear the stale state")
	}
	saveOffsiteReport(t, st, "stale-c", offsiteJSON(true, "escrowed", old, "ok", 1<<30, 50))
	oc.Check()
	if got := count(types, "offsite_stale"); got != 2 {
		t.Fatalf("re-staleness after recovery must alert again, got %d", got)
	}
	// never-ran escrowed customer is stale too
	saveOffsiteReport(t, st, "never-c", offsiteJSON(true, "escrowed", "", "", 0, 50))
	oc.Check()
	if got := count(types, "offsite_stale"); got != 3 {
		t.Fatalf("a never-ran escrowed target must be stale, got %d", got)
	}
}

// Scenario E (fill) — 90/95 of quota_gb; quota 0 never alerts; escalation-only; recovery re-arms.
func TestOffsite_FillAlert(t *testing.T) {
	st := newDiskStore(t)
	fresh := time.Now().UTC().Format(time.RFC3339)
	gb := int64(1) << 30

	var types, sevs []string
	onEvent := func(_, et, sev, _, _, _ string) { types = append(types, et); sevs = append(sevs, sev) }
	saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 40*gb, 50)) // 80% — under warn
	saveOffsiteReport(t, st, "noq-c", offsiteJSON(true, "escrowed", fresh, "ok", 900*gb, 0))  // dedicated: quota 0
	oc := NewOffsiteChecker(st, 48*time.Hour, onEvent, quietLog())
	oc.Check()
	if len(types) != 0 {
		t.Fatalf("80%% and quota-0 must not alert, got %v", types)
	}

	saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 46*gb, 50)) // 92% warn
	oc.Check()
	if count(types, "offsite_fill_warning") != 1 {
		t.Fatalf("92%% must warn once, got %v", types)
	}
	oc.Check() // same data — no re-page
	if len(types) != 1 {
		t.Fatalf("no re-page on an unchanged band, got %v", types)
	}
	saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 48*gb, 50)) // 96% crit
	oc.Check()
	if count(types, "offsite_fill_critical") != 1 || sevs[len(sevs)-1] != "critical" {
		t.Fatalf("96%% must escalate to critical, got types=%v sevs=%v", types, sevs)
	}
	// recovery re-arms
	saveOffsiteReport(t, st, "fill-c", offsiteJSON(true, "escrowed", fresh, "ok", 10*gb, 50))
	oc.Check()
	if oc.GetFillState("fill-c") != bandOK {
		t.Fatal("recovery must re-arm the fill state")
	}
}

// Nil-safety — reports without an offsite object (pre-v0.109 controllers / offbox disabled) are skipped:
// no alert, no state.
func TestOffsite_NilSafeOnOldReports(t *testing.T) {
	st := newDiskStore(t)
	saveOffsiteReport(t, st, "old-c", "") // no offsite key at all
	var types []string
	oc := NewOffsiteChecker(st, 48*time.Hour, func(_, et, _, _, _, _ string) { types = append(types, et) }, quietLog())
	oc.Check()
	if len(types) != 0 {
		t.Fatalf("a report without an offsite object must never alert, got %v", types)
	}
	if oc.GetStaleState("old-c") != "unknown" || oc.GetFillState("old-c") != "unknown" {
		t.Fatal("no state may exist for a customer without an offsite object")
	}
}

func count(list []string, want string) int {
	n := 0
	for _, s := range list {
		if s == want {
			n++
		}
	}
	return n
}

var _ = strings.Contains // keep strings import if unused by future edits