package web import ( "context" "fmt" "strings" "testing" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/store" "gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync" ) // R-1295: Tester4's fresh install on 2026-10-26 lost its whole-guest off-site tier and only the hub log said so. ep0's // self-check of the new token was refused twice within one second (PBS caches permissions for up to 5 s; reproduced on a // throwaway PBS 4.2.5: 9 of 20 first checks failed under load, the stale window 4.2–5.0 s) and the endpoint rolled the // token back. The consequences asserted: a rollback is retried after a pause and the tier IS set up; a final failure // reaches the operator (stored + dispatched, operator-only, a subject that says what to do). // RED-PROOF: call s.tenantsync.Provision/Reissue directly in pbsdrProvisionAtom (no tenancyWithRetry) → the first two // tests fail; drop the pbsdrProvisionFailed call → the third fails; drop the notify entries → TestR1295_TheMail fails. // selfCheckTenancy fails the first `fails` calls of the chosen op with the endpoint's self_check_failed outcome. type selfCheckTenancy struct { fakeTenancy provisionFails, reissueFails int } func selfCheckErr() error { return fmt.Errorf("%w: own-namespace list as felhom@pbs!peti failed twice — rolled back", tenantsync.ErrSelfCheckFailed) } func (f *selfCheckTenancy) Provision(ctx context.Context, id string) (*tenantsync.Result, error) { f.provisionCalls++ if f.provisionErr != nil { return nil, f.provisionErr } if f.provisionCalls <= f.provisionFails { return nil, selfCheckErr() } return f.result(id), nil } func (f *selfCheckTenancy) Reissue(ctx context.Context, id string) (*tenantsync.Result, error) { f.reissueCalls++ if f.reissueCalls <= f.reissueFails { return nil, selfCheckErr() } return f.result(id), nil } func r1295Server(t *testing.T, f *selfCheckTenancy) (*Server, *store.Store, *[]string) { t.Helper() old := pbsdrSelfCheckPause pbsdrSelfCheckPause = 0 t.Cleanup(func() { pbsdrSelfCheckPause = old }) s, st, _ := newPBSDRServer(t, nil) s.SetTenantSync(f) cfg, _ := st.GetCustomerConfig("peti") cfg.DRTier = true if err := st.SaveCustomerConfig(cfg); err != nil { t.Fatal(err) } var emitted []string s.emit = func(c, typ, sev, msg, details, src string) { emitted = append(emitted, c+"|"+typ+"|"+sev+"|"+msg) } return s, st, &emitted } func TestR1295_ARolledBackSelfCheckIsRetriedAndTheTierIsSetUp(t *testing.T) { f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295"}, provisionFails: 2} s, st, emitted := r1295Server(t, f) s.PBSDRAutoProvision(context.Background(), "peti") if desc, js, _ := hostState(t, st); desc == nil || !desc.Enabled || desc.Namespace != "peti" { t.Fatalf("after two rolled-back self-checks the tier must be set up on the third try: %+v (%s)", desc, js) } if f.provisionCalls != 3 { t.Errorf("provision calls = %d, want 3 (two rollbacks + the passing try)", f.provisionCalls) } if len(*emitted) != 0 { t.Errorf("a set-up that succeeded must not mail the operator: %v", *emitted) } } func TestR1295_TheF14ReissuePathIsRetriedToo(t *testing.T) { f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295", provisionErr: tenantsync.ErrTokenExists}, reissueFails: 1} s, st, _ := r1295Server(t, f) // The F-14 history (the Tester4 shape): the owning host was deleted through the escrow-ack flow. if err := st.UpsertHost(&store.Host{HostID: "peti-00-dead", CustomerID: "peti", APIKey: "old"}); err != nil { t.Fatal(err) } if _, _, err := st.SaveHostEscrow("peti-00-dead", []byte("opaque"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil { t.Fatal(err) } if err := st.DeleteHost("peti-00-dead", true); err != nil { t.Fatal(err) } s.PBSDRAutoProvision(context.Background(), "peti") if desc, js, _ := hostState(t, st); desc == nil || desc.Namespace != "peti" { t.Fatalf("the re-issue path must be retried after a rollback: %+v (%s)", desc, js) } if f.reissueCalls != 2 { t.Errorf("re-issue calls = %d, want 2", f.reissueCalls) } } func TestR1295_OtherFailuresAreNotRetried(t *testing.T) { f := &selfCheckTenancy{fakeTenancy: fakeTenancy{provisionErr: tenantsync.ErrTokenExists}} s, _, _ := r1295Server(t, f) s.PBSDRAutoProvision(context.Background(), "peti") // token_exists with no acked deletion: the manual-path refusal if f.provisionCalls != 1 || f.reissueCalls != 0 { t.Fatalf("only a self-check rollback is retried: provision %d, re-issue %d", f.provisionCalls, f.reissueCalls) } } func TestR1295_AFinalFailureReachesTheOperator(t *testing.T) { f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295"}, provisionFails: 99} s, st, emitted := r1295Server(t, f) s.PBSDRAutoProvision(context.Background(), "peti") if f.provisionCalls != pbsdrSelfCheckTries { t.Errorf("provision calls = %d, want the bound %d (never an endless loop)", f.provisionCalls, pbsdrSelfCheckTries) } if desc, _, _ := hostState(t, st); desc != nil && desc.Namespace != "" { t.Fatalf("a failed set-up must not leave a descriptor: %+v", desc) } if len(*emitted) != 1 || !strings.HasPrefix((*emitted)[0], "peti|pbsdr_provision_failed|error|") { t.Fatalf("the operator must get exactly one pbsdr_provision_failed event: %v", *emitted) } msg := (*emitted)[0] for _, want := range []string{"peti-01", "save the configuration", "NO whole-box off-site copy"} { if !strings.Contains(msg, want) { t.Errorf("the event must say %q: %s", want, msg) } } if strings.Contains(msg, "S-1295") { t.Error("the event carries a secret") } evs, err := st.GetRecentEvents("peti", 10) if err != nil { t.Fatal(err) } found := false for _, e := range evs { found = found || (e.EventType == "pbsdr_provision_failed" && e.Severity == "error" && e.Source == "hub") } if !found { t.Error("the event is not on the customer's timeline") } } func TestR1295_TheMail(t *testing.T) { if !notify.IsOperatorOnly("pbsdr_provision_failed") { t.Fatal("pbsdr_provision_failed must be operator-only (a household must never get it)") } subj, body := notify.FormatOperatorEmail("Tester4", "pbsdr_provision_failed", "error", "msg", "{}") if !strings.Contains(subj, "could not be set up") || !strings.Contains(subj, "save the customer config to retry") || !strings.Contains(subj, "Tester4") || !strings.Contains(subj, "🔴") { t.Fatalf("the subject must say what failed and what to do: %q", subj) } if !strings.Contains(body, "Dashboard: https://hub.felhom.eu/customers/Tester4") { t.Errorf("the body must link the customer: %s", body) } }