168 lines
6.5 KiB
Go
168 lines
6.5 KiB
Go
package web
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
|
||
)
|
||
|
||
// R-1295: Tester4's fresh install on 2026-10-26 lost its whole-guest off-site tier and only the hub log said so. ep0's
|
||
// self-check of the new token was refused twice within one second (PBS caches permissions for up to 5 s; reproduced on a
|
||
// throwaway PBS 4.2.5: 9 of 20 first checks failed under load, the stale window 4.2–5.0 s) and the endpoint rolled the
|
||
// token back. The consequences asserted: a rollback is retried after a pause and the tier IS set up; a final failure
|
||
// reaches the operator (stored + dispatched, operator-only, a subject that says what to do).
|
||
// RED-PROOF: call s.tenantsync.Provision/Reissue directly in pbsdrProvisionAtom (no tenancyWithRetry) → the first two
|
||
// tests fail; drop the pbsdrProvisionFailed call → the third fails; drop the notify entries → TestR1295_TheMail fails.
|
||
|
||
// selfCheckTenancy fails the first `fails` calls of the chosen op with the endpoint's self_check_failed outcome.
|
||
type selfCheckTenancy struct {
|
||
fakeTenancy
|
||
provisionFails, reissueFails int
|
||
}
|
||
|
||
func selfCheckErr() error {
|
||
return fmt.Errorf("%w: own-namespace list as felhom@pbs!peti failed twice — rolled back", tenantsync.ErrSelfCheckFailed)
|
||
}
|
||
|
||
func (f *selfCheckTenancy) Provision(ctx context.Context, id string) (*tenantsync.Result, error) {
|
||
f.provisionCalls++
|
||
if f.provisionErr != nil {
|
||
return nil, f.provisionErr
|
||
}
|
||
if f.provisionCalls <= f.provisionFails {
|
||
return nil, selfCheckErr()
|
||
}
|
||
return f.result(id), nil
|
||
}
|
||
|
||
func (f *selfCheckTenancy) Reissue(ctx context.Context, id string) (*tenantsync.Result, error) {
|
||
f.reissueCalls++
|
||
if f.reissueCalls <= f.reissueFails {
|
||
return nil, selfCheckErr()
|
||
}
|
||
return f.result(id), nil
|
||
}
|
||
|
||
func r1295Server(t *testing.T, f *selfCheckTenancy) (*Server, *store.Store, *[]string) {
|
||
t.Helper()
|
||
old := pbsdrSelfCheckPause
|
||
pbsdrSelfCheckPause = 0
|
||
t.Cleanup(func() { pbsdrSelfCheckPause = old })
|
||
s, st, _ := newPBSDRServer(t, nil)
|
||
s.SetTenantSync(f)
|
||
cfg, _ := st.GetCustomerConfig("peti")
|
||
cfg.DRTier = true
|
||
if err := st.SaveCustomerConfig(cfg); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var emitted []string
|
||
s.emit = func(c, typ, sev, msg, details, src string) { emitted = append(emitted, c+"|"+typ+"|"+sev+"|"+msg) }
|
||
return s, st, &emitted
|
||
}
|
||
|
||
func TestR1295_ARolledBackSelfCheckIsRetriedAndTheTierIsSetUp(t *testing.T) {
|
||
f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295"}, provisionFails: 2}
|
||
s, st, emitted := r1295Server(t, f)
|
||
|
||
s.PBSDRAutoProvision(context.Background(), "peti")
|
||
|
||
if desc, js, _ := hostState(t, st); desc == nil || !desc.Enabled || desc.Namespace != "peti" {
|
||
t.Fatalf("after two rolled-back self-checks the tier must be set up on the third try: %+v (%s)", desc, js)
|
||
}
|
||
if f.provisionCalls != 3 {
|
||
t.Errorf("provision calls = %d, want 3 (two rollbacks + the passing try)", f.provisionCalls)
|
||
}
|
||
if len(*emitted) != 0 {
|
||
t.Errorf("a set-up that succeeded must not mail the operator: %v", *emitted)
|
||
}
|
||
}
|
||
|
||
func TestR1295_TheF14ReissuePathIsRetriedToo(t *testing.T) {
|
||
f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295", provisionErr: tenantsync.ErrTokenExists}, reissueFails: 1}
|
||
s, st, _ := r1295Server(t, f)
|
||
// The F-14 history (the Tester4 shape): the owning host was deleted through the escrow-ack flow.
|
||
if err := st.UpsertHost(&store.Host{HostID: "peti-00-dead", CustomerID: "peti", APIKey: "old"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, _, err := st.SaveHostEscrow("peti-00-dead", []byte("opaque"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := st.DeleteHost("peti-00-dead", true); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
s.PBSDRAutoProvision(context.Background(), "peti")
|
||
|
||
if desc, js, _ := hostState(t, st); desc == nil || desc.Namespace != "peti" {
|
||
t.Fatalf("the re-issue path must be retried after a rollback: %+v (%s)", desc, js)
|
||
}
|
||
if f.reissueCalls != 2 {
|
||
t.Errorf("re-issue calls = %d, want 2", f.reissueCalls)
|
||
}
|
||
}
|
||
|
||
func TestR1295_OtherFailuresAreNotRetried(t *testing.T) {
|
||
f := &selfCheckTenancy{fakeTenancy: fakeTenancy{provisionErr: tenantsync.ErrTokenExists}}
|
||
s, _, _ := r1295Server(t, f)
|
||
s.PBSDRAutoProvision(context.Background(), "peti") // token_exists with no acked deletion: the manual-path refusal
|
||
if f.provisionCalls != 1 || f.reissueCalls != 0 {
|
||
t.Fatalf("only a self-check rollback is retried: provision %d, re-issue %d", f.provisionCalls, f.reissueCalls)
|
||
}
|
||
}
|
||
|
||
func TestR1295_AFinalFailureReachesTheOperator(t *testing.T) {
|
||
f := &selfCheckTenancy{fakeTenancy: fakeTenancy{secret: "S-1295"}, provisionFails: 99}
|
||
s, st, emitted := r1295Server(t, f)
|
||
|
||
s.PBSDRAutoProvision(context.Background(), "peti")
|
||
|
||
if f.provisionCalls != pbsdrSelfCheckTries {
|
||
t.Errorf("provision calls = %d, want the bound %d (never an endless loop)", f.provisionCalls, pbsdrSelfCheckTries)
|
||
}
|
||
if desc, _, _ := hostState(t, st); desc != nil && desc.Namespace != "" {
|
||
t.Fatalf("a failed set-up must not leave a descriptor: %+v", desc)
|
||
}
|
||
if len(*emitted) != 1 || !strings.HasPrefix((*emitted)[0], "peti|pbsdr_provision_failed|error|") {
|
||
t.Fatalf("the operator must get exactly one pbsdr_provision_failed event: %v", *emitted)
|
||
}
|
||
msg := (*emitted)[0]
|
||
for _, want := range []string{"peti-01", "save the configuration", "NO whole-box off-site copy"} {
|
||
if !strings.Contains(msg, want) {
|
||
t.Errorf("the event must say %q: %s", want, msg)
|
||
}
|
||
}
|
||
if strings.Contains(msg, "S-1295") {
|
||
t.Error("the event carries a secret")
|
||
}
|
||
evs, err := st.GetRecentEvents("peti", 10)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
found := false
|
||
for _, e := range evs {
|
||
found = found || (e.EventType == "pbsdr_provision_failed" && e.Severity == "error" && e.Source == "hub")
|
||
}
|
||
if !found {
|
||
t.Error("the event is not on the customer's timeline")
|
||
}
|
||
}
|
||
|
||
func TestR1295_TheMail(t *testing.T) {
|
||
if !notify.IsOperatorOnly("pbsdr_provision_failed") {
|
||
t.Fatal("pbsdr_provision_failed must be operator-only (a household must never get it)")
|
||
}
|
||
subj, body := notify.FormatOperatorEmail("Tester4", "pbsdr_provision_failed", "error", "msg", "{}")
|
||
if !strings.Contains(subj, "could not be set up") || !strings.Contains(subj, "save the customer config to retry") ||
|
||
!strings.Contains(subj, "Tester4") || !strings.Contains(subj, "🔴") {
|
||
t.Fatalf("the subject must say what failed and what to do: %q", subj)
|
||
}
|
||
if !strings.Contains(body, "Dashboard: https://hub.felhom.eu/customers/Tester4") {
|
||
t.Errorf("the body must link the customer: %s", body)
|
||
}
|
||
}
|