Files
felhom.eu/hub/internal/store/pbsdr_test.go
T
admin 6218e7919d hub v0.56.0: PBS-DR self-heal reconciler (re-stage a consumable secret)
Implements SPIKE-pbsdr-selfheal-2026-07-15 (e8f8c44). A box re-installed/rolled
back onto its stable host_id loses its agent-side converged marker; the hub
keeps the enabled descriptor + a CONSUMED one-time secret, the WG peer persists
(changed==false, cascade can't re-fire), so the agent sits in waiting_secret
forever. The missing piece is a consumable secret, not the descriptor.

New internal/pbsdrheal reconciler (5m, wgsync shape): for enabled+provisioned
hosts whose latest report pbs_dr.state is a stuck state past a >=2-distinct-report
debounce, re-stage the stored secret (store.RestageHostPBSSecret: clear
consumed_at, no ep0 call, NO generation bump); escalate to Re-issue (web
ReissuePBSDR) only when no secret is stored or the agent reports consumed_failed.
Converged/disabled/verify_failed/DR-OFF = no-op. PBSDRHEAL_ONLY_HOST scopes a
supervised rollout. Scenarios A-F + all six red-proofs verified. No agent change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HEPuEwyyGDJdcsXLFsTWJn
2026-07-15 18:27:39 +02:00

155 lines
6.0 KiB
Go

package store
import (
"database/sql"
"testing"
)
// RestageHostPBSSecret re-arms a stored (consumed) secret WITHOUT changing its value, inserting a
// row, or bumping any generation. Returns restaged=false when no row exists.
func TestRestageHostPBSSecret(t *testing.T) {
s := newTestStore(t)
// No row → restaged=false, no error (the caller escalates to Re-issue).
if restaged, err := s.RestageHostPBSSecret("h1"); err != nil || restaged {
t.Fatalf("restage with nothing stored = (%v, %v), want (false, nil)", restaged, err)
}
// Store + consume, then re-stage: the SAME value is served once more.
if err := s.SaveHostPBSSecret("h1", "the-secret"); err != nil {
t.Fatalf("save: %v", err)
}
if _, err := s.ConsumeHostPBSSecret("h1"); err != nil {
t.Fatalf("first consume: %v", err)
}
if _, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows {
t.Fatalf("pre-restage second consume = %v, want ErrNoRows", err)
}
restaged, err := s.RestageHostPBSSecret("h1")
if err != nil || !restaged {
t.Fatalf("restage of a stored secret = (%v, %v), want (true, nil)", restaged, err)
}
// Red-proof: dropping the `SET consumed_at = NULL` makes this consume return ErrNoRows.
got, err := s.ConsumeHostPBSSecret("h1")
if err != nil || got != "the-secret" {
t.Fatalf("post-restage consume = (%q, %v), want (the-secret, nil) — same value, re-armed", got, err)
}
}
// A-gen guard: a re-stage touches ONLY host_pbs_secrets — never the host generation (a bump would
// trigger an agent desired-state refetch loop). Red-proof: adding a SetHostDesired/gen bump to
// RestageHostPBSSecret makes this assert fail.
func TestRestageHostPBSSecret_NoGenerationBump(t *testing.T) {
s := newTestStore(t)
if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
t.Fatalf("host: %v", err)
}
genBefore, err := s.SetHostDesired("h1", []byte(`{"pbs_dr":{"enabled":true,"namespace":"c1"}}`))
if err != nil {
t.Fatalf("set desired: %v", err)
}
if err := s.SaveHostPBSSecret("h1", "s"); err != nil {
t.Fatalf("save secret: %v", err)
}
if _, err := s.RestageHostPBSSecret("h1"); err != nil {
t.Fatalf("restage: %v", err)
}
h, err := s.GetHost("h1")
if err != nil {
t.Fatalf("get host: %v", err)
}
if h.DesiredGeneration != genBefore {
t.Fatalf("generation changed by a re-stage: %d -> %d (a re-stage must never bump)", genBefore, h.DesiredGeneration)
}
}
// PBSDRHealStates joins each host's descriptor (enabled/provisioned) to its LATEST report's
// pbs_dr.state + report id.
func TestPBSDRHealStates(t *testing.T) {
s := newTestStore(t)
// Host A: enabled + provisioned, latest report waiting_secret (after an older applied report).
mustHost(t, s, "hA", "cA", `{"pbs_dr":{"enabled":true,"namespace":"cA","storage_id":"felhom-pbs"}}`)
mustReport(t, s, "hA", "cA", `{"pbs_dr":{"state":"applied"}}`)
mustReport(t, s, "hA", "cA", `{"pbs_dr":{"state":"waiting_secret"}}`) // newer → wins
// Host B: descriptor disabled.
mustHost(t, s, "hB", "cB", `{"pbs_dr":{"enabled":false,"namespace":"cB"}}`)
mustReport(t, s, "hB", "cB", `{"pbs_dr":{"state":"disabled"}}`)
// Host C: enabled but NOT provisioned (namespace empty); no report.
mustHost(t, s, "hC", "cC", `{"pbs_dr":{"enabled":true}}`)
rows, err := s.PBSDRHealStates()
if err != nil {
t.Fatalf("PBSDRHealStates: %v", err)
}
byHost := map[string]PBSDRHealRow{}
for _, r := range rows {
byHost[r.HostID] = r
}
a := byHost["hA"]
if !a.DescriptorEnabled || !a.DescriptorProvisioned || a.ReportedState != "waiting_secret" || a.ReportID == 0 {
t.Errorf("hA = %+v, want enabled+provisioned+waiting_secret+reportID>0", a)
}
b := byHost["hB"]
if b.DescriptorEnabled || b.ReportedState != "disabled" {
t.Errorf("hB = %+v, want disabled descriptor + state disabled", b)
}
c := byHost["hC"]
if !c.DescriptorEnabled || c.DescriptorProvisioned || c.ReportedState != "" || c.ReportID != 0 {
t.Errorf("hC = %+v, want enabled+unprovisioned+no-report", c)
}
}
func mustHost(t *testing.T, s *Store, hostID, customerID, desiredJSON string) {
t.Helper()
if err := s.UpsertHost(&Host{HostID: hostID, CustomerID: customerID, APIKey: "k-" + hostID}); err != nil {
t.Fatalf("UpsertHost %s: %v", hostID, err)
}
if _, err := s.SetHostDesired(hostID, []byte(desiredJSON)); err != nil {
t.Fatalf("SetHostDesired %s: %v", hostID, err)
}
}
func mustReport(t *testing.T, s *Store, hostID, customerID, reportJSON string) {
t.Helper()
if err := s.SaveHostReport(hostID, customerID, []byte(reportJSON), HostReportDenorm{AgentVersion: "0.88.0"}); err != nil {
t.Fatalf("SaveHostReport %s: %v", hostID, err)
}
}
// The host-scoped consume-once contract (PBS DR SLICE 1): exactly one read per stored value,
// a re-save resets the consumed flag (re-issue supersedes), absence is sql.ErrNoRows.
func TestHostPBSSecret_ConsumeOnce(t *testing.T) {
s := newTestStore(t)
if _, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows {
t.Fatalf("consume with nothing stored = %v, want sql.ErrNoRows", err)
}
if err := s.SaveHostPBSSecret("h1", "secret-1"); err != nil {
t.Fatalf("save: %v", err)
}
got, err := s.ConsumeHostPBSSecret("h1")
if err != nil || got != "secret-1" {
t.Fatalf("first consume = (%q, %v), want (secret-1, nil)", got, err)
}
// Single use: the second consume MUST be ErrNoRows (red-proof: dropping the consumed_at
// UPDATE in ConsumeHostPBSSecret makes this assert fail with the value returned again).
if got, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows {
t.Fatalf("second consume = (%q, %v), want sql.ErrNoRows — consume-once broken", got, err)
}
// Re-issue path: a fresh save resets consumption and serves the NEW value once.
if err := s.SaveHostPBSSecret("h1", "secret-2"); err != nil {
t.Fatalf("re-save: %v", err)
}
got, err = s.ConsumeHostPBSSecret("h1")
if err != nil || got != "secret-2" {
t.Fatalf("post-reissue consume = (%q, %v), want (secret-2, nil)", got, err)
}
// Host isolation: h2 never sees h1's rows.
if _, err := s.ConsumeHostPBSSecret("h2"); err != sql.ErrNoRows {
t.Fatalf("foreign host consume = %v, want sql.ErrNoRows", err)
}
}