Files
felhom.eu/hub/internal/web/pbsdr_test.go
T

458 lines
17 KiB
Go

package web
// PBS DR SLICE 1 — the provisioning flow behind the config form, against a FAKE tenancy
// provisioner (no SSH in CI). The load-bearing contracts: descriptor lands in the HOST
// desired_json + generation bump; the secret is stored consume-once and appears NOWHERE else
// (ConfigJSON, desired-state, logs); every failure is fail-closed (no descriptor, no bump, no
// secret); an already-provisioned re-save is a pure no-op (no re-key, no second secret, no
// spurious bump); re-issue rotates the secret + bumps.
import (
"bytes"
"context"
"database/sql"
"errors"
"log"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
"gitea.dooplex.hu/admin/felhom-hub/internal/tenantsync"
)
type fakeTenancy struct {
provisionCalls int
reissueCalls int
err error
secret string
}
func (f *fakeTenancy) result(customerID string) *tenantsync.Result {
return &tenantsync.Result{
TokenID: "felhom@pbs!" + customerID,
TokenSecret: f.secret,
Fingerprint: "aa:bb:cc",
Datastore: "felhom-offsite",
Namespace: customerID,
}
}
func (f *fakeTenancy) Provision(ctx context.Context, customerID string) (*tenantsync.Result, error) {
f.provisionCalls++
if f.err != nil {
return nil, f.err
}
return f.result(customerID), nil
}
func (f *fakeTenancy) Reissue(ctx context.Context, customerID string) (*tenantsync.Result, error) {
f.reissueCalls++
if f.err != nil {
return nil, f.err
}
return f.result(customerID), nil
}
// newPBSDRServer builds a server + store with the full provisioning preconditions satisfied:
// customer config, enrolled host, WG endpoint record, bound WG peer. The logger is captured so
// tests can grep-assert the secret never reaches it.
func newPBSDRServer(t *testing.T, fake *fakeTenancy) (*Server, *store.Store, *bytes.Buffer) {
t.Helper()
s, st := newTestServer(t)
logBuf := &bytes.Buffer{}
s.logger = log.New(logBuf, "", 0)
if fake != nil {
s.SetTenantSync(fake)
}
if err := st.SaveCustomerConfig(&store.CustomerConfig{
CustomerID: "peti", APIKey: "capi", RetrievalPassword: "pw",
}); err != nil {
t.Fatalf("seed config: %v", err)
}
if err := st.UpsertHost(&store.Host{HostID: "peti-01", CustomerID: "peti", APIKey: "hapi"}); err != nil {
t.Fatalf("seed host: %v", err)
}
if err := st.SetWGEndpoint(&store.WGEndpoint{
EndpointID: "ep0", DNSName: "ep0.felhom.eu", WGPort: 443,
ServerPubkey: "SPK", TunnelSubnet: "10.77.0.0/24", PBSTunnelIP: "10.77.0.1",
}); err != nil {
t.Fatalf("seed endpoint: %v", err)
}
if _, _, err := st.RegisterWGPeerForHost("peti-01", "PETIPUBKEY"); err != nil {
t.Fatalf("seed wg peer: %v", err)
}
return s, st, logBuf
}
// postUpdate drives the REAL handler pipeline (handleConfigUpdate → applyPBSDR → save).
func postUpdate(t *testing.T, s *Server, form url.Values) *httptest.ResponseRecorder {
t.Helper()
// v0.48.0 edit-a: the update handler now enforces the form's required fields server-side —
// supply them like every real submission does (the browser form marks both `required`).
if form.Get("customer_name") == "" {
form.Set("customer_name", "Peti")
}
if form.Get("domain") == "" {
form.Set("domain", "peti.example")
}
req := httptest.NewRequest("POST", "/configs/peti", strings.NewReader(form.Encode()))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
rr := httptest.NewRecorder()
s.handleConfigUpdate(rr, req, "peti")
return rr
}
func hostState(t *testing.T, st *store.Store) (desc *pbsDRDescriptor, desiredJSON string, gen int64) {
t.Helper()
h, err := st.GetHost("peti-01")
if err != nil || h == nil {
t.Fatalf("host read: %v", err)
}
return readPBSDR(h.DesiredJSON), h.DesiredJSON, h.DesiredGeneration
}
func TestPBSDR_ProvisionHappyPath(t *testing.T) {
fake := &fakeTenancy{secret: "SUPER-SECRET-TOKEN"}
s, st, logBuf := newPBSDRServer(t, fake)
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}, "pbsdr_storage_id": {"felhom-pbs"}})
if rr.Code != 303 {
t.Fatalf("save = %d (%s), want 303", rr.Code, rr.Body.String())
}
if fake.provisionCalls != 1 {
t.Errorf("provision calls = %d, want 1", fake.provisionCalls)
}
desc, desiredJSON, gen := hostState(t, st)
if desc == nil {
t.Fatalf("no pbs_dr descriptor in desired_json: %s", desiredJSON)
}
if !desc.Enabled || desc.StorageID != "felhom-pbs" || desc.PBSTunnelIP != "10.77.0.1" ||
desc.Datastore != "felhom-offsite" || desc.Namespace != "peti" ||
desc.TokenID != "felhom@pbs!peti" || desc.Fingerprint != "aa:bb:cc" {
t.Errorf("descriptor wrong: %+v", desc)
}
if gen != 1 {
t.Errorf("desired_generation = %d, want 1 (exactly one bump)", gen)
}
// The secret: stored consume-once for the HOST…
got, err := st.ConsumeHostPBSSecret("peti-01")
if err != nil || got != "SUPER-SECRET-TOKEN" {
t.Fatalf("consume = (%q, %v), want the stored secret", got, err)
}
// …and NOWHERE else: not in the desired-state, not in ConfigJSON, not in any log line.
if strings.Contains(desiredJSON, "SUPER-SECRET-TOKEN") {
t.Error("secret leaked into desired_json")
}
cfg, _ := st.GetCustomerConfig("peti")
if strings.Contains(cfg.ConfigJSON, "SUPER-SECRET-TOKEN") || strings.Contains(cfg.ConfigJSON, "pbs_dr") {
t.Errorf("ConfigJSON must carry neither the secret nor the descriptor: %s", cfg.ConfigJSON)
}
if strings.Contains(logBuf.String(), "SUPER-SECRET-TOKEN") {
t.Error("secret leaked into the hub log")
}
}
func TestPBSDR_ResaveIsNoOp(t *testing.T) {
fake := &fakeTenancy{secret: "SUPER-SECRET-TOKEN"}
s, st, _ := newPBSDRServer(t, fake)
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != nil {
t.Fatalf("first secret consume: %v", err)
}
// Idempotent re-save: NO re-key, NO second secret row, NO spurious generation bump.
// (Red-proof: dropping the already-provisioned short-circuit in applyPBSDR re-runs
// Provision → calls=2 + a fresh consumable secret → FAIL.)
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if rr.Code != 303 {
t.Fatalf("re-save = %d, want 303 (already-provisioned is success-no-op)", rr.Code)
}
if fake.provisionCalls != 1 {
t.Errorf("provision calls after re-save = %d, want 1 (token must not rotate)", fake.provisionCalls)
}
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows {
t.Errorf("re-save created a fresh secret (consume err %v, want ErrNoRows)", err)
}
if _, _, gen := hostState(t, st); gen != 1 {
t.Errorf("generation after re-save = %d, want 1 (no spurious bump)", gen)
}
}
// v0.51.0 (DR-tier-by-default): an UNMET PRECONDITION is an honest waiting stage, not a
// save-blocking error — the flag is stored and the cascade converges later. A REAL provisioning
// failure stays fail-closed. Red-proof partner for the coupling: drop the WG-peer stage check in
// pbsdrProvisionAtom → "no WG peer waits" fails (provision would be reached).
func TestPBSDR_FailClosed(t *testing.T) {
t.Run("no WG peer waits (stage, not error)", func(t *testing.T) {
fake := &fakeTenancy{secret: "S"}
s, st, logBuf := newPBSDRServer(t, fake)
// Kill the precondition: the host has never registered a WG key.
if err := st.RemoveWGPeer("PETIPUBKEY"); err != nil {
t.Fatalf("remove seed peer: %v", err)
}
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if rr.Code != 303 {
t.Fatalf("save without WG peer = %d (%s), want 303 — the flag stores, the cascade waits", rr.Code, rr.Body.String())
}
if fake.provisionCalls != 0 {
t.Errorf("provision reached despite missing peer (%d calls)", fake.provisionCalls)
}
h, _ := st.GetHost("peti-01")
if readPBSDR(h.DesiredJSON) != nil || h.DesiredGeneration != 0 {
t.Errorf("descriptor written while waiting: desc=%v gen=%d", readPBSDR(h.DesiredJSON), h.DesiredGeneration)
}
// The flag itself IS saved (the intent survives the wait).
cfg, _ := st.GetCustomerConfig("peti")
if !cfg.DRTier {
t.Error("DR flag not stored while the cascade waits")
}
if !strings.Contains(logBuf.String(), "has not reported a WG key yet") {
t.Error("the waiting stage (guard wording) was not logged")
}
})
t.Run("tenantsync error", func(t *testing.T) {
fake := &fakeTenancy{err: errors.New("ssh boom")}
s, st, _ := newPBSDRServer(t, fake)
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if rr.Code != 502 {
t.Fatalf("save with tenantsync error = %d, want 502", rr.Code)
}
desc, _, gen := hostState(t, st)
if desc != nil || gen != 0 {
t.Errorf("fail-closed violated: desc=%+v gen=%d", desc, gen)
}
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows {
t.Errorf("a secret exists after a failed provision (err %v)", err)
}
// Fail-closed also means the config save itself was aborted.
cfg, _ := st.GetCustomerConfig("peti")
if cfg.CustomerName == "half-saved" {
t.Error("config row saved despite provisioning failure")
}
})
t.Run("token_exists points at re-issue", func(t *testing.T) {
fake := &fakeTenancy{err: tenantsync.ErrTokenExists}
s, st, _ := newPBSDRServer(t, fake)
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if rr.Code != 502 || !strings.Contains(rr.Body.String(), "Re-issue") {
t.Fatalf("token_exists = %d (%s), want 502 mentioning Re-issue", rr.Code, rr.Body.String())
}
if desc, _, gen := hostState(t, st); desc != nil || gen != 0 {
t.Errorf("state written on token_exists: desc=%+v gen=%d", desc, gen)
}
_ = st
})
t.Run("not configured waits (stage, not error)", func(t *testing.T) {
s, st, _ := newPBSDRServer(t, nil) // no tenantsync
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if rr.Code != 303 {
t.Fatalf("save without tenantsync = %d, want 303 — the flag stores, provisioning waits", rr.Code)
}
if desc, _, gen := hostState(t, st); desc != nil || gen != 0 {
t.Errorf("state written without a provisioner: desc=%+v gen=%d", desc, gen)
}
})
}
// Scenario A (hands-free cascade): flag ON while the WG peer is missing → the save waits; the
// host's WG registration then fires PBSDRAutoProvision and the descriptor lands with ZERO
// operator steps. Red-proof partner: unhook the atom from PBSDRAutoProvision (early return) →
// this fails at "descriptor after WG registration".
func TestPBSDR_AutoProvisionOnWGRegistration(t *testing.T) {
fake := &fakeTenancy{secret: "AUTO-SECRET"}
s, st, logBuf := newPBSDRServer(t, fake)
if err := st.RemoveWGPeer("PETIPUBKEY"); err != nil {
t.Fatalf("remove seed peer: %v", err)
}
// 1. Flag ON, peer missing → stored intent, no descriptor.
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
if desc, _, _ := hostState(t, st); desc != nil {
t.Fatalf("descriptor exists before the WG peer: %+v", desc)
}
// 2. The agent registers its WG key (what the api handler does) → the hook fires.
if _, _, err := st.RegisterWGPeerForHost("peti-01", "PETIPUBKEY"); err != nil {
t.Fatalf("register peer: %v", err)
}
s.PBSDRAutoProvision(context.Background(), "peti")
desc, desiredJSON, _ := hostState(t, st)
if desc == nil || !desc.Enabled || desc.Namespace != "peti" {
t.Fatalf("descriptor after WG registration = %+v (json %s), want provisioned", desc, desiredJSON)
}
if fake.provisionCalls != 1 {
t.Errorf("provision calls = %d, want 1", fake.provisionCalls)
}
if got, err := st.ConsumeHostPBSSecret("peti-01"); err != nil || got != "AUTO-SECRET" {
t.Fatalf("consume-once secret after auto-provision = (%q, %v)", got, err)
}
if strings.Contains(logBuf.String(), "AUTO-SECRET") {
t.Error("secret leaked into the hub log")
}
// 3. Idempotent: a second hook firing (re-registration) must not re-key.
s.PBSDRAutoProvision(context.Background(), "peti")
if fake.provisionCalls != 1 {
t.Errorf("second hook firing re-provisioned (%d calls)", fake.provisionCalls)
}
// 4. Flag OFF → the hook never provisions (scenario B: off is off).
fake2 := &fakeTenancy{secret: "S2"}
s2, st2, _ := newPBSDRServer(t, fake2)
cfg, _ := st2.GetCustomerConfig("peti")
cfg.DRTier = false
if err := st2.SaveCustomerConfig(cfg); err != nil {
t.Fatalf("save flag-off config: %v", err)
}
s2.PBSDRAutoProvision(context.Background(), "peti")
if fake2.provisionCalls != 0 {
t.Errorf("hook provisioned with the DR flag OFF (%d calls)", fake2.provisionCalls)
}
}
func TestPBSDR_DisableKeepsTenancy(t *testing.T) {
fake := &fakeTenancy{secret: "S"}
s, st, _ := newPBSDRServer(t, fake)
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
// Unchecked box → descriptor enabled:false, coords kept, ONE bump; no endpoint mutation.
rr := postUpdate(t, s, url.Values{})
if rr.Code != 303 {
t.Fatalf("disable save = %d, want 303", rr.Code)
}
desc, _, gen := hostState(t, st)
if desc == nil || desc.Enabled || desc.Namespace != "peti" || desc.TokenID == "" {
t.Fatalf("disable must keep coords with enabled=false: %+v", desc)
}
if gen != 2 {
t.Errorf("generation = %d, want 2 (enable + disable)", gen)
}
// A second disabled save is a pure no-op.
postUpdate(t, s, url.Values{})
if _, _, gen := hostState(t, st); gen != 2 {
t.Errorf("second disabled save bumped generation to %d", gen)
}
if fake.provisionCalls != 1 || fake.reissueCalls != 0 {
t.Errorf("endpoint touched on disable: provision=%d reissue=%d", fake.provisionCalls, fake.reissueCalls)
}
}
func TestPBSDR_StorageIDChangeUpdatesDescriptorOnly(t *testing.T) {
fake := &fakeTenancy{secret: "S"}
s, st, _ := newPBSDRServer(t, fake)
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
st.ConsumeHostPBSSecret("peti-01") // spend the provision secret
rr := postUpdate(t, s, url.Values{"dr_tier": {"on"}, "pbsdr_storage_id": {"felhom-offsite"}})
if rr.Code != 303 {
t.Fatalf("storage-id change = %d, want 303", rr.Code)
}
desc, _, gen := hostState(t, st)
if desc.StorageID != "felhom-offsite" || !desc.Enabled {
t.Errorf("descriptor not updated: %+v", desc)
}
if gen != 2 {
t.Errorf("generation = %d, want 2", gen)
}
if fake.provisionCalls != 1 {
t.Errorf("a descriptor edit re-provisioned the tenancy (%d calls)", fake.provisionCalls)
}
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows {
t.Errorf("a descriptor edit staged a new secret (err %v)", err)
}
}
func TestPBSDR_Reissue(t *testing.T) {
fake := &fakeTenancy{secret: "OLD-SECRET"}
s, st, logBuf := newPBSDRServer(t, fake)
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
st.ConsumeHostPBSSecret("peti-01") // agent already consumed; the dead-end scenario
fake.secret = "FRESH-SECRET"
req := httptest.NewRequest("POST", "/configs/peti/pbsdr-reissue", nil)
rr := httptest.NewRecorder()
s.handlePBSDRReissue(rr, req, "peti")
if rr.Code != 303 {
t.Fatalf("reissue = %d (%s), want 303", rr.Code, rr.Body.String())
}
if fake.reissueCalls != 1 {
t.Errorf("reissue calls = %d, want 1", fake.reissueCalls)
}
got, err := st.ConsumeHostPBSSecret("peti-01")
if err != nil || got != "FRESH-SECRET" {
t.Fatalf("fresh secret consume = (%q, %v)", got, err)
}
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != sql.ErrNoRows {
t.Error("the old secret path survived the re-issue")
}
if _, _, gen := hostState(t, st); gen != 2 {
t.Errorf("generation = %d, want 2 (provision + reissue)", gen)
}
if strings.Contains(logBuf.String(), "FRESH-SECRET") || strings.Contains(logBuf.String(), "OLD-SECRET") {
t.Error("secret leaked into the hub log")
}
}
func TestPBSDR_ReissueRequiresProvisionedState(t *testing.T) {
fake := &fakeTenancy{secret: "S"}
s, _, _ := newPBSDRServer(t, fake)
req := httptest.NewRequest("POST", "/configs/peti/pbsdr-reissue", nil)
rr := httptest.NewRecorder()
s.handlePBSDRReissue(rr, req, "peti")
if rr.Code != 400 {
t.Fatalf("reissue without a provisioned tier = %d, want 400", rr.Code)
}
if fake.reissueCalls != 0 {
t.Errorf("endpoint touched without a descriptor (%d calls)", fake.reissueCalls)
}
}
// The form render leg: the section reflects the provisioned descriptor (checkbox + namespace line)
// and the fresh-form default storage id.
func TestPBSDR_FormRendersState(t *testing.T) {
fake := &fakeTenancy{secret: "S"}
s, _, _ := newPBSDRServer(t, fake)
postUpdate(t, s, url.Values{"dr_tier": {"on"}})
// v0.48.0 edit-a: the standalone GET edit page is a redirect now — the form renders embedded
// in the customer page's Edit tab (the same config_form_body sub-template), so assert there.
req := httptest.NewRequest("GET", "/customers/peti", nil)
rr := httptest.NewRecorder()
s.handleCustomerUnified(rr, req, "peti")
out := rr.Body.String()
if !strings.Contains(out, `name="dr_tier" checked`) {
t.Error("DR-tier checkbox not checked after provisioning")
}
if !strings.Contains(out, "descriptor provisioned (namespace peti, token felhom@pbs!peti)") {
t.Error("provisioned cascade stage line missing")
}
// The cascade (v0.51.0 scenario D): all four stages render as done on a fully-applied box.
if strings.Count(out, `class="badge badge-neutral">waiting</span>`) > 1 {
// escrow may still be waiting in this harness; every other stage must be done
t.Errorf("more than the escrow stage still waiting:\n%s", out[strings.Index(out, "DR tier (PBS, ep0)"):][:1200])
}
if !strings.Contains(out, "pbsdr-reissue") {
t.Error("re-issue button missing")
}
if strings.Contains(out, "S\"") && strings.Contains(out, "token_secret") {
t.Error("secret-ish content rendered")
}
// Deadline check on the detached-ctx behavior is out of scope here; the view path must
// stay read-only — a render must not have touched provisioning.
if fake.provisionCalls != 1 {
t.Errorf("rendering the form called provision (%d calls)", fake.provisionCalls)
}
_ = time.Second
}