admin/felhom.eu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hub v0.56.0: PBS-DR self-heal reconciler (re-stage a consumable secret)

Browse Source 
Implements SPIKE-pbsdr-selfheal-2026-07-15 (e8f8c44). A box re-installed/rolled
back onto its stable host_id loses its agent-side converged marker; the hub
keeps the enabled descriptor + a CONSUMED one-time secret, the WG peer persists
(changed==false, cascade can't re-fire), so the agent sits in waiting_secret
forever. The missing piece is a consumable secret, not the descriptor.

New internal/pbsdrheal reconciler (5m, wgsync shape): for enabled+provisioned
hosts whose latest report pbs_dr.state is a stuck state past a >=2-distinct-report
debounce, re-stage the stored secret (store.RestageHostPBSSecret: clear
consumed_at, no ep0 call, NO generation bump); escalate to Re-issue (web
ReissuePBSDR) only when no secret is stored or the agent reports consumed_failed.
Converged/disabled/verify_failed/DR-OFF = no-op. PBSDRHEAL_ONLY_HOST scopes a
supervised rollout. Scenarios A-F + all six red-proofs verified. No agent change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HEPuEwyyGDJdcsXLFsTWJn
This commit is contained in:
authentik Default Admin
2026-07-15 18:27:39 +02:00
parent e8f8c441fa
commit 6218e7919d
9 changed files with 878 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hub/internal/store/pbsdr.go
+91
View File
@@ -1,5 +1,10 @@
package store
import (
"database/sql"
"encoding/json"
)
// PBS DR tier (SLICE 1): the HOST-scoped one-time PBS token secret — the host/agent twin of the
// customer-scoped one_time_secrets pair (SaveOneTimeSecret/ConsumeOneTimeSecret). The hub stores
// the tenantsync-returned token secret here; the agent consumes it EXACTLY ONCE with its per-host
@@ -38,3 +43,89 @@ func (s *Store) ConsumeHostPBSSecret(hostID string) (string, error) {
}
return value, nil
}
// RestageHostPBSSecret re-arms an ALREADY-STORED one-time PBS secret for re-consumption by clearing
// its consumed flag — WITHOUT changing the secret value, WITHOUT inserting a row, and WITHOUT any
// endpoint/token interaction. This is the PBS-DR self-heal primitive
// (SPIKE-pbsdr-selfheal-2026-07-15, e8f8c44): a box that lost its converged marker (re-install /
// snapshot rollback) but whose ep0 token is still valid re-consumes the SAME secret and converges in
// one agent tick (spike SQ-2b′, zero ep0 churn). Returns restaged=true when a row existed (its
// consumed flag is now cleared, so ConsumeHostPBSSecret will serve it once); restaged=false when NO
// secret is stored for the host — the caller must escalate to Re-issue (a fresh mint). The value is
// never read or logged here, and NO host generation is bumped (a re-stage changes no descriptor
// content — a bump would trigger an agent desired-state refetch loop).
func (s *Store) RestageHostPBSSecret(hostID string) (bool, error) {
res, err := s.db.Exec(`UPDATE host_pbs_secrets SET consumed_at = NULL WHERE host_id = ?`, hostID)
if err != nil {
return false, err
}
n, err := res.RowsAffected()
if err != nil {
return false, err
}
return n > 0, nil
}
// PBSDRHealRow is the per-host self-heal decision input (SPIKE-pbsdr-selfheal-2026-07-15): the host's
// descriptor enable/provision state (parsed from desired_json) joined to the agent's latest reported
// pbs_dr.state (from the newest host_report). Mirrors GetHostOOBStates' latest-report-per-host shape.
type PBSDRHealRow struct {
HostID string
CustomerID string
DescriptorEnabled bool // desired_json pbs_dr.enabled — the DR-ON reality
DescriptorProvisioned bool // desired_json pbs_dr.namespace != "" (was provisioned, not a bare enable)
ReportedState string // latest report pbs_dr.state ("" = no report / no stanza)
ReportID int64 // id of that latest host_report (0 = none); the debounce distinctness key
}
// PBSDRHealStates returns one row per host: its descriptor enable/provision flags + the agent's
// latest reported pbs_dr.state + that report's id. Malformed JSON degrades to zero values (never an
// error) — the reconciler only acts on positively-parsed stuck states. One query for the whole fleet
// (the reconciler filters to enabled+provisioned hosts).
func (s *Store) PBSDRHealStates() ([]PBSDRHealRow, error) {
rows, err := s.db.Query(`
SELECT h.host_id, h.customer_id, h.desired_json, latest.mx, hr.report_json
FROM hosts h
LEFT JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
ON latest.host_id = h.host_id
LEFT JOIN host_reports hr ON hr.id = latest.mx`)
if err != nil {
return nil, err
}
defer rows.Close()
var out []PBSDRHealRow
for rows.Next() {
var hostID, customerID, desiredJSON string
var reportID sql.NullInt64
var reportJSON sql.NullString
if err := rows.Scan(&hostID, &customerID, &desiredJSON, &reportID, &reportJSON); err != nil {
return nil, err
}
r := PBSDRHealRow{HostID: hostID, CustomerID: customerID}
var dd struct {
PBSDR *struct {
Enabled bool `json:"enabled"`
Namespace string `json:"namespace"`
} `json:"pbs_dr"`
}
if json.Unmarshal([]byte(desiredJSON), &dd) == nil && dd.PBSDR != nil {
r.DescriptorEnabled = dd.PBSDR.Enabled
r.DescriptorProvisioned = dd.PBSDR.Namespace != ""
}
if reportID.Valid {
r.ReportID = reportID.Int64
}
if reportJSON.Valid {
var rr struct {
PBSDR *struct {
State string `json:"state"`
} `json:"pbs_dr"`
}
if json.Unmarshal([]byte(reportJSON.String), &rr) == nil && rr.PBSDR != nil {
r.ReportedState = rr.PBSDR.State
}
}
out = append(out, r)
}
return out, rows.Err()
}
hub/internal/store/pbsdr_test.go
+111
View File
@@ -5,6 +5,117 @@ import (
"testing"
)
// RestageHostPBSSecret re-arms a stored (consumed) secret WITHOUT changing its value, inserting a
// row, or bumping any generation. Returns restaged=false when no row exists.
func TestRestageHostPBSSecret(t *testing.T) {
s := newTestStore(t)
// No row → restaged=false, no error (the caller escalates to Re-issue).
if restaged, err := s.RestageHostPBSSecret("h1"); err != nil || restaged {
t.Fatalf("restage with nothing stored = (%v, %v), want (false, nil)", restaged, err)
}
// Store + consume, then re-stage: the SAME value is served once more.
if err := s.SaveHostPBSSecret("h1", "the-secret"); err != nil {
t.Fatalf("save: %v", err)
}
if _, err := s.ConsumeHostPBSSecret("h1"); err != nil {
t.Fatalf("first consume: %v", err)
}
if _, err := s.ConsumeHostPBSSecret("h1"); err != sql.ErrNoRows {
t.Fatalf("pre-restage second consume = %v, want ErrNoRows", err)
}
restaged, err := s.RestageHostPBSSecret("h1")
if err != nil || !restaged {
t.Fatalf("restage of a stored secret = (%v, %v), want (true, nil)", restaged, err)
}
// Red-proof: dropping the `SET consumed_at = NULL` makes this consume return ErrNoRows.
got, err := s.ConsumeHostPBSSecret("h1")
if err != nil || got != "the-secret" {
t.Fatalf("post-restage consume = (%q, %v), want (the-secret, nil) — same value, re-armed", got, err)
}
}
// A-gen guard: a re-stage touches ONLY host_pbs_secrets — never the host generation (a bump would
// trigger an agent desired-state refetch loop). Red-proof: adding a SetHostDesired/gen bump to
// RestageHostPBSSecret makes this assert fail.
func TestRestageHostPBSSecret_NoGenerationBump(t *testing.T) {
s := newTestStore(t)
if err := s.UpsertHost(&Host{HostID: "h1", CustomerID: "c1", APIKey: "k"}); err != nil {
t.Fatalf("host: %v", err)
}
genBefore, err := s.SetHostDesired("h1", []byte(`{"pbs_dr":{"enabled":true,"namespace":"c1"}}`))
if err != nil {
t.Fatalf("set desired: %v", err)
}
if err := s.SaveHostPBSSecret("h1", "s"); err != nil {
t.Fatalf("save secret: %v", err)
}
if _, err := s.RestageHostPBSSecret("h1"); err != nil {
t.Fatalf("restage: %v", err)
}
h, err := s.GetHost("h1")
if err != nil {
t.Fatalf("get host: %v", err)
}
if h.DesiredGeneration != genBefore {
t.Fatalf("generation changed by a re-stage: %d -> %d (a re-stage must never bump)", genBefore, h.DesiredGeneration)
}
}
// PBSDRHealStates joins each host's descriptor (enabled/provisioned) to its LATEST report's
// pbs_dr.state + report id.
func TestPBSDRHealStates(t *testing.T) {
s := newTestStore(t)
// Host A: enabled + provisioned, latest report waiting_secret (after an older applied report).
mustHost(t, s, "hA", "cA", `{"pbs_dr":{"enabled":true,"namespace":"cA","storage_id":"felhom-pbs"}}`)
mustReport(t, s, "hA", "cA", `{"pbs_dr":{"state":"applied"}}`)
mustReport(t, s, "hA", "cA", `{"pbs_dr":{"state":"waiting_secret"}}`) // newer → wins
// Host B: descriptor disabled.
mustHost(t, s, "hB", "cB", `{"pbs_dr":{"enabled":false,"namespace":"cB"}}`)
mustReport(t, s, "hB", "cB", `{"pbs_dr":{"state":"disabled"}}`)
// Host C: enabled but NOT provisioned (namespace empty); no report.
mustHost(t, s, "hC", "cC", `{"pbs_dr":{"enabled":true}}`)
rows, err := s.PBSDRHealStates()
if err != nil {
t.Fatalf("PBSDRHealStates: %v", err)
}
byHost := map[string]PBSDRHealRow{}
for _, r := range rows {
byHost[r.HostID] = r
}
a := byHost["hA"]
if !a.DescriptorEnabled || !a.DescriptorProvisioned || a.ReportedState != "waiting_secret" || a.ReportID == 0 {
t.Errorf("hA = %+v, want enabled+provisioned+waiting_secret+reportID>0", a)
}
b := byHost["hB"]
if b.DescriptorEnabled || b.ReportedState != "disabled" {
t.Errorf("hB = %+v, want disabled descriptor + state disabled", b)
}
c := byHost["hC"]
if !c.DescriptorEnabled || c.DescriptorProvisioned || c.ReportedState != "" || c.ReportID != 0 {
t.Errorf("hC = %+v, want enabled+unprovisioned+no-report", c)
}
}
func mustHost(t *testing.T, s *Store, hostID, customerID, desiredJSON string) {
t.Helper()
if err := s.UpsertHost(&Host{HostID: hostID, CustomerID: customerID, APIKey: "k-" + hostID}); err != nil {
t.Fatalf("UpsertHost %s: %v", hostID, err)
}
if _, err := s.SetHostDesired(hostID, []byte(desiredJSON)); err != nil {
t.Fatalf("SetHostDesired %s: %v", hostID, err)
}
}
func mustReport(t *testing.T, s *Store, hostID, customerID, reportJSON string) {
t.Helper()
if err := s.SaveHostReport(hostID, customerID, []byte(reportJSON), HostReportDenorm{AgentVersion: "0.88.0"}); err != nil {
t.Fatalf("SaveHostReport %s: %v", hostID, err)
}
}
// The host-scoped consume-once contract (PBS DR SLICE 1): exactly one read per stored value,
// a re-save resets the consumed flag (re-issue supersedes), absence is sql.ErrNoRows.
func TestHostPBSSecret_ConsumeOnce(t *testing.T) {