hub v0.56.0: PBS-DR self-heal reconciler (re-stage a consumable secret)

Implements SPIKE-pbsdr-selfheal-2026-07-15 (e8f8c44). A box re-installed/rolled back onto its stable host_id loses its agent-side converged marker; the hub keeps the enabled descriptor + a CONSUMED one-time secret, the WG peer persists (changed==false, cascade can't re-fire), so the agent sits in waiting_secret forever. The missing piece is a consumable secret, not the descriptor. New internal/pbsdrheal reconciler (5m, wgsync shape): for enabled+provisioned hosts whose latest report pbs_dr.state is a stuck state past a >=2-distinct-report debounce, re-stage the stored secret (store.RestageHostPBSSecret: clear consumed_at, no ep0 call, NO generation bump); escalate to Re-issue (web ReissuePBSDR) only when no secret is stored or the agent reports consumed_failed. Converged/disabled/verify_failed/DR-OFF = no-op. PBSDRHEAL_ONLY_HOST scopes a supervised rollout. Scenarios A-F + all six red-proofs verified. No agent change. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HEPuEwyyGDJdcsXLFsTWJn
2026-07-15 18:27:39 +02:00
parent e8f8c441fa
commit 6218e7919d
9 changed files with 878 additions and 43 deletions
@@ -1,5 +1,10 @@
 package store

+import (
+	"database/sql"
+	"encoding/json"
+)
+
 // PBS DR tier (SLICE 1): the HOST-scoped one-time PBS token secret — the host/agent twin of the
 // customer-scoped one_time_secrets pair (SaveOneTimeSecret/ConsumeOneTimeSecret). The hub stores
 // the tenantsync-returned token secret here; the agent consumes it EXACTLY ONCE with its per-host
@@ -38,3 +43,89 @@ func (s *Store) ConsumeHostPBSSecret(hostID string) (string, error) {
 	}
 	return value, nil
 }
+
+// RestageHostPBSSecret re-arms an ALREADY-STORED one-time PBS secret for re-consumption by clearing
+// its consumed flag — WITHOUT changing the secret value, WITHOUT inserting a row, and WITHOUT any
+// endpoint/token interaction. This is the PBS-DR self-heal primitive
+// (SPIKE-pbsdr-selfheal-2026-07-15, e8f8c44): a box that lost its converged marker (re-install /
+// snapshot rollback) but whose ep0 token is still valid re-consumes the SAME secret and converges in
+// one agent tick (spike SQ-2b′, zero ep0 churn). Returns restaged=true when a row existed (its
+// consumed flag is now cleared, so ConsumeHostPBSSecret will serve it once); restaged=false when NO
+// secret is stored for the host — the caller must escalate to Re-issue (a fresh mint). The value is
+// never read or logged here, and NO host generation is bumped (a re-stage changes no descriptor
+// content — a bump would trigger an agent desired-state refetch loop).
+func (s *Store) RestageHostPBSSecret(hostID string) (bool, error) {
+	res, err := s.db.Exec(`UPDATE host_pbs_secrets SET consumed_at = NULL WHERE host_id = ?`, hostID)
+	if err != nil {
+		return false, err
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return false, err
+	}
+	return n > 0, nil
+}
+
+// PBSDRHealRow is the per-host self-heal decision input (SPIKE-pbsdr-selfheal-2026-07-15): the host's
+// descriptor enable/provision state (parsed from desired_json) joined to the agent's latest reported
+// pbs_dr.state (from the newest host_report). Mirrors GetHostOOBStates' latest-report-per-host shape.
+type PBSDRHealRow struct {
+	HostID                string
+	CustomerID            string
+	DescriptorEnabled     bool   // desired_json pbs_dr.enabled — the DR-ON reality
+	DescriptorProvisioned bool   // desired_json pbs_dr.namespace != "" (was provisioned, not a bare enable)
+	ReportedState         string // latest report pbs_dr.state ("" = no report / no stanza)
+	ReportID              int64  // id of that latest host_report (0 = none); the debounce distinctness key
+}
+
+// PBSDRHealStates returns one row per host: its descriptor enable/provision flags + the agent's
+// latest reported pbs_dr.state + that report's id. Malformed JSON degrades to zero values (never an
+// error) — the reconciler only acts on positively-parsed stuck states. One query for the whole fleet
+// (the reconciler filters to enabled+provisioned hosts).
+func (s *Store) PBSDRHealStates() ([]PBSDRHealRow, error) {
+	rows, err := s.db.Query(`
+		SELECT h.host_id, h.customer_id, h.desired_json, latest.mx, hr.report_json
+		FROM hosts h
+		LEFT JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest
+		       ON latest.host_id = h.host_id
+		LEFT JOIN host_reports hr ON hr.id = latest.mx`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []PBSDRHealRow
+	for rows.Next() {
+		var hostID, customerID, desiredJSON string
+		var reportID sql.NullInt64
+		var reportJSON sql.NullString
+		if err := rows.Scan(&hostID, &customerID, &desiredJSON, &reportID, &reportJSON); err != nil {
+			return nil, err
+		}
+		r := PBSDRHealRow{HostID: hostID, CustomerID: customerID}
+		var dd struct {
+			PBSDR *struct {
+				Enabled   bool   `json:"enabled"`
+				Namespace string `json:"namespace"`
+			} `json:"pbs_dr"`
+		}
+		if json.Unmarshal([]byte(desiredJSON), &dd) == nil && dd.PBSDR != nil {
+			r.DescriptorEnabled = dd.PBSDR.Enabled
+			r.DescriptorProvisioned = dd.PBSDR.Namespace != ""
+		}
+		if reportID.Valid {
+			r.ReportID = reportID.Int64
+		}
+		if reportJSON.Valid {
+			var rr struct {
+				PBSDR *struct {
+					State string `json:"state"`
+				} `json:"pbs_dr"`
+			}
+			if json.Unmarshal([]byte(reportJSON.String), &rr) == nil && rr.PBSDR != nil {
+				r.ReportedState = rr.PBSDR.State
+			}
+		}
+		out = append(out, r)
+	}
+	return out, rows.Err()
+}