hub: R-39 core — stamp a secret GENERATION into the pbs_dr descriptor
The fleet half of R-39. An ep0 credential re-issue re-keys the SECRET of an existing token, so token_id, fingerprint, datastore and namespace all come back byte-identical. The agent re-applies on the descriptor's CONTENT HASH, so a re-issue was invisible to a converged box: it short-circuited, never consumed the fresh secret, and served a revoked credential while reporting `applied` — the N100 failure of 2026-07-18. host_pbs_secrets gains a monotonic per-host `generation`, advanced by every fresh MINT and by nothing else, stamped into the descriptor as `secret_generation`. That is now the only field a re-key moves, and it is what re-arms the agent. DEVIATION FROM SPEC, deliberate: the brief said to return "the new row's id (int64) … no schema change". There is no row id — host_pbs_secrets is keyed by host_id and UPSERTed last-write-wins, so a new row never exists, and created_at collides for two mints in the same second. An additive counter column is the only monotonic source; it uses the repo's existing idempotent ALTER-TABLE idiom. RestageHostPBSSecret deliberately does NOT advance it: a re-stage re-arms the SAME secret, the descriptor content genuinely has not changed, and a bump would cause a pointless agent refetch loop (that method's own contract says so). Also corrects a comment that asserted the re-issue refreshes the descriptor "with the NEW token_id/fingerprint". That is false for a re-key, and believing it is why the descriptor was never expected to be identical in the first place. omitempty is load-bearing: a zero generation must not start emitting a new key into every pre-existing descriptor, which would itself be a fleet-wide spurious re-apply. Compatibility: agents below 0.91.0 drop the unknown JSON key and behave exactly as today — inert, not breaking (Scenario C). Tests: store-level monotonicity + per-host isolation + restage-leaves-it-alone; descriptor byte-change, omitempty, and sibling-key round-trip; and a FLOW-level test driving ReissuePBSDR against a fake that models a real re-key. Red-proof run at the assertion level (not the compiler): commenting out the stamp makes the flow test fail with both byte-identical blocks printed.
This commit is contained in:
+42
-14
@@ -58,6 +58,19 @@ type pbsDRDescriptor struct {
|
||||
Namespace string `json:"namespace,omitempty"`
|
||||
TokenID string `json:"token_id,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// SecretGeneration is the monotonic per-host counter advanced by every fresh secret MINT
|
||||
// (store.SaveHostPBSSecret). It carries no secret material — only the fact that one rotated.
|
||||
//
|
||||
// It exists because a re-key changes NOTHING else in this struct: an ep0 re-issue re-keys the
|
||||
// secret of an EXISTING token, so token_id/fingerprint/datastore/namespace stay byte-identical.
|
||||
// The agent re-applies on the descriptor's content hash, so without this field a re-issue is
|
||||
// invisible to a converged box — it short-circuits, never consumes the fresh secret, and serves a
|
||||
// revoked credential while reporting `applied` (R-39, N100 2026-07-18).
|
||||
//
|
||||
// MUST stay field-exact with felhom-agent internal/hub.WirePBSDR. Agents below 0.91.0 do not
|
||||
// carry the field, drop the unknown JSON key, and behave exactly as they do today — the field is
|
||||
// inert to them, not breaking (Scenario C); the re-arm guarantee needs agent >= 0.91.0.
|
||||
SecretGeneration int64 `json:"secret_generation,omitempty"`
|
||||
}
|
||||
|
||||
// defaultPBSStorageID is the storage-entry id the agent bridge creates on a customer box. The DEMO
|
||||
@@ -254,17 +267,19 @@ func (s *Server) pbsdrProvisionAtom(ctx context.Context, customerID string, host
|
||||
}
|
||||
|
||||
// The atom: secret first (consume-once custody), then descriptor+bump (the agent's signal).
|
||||
if err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret); err != nil {
|
||||
secretGen, err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("pbsdr: store one-time token secret: %w", err)
|
||||
}
|
||||
desc := &pbsDRDescriptor{
|
||||
Enabled: true,
|
||||
StorageID: storageID,
|
||||
PBSTunnelIP: ep.PBSTunnelIP,
|
||||
Datastore: res.Datastore,
|
||||
Namespace: res.Namespace,
|
||||
TokenID: res.TokenID,
|
||||
Fingerprint: res.Fingerprint,
|
||||
Enabled: true,
|
||||
StorageID: storageID,
|
||||
PBSTunnelIP: ep.PBSTunnelIP,
|
||||
Datastore: res.Datastore,
|
||||
Namespace: res.Namespace,
|
||||
TokenID: res.TokenID,
|
||||
Fingerprint: res.Fingerprint,
|
||||
SecretGeneration: secretGen,
|
||||
}
|
||||
merged, err := mergePBSDR(host.DesiredJSON, desc)
|
||||
if err != nil {
|
||||
@@ -316,8 +331,14 @@ func (s *Server) PBSDRAutoProvision(ctx context.Context, customerID string) {
|
||||
// ReissuePBSDR re-keys the customer's ep0 PBS token and re-arms the agent — the non-HTTP core shared
|
||||
// by the operator button (handlePBSDRReissue) and the pbsdrheal self-heal reconciler (the escalation
|
||||
// path when no stored secret is re-stageable, or the agent burned one and reports consumed_failed).
|
||||
// tenantsync reissue → fresh consume-once secret (SaveHostPBSSecret) → descriptor refresh with the
|
||||
// NEW token_id/fingerprint + generation bump (the agent's re-consume signal). This reuses the same
|
||||
// tenantsync reissue → fresh consume-once secret (SaveHostPBSSecret) → descriptor refresh + host
|
||||
// generation bump (the agent's re-consume signal).
|
||||
//
|
||||
// CORRECTION (R-39, v0.68.0): this comment used to claim the refresh carried "the NEW
|
||||
// token_id/fingerprint". That is FALSE for a re-key — ep0 rotates the SECRET of an existing token,
|
||||
// so token_id and fingerprint come back byte-identical and the re-assignments below are no-ops. That
|
||||
// false belief is the whole reason the descriptor never moved and the agent never re-consumed
|
||||
// (N100, 2026-07-18). The thing that actually changes the descriptor is SecretGeneration. This reuses the same
|
||||
// reissue op the handler does — it is NOT a re-run of pbsdrProvisionAtom (which refuses ErrTokenExists
|
||||
// and would not re-key). The secret value is never logged. Keep this in lockstep with the tail of
|
||||
// handlePBSDRReissue.
|
||||
@@ -343,13 +364,18 @@ func (s *Server) ReissuePBSDR(ctx context.Context, customerID string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("pbsdr reissue for %s: %w", customerID, err)
|
||||
}
|
||||
if err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret); err != nil {
|
||||
secretGen, err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pbsdr reissue for %s: store secret: %w", customerID, err)
|
||||
}
|
||||
// These four are re-assigned for completeness but are byte-identical on a re-key (see the
|
||||
// correction above). SecretGeneration is the field that actually moves the descriptor hash and
|
||||
// therefore re-arms the agent.
|
||||
cur.TokenID = res.TokenID
|
||||
cur.Fingerprint = res.Fingerprint
|
||||
cur.Datastore = res.Datastore
|
||||
cur.Namespace = res.Namespace
|
||||
cur.SecretGeneration = secretGen
|
||||
merged, err := mergePBSDR(host.DesiredJSON, cur)
|
||||
if err != nil {
|
||||
return fmt.Errorf("pbsdr reissue for %s: merge descriptor: %w", customerID, err)
|
||||
@@ -395,7 +421,8 @@ func (s *Server) handlePBSDRReissue(w http.ResponseWriter, r *http.Request, cust
|
||||
http.Error(w, "PBS credential re-issue failed: "+err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret); err != nil {
|
||||
secretGen, err := s.store.SaveHostPBSSecret(host.HostID, res.TokenSecret)
|
||||
if err != nil {
|
||||
s.logger.Printf("[ERROR] pbsdr reissue for %s: secret store: %v", customerID, err)
|
||||
http.Error(w, "Re-issued on the endpoint but storing the secret failed — re-issue again", http.StatusInternalServerError)
|
||||
return
|
||||
@@ -404,6 +431,7 @@ func (s *Server) handlePBSDRReissue(w http.ResponseWriter, r *http.Request, cust
|
||||
cur.Fingerprint = res.Fingerprint
|
||||
cur.Datastore = res.Datastore
|
||||
cur.Namespace = res.Namespace
|
||||
cur.SecretGeneration = secretGen // the ONLY field a re-key actually moves — see ReissuePBSDR
|
||||
merged, err := mergePBSDR(host.DesiredJSON, cur)
|
||||
if err == nil {
|
||||
_, err = s.store.SetHostDesired(host.HostID, []byte(merged))
|
||||
@@ -422,8 +450,8 @@ func (s *Server) handlePBSDRReissue(w http.ResponseWriter, r *http.Request, cust
|
||||
// cascade stages (host → WG peer → descriptor → escrow) — one flag, ordered rollout, honest
|
||||
// intermediate states (scenario D).
|
||||
type pbsDRView struct {
|
||||
Supported bool // tenantsync configured on this hub
|
||||
NoHost bool // no enrolled host for the customer (cascade stage 1 waiting)
|
||||
Supported bool // tenantsync configured on this hub
|
||||
NoHost bool // no enrolled host for the customer (cascade stage 1 waiting)
|
||||
HostID string
|
||||
DRTier bool // the stored per-customer flag (operator INTENT)
|
||||
Enabled bool // descriptor enabled (host-side reality)
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-39, Scenario B — the hub half of the fix: a re-issue must change the DESCRIPTOR BYTES.
|
||||
//
|
||||
// The agent re-applies only when the descriptor's content hash moves (felhom-agent
|
||||
// internal/pbsdr/manager.go descriptorHash marshals the parsed struct). An ep0 re-key rotates the
|
||||
// secret of an EXISTING token, so token_id / fingerprint / datastore / namespace all come back
|
||||
// byte-identical — which is precisely why the 2026-07-18 N100 box short-circuited forever. The only
|
||||
// field that moves is SecretGeneration.
|
||||
//
|
||||
// COMPANION RED-PROOF (run + recorded in REPORT.md): delete the `SecretGeneration` field from
|
||||
// pbsDRDescriptor (or stop setting it in the re-issue path) → the two marshals below become
|
||||
// byte-identical and this test FAILS, reproducing the defect exactly.
|
||||
func TestPBSDRDescriptor_ReissueChangesTheBytes(t *testing.T) {
|
||||
// The descriptor as it stands after the FIRST provision.
|
||||
before := &pbsDRDescriptor{
|
||||
Enabled: true,
|
||||
StorageID: "felhom-pbs",
|
||||
PBSTunnelIP: "10.77.0.1",
|
||||
Datastore: "felhom-offsite",
|
||||
Namespace: "demo-felhom",
|
||||
TokenID: "felhom@pbs!demo-felhom",
|
||||
Fingerprint: "c6:07:28:3f",
|
||||
SecretGeneration: 1,
|
||||
}
|
||||
// After a RE-KEY: everything the endpoint returns is identical — only the secret rotated.
|
||||
after := *before
|
||||
after.SecretGeneration = 2
|
||||
|
||||
b1, err := json.Marshal(before)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal before: %v", err)
|
||||
}
|
||||
b2, err := json.Marshal(&after)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal after: %v", err)
|
||||
}
|
||||
if string(b1) == string(b2) {
|
||||
t.Fatalf("a re-issue left the descriptor BYTE-IDENTICAL — the agent will short-circuit and never "+
|
||||
"consume the fresh secret (this is the R-39 defect).\n bytes: %s", b1)
|
||||
}
|
||||
|
||||
// And the difference must be exactly the generation — not an accident of field ordering.
|
||||
var m1, m2 map[string]any
|
||||
if err := json.Unmarshal(b1, &m1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(b2, &m2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, v1 := range m1 {
|
||||
if k == "secret_generation" {
|
||||
continue
|
||||
}
|
||||
if v2, ok := m2[k]; !ok || string(mustJSON(t, v1)) != string(mustJSON(t, v2)) {
|
||||
t.Errorf("a re-key must change ONLY secret_generation, but %q moved: %v -> %v", k, v1, m2[k])
|
||||
}
|
||||
}
|
||||
if m2["secret_generation"] == m1["secret_generation"] {
|
||||
t.Error("secret_generation did not advance")
|
||||
}
|
||||
}
|
||||
|
||||
// The field must be OMITTED when zero, so a hub that has never minted for a host does not start
|
||||
// emitting a new key into every legacy descriptor (which would itself be a spurious re-apply).
|
||||
func TestPBSDRDescriptor_ZeroGenerationIsOmitted(t *testing.T) {
|
||||
b, err := json.Marshal(&pbsDRDescriptor{Enabled: true, StorageID: "felhom-pbs"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(b, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, present := m["secret_generation"]; present {
|
||||
t.Errorf("zero generation must be omitted (omitempty) — emitting it would move the hash of every "+
|
||||
"pre-existing descriptor and cause a fleet-wide spurious re-apply. got: %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
// An UNKNOWN key must survive a round-trip through readPBSDR/mergePBSDR untouched — the Scenario-C
|
||||
// compatibility direction, asserted from the hub side: an old agent's descriptor is not corrupted by
|
||||
// a hub that now writes the new field.
|
||||
func TestPBSDRDescriptor_RoundTripPreservesOtherKeys(t *testing.T) {
|
||||
desired := `{"operator":{"note":"keep me"},"pbs_dr":{"enabled":true,"storage_id":"felhom-pbs","secret_generation":7}}`
|
||||
cur := readPBSDR(desired)
|
||||
if cur == nil {
|
||||
t.Fatal("readPBSDR returned nil")
|
||||
}
|
||||
if cur.SecretGeneration != 7 {
|
||||
t.Fatalf("secret_generation round-trip = %d, want 7", cur.SecretGeneration)
|
||||
}
|
||||
cur.SecretGeneration = 8
|
||||
merged, err := mergePBSDR(desired, cur)
|
||||
if err != nil {
|
||||
t.Fatalf("merge: %v", err)
|
||||
}
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(merged), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := m["operator"]; !ok {
|
||||
t.Error("merge dropped a sibling key in desired_json")
|
||||
}
|
||||
if got := readPBSDR(merged); got == nil || got.SecretGeneration != 8 {
|
||||
t.Errorf("merged descriptor lost the advanced generation: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// R-39, Scenario B at the FLOW level — the assertion that actually guards the shipped behaviour.
|
||||
//
|
||||
// The struct test above proves the field moves the bytes; this proves ReissuePBSDR *stamps* it.
|
||||
// fakeTenancy returns an identical TokenID / Fingerprint / Datastore / Namespace on every call and
|
||||
// rotates only the secret — which is exactly what an ep0 re-key does, and exactly why the descriptor
|
||||
// used to come back byte-identical.
|
||||
//
|
||||
// COMPANION RED-PROOF (run + recorded): comment out `cur.SecretGeneration = secretGen` in
|
||||
// ReissuePBSDR (keep the field declared so it still compiles) → the pbs_dr block is unchanged across
|
||||
// the re-issue and this test FAILS with both identical blocks printed. That is the 2026-07-18 N100
|
||||
// behaviour reproduced in a unit test.
|
||||
func TestReissuePBSDR_ChangesTheStoredDescriptor(t *testing.T) {
|
||||
fake := &fakeTenancy{secret: "OLD"}
|
||||
s, st, _ := newPBSDRServer(t, fake)
|
||||
postUpdate(t, s, url.Values{"dr_tier": {"on"}}) // provision → descriptor + generation 1
|
||||
|
||||
pbsBlockOf := func(what string) string {
|
||||
t.Helper()
|
||||
h, err := st.GetHost("peti-01")
|
||||
if err != nil || h == nil {
|
||||
t.Fatalf("%s: get host: %v", what, err)
|
||||
}
|
||||
var doc map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(h.DesiredJSON), &doc); err != nil {
|
||||
t.Fatalf("%s: parse desired_json: %v", what, err)
|
||||
}
|
||||
return string(doc["pbs_dr"])
|
||||
}
|
||||
|
||||
before := pbsBlockOf("before")
|
||||
if before == "" {
|
||||
t.Fatal("no pbs_dr descriptor after provisioning")
|
||||
}
|
||||
|
||||
// The agent consumes it and converges — the box is now pinned to this exact descriptor hash.
|
||||
if _, err := st.ConsumeHostPBSSecret("peti-01"); err != nil {
|
||||
t.Fatalf("consume: %v", err)
|
||||
}
|
||||
|
||||
fake.secret = "FRESH" // the re-key: a new secret behind an unchanged token
|
||||
if err := s.ReissuePBSDR(context.Background(), "peti"); err != nil {
|
||||
t.Fatalf("ReissuePBSDR: %v", err)
|
||||
}
|
||||
after := pbsBlockOf("after")
|
||||
|
||||
if after == before {
|
||||
t.Fatalf("the re-issue left the pbs_dr descriptor BYTE-IDENTICAL — a converged agent will "+
|
||||
"short-circuit on its content hash and never consume the fresh secret (R-39).\n block: %s", before)
|
||||
}
|
||||
|
||||
// The fresh secret must genuinely be re-armed for consumption, and the generation advanced.
|
||||
var b, a pbsDRDescriptor
|
||||
if err := json.Unmarshal([]byte(before), &b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(after), &a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.SecretGeneration <= b.SecretGeneration {
|
||||
t.Errorf("secret_generation did not advance across a re-issue: %d -> %d", b.SecretGeneration, a.SecretGeneration)
|
||||
}
|
||||
// Everything else is identical — proving the generation is the ONLY thing carrying the signal.
|
||||
if a.TokenID != b.TokenID || a.Fingerprint != b.Fingerprint || a.Namespace != b.Namespace || a.Datastore != b.Datastore {
|
||||
t.Errorf("this fake models a re-key, so these must be unchanged; if they differ the test is no "+
|
||||
"longer exercising the defect shape.\n before=%+v\n after=%+v", b, a)
|
||||
}
|
||||
got, err := st.ConsumeHostPBSSecret("peti-01")
|
||||
if err != nil || got != "FRESH" {
|
||||
t.Errorf("post-reissue consume = (%q, %v), want (FRESH, nil) — the fresh secret must be consumable", got, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user