R-204 item 4 (hub half): the hub answers a rebuilt box's request (hub v0.96.0)
New internal/offsiteheal, the sibling of pbsdrheal: it acts ONLY on the state the box declares, sustained across two distinct reports, re-staging the stored credential before ever minting a new one. A healthy box is a pure no-op; it never blind-timer-reissues and never re-runs a provisioning step. RESTAGE IS POSSIBLE because the stored value survives a consume — established from the schema and ConsumeOneTimeSecret (which stamps consumed_at and nothing else), not inherited from the PBS analogy, and pinned by a test that asserts the SAME value comes back. reportHasOffsite is TIGHTENED to require enabled:true. Its comment asserted that presence == applied-on-the-box, and the declaration deliberately breaks that premise; left alone it would have read a request for help as proof the tier was applied. Provably a no-op for every report shape that existed before, because an attached object has always carried enabled:true. R-192's guard half is CLOSED BY REPLACEMENT: the delivery checker's counting inference read the OLDEST 500 reports after a consume — all predating a rebuild, which is why demo-hp sat stranded for 108 reports under a confident regressed-shape verdict. A declaration outranks both inferred shapes, and the checker stands down with a record so the two mechanisms cannot double-issue. No escrow ceremony is ever run or requested: credential automatic, key customer-present.
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
|
||||
"gitea.dooplex.hu/admin/felhom-hub/internal/offsiteheal"
|
||||
)
|
||||
|
||||
// R-192's guard half, closed by REPLACEMENT (R-204 item 4).
|
||||
//
|
||||
// The counting guard inferred the situation from how many of the OLDEST 500 reports after a consume
|
||||
// carried an offbox target. On demo-hp all 500 predated the rebuild, so the checker confidently
|
||||
// reported the REGRESSED shape and declined to heal — for 108 reports, while the box sat stranded.
|
||||
// A declaration needs no window and no count, and it OUTRANKS both inferred shapes.
|
||||
|
||||
func TestShapeOf_DeclarationOutranksBothInferredShapes(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
status offsite.DeliveryStatus
|
||||
declared bool
|
||||
want deliveryShape
|
||||
}{
|
||||
{"burned, undeclared", offsite.DeliveryStatus{OffsiteReportsSinceConsume: 0, ReportsSinceConsume: 9}, false, shapeBurned},
|
||||
{"regressed, undeclared", offsite.DeliveryStatus{OffsiteReportsSinceConsume: 500, ReportsSinceConsume: 500}, false, shapeRegressed},
|
||||
// THE demo-hp SHAPE: the counts say "regressed" from a window that predates the rebuild, and
|
||||
// the box says it needs a credential. The declaration wins.
|
||||
{"regressed counts BUT the box declares", offsite.DeliveryStatus{OffsiteReportsSinceConsume: 500, ReportsSinceConsume: 500}, true, shapeDeclared},
|
||||
{"burned counts AND the box declares", offsite.DeliveryStatus{OffsiteReportsSinceConsume: 0, ReportsSinceConsume: 9}, true, shapeDeclared},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := shapeOf(tc.status, tc.declared); got != tc.want {
|
||||
t.Errorf("%s: shapeOf = %q, want %q", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The declared-state string is duplicated in three packages (the controller declares it, this checker
|
||||
// recognises it, the reconciler acts on it). A silent drift between them would make the whole feature
|
||||
// inert with every test still green — so the two hub-side copies are pinned to each other here. The
|
||||
// controller's copy is pinned by its own report-shape test and by the live validation.
|
||||
func TestDeclaredStateStringMatchesTheReconciler(t *testing.T) {
|
||||
if declaredNeedsCredential != offsiteheal.StateNeedsCredential {
|
||||
t.Fatalf("declared-state drift: monitor has %q, offsiteheal has %q — the checker would never stand down and both mechanisms would heal the same customer",
|
||||
declaredNeedsCredential, offsiteheal.StateNeedsCredential)
|
||||
}
|
||||
}
|
||||
@@ -94,8 +94,19 @@ func (c *OffsiteDeliveryChecker) Check() {
|
||||
if age < stuckAfter {
|
||||
continue // normal convergence window
|
||||
}
|
||||
emitted := c.maybeEmitStuck(cfg.CustomerID, status, age)
|
||||
c.maybeHeal(cfg.CustomerID, status, emitted)
|
||||
// R-204 item 4 / R-192's guard half: does the BOX declare that it needs a credential? A
|
||||
// declaration is stronger evidence than anything this checker can infer, and it is owned by
|
||||
// internal/offsiteheal — see maybeHeal. A read error is treated as "no declaration", which is
|
||||
// the conservative direction: this checker keeps its pre-v0.96.0 behaviour rather than
|
||||
// silently standing down.
|
||||
declared := false
|
||||
if _, _, state, derr := c.store.LatestReportOffsiteDeclaration(cfg.CustomerID); derr != nil {
|
||||
c.logger.Printf("[WARN] offsite-delivery: %s: read declaration: %v (treating as undeclared)", cfg.CustomerID, derr)
|
||||
} else {
|
||||
declared = state == declaredNeedsCredential
|
||||
}
|
||||
emitted := c.maybeEmitStuck(cfg.CustomerID, status, age, declared)
|
||||
c.maybeHeal(cfg.CustomerID, status, emitted, declared)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,9 +122,25 @@ const (
|
||||
// the credential worked and the target was later lost (a guest rebuild does exactly this, R-193).
|
||||
// Re-issue is NOT indicated; it treats a symptom whose cause is elsewhere.
|
||||
shapeRegressed deliveryShape = "regressed"
|
||||
// shapeDeclared — the BOX itself says it needs a credential (controller >= v0.199.0). This
|
||||
// OUTRANKS both shapes above and is R-192's guard half closed by REPLACEMENT rather than repair:
|
||||
// the counting guard inferred the situation from how many of the OLDEST 500 reports after the
|
||||
// consume carried an offbox target — all of which predate a rebuild, which is why demo-hp sat
|
||||
// stranded for 108 reports while the checker confidently reported the regressed shape. A
|
||||
// declaration needs no window, no count and no inference. The remediation belongs to
|
||||
// internal/offsiteheal; this checker stands down and says so.
|
||||
shapeDeclared deliveryShape = "declared"
|
||||
)
|
||||
|
||||
func shapeOf(status offsite.DeliveryStatus) deliveryShape {
|
||||
// declaredNeedsCredential mirrors backup.OffsiteStateNeedsCredential / offsiteheal.StateNeedsCredential.
|
||||
// Duplicated as a literal rather than imported to avoid a monitor→offsiteheal dependency; the three
|
||||
// are pinned together by TestDeclaredStateStringMatchesTheReconciler.
|
||||
const declaredNeedsCredential = "needs_credential"
|
||||
|
||||
func shapeOf(status offsite.DeliveryStatus, declared bool) deliveryShape {
|
||||
if declared {
|
||||
return shapeDeclared
|
||||
}
|
||||
if status.OffsiteReportsSinceConsume == 0 {
|
||||
return shapeBurned
|
||||
}
|
||||
@@ -137,7 +164,7 @@ func shapeOf(status offsite.DeliveryStatus) deliveryShape {
|
||||
// assembled (R-199/R-200/R-201). Naming the window in the text is how it stays visible instead of
|
||||
// being laundered into a confident sentence — an instrument that can silently mis-scope its results
|
||||
// must say so where it reports them.
|
||||
func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration) bool {
|
||||
func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration, declared bool) bool {
|
||||
last, err := c.store.LastEventAt(customerID, eventDeliveryStuck)
|
||||
if err != nil {
|
||||
c.logger.Printf("[WARN] offsite-delivery: %s: cooldown read: %v", customerID, err)
|
||||
@@ -146,9 +173,12 @@ func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsit
|
||||
if !last.IsZero() && c.now().Sub(last) < stuckCooldown {
|
||||
return false
|
||||
}
|
||||
shape := shapeOf(status)
|
||||
shape := shapeOf(status, declared)
|
||||
var msg string
|
||||
switch shape {
|
||||
case shapeDeclared:
|
||||
msg = fmt.Sprintf("Offsite delivery stuck (DECLARED by the box): the one-time password was consumed %s ago and the box now reports `offsite.state=%s` — it has been rebuilt, holds no repository password, and the hub is holding a sealed recovery package for it. No inference was needed: the box said so. The offsite self-heal reconciler owns this remediation (it re-arms the stored credential before minting a new one); no operator action is indicated unless this repeats.",
|
||||
age.Round(time.Minute), declaredNeedsCredential)
|
||||
case shapeBurned:
|
||||
msg = fmt.Sprintf("Offsite delivery stuck (BURNED-credential shape): the one-time password was consumed %s ago; of the first %d report(s) after that consume, NONE carried an offbox target, and the latest report carries none either. The credential never reached a persisted apply. Re-issue delivers a fresh one.",
|
||||
age.Round(time.Minute), status.ReportsSinceConsume)
|
||||
@@ -185,10 +215,19 @@ func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsit
|
||||
// the set of situations in which the heal fires is byte-for-byte what it was; only the silence is
|
||||
// gone. The two conditions are split into separate branches solely so each refusal can name its own
|
||||
// reason.
|
||||
func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus, recordRefusal bool) {
|
||||
func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus, recordRefusal bool, declared bool) {
|
||||
if c.reissuer == nil {
|
||||
return // no provisioner configured: the heal does not exist on this hub, so there is nothing to explain
|
||||
}
|
||||
// R-204 item 4: a DECLARING box belongs to internal/offsiteheal, which re-arms the stored
|
||||
// credential before minting a new one. Two mechanisms healing the same customer would double-issue
|
||||
// — and this one can only mint, so it would also skip the cheap path. Stand down, with a record:
|
||||
// "we chose not to act" and "the heal never ran" must not look identical (R-192 defect (b)).
|
||||
if declared {
|
||||
c.recordHealRefusal(customerID, recordRefusal,
|
||||
"the box DECLARES offsite.state=needs_credential; internal/offsiteheal owns this remediation (it re-stages the stored credential before minting). A second mechanism minting here would double-issue.")
|
||||
return
|
||||
}
|
||||
if status.OffsiteReportsSinceConsume != 0 {
|
||||
c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf(
|
||||
"regressed-apply shape: %d of the first %d report(s) after the consume DID carry an offbox target, so a burned credential is ruled out — a restage would treat a symptom whose cause is elsewhere. Operator's call (R-193).",
|
||||
|
||||
Reference in New Issue
Block a user