package monitor import ( "context" "encoding/json" "fmt" "log" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-70 + R-71(c): the offsite delivery-state checker. Reads the shared detector // (offsite.DeliveryStateFor) for every offsite-enabled customer and drives two consumers: // // - the LOUD EVENT: `offsite_delivery_stuck` (warning → operator email per existing dispatcher // rules) when the burned-credential shape persists past stuckAfter; // - the SELF-HEAL (R-71c): invoke the EXISTING Re-issue path — never a second delivery // mechanism — when the shape is unambiguous, then surface `offsite_credential_restaged` // (warning) so the operator ALWAYS knows it fired. // // Cooldowns are durable: both events rate-limit off store.LastEventAt (the events table), so a hub // restart cannot flood or silently re-heal. A repeating pattern surfaces as repeating events on a // 24 h cadence, never as a silent retry loop. // // THE R-39(a) GUARD (mandatory, enforced HERE because the store deliberately clobbers): // SaveOneTimeSecret is last-write-wins by design — Re-issue depends on supersede. Restaging on top // of an UNCONSUMED secret would clobber a password a box may be about to consume (the operator may // have clicked Re-issue between this checker's derive and its act). So the heal re-reads the // secret row IMMEDIATELY before acting and refuses unless it is still a CONSUMED row. const ( eventDeliveryStuck = "offsite_delivery_stuck" // hub-internal (not in allowedEventTypes, like pbsdr_*) eventCredentialRestaged = "offsite_credential_restaged" // hub-internal // stuckAfter: consumed_awaiting_apply is normal for seconds (F10 repair: consume→applied in // 4 s). An hour of it means the apply will never come without intervention. stuckAfter = time.Hour // stuckCooldown / healCooldown: per-customer, durable via the events table. stuckCooldown = 24 * time.Hour healCooldown = 24 * time.Hour // healMinReports: at least this many consecutive offbox-less reports after consumed_at before // the self-heal may fire — the box must be alive and reporting, just not applied. healMinReports = 4 ) // OffsiteReissuer is the narrow reissue surface the self-heal needs — satisfied by // *web.Server.ReissueOffsiteForCustomer (the pbsdrheal.Reissuer precedent; avoids an import // cycle and guarantees the heal IS the designed Re-issue path, not a sibling mechanism). type OffsiteReissuer interface { ReissueOffsiteForCustomer(ctx context.Context, customerID string) error } // OffsiteDeliveryChecker runs on the shared monitor ticker. type OffsiteDeliveryChecker struct { store *store.Store reissuer OffsiteReissuer // nil → self-heal disabled (no provisioner configured); detector+event still run onEvent EventNotifyFunc logger *log.Logger now func() time.Time } // NewOffsiteDeliveryChecker constructs the checker. reissuer may be nil (heal disabled). func NewOffsiteDeliveryChecker(s *store.Store, reissuer OffsiteReissuer, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteDeliveryChecker { return &OffsiteDeliveryChecker{store: s, reissuer: reissuer, onEvent: onEvent, logger: logger, now: time.Now} } // Check derives the delivery state for every offsite-enabled active customer and applies the // event + self-heal rules. Never returns an error — a checker failure must not take the ticker // down (log-and-continue, like every sibling checker). func (c *OffsiteDeliveryChecker) Check() { configs, err := c.store.ListCustomerConfigs() if err != nil { c.logger.Printf("[WARN] offsite-delivery: list configs: %v", err) return } for _, cfg := range configs { if cfg.Status != "active" { continue // blocked/inactive customers are not delivery-monitored (and never healed) } d, err := offsite.ReadDescriptor(cfg.ConfigJSON) if err != nil || d == nil || !d.Enabled { continue // unparseable config never drives a heal; the config UI owns that failure } status, err := offsite.DeliveryStateFor(c.store, cfg.CustomerID) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: derive: %v", cfg.CustomerID, err) continue } if status.State != offsite.DeliveryConsumedAwaitingApply { continue // applied / staged / no_secret: card-rendered states, no event or heal (yet) } age := c.now().Sub(status.Since) if age < stuckAfter { continue // normal convergence window } // R-204 item 4 / R-192's guard half: does the BOX declare that it needs a credential? A // declaration is stronger evidence than anything this checker can infer, and it is owned by // internal/offsiteheal — see maybeHeal. A read error is treated as "no declaration", which is // the conservative direction: this checker keeps its pre-v0.96.0 behaviour rather than // silently standing down. declared := false if _, _, state, derr := c.store.LatestReportOffsiteDeclaration(cfg.CustomerID); derr != nil { c.logger.Printf("[WARN] offsite-delivery: %s: read declaration: %v (treating as undeclared)", cfg.CustomerID, derr) } else { declared = state == declaredNeedsCredential } emitted := c.maybeEmitStuck(cfg.CustomerID, status, age, declared) c.maybeHeal(cfg.CustomerID, status, emitted, declared) } } // deliveryShape names the two situations the ONE stuck state actually covers. They need different // text and different advice, and conflating them is R-192's defect (a). type deliveryShape string const ( // shapeBurned — NO report since the consume carried an offbox target: the apply never persisted. // Re-issue is the indicated action. shapeBurned deliveryShape = "burned" // shapeRegressed — reports since the consume DID carry an offbox target and the latest does not: // the credential worked and the target was later lost (a guest rebuild does exactly this, R-193). // Re-issue is NOT indicated; it treats a symptom whose cause is elsewhere. shapeRegressed deliveryShape = "regressed" // shapeDeclared — the BOX itself says it needs a credential (controller >= v0.199.0). This // OUTRANKS both shapes above and is R-192's guard half closed by REPLACEMENT rather than repair: // the counting guard inferred the situation from how many of the OLDEST 500 reports after the // consume carried an offbox target — all of which predate a rebuild, which is why demo-hp sat // stranded for 108 reports while the checker confidently reported the regressed shape. A // declaration needs no window, no count and no inference. The remediation belongs to // internal/offsiteheal; this checker stands down and says so. shapeDeclared deliveryShape = "declared" ) // declaredNeedsCredential mirrors backup.OffsiteStateNeedsCredential / offsiteheal.StateNeedsCredential. // Duplicated as a literal rather than imported to avoid a monitor→offsiteheal dependency; the three // are pinned together by TestDeclaredStateStringMatchesTheReconciler. const declaredNeedsCredential = "needs_credential" func shapeOf(status offsite.DeliveryStatus, declared bool) deliveryShape { if declared { return shapeDeclared } if status.OffsiteReportsSinceConsume == 0 { return shapeBurned } return shapeRegressed } // maybeEmitStuck emits offsite_delivery_stuck (warning) once per stuckCooldown per customer. Returns // whether it emitted, so the heal's refusal record rides the same cadence rather than inventing one. // // R-192 defect (a), fixed here: the message used to interpolate ReportsSinceConsume (the TOTAL) into // a hardcoded phrase "report(s) since carry no offbox target", and never consulted // OffsiteReportsSinceConsume — the field that says the opposite. On demo-hp it stated, daily, that // 500 reports carried no offbox target when all 500 of them did, and prescribed Re-issue for a // failure mode that had not occurred. The message now STATES WHAT WAS MEASURED and lets the operator // read it; the recommendation follows the shape rather than being hardcoded. // // THE WINDOW IS NAMED ON PURPOSE. CountReportsOffsiteSince reads `ORDER BY id LIMIT 500` — the OLDEST // 500 reports after the consume, not the newest — so on a long-lived customer these counts describe // the beginning of the window and not the present. That is a real scoping defect (R-192's other half) // and it stays OPEN because its correct shape depends on the recovery chain that is not yet // assembled (R-199/R-200/R-201). Naming the window in the text is how it stays visible instead of // being laundered into a confident sentence — an instrument that can silently mis-scope its results // must say so where it reports them. func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration, declared bool) bool { last, err := c.store.LastEventAt(customerID, eventDeliveryStuck) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: cooldown read: %v", customerID, err) return false } if !last.IsZero() && c.now().Sub(last) < stuckCooldown { return false } shape := shapeOf(status, declared) var msg string switch shape { case shapeDeclared: msg = fmt.Sprintf("Offsite delivery stuck (DECLARED by the box): the one-time password was consumed %s ago and the box now reports `offsite.state=%s` — it has been rebuilt, holds no repository password, and the hub is holding a sealed recovery package for it. No inference was needed: the box said so. The offsite self-heal reconciler owns this remediation (it re-arms the stored credential before minting a new one); no operator action is indicated unless this repeats.", age.Round(time.Minute), declaredNeedsCredential) case shapeBurned: msg = fmt.Sprintf("Offsite delivery stuck (BURNED-credential shape): the one-time password was consumed %s ago; of the first %d report(s) after that consume, NONE carried an offbox target, and the latest report carries none either. The credential never reached a persisted apply. Re-issue delivers a fresh one.", age.Round(time.Minute), status.ReportsSinceConsume) default: msg = fmt.Sprintf("Offsite delivery stuck (REGRESSED-apply shape): the one-time password was consumed %s ago; of the first %d report(s) after that consume, %d DID carry an offbox target — and the latest report carries none. The credential was applied and worked; the target was lost afterwards. Re-issue is NOT the indicated action: find what removed the offbox target (a guest rebuild does, R-193). Automatic restage is deliberately withheld for this shape. NOTE: the counts cover at most the first 500 reports after the consume, so on a long-lived box they describe the start of the window, not now (R-192, open).", age.Round(time.Minute), status.ReportsSinceConsume, status.OffsiteReportsSinceConsume) } details, _ := json.Marshal(map[string]any{ "state": string(status.State), "shape": string(shape), "consumed_at": status.Since.UTC().Format(time.RFC3339), "reports_since_consume": status.ReportsSinceConsume, "offsite_reports_since_consume": status.OffsiteReportsSinceConsume, "count_window": "oldest 500 reports after consumed_at (R-192, open)", }) c.emit(customerID, eventDeliveryStuck, "warning", msg, string(details)) return true } // maybeHeal fires the R-71c self-heal when the burned-credential shape is unambiguous: // consumed ≥ stuckAfter ago, ≥ healMinReports consecutive reports since with ZERO offbox evidence, // one heal per healCooldown — and the R-39(a) guard holds at act time. // // R-192 defect (b), fixed here: every refusal above the "not configured" line now leaves a RECORD. // The regressed-shape branch used to be a bare `return`, so the operator received a daily e-mail with // the wrong story, no heal, and nothing anywhere saying why the heal declined — "we chose not to act" // and "the heal never ran" looked identical. `offsite_credential_restaged` has never fired for any // customer, and until now that fact was indistinguishable from the checker being dead. // // The record is a notification_log row (the dispatcher's suppressed-operator-e-mail precedent, R-182: // a decision not to act is written down on the channel it would have used). It rides `recordRefusal` // — true only when the stuck event was emitted this pass — so it appears once per stuckCooldown // beside the e-mail it explains, rather than once per monitor tick. The GUARD ITSELF IS UNCHANGED: // the set of situations in which the heal fires is byte-for-byte what it was; only the silence is // gone. The two conditions are split into separate branches solely so each refusal can name its own // reason. func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus, recordRefusal bool, declared bool) { if c.reissuer == nil { return // no provisioner configured: the heal does not exist on this hub, so there is nothing to explain } // R-204 item 4: a DECLARING box belongs to internal/offsiteheal, which re-arms the stored // credential before minting a new one. Two mechanisms healing the same customer would double-issue // — and this one can only mint, so it would also skip the cheap path. Stand down, with a record: // "we chose not to act" and "the heal never ran" must not look identical (R-192 defect (b)). if declared { c.recordHealRefusal(customerID, recordRefusal, "the box DECLARES offsite.state=needs_credential; internal/offsiteheal owns this remediation (it re-stages the stored credential before minting). A second mechanism minting here would double-issue.") return } if status.OffsiteReportsSinceConsume != 0 { c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf( "regressed-apply shape: %d of the first %d report(s) after the consume DID carry an offbox target, so a burned credential is ruled out — a restage would treat a symptom whose cause is elsewhere. Operator's call (R-193).", status.OffsiteReportsSinceConsume, status.ReportsSinceConsume)) return } if status.ReportsSinceConsume < healMinReports { c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf( "only %d report(s) since the consume (need %d): the box has not reported enough for the burned shape to be unambiguous.", status.ReportsSinceConsume, healMinReports)) return } last, err := c.store.LastEventAt(customerID, eventCredentialRestaged) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: heal rate-limit read: %v", customerID, err) return } if !last.IsZero() && c.now().Sub(last) < healCooldown { return // one restage per customer per 24 h — repeats surface as repeated events only } // THE R-39(a) GUARD — re-read the secret row immediately before acting. The derive above is a // snapshot; an operator Re-issue may have staged a FRESH UNCONSUMED secret since (TOCTOU). // SaveOneTimeSecret clobbers by design, so acting now would burn that fresh password. info, err := c.store.GetOneTimeSecretInfo(customerID) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: guard read: %v", customerID, err) return } if info == nil || info.ConsumedAt.IsZero() { c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf( "R-39(a) guard: the secret row is now %s — restaging over an unconsumed secret would clobber a password the box may be about to consume.", secretShape(info))) return } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() if err := c.reissuer.ReissueOffsiteForCustomer(ctx, customerID); err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: self-heal reissue failed: %v", customerID, err) return } c.logger.Printf("[INFO] offsite-delivery: %s: self-heal restage fired (consumed_at %s, %d offbox-less reports since)", customerID, status.Since.UTC().Format(time.RFC3339), status.ReportsSinceConsume) details, _ := json.Marshal(map[string]any{ "burned_consumed_at": status.Since.UTC().Format(time.RFC3339), "reports_since_consume": status.ReportsSinceConsume, }) c.emit(customerID, eventCredentialRestaged, "warning", "Offsite credential re-staged automatically: the previous one-time password was consumed but never applied (burned mid-delivery). The box picks the fresh password up on its next config refresh.", string(details)) } // recordHealRefusal makes a decision NOT to self-heal visible. Always logs; additionally writes a // notification_log row on the "operator" channel with status "refused" when `record` is set (the // stuck event was emitted this pass), so the refusal sits next to the e-mail that prompted the // question. A LogNotification failure is logged, never swallowed, and never blocks the refusal — the // refusal is the primary effect. func (c *OffsiteDeliveryChecker) recordHealRefusal(customerID string, record bool, reason string) { c.logger.Printf("[INFO] offsite-delivery: %s: self-heal REFUSED — %s", customerID, reason) if !record { return } if err := c.store.LogNotification(customerID, eventCredentialRestaged, "warning", "Automatic offsite credential restage was NOT performed.", "refused", reason, "operator"); err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: could not record the heal refusal: %v", customerID, err) } } func secretShape(info *store.OneTimeSecretInfo) string { if info == nil { return "absent" } return "unconsumed (staged " + info.CreatedAt.UTC().Format(time.RFC3339) + ")" } // emit saves the event (audit trail first) and then notifies — the OffsiteChecker convention: // SaveEvent failure logs and SKIPS the notification (an email without its audit row lies). func (c *OffsiteDeliveryChecker) emit(customerID, eventType, severity, message, details string) { if _, err := c.store.SaveEvent(customerID, eventType, severity, message, details, "hub"); err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: save %s: %v", customerID, eventType, err) return } c.logger.Printf("[INFO] offsite-delivery: %s: %s (%s)", customerID, eventType, severity) if c.onEvent != nil { c.onEvent(customerID, eventType, severity, message, details, "hub") } }