package monitor import ( "context" "encoding/json" "fmt" "log" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-70 + R-71(c): the offsite delivery-state checker. Reads the shared detector // (offsite.DeliveryStateFor) for every offsite-enabled customer and drives two consumers: // // - the LOUD EVENT: `offsite_delivery_stuck` (warning → operator email per existing dispatcher // rules) when the burned-credential shape persists past stuckAfter; // - the SELF-HEAL (R-71c): invoke the EXISTING Re-issue path — never a second delivery // mechanism — when the shape is unambiguous, then surface `offsite_credential_restaged` // (warning) so the operator ALWAYS knows it fired. // // Cooldowns are durable: both events rate-limit off store.LastEventAt (the events table), so a hub // restart cannot flood or silently re-heal. A repeating pattern surfaces as repeating events on a // 24 h cadence, never as a silent retry loop. // // THE R-39(a) GUARD (mandatory, enforced HERE because the store deliberately clobbers): // SaveOneTimeSecret is last-write-wins by design — Re-issue depends on supersede. Restaging on top // of an UNCONSUMED secret would clobber a password a box may be about to consume (the operator may // have clicked Re-issue between this checker's derive and its act). So the heal re-reads the // secret row IMMEDIATELY before acting and refuses unless it is still a CONSUMED row. const ( eventDeliveryStuck = "offsite_delivery_stuck" // hub-internal (not in allowedEventTypes, like pbsdr_*) eventCredentialRestaged = "offsite_credential_restaged" // hub-internal // stuckAfter: consumed_awaiting_apply is normal for seconds (F10 repair: consume→applied in // 4 s). An hour of it means the apply will never come without intervention. stuckAfter = time.Hour // stuckCooldown / healCooldown: per-customer, durable via the events table. stuckCooldown = 24 * time.Hour healCooldown = 24 * time.Hour // healMinReports: at least this many consecutive offbox-less reports after consumed_at before // the self-heal may fire — the box must be alive and reporting, just not applied. healMinReports = 4 ) // OffsiteReissuer is the narrow reissue surface the self-heal needs — satisfied by // *web.Server.ReissueOffsiteForCustomer (the pbsdrheal.Reissuer precedent; avoids an import // cycle and guarantees the heal IS the designed Re-issue path, not a sibling mechanism). type OffsiteReissuer interface { ReissueOffsiteForCustomer(ctx context.Context, customerID string) error } // OffsiteDeliveryChecker runs on the shared monitor ticker. type OffsiteDeliveryChecker struct { store *store.Store reissuer OffsiteReissuer // nil → self-heal disabled (no provisioner configured); detector+event still run onEvent EventNotifyFunc logger *log.Logger now func() time.Time } // NewOffsiteDeliveryChecker constructs the checker. reissuer may be nil (heal disabled). func NewOffsiteDeliveryChecker(s *store.Store, reissuer OffsiteReissuer, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteDeliveryChecker { return &OffsiteDeliveryChecker{store: s, reissuer: reissuer, onEvent: onEvent, logger: logger, now: time.Now} } // Check derives the delivery state for every offsite-enabled active customer and applies the // event + self-heal rules. Never returns an error — a checker failure must not take the ticker // down (log-and-continue, like every sibling checker). func (c *OffsiteDeliveryChecker) Check() { configs, err := c.store.ListCustomerConfigs() if err != nil { c.logger.Printf("[WARN] offsite-delivery: list configs: %v", err) return } for _, cfg := range configs { if cfg.Status != "active" { continue // blocked/inactive customers are not delivery-monitored (and never healed) } d, err := offsite.ReadDescriptor(cfg.ConfigJSON) if err != nil || d == nil || !d.Enabled { continue // unparseable config never drives a heal; the config UI owns that failure } status, err := offsite.DeliveryStateFor(c.store, cfg.CustomerID) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: derive: %v", cfg.CustomerID, err) continue } if status.State != offsite.DeliveryConsumedAwaitingApply { continue // applied / staged / no_secret: card-rendered states, no event or heal (yet) } age := c.now().Sub(status.Since) if age < stuckAfter { continue // normal convergence window } c.maybeEmitStuck(cfg.CustomerID, status, age) c.maybeHeal(cfg.CustomerID, status) } } // maybeEmitStuck emits offsite_delivery_stuck (warning) once per stuckCooldown per customer. func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration) { last, err := c.store.LastEventAt(customerID, eventDeliveryStuck) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: cooldown read: %v", customerID, err) return } if !last.IsZero() && c.now().Sub(last) < stuckCooldown { return } msg := fmt.Sprintf("Offsite delivery stuck: one-time password consumed %s ago and %d report(s) since carry no offbox target — the credential is likely burned (apply died between consume and persist). Re-issue delivers a fresh one.", age.Round(time.Minute), status.ReportsSinceConsume) details, _ := json.Marshal(map[string]any{ "state": string(status.State), "consumed_at": status.Since.UTC().Format(time.RFC3339), "reports_since_consume": status.ReportsSinceConsume, }) c.emit(customerID, eventDeliveryStuck, "warning", msg, string(details)) } // maybeHeal fires the R-71c self-heal when the burned-credential shape is unambiguous: // consumed ≥ stuckAfter ago, ≥ healMinReports consecutive reports since with ZERO offbox evidence, // one heal per healCooldown — and the R-39(a) guard holds at act time. func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus) { if c.reissuer == nil { return } if status.ReportsSinceConsume < healMinReports || status.OffsiteReportsSinceConsume != 0 { return // box not reporting enough, or offbox evidence exists (regressed-apply shape) → operator's call } last, err := c.store.LastEventAt(customerID, eventCredentialRestaged) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: heal rate-limit read: %v", customerID, err) return } if !last.IsZero() && c.now().Sub(last) < healCooldown { return // one restage per customer per 24 h — repeats surface as repeated events only } // THE R-39(a) GUARD — re-read the secret row immediately before acting. The derive above is a // snapshot; an operator Re-issue may have staged a FRESH UNCONSUMED secret since (TOCTOU). // SaveOneTimeSecret clobbers by design, so acting now would burn that fresh password. info, err := c.store.GetOneTimeSecretInfo(customerID) if err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: guard read: %v", customerID, err) return } if info == nil || info.ConsumedAt.IsZero() { c.logger.Printf("[INFO] offsite-delivery: %s: heal refused — secret row is now %s (R-39(a) guard: never restage over an unconsumed secret)", customerID, secretShape(info)) return } ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() if err := c.reissuer.ReissueOffsiteForCustomer(ctx, customerID); err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: self-heal reissue failed: %v", customerID, err) return } c.logger.Printf("[INFO] offsite-delivery: %s: self-heal restage fired (consumed_at %s, %d offbox-less reports since)", customerID, status.Since.UTC().Format(time.RFC3339), status.ReportsSinceConsume) details, _ := json.Marshal(map[string]any{ "burned_consumed_at": status.Since.UTC().Format(time.RFC3339), "reports_since_consume": status.ReportsSinceConsume, }) c.emit(customerID, eventCredentialRestaged, "warning", "Offsite credential re-staged automatically: the previous one-time password was consumed but never applied (burned mid-delivery). The box picks the fresh password up on its next config refresh.", string(details)) } func secretShape(info *store.OneTimeSecretInfo) string { if info == nil { return "absent" } return "unconsumed (staged " + info.CreatedAt.UTC().Format(time.RFC3339) + ")" } // emit saves the event (audit trail first) and then notifies — the OffsiteChecker convention: // SaveEvent failure logs and SKIPS the notification (an email without its audit row lies). func (c *OffsiteDeliveryChecker) emit(customerID, eventType, severity, message, details string) { if _, err := c.store.SaveEvent(customerID, eventType, severity, message, details, "hub"); err != nil { c.logger.Printf("[WARN] offsite-delivery: %s: save %s: %v", customerID, eventType, err) return } c.logger.Printf("[INFO] offsite-delivery: %s: %s (%s)", customerID, eventType, severity) if c.onEvent != nil { c.onEvent(customerID, eventType, severity, message, details, "hub") } }