felhom.eu/hub/internal/monitor/offsite_delivery.go

package monitor

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-70 + R-71(c): the offsite delivery-state checker. Reads the shared detector
// (offsite.DeliveryStateFor) for every offsite-enabled customer and drives two consumers:
//
//   - the LOUD EVENT: `offsite_delivery_stuck` (warning → operator email per existing dispatcher
//     rules) when the burned-credential shape persists past stuckAfter;
//   - the SELF-HEAL (R-71c): invoke the EXISTING Re-issue path — never a second delivery
//     mechanism — when the shape is unambiguous, then surface `offsite_credential_restaged`
//     (warning) so the operator ALWAYS knows it fired.
//
// Cooldowns are durable: both events rate-limit off store.LastEventAt (the events table), so a hub
// restart cannot flood or silently re-heal. A repeating pattern surfaces as repeating events on a
// 24 h cadence, never as a silent retry loop.
//
// THE R-39(a) GUARD (mandatory, enforced HERE because the store deliberately clobbers):
// SaveOneTimeSecret is last-write-wins by design — Re-issue depends on supersede. Restaging on top
// of an UNCONSUMED secret would clobber a password a box may be about to consume (the operator may
// have clicked Re-issue between this checker's derive and its act). So the heal re-reads the
// secret row IMMEDIATELY before acting and refuses unless it is still a CONSUMED row.

const (
	eventDeliveryStuck      = "offsite_delivery_stuck"      // hub-internal (not in allowedEventTypes, like pbsdr_*)
	eventCredentialRestaged = "offsite_credential_restaged" // hub-internal
	// stuckAfter: consumed_awaiting_apply is normal for seconds (F10 repair: consume→applied in
	// 4 s). An hour of it means the apply will never come without intervention.
	stuckAfter = time.Hour
	// stuckCooldown / healCooldown: per-customer, durable via the events table.
	stuckCooldown = 24 * time.Hour
	healCooldown  = 24 * time.Hour
	// healMinReports: at least this many consecutive offbox-less reports after consumed_at before
	// the self-heal may fire — the box must be alive and reporting, just not applied.
	healMinReports = 4
)

// OffsiteReissuer is the narrow reissue surface the self-heal needs — satisfied by
// *web.Server.ReissueOffsiteForCustomer (the pbsdrheal.Reissuer precedent; avoids an import
// cycle and guarantees the heal IS the designed Re-issue path, not a sibling mechanism).
type OffsiteReissuer interface {
	ReissueOffsiteForCustomer(ctx context.Context, customerID string) error
}

// OffsiteDeliveryChecker runs on the shared monitor ticker.
type OffsiteDeliveryChecker struct {
	store    *store.Store
	reissuer OffsiteReissuer // nil → self-heal disabled (no provisioner configured); detector+event still run
	onEvent  EventNotifyFunc
	logger   *log.Logger
	now      func() time.Time
}

// NewOffsiteDeliveryChecker constructs the checker. reissuer may be nil (heal disabled).
func NewOffsiteDeliveryChecker(s *store.Store, reissuer OffsiteReissuer, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteDeliveryChecker {
	return &OffsiteDeliveryChecker{store: s, reissuer: reissuer, onEvent: onEvent, logger: logger, now: time.Now}
}

// Check derives the delivery state for every offsite-enabled active customer and applies the
// event + self-heal rules. Never returns an error — a checker failure must not take the ticker
// down (log-and-continue, like every sibling checker).
func (c *OffsiteDeliveryChecker) Check() {
	configs, err := c.store.ListCustomerConfigs()
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: list configs: %v", err)
		return
	}
	for _, cfg := range configs {
		if cfg.Status != "active" {
			continue // blocked/inactive customers are not delivery-monitored (and never healed)
		}
		d, err := offsite.ReadDescriptor(cfg.ConfigJSON)
		if err != nil || d == nil || !d.Enabled {
			continue // unparseable config never drives a heal; the config UI owns that failure
		}
		status, err := offsite.DeliveryStateFor(c.store, cfg.CustomerID)
		if err != nil {
			c.logger.Printf("[WARN] offsite-delivery: %s: derive: %v", cfg.CustomerID, err)
			continue
		}
		if status.State != offsite.DeliveryConsumedAwaitingApply {
			continue // applied / staged / no_secret: card-rendered states, no event or heal (yet)
		}
		age := c.now().Sub(status.Since)
		if age < stuckAfter {
			continue // normal convergence window
		}
		// R-204 item 4 / R-192's guard half: does the BOX declare that it needs a credential? A
		// declaration is stronger evidence than anything this checker can infer, and it is owned by
		// internal/offsiteheal — see maybeHeal. A read error is treated as "no declaration", which is
		// the conservative direction: this checker keeps its pre-v0.96.0 behaviour rather than
		// silently standing down.
		declared := false
		if _, _, state, derr := c.store.LatestReportOffsiteDeclaration(cfg.CustomerID); derr != nil {
			c.logger.Printf("[WARN] offsite-delivery: %s: read declaration: %v (treating as undeclared)", cfg.CustomerID, derr)
		} else {
			declared = state == declaredNeedsCredential
		}
		emitted := c.maybeEmitStuck(cfg.CustomerID, status, age, declared)
		c.maybeHeal(cfg.CustomerID, status, emitted, declared)
	}
}

// deliveryShape names the two situations the ONE stuck state actually covers. They need different
// text and different advice, and conflating them is R-192's defect (a).
type deliveryShape string

const (
	// shapeBurned — NO report since the consume carried an offbox target: the apply never persisted.
	// Re-issue is the indicated action.
	shapeBurned deliveryShape = "burned"
	// shapeRegressed — reports since the consume DID carry an offbox target and the latest does not:
	// the credential worked and the target was later lost (a guest rebuild does exactly this, R-193).
	// Re-issue is NOT indicated; it treats a symptom whose cause is elsewhere.
	shapeRegressed deliveryShape = "regressed"
	// shapeDeclared — the BOX itself says it needs a credential (controller >= v0.199.0). This
	// OUTRANKS both shapes above and is R-192's guard half closed by REPLACEMENT rather than repair:
	// the counting guard inferred the situation from how many of the OLDEST 500 reports after the
	// consume carried an offbox target — all of which predate a rebuild, which is why demo-hp sat
	// stranded for 108 reports while the checker confidently reported the regressed shape. A
	// declaration needs no window, no count and no inference. The remediation belongs to
	// internal/offsiteheal; this checker stands down and says so.
	shapeDeclared deliveryShape = "declared"
)

// declaredNeedsCredential mirrors backup.OffsiteStateNeedsCredential / offsiteheal.StateNeedsCredential.
// Duplicated as a literal rather than imported to avoid a monitor→offsiteheal dependency; the three
// are pinned together by TestDeclaredStateStringMatchesTheReconciler.
const declaredNeedsCredential = "needs_credential"

func shapeOf(status offsite.DeliveryStatus, declared bool) deliveryShape {
	if declared {
		return shapeDeclared
	}
	if status.OffsiteReportsSinceConsume == 0 {
		return shapeBurned
	}
	return shapeRegressed
}

// maybeEmitStuck emits offsite_delivery_stuck (warning) once per stuckCooldown per customer. Returns
// whether it emitted, so the heal's refusal record rides the same cadence rather than inventing one.
//
// R-192 defect (a), fixed here: the message used to interpolate ReportsSinceConsume (the TOTAL) into
// a hardcoded phrase "report(s) since carry no offbox target", and never consulted
// OffsiteReportsSinceConsume — the field that says the opposite. On demo-hp it stated, daily, that
// 500 reports carried no offbox target when all 500 of them did, and prescribed Re-issue for a
// failure mode that had not occurred. The message now STATES WHAT WAS MEASURED and lets the operator
// read it; the recommendation follows the shape rather than being hardcoded.
//
// THE WINDOW IS NAMED ON PURPOSE. CountReportsOffsiteSince reads `ORDER BY id LIMIT 500` — the OLDEST
// 500 reports after the consume, not the newest — so on a long-lived customer these counts describe
// the beginning of the window and not the present. That is a real scoping defect (R-192's other half)
// and it stays OPEN because its correct shape depends on the recovery chain that is not yet
// assembled (R-199/R-200/R-201). Naming the window in the text is how it stays visible instead of
// being laundered into a confident sentence — an instrument that can silently mis-scope its results
// must say so where it reports them.
func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration, declared bool) bool {
	last, err := c.store.LastEventAt(customerID, eventDeliveryStuck)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: cooldown read: %v", customerID, err)
		return false
	}
	if !last.IsZero() && c.now().Sub(last) < stuckCooldown {
		return false
	}
	shape := shapeOf(status, declared)
	var msg string
	switch shape {
	case shapeDeclared:
		msg = fmt.Sprintf("Offsite delivery stuck (DECLARED by the box): the one-time password was consumed %s ago and the box now reports `offsite.state=%s` — it has been rebuilt, holds no repository password, and the hub is holding a sealed recovery package for it. No inference was needed: the box said so. The offsite self-heal reconciler owns this remediation (it re-arms the stored credential before minting a new one); no operator action is indicated unless this repeats.",
			age.Round(time.Minute), declaredNeedsCredential)
	case shapeBurned:
		msg = fmt.Sprintf("Offsite delivery stuck (BURNED-credential shape): the one-time password was consumed %s ago; of the first %d report(s) after that consume, NONE carried an offbox target, and the latest report carries none either. The credential never reached a persisted apply. Re-issue delivers a fresh one.",
			age.Round(time.Minute), status.ReportsSinceConsume)
	default:
		msg = fmt.Sprintf("Offsite delivery stuck (REGRESSED-apply shape): the one-time password was consumed %s ago; of the first %d report(s) after that consume, %d DID carry an offbox target — and the latest report carries none. The credential was applied and worked; the target was lost afterwards. Re-issue is NOT the indicated action: find what removed the offbox target (a guest rebuild does, R-193). Automatic restage is deliberately withheld for this shape. NOTE: the counts cover at most the first 500 reports after the consume, so on a long-lived box they describe the start of the window, not now (R-192, open).",
			age.Round(time.Minute), status.ReportsSinceConsume, status.OffsiteReportsSinceConsume)
	}
	details, _ := json.Marshal(map[string]any{
		"state":                         string(status.State),
		"shape":                         string(shape),
		"consumed_at":                   status.Since.UTC().Format(time.RFC3339),
		"reports_since_consume":         status.ReportsSinceConsume,
		"offsite_reports_since_consume": status.OffsiteReportsSinceConsume,
		"count_window":                  "oldest 500 reports after consumed_at (R-192, open)",
	})
	c.emit(customerID, eventDeliveryStuck, "warning", msg, string(details))
	return true
}

// maybeHeal fires the R-71c self-heal when the burned-credential shape is unambiguous:
// consumed ≥ stuckAfter ago, ≥ healMinReports consecutive reports since with ZERO offbox evidence,
// one heal per healCooldown — and the R-39(a) guard holds at act time.
//
// R-192 defect (b), fixed here: every refusal above the "not configured" line now leaves a RECORD.
// The regressed-shape branch used to be a bare `return`, so the operator received a daily e-mail with
// the wrong story, no heal, and nothing anywhere saying why the heal declined — "we chose not to act"
// and "the heal never ran" looked identical. `offsite_credential_restaged` has never fired for any
// customer, and until now that fact was indistinguishable from the checker being dead.
//
// The record is a notification_log row (the dispatcher's suppressed-operator-e-mail precedent, R-182:
// a decision not to act is written down on the channel it would have used). It rides `recordRefusal`
// — true only when the stuck event was emitted this pass — so it appears once per stuckCooldown
// beside the e-mail it explains, rather than once per monitor tick. The GUARD ITSELF IS UNCHANGED:
// the set of situations in which the heal fires is byte-for-byte what it was; only the silence is
// gone. The two conditions are split into separate branches solely so each refusal can name its own
// reason.
func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus, recordRefusal bool, declared bool) {
	if c.reissuer == nil {
		return // no provisioner configured: the heal does not exist on this hub, so there is nothing to explain
	}
	// R-204 item 4: a DECLARING box belongs to internal/offsiteheal, which re-arms the stored
	// credential before minting a new one. Two mechanisms healing the same customer would double-issue
	// — and this one can only mint, so it would also skip the cheap path. Stand down, with a record:
	// "we chose not to act" and "the heal never ran" must not look identical (R-192 defect (b)).
	if declared {
		c.recordHealRefusal(customerID, recordRefusal,
			"the box DECLARES offsite.state=needs_credential; internal/offsiteheal owns this remediation (it re-stages the stored credential before minting). A second mechanism minting here would double-issue.")
		return
	}
	if status.OffsiteReportsSinceConsume != 0 {
		c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf(
			"regressed-apply shape: %d of the first %d report(s) after the consume DID carry an offbox target, so a burned credential is ruled out — a restage would treat a symptom whose cause is elsewhere. Operator's call (R-193).",
			status.OffsiteReportsSinceConsume, status.ReportsSinceConsume))
		return
	}
	if status.ReportsSinceConsume < healMinReports {
		c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf(
			"only %d report(s) since the consume (need %d): the box has not reported enough for the burned shape to be unambiguous.",
			status.ReportsSinceConsume, healMinReports))
		return
	}
	last, err := c.store.LastEventAt(customerID, eventCredentialRestaged)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: heal rate-limit read: %v", customerID, err)
		return
	}
	if !last.IsZero() && c.now().Sub(last) < healCooldown {
		return // one restage per customer per 24 h — repeats surface as repeated events only
	}
	// THE R-39(a) GUARD — re-read the secret row immediately before acting. The derive above is a
	// snapshot; an operator Re-issue may have staged a FRESH UNCONSUMED secret since (TOCTOU).
	// SaveOneTimeSecret clobbers by design, so acting now would burn that fresh password.
	info, err := c.store.GetOneTimeSecretInfo(customerID)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: guard read: %v", customerID, err)
		return
	}
	if info == nil || info.ConsumedAt.IsZero() {
		c.recordHealRefusal(customerID, recordRefusal, fmt.Sprintf(
			"R-39(a) guard: the secret row is now %s — restaging over an unconsumed secret would clobber a password the box may be about to consume.", secretShape(info)))
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()
	if err := c.reissuer.ReissueOffsiteForCustomer(ctx, customerID); err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: self-heal reissue failed: %v", customerID, err)
		return
	}
	c.logger.Printf("[INFO] offsite-delivery: %s: self-heal restage fired (consumed_at %s, %d offbox-less reports since)",
		customerID, status.Since.UTC().Format(time.RFC3339), status.ReportsSinceConsume)
	details, _ := json.Marshal(map[string]any{
		"burned_consumed_at":    status.Since.UTC().Format(time.RFC3339),
		"reports_since_consume": status.ReportsSinceConsume,
	})
	c.emit(customerID, eventCredentialRestaged, "warning",
		"Offsite credential re-staged automatically: the previous one-time password was consumed but never applied (burned mid-delivery). The box picks the fresh password up on its next config refresh.",
		string(details))
}

// recordHealRefusal makes a decision NOT to self-heal visible. Always logs; additionally writes a
// notification_log row on the "operator" channel with status "refused" when `record` is set (the
// stuck event was emitted this pass), so the refusal sits next to the e-mail that prompted the
// question. A LogNotification failure is logged, never swallowed, and never blocks the refusal — the
// refusal is the primary effect.
func (c *OffsiteDeliveryChecker) recordHealRefusal(customerID string, record bool, reason string) {
	c.logger.Printf("[INFO] offsite-delivery: %s: self-heal REFUSED — %s", customerID, reason)
	if !record {
		return
	}
	if err := c.store.LogNotification(customerID, eventCredentialRestaged, "warning",
		"Automatic offsite credential restage was NOT performed.", "refused", reason, "operator"); err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: could not record the heal refusal: %v", customerID, err)
	}
}

func secretShape(info *store.OneTimeSecretInfo) string {
	if info == nil {
		return "absent"
	}
	return "unconsumed (staged " + info.CreatedAt.UTC().Format(time.RFC3339) + ")"
}

// emit saves the event (audit trail first) and then notifies — the OffsiteChecker convention:
// SaveEvent failure logs and SKIPS the notification (an email without its audit row lies).
func (c *OffsiteDeliveryChecker) emit(customerID, eventType, severity, message, details string) {
	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, details, "hub"); err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: save %s: %v", customerID, eventType, err)
		return
	}
	c.logger.Printf("[INFO] offsite-delivery: %s: %s (%s)", customerID, eventType, severity)
	if c.onEvent != nil {
		c.onEvent(customerID, eventType, severity, message, details, "hub")
	}
}