felhom.eu/hub/internal/monitor/offsite_delivery.go

package monitor

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-70 + R-71(c): the offsite delivery-state checker. Reads the shared detector
// (offsite.DeliveryStateFor) for every offsite-enabled customer and drives two consumers:
//
//   - the LOUD EVENT: `offsite_delivery_stuck` (warning → operator email per existing dispatcher
//     rules) when the burned-credential shape persists past stuckAfter;
//   - the SELF-HEAL (R-71c): invoke the EXISTING Re-issue path — never a second delivery
//     mechanism — when the shape is unambiguous, then surface `offsite_credential_restaged`
//     (warning) so the operator ALWAYS knows it fired.
//
// Cooldowns are durable: both events rate-limit off store.LastEventAt (the events table), so a hub
// restart cannot flood or silently re-heal. A repeating pattern surfaces as repeating events on a
// 24 h cadence, never as a silent retry loop.
//
// THE R-39(a) GUARD (mandatory, enforced HERE because the store deliberately clobbers):
// SaveOneTimeSecret is last-write-wins by design — Re-issue depends on supersede. Restaging on top
// of an UNCONSUMED secret would clobber a password a box may be about to consume (the operator may
// have clicked Re-issue between this checker's derive and its act). So the heal re-reads the
// secret row IMMEDIATELY before acting and refuses unless it is still a CONSUMED row.

const (
	eventDeliveryStuck      = "offsite_delivery_stuck"      // hub-internal (not in allowedEventTypes, like pbsdr_*)
	eventCredentialRestaged = "offsite_credential_restaged" // hub-internal
	// stuckAfter: consumed_awaiting_apply is normal for seconds (F10 repair: consume→applied in
	// 4 s). An hour of it means the apply will never come without intervention.
	stuckAfter = time.Hour
	// stuckCooldown / healCooldown: per-customer, durable via the events table.
	stuckCooldown = 24 * time.Hour
	healCooldown  = 24 * time.Hour
	// healMinReports: at least this many consecutive offbox-less reports after consumed_at before
	// the self-heal may fire — the box must be alive and reporting, just not applied.
	healMinReports = 4
)

// OffsiteReissuer is the narrow reissue surface the self-heal needs — satisfied by
// *web.Server.ReissueOffsiteForCustomer (the pbsdrheal.Reissuer precedent; avoids an import
// cycle and guarantees the heal IS the designed Re-issue path, not a sibling mechanism).
type OffsiteReissuer interface {
	ReissueOffsiteForCustomer(ctx context.Context, customerID string) error
}

// OffsiteDeliveryChecker runs on the shared monitor ticker.
type OffsiteDeliveryChecker struct {
	store    *store.Store
	reissuer OffsiteReissuer // nil → self-heal disabled (no provisioner configured); detector+event still run
	onEvent  EventNotifyFunc
	logger   *log.Logger
	now      func() time.Time
}

// NewOffsiteDeliveryChecker constructs the checker. reissuer may be nil (heal disabled).
func NewOffsiteDeliveryChecker(s *store.Store, reissuer OffsiteReissuer, onEvent EventNotifyFunc, logger *log.Logger) *OffsiteDeliveryChecker {
	return &OffsiteDeliveryChecker{store: s, reissuer: reissuer, onEvent: onEvent, logger: logger, now: time.Now}
}

// Check derives the delivery state for every offsite-enabled active customer and applies the
// event + self-heal rules. Never returns an error — a checker failure must not take the ticker
// down (log-and-continue, like every sibling checker).
func (c *OffsiteDeliveryChecker) Check() {
	configs, err := c.store.ListCustomerConfigs()
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: list configs: %v", err)
		return
	}
	for _, cfg := range configs {
		if cfg.Status != "active" {
			continue // blocked/inactive customers are not delivery-monitored (and never healed)
		}
		d, err := offsite.ReadDescriptor(cfg.ConfigJSON)
		if err != nil || d == nil || !d.Enabled {
			continue // unparseable config never drives a heal; the config UI owns that failure
		}
		status, err := offsite.DeliveryStateFor(c.store, cfg.CustomerID)
		if err != nil {
			c.logger.Printf("[WARN] offsite-delivery: %s: derive: %v", cfg.CustomerID, err)
			continue
		}
		if status.State != offsite.DeliveryConsumedAwaitingApply {
			continue // applied / staged / no_secret: card-rendered states, no event or heal (yet)
		}
		age := c.now().Sub(status.Since)
		if age < stuckAfter {
			continue // normal convergence window
		}
		c.maybeEmitStuck(cfg.CustomerID, status, age)
		c.maybeHeal(cfg.CustomerID, status)
	}
}

// maybeEmitStuck emits offsite_delivery_stuck (warning) once per stuckCooldown per customer.
func (c *OffsiteDeliveryChecker) maybeEmitStuck(customerID string, status offsite.DeliveryStatus, age time.Duration) {
	last, err := c.store.LastEventAt(customerID, eventDeliveryStuck)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: cooldown read: %v", customerID, err)
		return
	}
	if !last.IsZero() && c.now().Sub(last) < stuckCooldown {
		return
	}
	msg := fmt.Sprintf("Offsite delivery stuck: one-time password consumed %s ago and %d report(s) since carry no offbox target — the credential is likely burned (apply died between consume and persist). Re-issue delivers a fresh one.",
		age.Round(time.Minute), status.ReportsSinceConsume)
	details, _ := json.Marshal(map[string]any{
		"state":                 string(status.State),
		"consumed_at":           status.Since.UTC().Format(time.RFC3339),
		"reports_since_consume": status.ReportsSinceConsume,
	})
	c.emit(customerID, eventDeliveryStuck, "warning", msg, string(details))
}

// maybeHeal fires the R-71c self-heal when the burned-credential shape is unambiguous:
// consumed ≥ stuckAfter ago, ≥ healMinReports consecutive reports since with ZERO offbox evidence,
// one heal per healCooldown — and the R-39(a) guard holds at act time.
func (c *OffsiteDeliveryChecker) maybeHeal(customerID string, status offsite.DeliveryStatus) {
	if c.reissuer == nil {
		return
	}
	if status.ReportsSinceConsume < healMinReports || status.OffsiteReportsSinceConsume != 0 {
		return // box not reporting enough, or offbox evidence exists (regressed-apply shape) → operator's call
	}
	last, err := c.store.LastEventAt(customerID, eventCredentialRestaged)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: heal rate-limit read: %v", customerID, err)
		return
	}
	if !last.IsZero() && c.now().Sub(last) < healCooldown {
		return // one restage per customer per 24 h — repeats surface as repeated events only
	}
	// THE R-39(a) GUARD — re-read the secret row immediately before acting. The derive above is a
	// snapshot; an operator Re-issue may have staged a FRESH UNCONSUMED secret since (TOCTOU).
	// SaveOneTimeSecret clobbers by design, so acting now would burn that fresh password.
	info, err := c.store.GetOneTimeSecretInfo(customerID)
	if err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: guard read: %v", customerID, err)
		return
	}
	if info == nil || info.ConsumedAt.IsZero() {
		c.logger.Printf("[INFO] offsite-delivery: %s: heal refused — secret row is now %s (R-39(a) guard: never restage over an unconsumed secret)",
			customerID, secretShape(info))
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
	defer cancel()
	if err := c.reissuer.ReissueOffsiteForCustomer(ctx, customerID); err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: self-heal reissue failed: %v", customerID, err)
		return
	}
	c.logger.Printf("[INFO] offsite-delivery: %s: self-heal restage fired (consumed_at %s, %d offbox-less reports since)",
		customerID, status.Since.UTC().Format(time.RFC3339), status.ReportsSinceConsume)
	details, _ := json.Marshal(map[string]any{
		"burned_consumed_at":    status.Since.UTC().Format(time.RFC3339),
		"reports_since_consume": status.ReportsSinceConsume,
	})
	c.emit(customerID, eventCredentialRestaged, "warning",
		"Offsite credential re-staged automatically: the previous one-time password was consumed but never applied (burned mid-delivery). The box picks the fresh password up on its next config refresh.",
		string(details))
}

func secretShape(info *store.OneTimeSecretInfo) string {
	if info == nil {
		return "absent"
	}
	return "unconsumed (staged " + info.CreatedAt.UTC().Format(time.RFC3339) + ")"
}

// emit saves the event (audit trail first) and then notifies — the OffsiteChecker convention:
// SaveEvent failure logs and SKIPS the notification (an email without its audit row lies).
func (c *OffsiteDeliveryChecker) emit(customerID, eventType, severity, message, details string) {
	if _, err := c.store.SaveEvent(customerID, eventType, severity, message, details, "hub"); err != nil {
		c.logger.Printf("[WARN] offsite-delivery: %s: save %s: %v", customerID, eventType, err)
		return
	}
	c.logger.Printf("[INFO] offsite-delivery: %s: %s (%s)", customerID, eventType, severity)
	if c.onEvent != nil {
		c.onEvent(customerID, eventType, severity, message, details, "hub")
	}
}