felhom.eu/hub/internal/offsiteheal/reconciler.go

// Package offsiteheal is the hub-side OFF-SITE credential self-heal reconciler (R-193 / R-204 item 4,
// hub v0.96.0). It is the sibling of internal/pbsdrheal, deliberately: same shape, same restraint,
// same rule that one thing is left loud rather than healed.
//
// THE PROBLEM. The 2026-08-04 drill (R-201) proved a customer's file survives a machine rebuild and
// comes back — with a person present for four manual interventions. Three were closed in controller
// v0.198.0 / hub v0.95.0. The fourth is this one: a REBUILT box has no off-site credential of its own,
// because the one-time provider password was spent by its predecessor. Everything downstream is
// self-service; nothing gets the box past that first step.
//
// WHY THE TRIGGER IS A DECLARATION AND NOT AN INFERENCE (the operator ruling, 2026-08-05, and the
// whole design). From the hub, an ABSENT off-site object has FOUR meanings — never configured,
// mid-restart, a transient config read failure, and rebuilt-and-stranded — and the hub cannot tell
// them apart. The BOX can, from two local facts it holds with certainty: its data area is fresh (no
// repository password) AND the hub is holding a sealed recovery package for it. So the box says so,
// in its ordinary report, and this reconciler acts on a stated request rather than on a silence.
// That is also why R-192's counting guard is REPLACED rather than repaired: it inferred the same
// thing by counting reports over the OLDEST 500 after a consume, all of which predate a rebuild.
//
// WHAT IT DOES, once a declaration has held across a debounce: RE-STAGES the customer's stored
// one-time secret (store.RestageOneTimeSecret — no provider call, no new password) and escalates to
// the existing Re-issue ONLY when there is nothing stored to re-arm. It NEVER blind-timer-reissues
// and NEVER re-runs a provisioning step. A box that is not declaring is a pure no-op.
//
// WHAT IT DELIBERATELY DOES NOT DO: it never runs, or asks for, an escrow ceremony. A credential is
// replaceable; the recovery code is not, because only the customer holds it. Credential automatic,
// key customer-present — the ruling this session implements and must not quietly widen.
package offsiteheal

import (
	"context"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// StateNeedsCredential is the ONE state acted on — the string controller v0.199.0 declares on its
// report's `offsite` object (backup.OffsiteStateNeedsCredential). Everything else, including "" (an
// older controller, a healthy box, or a box with no off-site object at all), is a no-op.
//
// THE §8.4 QUESTION — IS THERE AN EQUIVALENT OF pbsdrheal's DELIBERATELY-UNHEALED `verify_failed`?
// Yes, and it is handled by CONSTRUCTION rather than by a case here, which is worth stating plainly
// because "there is nothing like that here" is usually wrong. The analogue is the REGRESSED shape:
// a box that HAD a working off-site tier and lost its target while still holding its repository
// password. Re-arming a credential would not help it — its problem is whatever removed the target —
// and R-192's existing checker already refuses to heal that shape for exactly this reason. It cannot
// reach this reconciler at all, because the controller's declaration predicate requires the
// repository password to be ABSENT (backup.needsOffsiteCredential). So the unhealable case is
// excluded upstream, by the declaration itself, rather than filtered out here.
const StateNeedsCredential = "needs_credential"

// Audit event types (store.SaveEvent; hub-internal, not gated by allowedEventTypes — the pbsdr_*
// precedent). Distinct per remediation so the operator sees exactly what was auto-done.
const (
	eventRestaged = "offsite_selfheal_restaged" // re-armed the stored secret (routine, no provider call)
	eventReissued = "offsite_selfheal_reissued" // nothing stored to re-arm → minted a fresh credential
)

// debounceReportsDefault — how many DISTINCT reports must carry the declaration before acting.
//
// TWO, and the interval is derived rather than chosen. The controller reports every ~15 minutes, so
// two distinct declarations mean the state has survived at least one full report cycle: a restart, a
// slow first report or a transient config read cannot produce it, because each of those resolves
// well inside one cycle. One report would act on a blip; three would leave a genuinely stranded
// customer waiting ~45 minutes for a credential they cannot obtain any other way. The reconciler's
// own tick (below) is deliberately FASTER than the report cadence so it never adds latency of its
// own — the debounce is counted in fresh evidence, not in ticks.
const debounceReportsDefault = 2

// tickIntervalDefault — how often the fleet is swept. Shorter than the 15-minute report cadence on
// purpose (see above); a sweep that finds nothing writes nothing.
const tickIntervalDefault = 5 * time.Minute

// Actions is the mutation seam — fakes in tests count calls without touching a provider. Restage
// flips a stored secret's consumed flag (false when NO row exists → the caller escalates); Reissue
// mints a fresh provider credential and stores a fresh consume-once secret.
type Actions interface {
	Restage(customerID string) (restaged bool, err error)
	Reissue(ctx context.Context, customerID string) error
}

// Reissuer is satisfied by *web.Server (its ReissueOffsiteForCustomer) — the same indirection
// pbsdrheal uses, so the escalation IS the designed Re-issue path and not a sibling mechanism.
type Reissuer interface {
	ReissueOffsiteForCustomer(ctx context.Context, customerID string) error
}

type storeActions struct {
	st       *store.Store
	reissuer Reissuer
}

func (a storeActions) Restage(customerID string) (bool, error) {
	return a.st.RestageOneTimeSecret(customerID)
}
func (a storeActions) Reissue(ctx context.Context, customerID string) error {
	return a.reissuer.ReissueOffsiteForCustomer(ctx, customerID)
}

// NewActions builds the production mutation seam.
func NewActions(st *store.Store, reissuer Reissuer) Actions { return storeActions{st: st, reissuer: reissuer} }

// debounceState tracks, per customer, the last DISTINCT report seen and how many consecutive
// distinct reports have carried the declaration.
type debounceState struct {
	reportID int64
	state    string
	streak   int
}

// Reconciler re-arms stranded boxes. DECLARATIVE + IDEMPOTENT: a tick over a healthy fleet writes
// nothing. It reads the hub DB — never the box.
type Reconciler struct {
	store           *store.Store
	act             Actions
	interval        time.Duration
	debounceReports int
	onlyCustomer    string // "" = whole fleet; non-empty restricts the work set (supervised rollout)
	trigger         chan struct{}
	logger          *log.Logger

	deb map[string]debounceState
}

// NewReconciler builds the reconciler with the derived defaults.
func NewReconciler(st *store.Store, act Actions, logger *log.Logger) *Reconciler {
	if logger == nil {
		logger = log.Default()
	}
	return &Reconciler{
		store:           st,
		act:             act,
		interval:        tickIntervalDefault,
		debounceReports: debounceReportsDefault,
		trigger:         make(chan struct{}, 1),
		logger:          logger,
		deb:             map[string]debounceState{},
	}
}

// RestrictToCustomer scopes the work set to one customer (empty = whole fleet) for a supervised first
// rollout — the pbsdrheal precedent. Set before Run.
func (r *Reconciler) RestrictToCustomer(customerID string) { r.onlyCustomer = customerID }

// SetDebounceReports overrides the debounce (tests). Values < 1 are ignored.
func (r *Reconciler) SetDebounceReports(n int) {
	if n >= 1 {
		r.debounceReports = n
	}
}

// Trigger requests an immediate reconcile. Non-blocking.
func (r *Reconciler) Trigger() {
	select {
	case r.trigger <- struct{}{}:
	default:
	}
}

// Run loops until ctx is done. Never exits on an error.
func (r *Reconciler) Run(ctx context.Context) {
	ticker := time.NewTicker(r.interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-r.trigger:
		case <-ticker.C:
		}
		r.ReconcileOnce(ctx)
	}
}

// ReconcileOnce sweeps the fleet once. Exported so tests drive it directly instead of racing a
// ticker, and so the wiring can trigger it. Errors are logged and retried next tick; a read failure
// means this reconciler does NOTHING rather than acting on a partial view.
func (r *Reconciler) ReconcileOnce(ctx context.Context) {
	configs, err := r.store.ListCustomerConfigs()
	if err != nil {
		r.logger.Printf("[ERROR] offsiteheal: list customer configs: %v (retry next tick)", err)
		return
	}
	seen := make(map[string]bool, len(configs))
	for _, cfg := range configs {
		if r.onlyCustomer != "" && cfg.CustomerID != r.onlyCustomer {
			continue
		}
		if cfg.Status != "active" {
			continue // blocked/inactive customers are never healed
		}
		// The descriptor must still say the customer HAS an off-site tier. A deliberately disabled
		// tier must not be re-credentialed behind the operator's back; an unparseable config never
		// drives a heal (the config UI owns that failure).
		d, derr := offsite.ReadDescriptor(cfg.ConfigJSON)
		if derr != nil || d == nil || !d.Enabled {
			delete(r.deb, cfg.CustomerID)
			continue
		}
		found, reportID, state, err := r.store.LatestReportOffsiteDeclaration(cfg.CustomerID)
		if err != nil {
			r.logger.Printf("[ERROR] offsiteheal: %s: read latest declaration: %v (retry next tick)", cfg.CustomerID, err)
			continue // never act on a partial view
		}
		seen[cfg.CustomerID] = true
		if !found || state != StateNeedsCredential {
			delete(r.deb, cfg.CustomerID) // healthy, silent, or an older controller → pure no-op
			continue
		}
		if !r.confirm(cfg.CustomerID, reportID, state) {
			continue // one declaration is not evidence — wait for a second distinct report
		}
		r.heal(ctx, cfg.CustomerID, reportID, state)
	}
	for c := range r.deb {
		if !seen[c] {
			delete(r.deb, c)
		}
	}
}

// confirm advances the per-customer debounce and reports whether the declaration has held across
// >= debounceReports DISTINCT reports. A re-observed same report never advances the streak — the
// debounce counts fresh evidence, not reconciler ticks, so a fast tick cannot shorten it.
func (r *Reconciler) confirm(customerID string, reportID int64, state string) bool {
	st := r.deb[customerID]
	if reportID != st.reportID {
		if state == st.state {
			st.streak++
		} else {
			st.streak = 1
		}
		st.state = state
		st.reportID = reportID
		r.deb[customerID] = st
	}
	return st.streak >= r.debounceReports
}

// heal re-arms the stored secret; with nothing stored, escalates to a fresh mint.
//
// RESTAGE BEFORE MINT, because re-arming costs no provider call and converges in one report tick,
// while a mint is external churn for a credential the hub already holds. The stored value survives a
// consume (store.RestageOneTimeSecret documents how that was established), which is what makes the
// cheap path possible at all.
func (r *Reconciler) heal(ctx context.Context, customerID string, reportID int64, state string) {
	restaged, err := r.act.Restage(customerID)
	if err != nil {
		r.logger.Printf("[ERROR] offsiteheal: re-stage %s: %v (retry next tick)", customerID, err)
		return
	}
	if restaged {
		r.logger.Printf("[INFO] offsiteheal: re-staged the stored one-time offsite secret for customer %s (declared %s across %d reports) — the box re-consumes on its next cycle; no provider credential was minted",
			customerID, state, r.debounceReports)
		r.event(customerID, eventRestaged, "info",
			"Offsite self-heal: a rebuilt box asked for its storage credential and the stored one-time password was re-armed. No new credential was created at the storage provider.")
		r.resetAfterHeal(customerID, reportID, state)
		return
	}
	r.logger.Printf("[INFO] offsiteheal: customer %s declares %s with NO stored one-time secret — escalating to Re-issue", customerID, state)
	if err := r.act.Reissue(ctx, customerID); err != nil {
		r.logger.Printf("[ERROR] offsiteheal: re-issue for customer %s: %v (retry next tick)", customerID, err)
		return
	}
	r.event(customerID, eventReissued, "warning",
		"Offsite self-heal: a rebuilt box asked for its storage credential and the hub had none stored, so fresh credentials were issued at the storage provider.")
	r.resetAfterHeal(customerID, reportID, state)
}

// resetAfterHeal clears the streak (keeping the report id) so the SAME report cannot re-heal on the
// next tick — a fresh report must re-confirm the box is still stranded before acting again. This is
// what makes Scenario E's "one mint, not repeated" true.
func (r *Reconciler) resetAfterHeal(customerID string, reportID int64, state string) {
	r.deb[customerID] = debounceState{reportID: reportID, state: state, streak: 0}
}

// event records an audit row; a failure to write it must never break the heal loop.
func (r *Reconciler) event(customerID, eventType, severity, message string) {
	if _, err := r.store.SaveEvent(customerID, eventType, severity, message, "", "hub"); err != nil {
		r.logger.Printf("[WARN] offsiteheal: audit event %s for %s not stored: %v", eventType, customerID, err)
	}
}