felhom.eu/hub/internal/offsiteheal/reconciler.go

// Package offsiteheal is the hub-side OFF-SITE credential self-heal reconciler (R-193 / R-204 item 4,
// hub v0.96.0). It is the sibling of internal/pbsdrheal, deliberately: same shape, same restraint,
// same rule that one thing is left loud rather than healed.
//
// THE PROBLEM. The 2026-08-04 drill (R-201) proved a customer's file survives a machine rebuild and
// comes back — with a person present for four manual interventions. Three were closed in controller
// v0.198.0 / hub v0.95.0. The fourth is this one: a REBUILT box has no off-site credential of its own,
// because the one-time provider password was spent by its predecessor. Everything downstream is
// self-service; nothing gets the box past that first step.
//
// WHY THE TRIGGER IS A DECLARATION AND NOT AN INFERENCE (the operator ruling, 2026-08-05, and the
// whole design). From the hub, an ABSENT off-site object has FOUR meanings — never configured,
// mid-restart, a transient config read failure, and rebuilt-and-stranded — and the hub cannot tell
// them apart. The BOX can, from two local facts it holds with certainty: its data area is fresh (no
// repository password) AND the hub is holding a sealed recovery package for it. So the box says so,
// in its ordinary report, and this reconciler acts on a stated request rather than on a silence.
// That is also why R-192's counting guard is REPLACED rather than repaired: it inferred the same
// thing by counting reports over the OLDEST 500 after a consume, all of which predate a rebuild.
//
// WHAT IT DOES, once a declaration has held across a debounce: RE-STAGES the customer's stored
// one-time secret (store.RestageOneTimeSecret — no provider call, no new password) and escalates to
// the existing Re-issue ONLY when there is nothing stored to re-arm. It NEVER blind-timer-reissues
// and NEVER re-runs a provisioning step. A box that is not declaring is a pure no-op.
//
// WHAT IT DELIBERATELY DOES NOT DO: it never runs, or asks for, an escrow ceremony. A credential is
// replaceable; the recovery code is not, because only the customer holds it. Credential automatic,
// key customer-present — the ruling this session implements and must not quietly widen.
package offsiteheal

import (
	"context"
	"log"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/offsite"
	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// StateNeedsCredential is the ONE state acted on — the string controller v0.199.0 declares on its
// report's `offsite` object (backup.OffsiteStateNeedsCredential). Everything else, including "" (an
// older controller, a healthy box, or a box with no off-site object at all), is a no-op.
//
// THE §8.4 QUESTION — IS THERE AN EQUIVALENT OF pbsdrheal's DELIBERATELY-UNHEALED `verify_failed`?
// Yes, and it is handled by CONSTRUCTION rather than by a case here, which is worth stating plainly
// because "there is nothing like that here" is usually wrong. The analogue is the REGRESSED shape:
// a box that HAD a working off-site tier and lost its target while still holding its repository
// password. Re-arming a credential would not help it — its problem is whatever removed the target —
// and R-192's existing checker already refuses to heal that shape for exactly this reason. It cannot
// reach this reconciler at all, because the controller's declaration predicate requires the
// repository password to be ABSENT (backup.needsOffsiteCredential). So the unhealable case is
// excluded upstream, by the declaration itself, rather than filtered out here.
const StateNeedsCredential = "needs_credential"

// Audit event types (store.SaveEvent; hub-internal, not gated by allowedEventTypes — the pbsdr_*
// precedent). Distinct per remediation so the operator sees exactly what was auto-done.
const (
	eventRestaged = "offsite_selfheal_restaged" // re-armed the stored secret (routine, no provider call)
	eventReissued = "offsite_selfheal_reissued" // nothing stored to re-arm → minted a fresh credential
)

// debounceReportsDefault — how many DISTINCT reports must carry the declaration before acting.
//
// TWO, and the interval is derived rather than chosen. The controller reports every ~15 minutes, so
// two distinct declarations mean the state has survived at least one full report cycle: a restart, a
// slow first report or a transient config read cannot produce it, because each of those resolves
// well inside one cycle. One report would act on a blip; three would leave a genuinely stranded
// customer waiting ~45 minutes for a credential they cannot obtain any other way. The reconciler's
// own tick (below) is deliberately FASTER than the report cadence so it never adds latency of its
// own — the debounce is counted in fresh evidence, not in ticks.
const debounceReportsDefault = 2

// tickIntervalDefault — how often the fleet is swept. Shorter than the 15-minute report cadence on
// purpose (see above); a sweep that finds nothing writes nothing.
const tickIntervalDefault = 5 * time.Minute

// Actions is the mutation seam — fakes in tests count calls without touching a provider. Restage
// flips a stored secret's consumed flag (false when NO row exists → the caller escalates); Reissue
// mints a fresh provider credential and stores a fresh consume-once secret.
type Actions interface {
	Restage(customerID string) (restaged bool, err error)
	Reissue(ctx context.Context, customerID string) error
}

// Reissuer is satisfied by *web.Server (its ReissueOffsiteForCustomer) — the same indirection
// pbsdrheal uses, so the escalation IS the designed Re-issue path and not a sibling mechanism.
type Reissuer interface {
	ReissueOffsiteForCustomer(ctx context.Context, customerID string) error
}

type storeActions struct {
	st       *store.Store
	reissuer Reissuer
}

func (a storeActions) Restage(customerID string) (bool, error) {
	return a.st.RestageOneTimeSecret(customerID)
}
func (a storeActions) Reissue(ctx context.Context, customerID string) error {
	return a.reissuer.ReissueOffsiteForCustomer(ctx, customerID)
}

// NewActions builds the production mutation seam.
func NewActions(st *store.Store, reissuer Reissuer) Actions { return storeActions{st: st, reissuer: reissuer} }

// debounceState tracks, per customer, the last DISTINCT report seen and how many consecutive
// distinct reports have carried the declaration.
type debounceState struct {
	reportID int64
	state    string
	streak   int
}

// Reconciler re-arms stranded boxes. DECLARATIVE + IDEMPOTENT: a tick over a healthy fleet writes
// nothing. It reads the hub DB — never the box.
type Reconciler struct {
	store           *store.Store
	act             Actions
	interval        time.Duration
	debounceReports int
	onlyCustomer    string // "" = whole fleet; non-empty restricts the work set (supervised rollout)
	trigger         chan struct{}
	logger          *log.Logger

	deb map[string]debounceState
}

// NewReconciler builds the reconciler with the derived defaults.
func NewReconciler(st *store.Store, act Actions, logger *log.Logger) *Reconciler {
	if logger == nil {
		logger = log.Default()
	}
	return &Reconciler{
		store:           st,
		act:             act,
		interval:        tickIntervalDefault,
		debounceReports: debounceReportsDefault,
		trigger:         make(chan struct{}, 1),
		logger:          logger,
		deb:             map[string]debounceState{},
	}
}

// RestrictToCustomer scopes the work set to one customer (empty = whole fleet) for a supervised first
// rollout — the pbsdrheal precedent. Set before Run.
func (r *Reconciler) RestrictToCustomer(customerID string) { r.onlyCustomer = customerID }

// SetDebounceReports overrides the debounce (tests). Values < 1 are ignored.
func (r *Reconciler) SetDebounceReports(n int) {
	if n >= 1 {
		r.debounceReports = n
	}
}

// Trigger requests an immediate reconcile. Non-blocking.
func (r *Reconciler) Trigger() {
	select {
	case r.trigger <- struct{}{}:
	default:
	}
}

// Run loops until ctx is done. Never exits on an error.
func (r *Reconciler) Run(ctx context.Context) {
	ticker := time.NewTicker(r.interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-r.trigger:
		case <-ticker.C:
		}
		r.ReconcileOnce(ctx)
	}
}

// ReconcileOnce sweeps the fleet once. Exported so tests drive it directly instead of racing a
// ticker, and so the wiring can trigger it. Errors are logged and retried next tick; a read failure
// means this reconciler does NOTHING rather than acting on a partial view.
func (r *Reconciler) ReconcileOnce(ctx context.Context) {
	configs, err := r.store.ListCustomerConfigs()
	if err != nil {
		r.logger.Printf("[ERROR] offsiteheal: list customer configs: %v (retry next tick)", err)
		return
	}
	seen := make(map[string]bool, len(configs))
	for _, cfg := range configs {
		if r.onlyCustomer != "" && cfg.CustomerID != r.onlyCustomer {
			continue
		}
		if cfg.Status != "active" {
			continue // blocked/inactive customers are never healed
		}
		// The descriptor must still say the customer HAS an off-site tier. A deliberately disabled
		// tier must not be re-credentialed behind the operator's back; an unparseable config never
		// drives a heal (the config UI owns that failure).
		d, derr := offsite.ReadDescriptor(cfg.ConfigJSON)
		if derr != nil || d == nil || !d.Enabled {
			delete(r.deb, cfg.CustomerID)
			continue
		}
		found, reportID, state, err := r.store.LatestReportOffsiteDeclaration(cfg.CustomerID)
		if err != nil {
			r.logger.Printf("[ERROR] offsiteheal: %s: read latest declaration: %v (retry next tick)", cfg.CustomerID, err)
			continue // never act on a partial view
		}
		seen[cfg.CustomerID] = true
		if !found || state != StateNeedsCredential {
			delete(r.deb, cfg.CustomerID) // healthy, silent, or an older controller → pure no-op
			continue
		}
		if !r.confirm(cfg.CustomerID, reportID, state) {
			continue // one declaration is not evidence — wait for a second distinct report
		}
		r.heal(ctx, cfg.CustomerID, reportID, state)
	}
	for c := range r.deb {
		if !seen[c] {
			delete(r.deb, c)
		}
	}
}

// confirm advances the per-customer debounce and reports whether the declaration has held across
// >= debounceReports DISTINCT reports. A re-observed same report never advances the streak — the
// debounce counts fresh evidence, not reconciler ticks, so a fast tick cannot shorten it.
func (r *Reconciler) confirm(customerID string, reportID int64, state string) bool {
	st := r.deb[customerID]
	if reportID != st.reportID {
		if state == st.state {
			st.streak++
		} else {
			st.streak = 1
		}
		st.state = state
		st.reportID = reportID
		r.deb[customerID] = st
	}
	return st.streak >= r.debounceReports
}

// heal re-arms the stored secret; with nothing stored, escalates to a fresh mint.
//
// RESTAGE BEFORE MINT, because re-arming costs no provider call and converges in one report tick,
// while a mint is external churn for a credential the hub already holds. The stored value survives a
// consume (store.RestageOneTimeSecret documents how that was established), which is what makes the
// cheap path possible at all.
func (r *Reconciler) heal(ctx context.Context, customerID string, reportID int64, state string) {
	restaged, err := r.act.Restage(customerID)
	if err != nil {
		r.logger.Printf("[ERROR] offsiteheal: re-stage %s: %v (retry next tick)", customerID, err)
		return
	}
	if restaged {
		// v0.127.0 (decision 69): the box no longer receives the password. Re-arming only resets the
		// delivery ledger; the box gets access by registering its PUBLIC key, which the hub installs with
		// the credential it keeps sealed. Nothing here reaches the box.
		r.logger.Printf("[INFO] offsiteheal: customer %s declared %s across %d reports — the stored credential is present (sealed); the box re-registers its public key and the hub installs it; no provider credential was minted",
			customerID, state, r.debounceReports)
		r.event(customerID, eventRestaged, "info",
			"Offsite self-heal: a rebuilt box asked for off-site access; the hub still holds the storage credential (sealed) and will install the box's key when it registers. No new credential was created at the storage provider, and none was sent to the box.")
		r.resetAfterHeal(customerID, reportID, state)
		return
	}
	r.logger.Printf("[INFO] offsiteheal: customer %s declares %s with NO stored one-time secret — escalating to Re-issue", customerID, state)
	if err := r.act.Reissue(ctx, customerID); err != nil {
		r.logger.Printf("[ERROR] offsiteheal: re-issue for customer %s: %v (retry next tick)", customerID, err)
		return
	}
	r.event(customerID, eventReissued, "warning",
		"Offsite self-heal: a rebuilt box asked for its storage credential and the hub had none stored, so fresh credentials were issued at the storage provider.")
	r.resetAfterHeal(customerID, reportID, state)
}

// resetAfterHeal clears the streak (keeping the report id) so the SAME report cannot re-heal on the
// next tick — a fresh report must re-confirm the box is still stranded before acting again. This is
// what makes Scenario E's "one mint, not repeated" true.
func (r *Reconciler) resetAfterHeal(customerID string, reportID int64, state string) {
	r.deb[customerID] = debounceState{reportID: reportID, state: state, streak: 0}
}

// event records an audit row; a failure to write it must never break the heal loop.
func (r *Reconciler) event(customerID, eventType, severity, message string) {
	if _, err := r.store.SaveEvent(customerID, eventType, severity, message, "", "hub"); err != nil {
		r.logger.Printf("[WARN] offsiteheal: audit event %s for %s not stored: %v", eventType, customerID, err)
	}
}