felhom-controller/controller/internal/offsiteapply/offsiteapply.go

// Package offsiteapply is the controller-side apply-bridge (SLICE 2): it turns the hub-served offsite
// descriptor + the one-time password into a working key-only offbox target. On config apply it consumes the
// one-time password, VERIFIES the box host key against the hub-captured fingerprint (no blind TOFU), pins it,
// installs the controller's own key, and configures the offbox target → EscrowState="pending" (the fork-4
// enable path). Idempotent (a descriptor hash marker prevents re-consuming a spent password) and fail-safe
// (any step fails → nothing persisted, retried next cycle; never a half-configured offbox).
package offsiteapply

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)

// The apply-bridge seams (tests inject fakes — no live SSH / hub calls in unit tests).
type (
	// PasswordConsumer fetches the one-time transient password from the hub (single-use).
	PasswordConsumer interface {
		Consume(ctx context.Context) (string, error)
	}
	// HostKeyScanner returns the box's host-key fingerprint (SHA256:…) + the known_hosts line to pin.
	HostKeyScanner interface {
		Scan(ctx context.Context, host string, port int) (fingerprint, knownHostsLine string, err error)
	}
	// KeyGenerator produces a fresh keypair: the private key (PEM) and the authorized_keys pub line.
	KeyGenerator interface {
		Generate() (privPEM, pubAuthorized string, err error)
	}
	// KeyInstaller installs the pub line on the box using the one-time password, then verifies passwordless
	// key auth with the private key. It MUST pin the VERIFIED knownHosts line (from the scan) on the
	// connection — never blind-TOFU — so a MITM cannot swap the key between the scan and the install.
	KeyInstaller interface {
		Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error
	}
	// OffboxEnabler configures the offbox target (key + known_hosts + target + soft quota) and goes
	// EscrowState="pending" (the fork-4 enable path). quotaGB=0 = no soft limit (dedicated boxes).
	OffboxEnabler interface {
		ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
	}
	// KeyAuthProber checks whether an ALREADY-INSTALLED key authenticates to the target (pinned to the
	// freshly-scanned knownHosts). ok=true returns that key's PEM so the descriptor change is applied by
	// re-pinning + reconfiguring WITHOUT consuming a one-time password (key-auth-first — kills the
	// stale-descriptor consume-404 loop and shrinks the re-issue blast radius to genuinely-fresh guests).
	// ok=false (no key / auth refused) → the caller falls through to the full consume+install path.
	KeyAuthProber interface {
		Probe(ctx context.Context, host, user string, port int, knownHosts string) (privPEM string, ok bool)
	}
	// SettleProvider reports the managed-update settle state so the bridge can DEFER consuming the
	// one-time password until any imminent managed floor-update has converged (R-71a — the structural
	// fix for the F10 day-0 race). The failure it prevents: a fresh box boots below the operator floor,
	// the bridge consumes the single-use password, then ~35 s later the managed auto-floor update
	// replaces the container mid-install → the new process finds no installed key → consume → 404 →
	// offsite dead until an operator Re-issue. version = the running controller version; floor = the
	// operator-enforced minimum (from the hub report ACK); updateRunning = a swap is in flight;
	// floorKnown = the floor has been learned yet (false = the first report ACK hasn't landed). A hub
	// that hasn't served a floor cannot serve a consume either, so !floorKnown carries NO burn risk.
	SettleProvider interface {
		SettleState() (version, floor string, updateRunning, floorKnown bool)
	}
)

// Settle-gate timing (R-71a). Named constants with rationale so the trade-offs stay visible.
const (
	// reconcileTimeout bounds the actual Reconcile once the gate releases (moved here from main.go so
	// ReconcileWhenSettled owns the whole "gate THEN reconcile" contract). The gate's own wait must NOT
	// eat this budget — the reconcile context is created only after the gate returns.
	reconcileTimeout = 3 * time.Minute
	// settlePoll is the gate's poll cadence. The bridge is a background reconcile with no user-visible
	// latency, so a coarse poll is free; the at/above-floor happy path returns on the FIRST evaluation
	// with no sleep at all (the B′ invariant), so this cadence only ever paces the deferral cases.
	settlePoll = 10 * time.Second
	// settleFloorSubBound: if the hub never tells us the floor (report ACK), stop waiting and GO+WARN.
	// Sized to the report-ACK latency observed at source: the startup report fires ~5 s after boot
	// (main.go's startup goroutine sleeps 5 s), and SetFloor runs SYNCHRONOUSLY inside that report's
	// ACK handler (main.go OnPushResponse → updater.SetFloor), so the floor is normally known within
	// ~5–10 s. The startup report retries up to 3× with 15 s gaps, so a slow first report can push
	// floor-knowledge to ~45 s; 90 s is generous headroom over that worst case. Proceeding here cannot
	// burn a one-time password: a hub that cannot serve a floor cannot serve a consume.
	settleFloorSubBound = 90 * time.Second
	// settleOverallBound: the absolute cap. If we are still below floor after this (a managed
	// floor-update that never lands), proceed anyway — the R-71c hub self-heal restage is the belt for
	// a consume that a genuinely stuck update might later burn.
	settleOverallBound = 5 * time.Minute
)

// belowFloor reports whether the running version is strictly below the operator floor. An
// unparseable version or floor (a dev build, or a malformed floor) is treated as NOT below — the gate
// must never wedge on a version it cannot compare (and a dev build never auto-floor-updates anyway).
func belowFloor(version, floor string) bool {
	v, err1 := util.ParseVersion(version)
	f, err2 := util.ParseVersion(floor)
	if err1 != nil || err2 != nil {
		return false
	}
	return v.Compare(f) < 0
}

// Bridge reconciles the offsite descriptor into a configured offbox target.
type Bridge struct {
	Cfg        *config.Config
	Consumer   PasswordConsumer
	Scanner    HostKeyScanner
	KeyGen     KeyGenerator
	Installer  KeyInstaller
	Enabler    OffboxEnabler
	Prober     KeyAuthProber // optional: key-auth-first (nil → always the full consume+install path)
	MarkerPath string        // where the applied-descriptor-hash is persisted (e.g. <dataDir>/offbox/applied_marker)
	Logger     *log.Logger

	// Settle gates the consume/install path behind managed-update convergence (R-71a). nil → no gate
	// (old behavior: reconcile immediately). Wired only when a self-updater exists — with no update
	// mechanism there is no floor-update to race, so no gate is needed.
	Settle SettleProvider
	// Now/Sleep are clock seams for the settle-gate ONLY (tests inject a fake clock so the bounds are
	// exercised with zero real sleeps). nil → the real wall clock and a context-aware sleep.
	Now   func() time.Time
	Sleep func(ctx context.Context, d time.Duration)
}

func (b *Bridge) nowFn() func() time.Time {
	if b.Now != nil {
		return b.Now
	}
	return time.Now
}

func (b *Bridge) sleepFn() func(context.Context, time.Duration) {
	if b.Sleep != nil {
		return b.Sleep
	}
	return func(ctx context.Context, d time.Duration) {
		t := time.NewTimer(d)
		defer t.Stop()
		select {
		case <-ctx.Done():
		case <-t.C:
		}
	}
}

func (b *Bridge) logf(f string, a ...any) {
	if b.Logger != nil {
		b.Logger.Printf(f, a...)
	}
}

// descriptorHash is the applied-marker key: a hash of the APPLY-RELEVANT descriptor fields. A change
// (re-provision → new host/user/fingerprint, or a quota adjustment) yields a new hash → the bridge
// re-applies. QuotaGB is included (SLICE 4) so a hub-side quota raise reaches the target — on an
// already-provisioned guest that re-apply is a cheap key-auth-first re-pin (no password consumed).
func descriptorHash(o config.OffsiteConfig) string {
	s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)
	sum := sha256.Sum256([]byte(s))
	return hex.EncodeToString(sum[:])
}

func (b *Bridge) readMarker() string {
	data, err := os.ReadFile(b.MarkerPath)
	if err != nil {
		return ""
	}
	return string(data)
}

func (b *Bridge) writeMarker(h string) error {
	if err := os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700); err != nil {
		return err
	}
	tmp := b.MarkerPath + ".tmp"
	if err := os.WriteFile(tmp, []byte(h), 0o600); err != nil {
		return err
	}
	return os.Rename(tmp, b.MarkerPath)
}

// Reconcile applies the offsite descriptor. Safe to call repeatedly (idempotent) and on any error leaves
// nothing half-configured (fail-safe). Returns an error for logging; callers run it async and retry.
func (b *Bridge) Reconcile(ctx context.Context) error {
	o := b.Cfg.Offsite
	if !o.Enabled {
		return nil // disabled → the fork-4 gate blocks runs; nothing to apply
	}
	port := o.Port
	if port == 0 {
		port = 23
	}
	h := descriptorHash(o)
	if b.readMarker() == h {
		return nil // already applied for this descriptor (idempotent) — do NOT re-consume a spent password
	}
	if o.HostFingerprint == "" {
		return fmt.Errorf("offsite-apply: descriptor has no host_fingerprint — refusing (no blind TOFU)")
	}
	if o.Host == "" || o.User == "" || o.RepoPath == "" {
		return fmt.Errorf("offsite-apply: descriptor missing host/user/repo_path")
	}

	// 1) Scan + VERIFY the host key BEFORE consuming the password (don't waste it on a mismatch).
	scannedFP, knownHostsLine, err := b.Scanner.Scan(ctx, o.Host, port)
	if err != nil {
		return fmt.Errorf("offsite-apply: host-key scan: %w", err)
	}
	if scannedFP != o.HostFingerprint {
		return fmt.Errorf("offsite-apply: host-key MISMATCH for %s (got %s, want %s) — refusing to pin/install (possible MITM)", o.Host, scannedFP, o.HostFingerprint)
	}

	// 1b) Key-auth-first: if an already-installed key still authenticates (pinned to the key we JUST
	// verified — the probe never weakens the identity check), the descriptor change is applied by
	// re-pinning + reconfiguring alone. NO one-time password is consumed — a stale/re-scanned descriptor
	// on an already-provisioned guest no longer loops on consume-404.
	if b.Prober != nil {
		if privPEM, ok := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine); ok {
			if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
				return fmt.Errorf("offsite-apply: reconfigure (key-auth-first): %w", err)
			}
			if err := b.writeMarker(h); err != nil {
				b.logf("[WARN] [offsite-apply] key-auth-first applied for %s but failed to persist the marker: %v", o.Host, err)
				return err
			}
			b.logf("[INFO] [offsite-apply] existing key still authenticates to %s@%s — re-pinned + reconfigured without consuming a password", o.User, o.Host)
			return nil
		}
	}

	// 2) Generate the controller keypair.
	privPEM, pubAuthorized, err := b.KeyGen.Generate()
	if err != nil {
		return fmt.Errorf("offsite-apply: keygen: %w", err)
	}

	// 3) Consume the one-time password (single-use). After this the password is SPENT.
	password, err := b.Consumer.Consume(ctx)
	if err != nil {
		return fmt.Errorf("offsite-apply: consume one-time password: %w", err)
	}

	// 4) Install the pubkey using the password (proven ssh-copy-id -s -f), verify key auth. Pin the
	//    scanner-VERIFIED known_hosts line on the install/verify connections — never accept-new — so a MITM
	//    cannot substitute a different key in the gap between the scan and the install.
	if err := b.Installer.Install(ctx, o.Host, o.User, port, password, privPEM, pubAuthorized, knownHostsLine); err != nil {
		// The password is now SPENT but install failed — a loud, distinct signal: the operator must reset
		// the box password on the hub and let the bridge retry. Do NOT mark applied.
		b.logf("[ERROR] [offsite-apply] key install FAILED after consuming the one-time password for %s@%s — the password is spent; reset it on the hub to retry: %v", o.User, o.Host, err)
		return fmt.Errorf("offsite-apply: install key (password spent — needs hub reset): %w", err)
	}

	// 5) Configure the offbox target + go EscrowState="pending" (fork-4 enable path).
	if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
		return fmt.Errorf("offsite-apply: configure offbox: %w", err)
	}

	// 6) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe).
	if err := b.writeMarker(h); err != nil {
		b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (will re-apply next cycle — the password is spent, needs reset): %v", o.Host, err)
		return err
	}
	b.logf("[INFO] [offsite-apply] offsite configured for %s@%s:%s (pending key escrow)", o.User, o.Host, o.RepoPath)
	return nil
}

// AwaitSettle blocks until it is safe to run the apply-bridge, then returns (R-71a). It removes the
// SYSTEMATIC trigger for the F10 day-0 race by refusing to consume the one-time password while a
// managed floor-update is in flight or imminent (we are below the floor): that update's restart would
// supersede the bridge and kill it mid-install, spending the password for nothing. It NEVER blocks
// when it is genuinely safe — the overwhelmingly common shape (a restart of an at/above-floor box)
// evaluates GO on the first poll with no sleep at all (the B′ invariant: zero new latency).
//
// It is a strict no-op unless a SettleProvider is wired (nil → old behavior, reconcile immediately).
// The gate has its own bounds (settlePoll/settleFloorSubBound/settleOverallBound) and its own context
// so the deferral never eats the reconcile budget.
func (b *Bridge) AwaitSettle(ctx context.Context) {
	if b.Settle == nil {
		return
	}
	now, sleep := b.nowFn(), b.sleepFn()
	start := now()
	var loggedUpdate, loggedDefer, loggedFloorWait bool
	for {
		version, floor, updateRunning, floorKnown := b.Settle.SettleState()
		elapsed := now().Sub(start)
		switch {
		case updateRunning:
			// A swap is in flight; its restart supersedes us. Wait it out.
			if !loggedUpdate {
				b.logf("[INFO] [offsite-apply] settle-gate: a managed update is in progress — deferring offsite apply until it converges")
				loggedUpdate = true
			}
		case floorKnown && belowFloor(version, floor):
			// The auto-floor update is imminent (below floor + floor known). Do NOT consume — the
			// update's restart would burn the password. Wait for the update to land (which restarts us
			// at floor → the GO branch below).
			if !loggedDefer {
				b.logf("[INFO] [offsite-apply] deferring offsite apply: managed update to floor %s pending (we are %s)", floor, version)
				loggedDefer = true
			}
		case floorKnown:
			// At/above floor, no update running — the safe steady state. GO.
			b.logf("[INFO] [offsite-apply] settle-gate: GO — at/above floor %s (we are %s), no managed update running", floor, version)
			return
		case elapsed >= settleFloorSubBound:
			// Floor never became known within the sub-bound. A hub that will not tell us the floor
			// cannot serve a consume either, so the burn risk is nil — don't hold offsite hostage.
			b.logf("[WARN] [offsite-apply] settle-gate: GO — floor still unknown after %s; a hub that cannot serve a floor cannot serve a consume (no burn risk)", settleFloorSubBound)
			return
		default:
			// Floor not known yet, still inside the sub-bound — wait for the report ACK.
			if !loggedFloorWait {
				b.logf("[INFO] [offsite-apply] settle-gate: awaiting floor knowledge (first report ACK) before offsite apply")
				loggedFloorWait = true
			}
		}
		if elapsed >= settleOverallBound {
			b.logf("[WARN] [offsite-apply] settle-gate bound exhausted after %s — proceeding; R-71c self-heal is the belt", settleOverallBound)
			return
		}
		sleep(ctx, settlePoll)
		if ctx.Err() != nil {
			return // shutdown / cancellation — abandon the gate (the next start retries)
		}
	}
}

// ReconcileWhenSettled runs the settle-gate (R-71a) and THEN Reconcile under a FRESH reconcile
// context. The gate's deferral must not eat the reconcile budget, so the reconcile timeout starts
// only after the gate releases. gateCtx bounds the gate (e.g. process shutdown); a cancelled gate
// skips the reconcile (the next start retries).
func (b *Bridge) ReconcileWhenSettled(gateCtx context.Context) error {
	b.AwaitSettle(gateCtx)
	if gateCtx.Err() != nil {
		return gateCtx.Err()
	}
	ctx, cancel := context.WithTimeout(context.Background(), reconcileTimeout)
	defer cancel()
	return b.Reconcile(ctx)
}

// ── R-218, THE CONSUME HALF ──────────────────────────────────────────────────────────────────────
//
// `Reconcile` was correct from the day it shipped and was simply never run again. It fires at
// start-up and once more when the recovery screen drives it (R-219) — and BOTH precede the moment the
// hub has anything staged, because the hub stages in RESPONSE to the declaration those runs come
// before. So the hub held a credential the box would never fetch.
//
// Measured on the R-201 re-walk, 2026-08-06: unlock reconcile 11:43:07 · hub staged 11:44:57 saying
// "the box re-consumes on its next cycle" · a full report cycle ran 11:55:46 · still unconsumed at
// 12:06. A guest command line moved it in 18 seconds — everything was fine except the trigger.

// NeedsCredentialFunc reports whether the box STILL declares it needs a transport credential. It is
// deliberately the box's own published declaration (`backup.OffboxReportStatus().State`) rather than a
// second predicate: the hub acts on that statement, so driving the retry from anything else would let
// the two disagree about whether a retry is wanted.
type NeedsCredentialFunc func() bool

// RetryIfDeclared is ONE tick of the consume half.
//
// It reconciles **only while the box declares a need**, which is what makes it stop: the instant a
// target exists the declaration goes false, this returns immediately, and a healthy box does no work
// and logs nothing. The settle gate is deliberately preserved — `ReconcileWhenSettled` waits for floor
// knowledge exactly as the start-up path does, because the day-0 race it guards is unchanged.
//
// Returns whether a reconcile was ATTEMPTED, so a caller (and a test) can tell "declined to run" from
// "ran and failed" without reading the log.
func (b *Bridge) RetryIfDeclared(ctx context.Context, declared NeedsCredentialFunc) (attempted bool, err error) {
	if b == nil || declared == nil || !declared() {
		return false, nil
	}
	return true, b.ReconcileWhenSettled(ctx)
}