// Package offsiteapply is the controller-side apply-bridge (SLICE 2): it turns the hub-served offsite // descriptor + the one-time password into a working key-only offbox target. On config apply it consumes the // one-time password, VERIFIES the box host key against the hub-captured fingerprint (no blind TOFU), pins it, // installs the controller's own key, and configures the offbox target → EscrowState="pending" (the fork-4 // enable path). Idempotent (a descriptor hash marker prevents re-consuming a spent password) and fail-safe // (any step fails → nothing persisted, retried next cycle; never a half-configured offbox). package offsiteapply import ( "context" "crypto/sha256" "encoding/hex" "fmt" "log" "os" "path/filepath" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/config" "gitea.dooplex.hu/admin/felhom-controller/internal/util" ) // The apply-bridge seams (tests inject fakes — no live SSH / hub calls in unit tests). type ( // PasswordConsumer fetches the one-time transient password from the hub (single-use). PasswordConsumer interface { Consume(ctx context.Context) (string, error) } // HostKeyScanner returns the box's host-key fingerprint (SHA256:…) + the known_hosts line to pin. HostKeyScanner interface { Scan(ctx context.Context, host string, port int) (fingerprint, knownHostsLine string, err error) } // KeyGenerator produces a fresh keypair: the private key (PEM) and the authorized_keys pub line. KeyGenerator interface { Generate() (privPEM, pubAuthorized string, err error) } // KeyInstaller installs the pub line on the box using the one-time password, then verifies passwordless // key auth with the private key. It MUST pin the VERIFIED knownHosts line (from the scan) on the // connection — never blind-TOFU — so a MITM cannot swap the key between the scan and the install. KeyInstaller interface { Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error } // OffboxEnabler configures the offbox target (key + known_hosts + target + soft quota) and goes // EscrowState="pending" (the fork-4 enable path). quotaGB=0 = no soft limit (dedicated boxes). OffboxEnabler interface { ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error } // KeyAuthProber checks whether an ALREADY-INSTALLED key authenticates to the target (pinned to the // freshly-scanned knownHosts). ok=true returns that key's PEM so the descriptor change is applied by // re-pinning + reconfiguring WITHOUT consuming a one-time password (key-auth-first — kills the // stale-descriptor consume-404 loop and shrinks the re-issue blast radius to genuinely-fresh guests). // ok=false (no key / auth refused) → the caller falls through to the full consume+install path. KeyAuthProber interface { Probe(ctx context.Context, host, user string, port int, knownHosts string) (privPEM string, ok bool) } // SettleProvider reports the managed-update settle state so the bridge can DEFER consuming the // one-time password until any imminent managed floor-update has converged (R-71a — the structural // fix for the F10 day-0 race). The failure it prevents: a fresh box boots below the operator floor, // the bridge consumes the single-use password, then ~35 s later the managed auto-floor update // replaces the container mid-install → the new process finds no installed key → consume → 404 → // offsite dead until an operator Re-issue. version = the running controller version; floor = the // operator-enforced minimum (from the hub report ACK); updateRunning = a swap is in flight; // floorKnown = the floor has been learned yet (false = the first report ACK hasn't landed). A hub // that hasn't served a floor cannot serve a consume either, so !floorKnown carries NO burn risk. SettleProvider interface { SettleState() (version, floor string, updateRunning, floorKnown bool) } ) // Settle-gate timing (R-71a). Named constants with rationale so the trade-offs stay visible. const ( // reconcileTimeout bounds the actual Reconcile once the gate releases (moved here from main.go so // ReconcileWhenSettled owns the whole "gate THEN reconcile" contract). The gate's own wait must NOT // eat this budget — the reconcile context is created only after the gate returns. reconcileTimeout = 3 * time.Minute // settlePoll is the gate's poll cadence. The bridge is a background reconcile with no user-visible // latency, so a coarse poll is free; the at/above-floor happy path returns on the FIRST evaluation // with no sleep at all (the B′ invariant), so this cadence only ever paces the deferral cases. settlePoll = 10 * time.Second // settleFloorSubBound: if the hub never tells us the floor (report ACK), stop waiting and GO+WARN. // Sized to the report-ACK latency observed at source: the startup report fires ~5 s after boot // (main.go's startup goroutine sleeps 5 s), and SetFloor runs SYNCHRONOUSLY inside that report's // ACK handler (main.go OnPushResponse → updater.SetFloor), so the floor is normally known within // ~5–10 s. The startup report retries up to 3× with 15 s gaps, so a slow first report can push // floor-knowledge to ~45 s; 90 s is generous headroom over that worst case. Proceeding here cannot // burn a one-time password: a hub that cannot serve a floor cannot serve a consume. settleFloorSubBound = 90 * time.Second // settleOverallBound: the absolute cap. If we are still below floor after this (a managed // floor-update that never lands), proceed anyway — the R-71c hub self-heal restage is the belt for // a consume that a genuinely stuck update might later burn. settleOverallBound = 5 * time.Minute ) // belowFloor reports whether the running version is strictly below the operator floor. An // unparseable version or floor (a dev build, or a malformed floor) is treated as NOT below — the gate // must never wedge on a version it cannot compare (and a dev build never auto-floor-updates anyway). func belowFloor(version, floor string) bool { v, err1 := util.ParseVersion(version) f, err2 := util.ParseVersion(floor) if err1 != nil || err2 != nil { return false } return v.Compare(f) < 0 } // Bridge reconciles the offsite descriptor into a configured offbox target. type Bridge struct { Cfg *config.Config Consumer PasswordConsumer Scanner HostKeyScanner KeyGen KeyGenerator Installer KeyInstaller Enabler OffboxEnabler Prober KeyAuthProber // optional: key-auth-first (nil → always the full consume+install path) MarkerPath string // where the applied-descriptor-hash is persisted (e.g. /offbox/applied_marker) Logger *log.Logger // Settle gates the consume/install path behind managed-update convergence (R-71a). nil → no gate // (old behavior: reconcile immediately). Wired only when a self-updater exists — with no update // mechanism there is no floor-update to race, so no gate is needed. Settle SettleProvider // Now/Sleep are clock seams for the settle-gate ONLY (tests inject a fake clock so the bounds are // exercised with zero real sleeps). nil → the real wall clock and a context-aware sleep. Now func() time.Time Sleep func(ctx context.Context, d time.Duration) } func (b *Bridge) nowFn() func() time.Time { if b.Now != nil { return b.Now } return time.Now } func (b *Bridge) sleepFn() func(context.Context, time.Duration) { if b.Sleep != nil { return b.Sleep } return func(ctx context.Context, d time.Duration) { t := time.NewTimer(d) defer t.Stop() select { case <-ctx.Done(): case <-t.C: } } } func (b *Bridge) logf(f string, a ...any) { if b.Logger != nil { b.Logger.Printf(f, a...) } } // descriptorHash is the applied-marker key: a hash of the APPLY-RELEVANT descriptor fields. A change // (re-provision → new host/user/fingerprint, or a quota adjustment) yields a new hash → the bridge // re-applies. QuotaGB is included (SLICE 4) so a hub-side quota raise reaches the target — on an // already-provisioned guest that re-apply is a cheap key-auth-first re-pin (no password consumed). func descriptorHash(o config.OffsiteConfig) string { s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB) sum := sha256.Sum256([]byte(s)) return hex.EncodeToString(sum[:]) } func (b *Bridge) readMarker() string { data, err := os.ReadFile(b.MarkerPath) if err != nil { return "" } return string(data) } func (b *Bridge) writeMarker(h string) error { if err := os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700); err != nil { return err } tmp := b.MarkerPath + ".tmp" if err := os.WriteFile(tmp, []byte(h), 0o600); err != nil { return err } return os.Rename(tmp, b.MarkerPath) } // Reconcile applies the offsite descriptor. Safe to call repeatedly (idempotent) and on any error leaves // nothing half-configured (fail-safe). Returns an error for logging; callers run it async and retry. func (b *Bridge) Reconcile(ctx context.Context) error { o := b.Cfg.Offsite if !o.Enabled { return nil // disabled → the fork-4 gate blocks runs; nothing to apply } port := o.Port if port == 0 { port = 23 } h := descriptorHash(o) if b.readMarker() == h { return nil // already applied for this descriptor (idempotent) — do NOT re-consume a spent password } if o.HostFingerprint == "" { return fmt.Errorf("offsite-apply: descriptor has no host_fingerprint — refusing (no blind TOFU)") } if o.Host == "" || o.User == "" || o.RepoPath == "" { return fmt.Errorf("offsite-apply: descriptor missing host/user/repo_path") } // 1) Scan + VERIFY the host key BEFORE consuming the password (don't waste it on a mismatch). scannedFP, knownHostsLine, err := b.Scanner.Scan(ctx, o.Host, port) if err != nil { return fmt.Errorf("offsite-apply: host-key scan: %w", err) } if scannedFP != o.HostFingerprint { return fmt.Errorf("offsite-apply: host-key MISMATCH for %s (got %s, want %s) — refusing to pin/install (possible MITM)", o.Host, scannedFP, o.HostFingerprint) } // 1b) Key-auth-first: if an already-installed key still authenticates (pinned to the key we JUST // verified — the probe never weakens the identity check), the descriptor change is applied by // re-pinning + reconfiguring alone. NO one-time password is consumed — a stale/re-scanned descriptor // on an already-provisioned guest no longer loops on consume-404. if b.Prober != nil { if privPEM, ok := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine); ok { if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil { return fmt.Errorf("offsite-apply: reconfigure (key-auth-first): %w", err) } if err := b.writeMarker(h); err != nil { b.logf("[WARN] [offsite-apply] key-auth-first applied for %s but failed to persist the marker: %v", o.Host, err) return err } b.logf("[INFO] [offsite-apply] existing key still authenticates to %s@%s — re-pinned + reconfigured without consuming a password", o.User, o.Host) return nil } } // 2) Generate the controller keypair. privPEM, pubAuthorized, err := b.KeyGen.Generate() if err != nil { return fmt.Errorf("offsite-apply: keygen: %w", err) } // 3) Consume the one-time password (single-use). After this the password is SPENT. password, err := b.Consumer.Consume(ctx) if err != nil { return fmt.Errorf("offsite-apply: consume one-time password: %w", err) } // 4) Install the pubkey using the password (proven ssh-copy-id -s -f), verify key auth. Pin the // scanner-VERIFIED known_hosts line on the install/verify connections — never accept-new — so a MITM // cannot substitute a different key in the gap between the scan and the install. if err := b.Installer.Install(ctx, o.Host, o.User, port, password, privPEM, pubAuthorized, knownHostsLine); err != nil { // The password is now SPENT but install failed — a loud, distinct signal: the operator must reset // the box password on the hub and let the bridge retry. Do NOT mark applied. b.logf("[ERROR] [offsite-apply] key install FAILED after consuming the one-time password for %s@%s — the password is spent; reset it on the hub to retry: %v", o.User, o.Host, err) return fmt.Errorf("offsite-apply: install key (password spent — needs hub reset): %w", err) } // 5) Configure the offbox target + go EscrowState="pending" (fork-4 enable path). if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil { return fmt.Errorf("offsite-apply: configure offbox: %w", err) } // 6) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe). if err := b.writeMarker(h); err != nil { b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (will re-apply next cycle — the password is spent, needs reset): %v", o.Host, err) return err } b.logf("[INFO] [offsite-apply] offsite configured for %s@%s:%s (pending key escrow)", o.User, o.Host, o.RepoPath) return nil } // AwaitSettle blocks until it is safe to run the apply-bridge, then returns (R-71a). It removes the // SYSTEMATIC trigger for the F10 day-0 race by refusing to consume the one-time password while a // managed floor-update is in flight or imminent (we are below the floor): that update's restart would // supersede the bridge and kill it mid-install, spending the password for nothing. It NEVER blocks // when it is genuinely safe — the overwhelmingly common shape (a restart of an at/above-floor box) // evaluates GO on the first poll with no sleep at all (the B′ invariant: zero new latency). // // It is a strict no-op unless a SettleProvider is wired (nil → old behavior, reconcile immediately). // The gate has its own bounds (settlePoll/settleFloorSubBound/settleOverallBound) and its own context // so the deferral never eats the reconcile budget. func (b *Bridge) AwaitSettle(ctx context.Context) { if b.Settle == nil { return } now, sleep := b.nowFn(), b.sleepFn() start := now() var loggedUpdate, loggedDefer, loggedFloorWait bool for { version, floor, updateRunning, floorKnown := b.Settle.SettleState() elapsed := now().Sub(start) switch { case updateRunning: // A swap is in flight; its restart supersedes us. Wait it out. if !loggedUpdate { b.logf("[INFO] [offsite-apply] settle-gate: a managed update is in progress — deferring offsite apply until it converges") loggedUpdate = true } case floorKnown && belowFloor(version, floor): // The auto-floor update is imminent (below floor + floor known). Do NOT consume — the // update's restart would burn the password. Wait for the update to land (which restarts us // at floor → the GO branch below). if !loggedDefer { b.logf("[INFO] [offsite-apply] deferring offsite apply: managed update to floor %s pending (we are %s)", floor, version) loggedDefer = true } case floorKnown: // At/above floor, no update running — the safe steady state. GO. b.logf("[INFO] [offsite-apply] settle-gate: GO — at/above floor %s (we are %s), no managed update running", floor, version) return case elapsed >= settleFloorSubBound: // Floor never became known within the sub-bound. A hub that will not tell us the floor // cannot serve a consume either, so the burn risk is nil — don't hold offsite hostage. b.logf("[WARN] [offsite-apply] settle-gate: GO — floor still unknown after %s; a hub that cannot serve a floor cannot serve a consume (no burn risk)", settleFloorSubBound) return default: // Floor not known yet, still inside the sub-bound — wait for the report ACK. if !loggedFloorWait { b.logf("[INFO] [offsite-apply] settle-gate: awaiting floor knowledge (first report ACK) before offsite apply") loggedFloorWait = true } } if elapsed >= settleOverallBound { b.logf("[WARN] [offsite-apply] settle-gate bound exhausted after %s — proceeding; R-71c self-heal is the belt", settleOverallBound) return } sleep(ctx, settlePoll) if ctx.Err() != nil { return // shutdown / cancellation — abandon the gate (the next start retries) } } } // ReconcileWhenSettled runs the settle-gate (R-71a) and THEN Reconcile under a FRESH reconcile // context. The gate's deferral must not eat the reconcile budget, so the reconcile timeout starts // only after the gate releases. gateCtx bounds the gate (e.g. process shutdown); a cancelled gate // skips the reconcile (the next start retries). func (b *Bridge) ReconcileWhenSettled(gateCtx context.Context) error { b.AwaitSettle(gateCtx) if gateCtx.Err() != nil { return gateCtx.Err() } ctx, cancel := context.WithTimeout(context.Background(), reconcileTimeout) defer cancel() return b.Reconcile(ctx) }