v0.289.0: off-site key cannot delete — append-only rclone transport, box sends only its public key (hub registrar), retention only inside a hub window behind the fake-snapshot guard (decisions 68-69, R-820, R-822)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/offsiteapply/offsiteapply.go

@@ -1,9 +1,11 @@
-// Package offsiteapply is the controller-side apply-bridge (SLICE 2): it turns the hub-served offsite
-// descriptor + the one-time password into a working key-only offbox target. On config apply it consumes the
-// one-time password, VERIFIES the box host key against the hub-captured fingerprint (no blind TOFU), pins it,
-// installs the controller's own key, and configures the offbox target → EscrowState="pending" (the fork-4
-// enable path). Idempotent (a descriptor hash marker prevents re-consuming a spent password) and fail-safe
-// (any step fails → nothing persisted, retried next cycle; never a half-configured offbox).
+// Package offsiteapply is the controller-side apply-bridge: it turns the hub-served offsite descriptor into
+// a working off-site target reached through an APPEND-ONLY key. It VERIFIES the Storage Box host key against
+// the hub-captured fingerprint (no blind TOFU), sends the box's PUBLIC key to the hub's key registrar, which
+// writes it into the sub-account pinned to `rclone serve restic --stdio --append-only <repo>` (decision 69,
+// controller v0.289.0), proves the key reaches the pinned server, and configures the offbox target →
+// EscrowState="pending" (the fork-4 enable path). The box NEVER receives the sub-account password any more —
+// that password can rewrite authorized_keys and remove the pin (R-820). Idempotent and fail-safe (any step
+// fails → nothing persisted, retried next cycle).
 package offsiteapply
 
 import (
@@ -22,9 +24,11 @@ import (
 
 // The apply-bridge seams (tests inject fakes — no live SSH / hub calls in unit tests).
 type (
-	// PasswordConsumer fetches the one-time transient password from the hub (single-use).
-	PasswordConsumer interface {
-		Consume(ctx context.Context) (string, error)
+	// KeyRegistrar is the hub's key registrar (decision 69): install a PUBLIC key pinned append-only,
+	// then confirm the one the box uses (the hub drops every other line).
+	KeyRegistrar interface {
+		Register(ctx context.Context, pub string) (fingerprint string, err error)
+		Confirm(ctx context.Context, fingerprint string) error
 	}
 	// HostKeyScanner returns the box's host-key fingerprint (SHA256:…) + the known_hosts line to pin.
 	HostKeyScanner interface {
@@ -34,25 +38,18 @@ type (
 	KeyGenerator interface {
 		Generate() (privPEM, pubAuthorized string, err error)
 	}
-	// KeyInstaller installs the pub line on the box using the one-time password, then verifies passwordless
-	// key auth with the private key. It MUST pin the VERIFIED knownHosts line (from the scan) on the
-	// connection — never blind-TOFU — so a MITM cannot swap the key between the scan and the install.
-	KeyInstaller interface {
-		Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error
-	}
 	// OffboxEnabler configures the offbox target (key + known_hosts + target + soft quota) and goes
 	// EscrowState="pending" (the fork-4 enable path). quotaGB=0 = no soft limit (dedicated boxes).
 	OffboxEnabler interface {
 		ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
 	}
-	// KeyAuthProber checks whether an ALREADY-INSTALLED key authenticates to the target (pinned to the
-	// freshly-scanned knownHosts). ok=true returns that key's PEM so the descriptor change is applied by
-	// re-pinning + reconfiguring WITHOUT consuming a one-time password (key-auth-first — kills the
-	// stale-descriptor consume-404 loop and shrinks the re-issue blast radius to genuinely-fresh guests).
-	// ok=false (no key / auth refused) → the caller falls through to the full consume+install path.
-	KeyAuthProber interface {
-		Probe(ctx context.Context, host, user string, port int, knownHosts string) (privPEM string, ok bool)
+	// PinnedKeyProber proves a key reaches the PINNED append-only server (not merely that it
+	// authenticates — an unpinned key authenticates too, and can delete).
+	PinnedKeyProber interface {
+		Probe(ctx context.Context, host, user string, port int, knownHosts, privPEM string) bool
 	}
+	// ExistingKey returns the box's installed off-site private key, "" when none.
+	ExistingKey func() string
 	// SettleProvider reports the managed-update settle state so the bridge can DEFER consuming the
 	// one-time password until any imminent managed floor-update has converged (R-71a — the structural
 	// fix for the F10 day-0 race). The failure it prevents: a fresh box boots below the operator floor,
@@ -106,13 +103,13 @@ func belowFloor(version, floor string) bool {
 // Bridge reconciles the offsite descriptor into a configured offbox target.
 type Bridge struct {
 	Cfg        *config.Config
-	Consumer   PasswordConsumer
+	Registrar  KeyRegistrar
 	Scanner    HostKeyScanner
 	KeyGen     KeyGenerator
-	Installer  KeyInstaller
 	Enabler    OffboxEnabler
-	Prober     KeyAuthProber // optional: key-auth-first (nil → always the full consume+install path)
-	MarkerPath string        // where the applied-descriptor-hash is persisted (e.g. <dataDir>/offbox/applied_marker)
+	Prober     PinnedKeyProber
+	Existing   ExistingKey // nil → no existing key (always a fresh keypair)
+	MarkerPath string      // where the applied-descriptor-hash is persisted (e.g. <dataDir>/offbox/applied_marker)
 	Logger     *log.Logger
 
 	// Settle gates the consume/install path behind managed-update convergence (R-71a). nil → no gate
@@ -157,7 +154,9 @@ func (b *Bridge) logf(f string, a ...any) {
 // re-applies. QuotaGB is included (SLICE 4) so a hub-side quota raise reaches the target — on an
 // already-provisioned guest that re-apply is a cheap key-auth-first re-pin (no password consumed).
 func descriptorHash(o config.OffsiteConfig) string {
-	s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)
+	// "|pinned-v1" (v0.289.0): a box upgraded from the password era re-applies ONCE, which is what moves
+	// its key to the pinned append-only line. Without it the old marker matches and nothing migrates.
+	s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d|pinned-v1", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)
 	sum := sha256.Sum256([]byte(s))
 	return hex.EncodeToString(sum[:])
 }
@@ -212,57 +211,64 @@ func (b *Bridge) Reconcile(ctx context.Context) error {
 		return fmt.Errorf("offsite-apply: host-key MISMATCH for %s (got %s, want %s) — refusing to pin/install (possible MITM)", o.Host, scannedFP, o.HostFingerprint)
 	}
 
-	// 1b) Key-auth-first: if an already-installed key still authenticates (pinned to the key we JUST
-	// verified — the probe never weakens the identity check), the descriptor change is applied by
-	// re-pinning + reconfiguring alone. NO one-time password is consumed — a stale/re-scanned descriptor
-	// on an already-provisioned guest no longer loops on consume-404.
-	if b.Prober != nil {
-		if privPEM, ok := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine); ok {
-			if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
-				return fmt.Errorf("offsite-apply: reconfigure (key-auth-first): %w", err)
+	// 2) The key: the one already installed (a running box, or one upgraded from the password era — the
+	//    SAME key is re-registered and comes back pinned), else a fresh pair.
+	privPEM, pub, fresh := "", "", false
+	if b.Existing != nil {
+		if pem := b.Existing(); pem != "" {
+			if p, perr := PublicKeyOf(pem); perr == nil {
+				privPEM, pub = pem, p
+			} else {
+				b.logf("[WARN] [offsite-apply] the installed key does not parse (%v) — generating a fresh one", perr)
 			}
-			if err := b.writeMarker(h); err != nil {
-				b.logf("[WARN] [offsite-apply] key-auth-first applied for %s but failed to persist the marker: %v", o.Host, err)
-				return err
-			}
-			b.logf("[INFO] [offsite-apply] existing key still authenticates to %s@%s — re-pinned + reconfigured without consuming a password", o.User, o.Host)
-			return nil
 		}
 	}
-
-	// 2) Generate the controller keypair.
-	privPEM, pubAuthorized, err := b.KeyGen.Generate()
+	if privPEM == "" {
+		if privPEM, pub, err = b.KeyGen.Generate(); err != nil {
+			return fmt.Errorf("offsite-apply: keygen: %w", err)
+		}
+		fresh = true
+	}
+	fp, err := FingerprintOf(pub)
 	if err != nil {
-		return fmt.Errorf("offsite-apply: keygen: %w", err)
+		return fmt.Errorf("offsite-apply: own key: %w", err)
 	}
 
-	// 3) Consume the one-time password (single-use). After this the password is SPENT.
-	password, err := b.Consumer.Consume(ctx)
-	if err != nil {
-		return fmt.Errorf("offsite-apply: consume one-time password: %w", err)
+	// 3) Already pinned? (a descriptor change on a healthy box, or a restart) → no hub write needed.
+	pinned := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine, privPEM)
+	if !pinned {
+		// 4) Ask the hub's registrar to install it pinned append-only. NO password crosses this call.
+		hubFP, rerr := b.Registrar.Register(ctx, pub)
+		if rerr != nil {
+			return fmt.Errorf("offsite-apply: register key with the hub: %w", rerr)
+		}
+		if hubFP != fp {
+			return fmt.Errorf("offsite-apply: hub installed key %s, ours is %s — refusing", hubFP, fp)
+		}
+		// 5) Prove the key now reaches the PINNED server (positive observable, not just auth).
+		if !b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine, privPEM) {
+			return fmt.Errorf("offsite-apply: key %s registered but does not reach the pinned append-only server", fp)
+		}
+		b.logf("[INFO] [offsite-apply] the hub installed key %s append-only on %s@%s (fresh=%v)", fp, o.User, o.Host, fresh)
 	}
 
-	// 4) Install the pubkey using the password (proven ssh-copy-id -s -f), verify key auth. Pin the
-	//    scanner-VERIFIED known_hosts line on the install/verify connections — never accept-new — so a MITM
-	//    cannot substitute a different key in the gap between the scan and the install.
-	if err := b.Installer.Install(ctx, o.Host, o.User, port, password, privPEM, pubAuthorized, knownHostsLine); err != nil {
-		// The password is now SPENT but install failed — a loud, distinct signal: the operator must reset
-		// the box password on the hub and let the bridge retry. Do NOT mark applied.
-		b.logf("[ERROR] [offsite-apply] key install FAILED after consuming the one-time password for %s@%s — the password is spent; reset it on the hub to retry: %v", o.User, o.Host, err)
-		return fmt.Errorf("offsite-apply: install key (password spent — needs hub reset): %w", err)
-	}
-
-	// 5) Configure the offbox target + go EscrowState="pending" (fork-4 enable path).
+	// 6) Configure the offbox target (writes the key) + go EscrowState="pending" when new (fork-4).
 	if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
 		return fmt.Errorf("offsite-apply: configure offbox: %w", err)
 	}
 
-	// 6) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe).
+	// 7) Confirm: the hub removes every other key line (rotation's last step). Best effort — a failed
+	//    confirm leaves an extra PINNED line, which cannot delete; the next apply retries it.
+	if err := b.Registrar.Confirm(ctx, fp); err != nil {
+		b.logf("[WARN] [offsite-apply] confirm key %s with the hub failed (an extra pinned line may remain; retried on the next apply): %v", fp, err)
+	}
+
+	// 8) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe).
 	if err := b.writeMarker(h); err != nil {
-		b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (will re-apply next cycle — the password is spent, needs reset): %v", o.Host, err)
+		b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (re-applies next cycle; harmless — no password is involved): %v", o.Host, err)
 		return err
 	}
-	b.logf("[INFO] [offsite-apply] offsite configured for %s@%s:%s (pending key escrow)", o.User, o.Host, o.RepoPath)
+	b.logf("[INFO] [offsite-apply] offsite configured append-only for %s@%s:%s (key %s)", o.User, o.Host, o.RepoPath, fp)
 	return nil
 }