v0.289.0: off-site key cannot delete — append-only rclone transport, box sends only its public key (hub registrar), retention only inside a hub window behind the fake-snapshot guard (decisions 68-69, R-820, R-822)
gates / gates (push) Successful in 27s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-03 17:01:36 +02:00
parent 09453325d1
commit 55bb6c3d32
15 changed files with 1039 additions and 386 deletions
@@ -1,9 +1,11 @@
// Package offsiteapply is the controller-side apply-bridge (SLICE 2): it turns the hub-served offsite
// descriptor + the one-time password into a working key-only offbox target. On config apply it consumes the
// one-time password, VERIFIES the box host key against the hub-captured fingerprint (no blind TOFU), pins it,
// installs the controller's own key, and configures the offbox target → EscrowState="pending" (the fork-4
// enable path). Idempotent (a descriptor hash marker prevents re-consuming a spent password) and fail-safe
// (any step fails → nothing persisted, retried next cycle; never a half-configured offbox).
// Package offsiteapply is the controller-side apply-bridge: it turns the hub-served offsite descriptor into
// a working off-site target reached through an APPEND-ONLY key. It VERIFIES the Storage Box host key against
// the hub-captured fingerprint (no blind TOFU), sends the box's PUBLIC key to the hub's key registrar, which
// writes it into the sub-account pinned to `rclone serve restic --stdio --append-only <repo>` (decision 69,
// controller v0.289.0), proves the key reaches the pinned server, and configures the offbox target →
// EscrowState="pending" (the fork-4 enable path). The box NEVER receives the sub-account password any more —
// that password can rewrite authorized_keys and remove the pin (R-820). Idempotent and fail-safe (any step
// fails → nothing persisted, retried next cycle).
package offsiteapply
import (
@@ -22,9 +24,11 @@ import (
// The apply-bridge seams (tests inject fakes — no live SSH / hub calls in unit tests).
type (
// PasswordConsumer fetches the one-time transient password from the hub (single-use).
PasswordConsumer interface {
Consume(ctx context.Context) (string, error)
// KeyRegistrar is the hub's key registrar (decision 69): install a PUBLIC key pinned append-only,
// then confirm the one the box uses (the hub drops every other line).
KeyRegistrar interface {
Register(ctx context.Context, pub string) (fingerprint string, err error)
Confirm(ctx context.Context, fingerprint string) error
}
// HostKeyScanner returns the box's host-key fingerprint (SHA256:…) + the known_hosts line to pin.
HostKeyScanner interface {
@@ -34,25 +38,18 @@ type (
KeyGenerator interface {
Generate() (privPEM, pubAuthorized string, err error)
}
// KeyInstaller installs the pub line on the box using the one-time password, then verifies passwordless
// key auth with the private key. It MUST pin the VERIFIED knownHosts line (from the scan) on the
// connection — never blind-TOFU — so a MITM cannot swap the key between the scan and the install.
KeyInstaller interface {
Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error
}
// OffboxEnabler configures the offbox target (key + known_hosts + target + soft quota) and goes
// EscrowState="pending" (the fork-4 enable path). quotaGB=0 = no soft limit (dedicated boxes).
OffboxEnabler interface {
ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
}
// KeyAuthProber checks whether an ALREADY-INSTALLED key authenticates to the target (pinned to the
// freshly-scanned knownHosts). ok=true returns that key's PEM so the descriptor change is applied by
// re-pinning + reconfiguring WITHOUT consuming a one-time password (key-auth-first — kills the
// stale-descriptor consume-404 loop and shrinks the re-issue blast radius to genuinely-fresh guests).
// ok=false (no key / auth refused) → the caller falls through to the full consume+install path.
KeyAuthProber interface {
Probe(ctx context.Context, host, user string, port int, knownHosts string) (privPEM string, ok bool)
// PinnedKeyProber proves a key reaches the PINNED append-only server (not merely that it
// authenticates — an unpinned key authenticates too, and can delete).
PinnedKeyProber interface {
Probe(ctx context.Context, host, user string, port int, knownHosts, privPEM string) bool
}
// ExistingKey returns the box's installed off-site private key, "" when none.
ExistingKey func() string
// SettleProvider reports the managed-update settle state so the bridge can DEFER consuming the
// one-time password until any imminent managed floor-update has converged (R-71a — the structural
// fix for the F10 day-0 race). The failure it prevents: a fresh box boots below the operator floor,
@@ -106,13 +103,13 @@ func belowFloor(version, floor string) bool {
// Bridge reconciles the offsite descriptor into a configured offbox target.
type Bridge struct {
Cfg *config.Config
Consumer PasswordConsumer
Registrar KeyRegistrar
Scanner HostKeyScanner
KeyGen KeyGenerator
Installer KeyInstaller
Enabler OffboxEnabler
Prober KeyAuthProber // optional: key-auth-first (nil → always the full consume+install path)
MarkerPath string // where the applied-descriptor-hash is persisted (e.g. <dataDir>/offbox/applied_marker)
Prober PinnedKeyProber
Existing ExistingKey // nil → no existing key (always a fresh keypair)
MarkerPath string // where the applied-descriptor-hash is persisted (e.g. <dataDir>/offbox/applied_marker)
Logger *log.Logger
// Settle gates the consume/install path behind managed-update convergence (R-71a). nil → no gate
@@ -157,7 +154,9 @@ func (b *Bridge) logf(f string, a ...any) {
// re-applies. QuotaGB is included (SLICE 4) so a hub-side quota raise reaches the target — on an
// already-provisioned guest that re-apply is a cheap key-auth-first re-pin (no password consumed).
func descriptorHash(o config.OffsiteConfig) string {
s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)
// "|pinned-v1" (v0.289.0): a box upgraded from the password era re-applies ONCE, which is what moves
// its key to the pinned append-only line. Without it the old marker matches and nothing migrates.
s := fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d|pinned-v1", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)
sum := sha256.Sum256([]byte(s))
return hex.EncodeToString(sum[:])
}
@@ -212,57 +211,64 @@ func (b *Bridge) Reconcile(ctx context.Context) error {
return fmt.Errorf("offsite-apply: host-key MISMATCH for %s (got %s, want %s) — refusing to pin/install (possible MITM)", o.Host, scannedFP, o.HostFingerprint)
}
// 1b) Key-auth-first: if an already-installed key still authenticates (pinned to the key we JUST
// verified — the probe never weakens the identity check), the descriptor change is applied by
// re-pinning + reconfiguring alone. NO one-time password is consumed — a stale/re-scanned descriptor
// on an already-provisioned guest no longer loops on consume-404.
if b.Prober != nil {
if privPEM, ok := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine); ok {
if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
return fmt.Errorf("offsite-apply: reconfigure (key-auth-first): %w", err)
// 2) The key: the one already installed (a running box, or one upgraded from the password era — the
// SAME key is re-registered and comes back pinned), else a fresh pair.
privPEM, pub, fresh := "", "", false
if b.Existing != nil {
if pem := b.Existing(); pem != "" {
if p, perr := PublicKeyOf(pem); perr == nil {
privPEM, pub = pem, p
} else {
b.logf("[WARN] [offsite-apply] the installed key does not parse (%v) — generating a fresh one", perr)
}
if err := b.writeMarker(h); err != nil {
b.logf("[WARN] [offsite-apply] key-auth-first applied for %s but failed to persist the marker: %v", o.Host, err)
return err
}
b.logf("[INFO] [offsite-apply] existing key still authenticates to %s@%s — re-pinned + reconfigured without consuming a password", o.User, o.Host)
return nil
}
}
// 2) Generate the controller keypair.
privPEM, pubAuthorized, err := b.KeyGen.Generate()
if privPEM == "" {
if privPEM, pub, err = b.KeyGen.Generate(); err != nil {
return fmt.Errorf("offsite-apply: keygen: %w", err)
}
fresh = true
}
fp, err := FingerprintOf(pub)
if err != nil {
return fmt.Errorf("offsite-apply: keygen: %w", err)
return fmt.Errorf("offsite-apply: own key: %w", err)
}
// 3) Consume the one-time password (single-use). After this the password is SPENT.
password, err := b.Consumer.Consume(ctx)
if err != nil {
return fmt.Errorf("offsite-apply: consume one-time password: %w", err)
// 3) Already pinned? (a descriptor change on a healthy box, or a restart) → no hub write needed.
pinned := b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine, privPEM)
if !pinned {
// 4) Ask the hub's registrar to install it pinned append-only. NO password crosses this call.
hubFP, rerr := b.Registrar.Register(ctx, pub)
if rerr != nil {
return fmt.Errorf("offsite-apply: register key with the hub: %w", rerr)
}
if hubFP != fp {
return fmt.Errorf("offsite-apply: hub installed key %s, ours is %s — refusing", hubFP, fp)
}
// 5) Prove the key now reaches the PINNED server (positive observable, not just auth).
if !b.Prober.Probe(ctx, o.Host, o.User, port, knownHostsLine, privPEM) {
return fmt.Errorf("offsite-apply: key %s registered but does not reach the pinned append-only server", fp)
}
b.logf("[INFO] [offsite-apply] the hub installed key %s append-only on %s@%s (fresh=%v)", fp, o.User, o.Host, fresh)
}
// 4) Install the pubkey using the password (proven ssh-copy-id -s -f), verify key auth. Pin the
// scanner-VERIFIED known_hosts line on the install/verify connections — never accept-new — so a MITM
// cannot substitute a different key in the gap between the scan and the install.
if err := b.Installer.Install(ctx, o.Host, o.User, port, password, privPEM, pubAuthorized, knownHostsLine); err != nil {
// The password is now SPENT but install failed — a loud, distinct signal: the operator must reset
// the box password on the hub and let the bridge retry. Do NOT mark applied.
b.logf("[ERROR] [offsite-apply] key install FAILED after consuming the one-time password for %s@%s — the password is spent; reset it on the hub to retry: %v", o.User, o.Host, err)
return fmt.Errorf("offsite-apply: install key (password spent — needs hub reset): %w", err)
}
// 5) Configure the offbox target + go EscrowState="pending" (fork-4 enable path).
// 6) Configure the offbox target (writes the key) + go EscrowState="pending" when new (fork-4).
if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine, o.QuotaGB); err != nil {
return fmt.Errorf("offsite-apply: configure offbox: %w", err)
}
// 6) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe).
// 7) Confirm: the hub removes every other key line (rotation's last step). Best effort — a failed
// confirm leaves an extra PINNED line, which cannot delete; the next apply retries it.
if err := b.Registrar.Confirm(ctx, fp); err != nil {
b.logf("[WARN] [offsite-apply] confirm key %s with the hub failed (an extra pinned line may remain; retried on the next apply): %v", fp, err)
}
// 8) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe).
if err := b.writeMarker(h); err != nil {
b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (will re-apply next cycle — the password is spent, needs reset): %v", o.Host, err)
b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (re-applies next cycle; harmless — no password is involved): %v", o.Host, err)
return err
}
b.logf("[INFO] [offsite-apply] offsite configured for %s@%s:%s (pending key escrow)", o.User, o.Host, o.RepoPath)
b.logf("[INFO] [offsite-apply] offsite configured append-only for %s@%s:%s (key %s)", o.User, o.Host, o.RepoPath, fp)
return nil
}
@@ -3,7 +3,10 @@ package offsiteapply
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"log"
"os"
"path/filepath"
@@ -15,19 +18,36 @@ import (
// --- fakes ---
type fakeConsumer struct {
pw string
err error
calls int
panics bool
// fakeRegistrar is the hub's key registrar. `calls` counts Register — the one call that writes to the
// provider; it is the "did the bridge go to the hub" signal the settle-gate and retry tests count.
type fakeRegistrar struct {
calls int
confirms []string
gotPub string
wrongFP bool
err error
confirmEr error
panics bool
}
func (f *fakeConsumer) Consume(_ context.Context) (string, error) {
func (f *fakeRegistrar) Register(_ context.Context, pub string) (string, error) {
if f.panics {
panic("consume must NOT be called (idempotent no-op)")
panic("register must NOT be called")
}
f.calls++
return f.pw, f.err
f.gotPub = pub
if f.err != nil {
return "", f.err
}
if f.wrongFP {
return "SHA256:somebody-else", nil
}
return FingerprintOf(pub)
}
func (f *fakeRegistrar) Confirm(_ context.Context, fp string) error {
f.confirms = append(f.confirms, fp)
return f.confirmEr
}
type fakeScanner struct {
@@ -39,40 +59,25 @@ func (f *fakeScanner) Scan(_ context.Context, _ string, _ int) (string, string,
return f.fp, f.line, f.err
}
type fakeKeyGen struct{ priv, pub string }
// realKeyGen returns ONE real pair (the bridge parses keys now — fingerprints are real).
type realKeyGen struct{ priv, pub string }
func (f *fakeKeyGen) Generate() (string, string, error) { return f.priv, f.pub, nil }
type fakeInstaller struct {
err error
calls int
gotPub string
gotPriv string
gotPw string
gotKnownHost string
}
func (f *fakeInstaller) Install(_ context.Context, _, _ string, _ int, password, privPEM, pub, knownHosts string) error {
f.calls++
f.gotPub, f.gotPriv, f.gotPw, f.gotKnownHost = pub, privPEM, password, knownHosts
return f.err
}
func (f *realKeyGen) Generate() (string, string, error) { return f.priv, f.pub, nil }
// fakeProber models the provider: a key reaches the pinned server once the registrar installed it
// (pinnedAfterRegister) or from the start (pinnedInitially).
type fakeProber struct {
pem string
ok bool
panics bool
calls int
gotKH string
reg *fakeRegistrar
pinnedInitially bool
pinnedAfterRegister bool
calls int
gotKH, gotPriv string
}
func (f *fakeProber) Probe(_ context.Context, _, _ string, _ int, kh string) (string, bool) {
if f.panics {
panic("prober must NOT be called (verify must precede the probe)")
}
func (f *fakeProber) Probe(_ context.Context, _, _ string, _ int, kh, priv string) bool {
f.calls++
f.gotKH = kh
return f.pem, f.ok
f.gotKH, f.gotPriv = kh, priv
return f.pinnedInitially || (f.pinnedAfterRegister && f.reg.calls > 0)
}
type fakeEnabler struct {
@@ -90,187 +95,200 @@ func (f *fakeEnabler) ConfigureOffbox(_ context.Context, host, _ string, _ int,
return f.err
}
func newBridge(t *testing.T, o config.OffsiteConfig) (*Bridge, *fakeConsumer, *fakeInstaller, *fakeEnabler, *bytes.Buffer) {
var testPriv, testPub = func() (string, string) {
p, q, err := ED25519KeyGen{}.Generate()
if err != nil {
panic(err)
}
return p, q
}()
var otherPriv, otherPub = func() (string, string) {
p, q, _ := ED25519KeyGen{}.Generate()
return p, q
}()
func newBridge(t *testing.T, o config.OffsiteConfig) (*Bridge, *fakeRegistrar, *fakeProber, *fakeEnabler, *bytes.Buffer) {
t.Helper()
cfg := &config.Config{}
cfg.Offsite = o
cons := &fakeConsumer{pw: "the-transient-pw"}
inst := &fakeInstaller{}
reg := &fakeRegistrar{}
pr := &fakeProber{reg: reg, pinnedAfterRegister: true}
en := &fakeEnabler{}
var logbuf bytes.Buffer
b := &Bridge{
Cfg: cfg,
Consumer: cons,
Registrar: reg,
Scanner: &fakeScanner{fp: "SHA256:goodfp", line: "[h]:23 ssh-ed25519 AAAAKEY"},
KeyGen: &fakeKeyGen{priv: "PRIVPEM", pub: "ssh-ed25519 AAAAPUB felhom"},
Installer: inst,
KeyGen: &realKeyGen{priv: testPriv, pub: testPub},
Prober: pr,
Enabler: en,
MarkerPath: filepath.Join(t.TempDir(), "offbox", "applied_marker"),
Logger: log.New(&logbuf, "", 0),
}
return b, cons, inst, en, &logbuf
return b, reg, pr, en, &logbuf
}
func goodOffsite() config.OffsiteConfig {
return config.OffsiteConfig{Enabled: true, Type: "shared", Host: "h", User: "u", Port: 23, RepoPath: "/home/felhom-repo", QuotaGB: 50, HostFingerprint: "SHA256:goodfp"}
}
// Scenario A — full apply: consume → verify-pin → install → configure offbox → marker persisted; pw not logged.
func TestBridge_AppliesEndToEnd(t *testing.T) {
b, cons, inst, en, logbuf := newBridge(t, goodOffsite())
// A fresh box: verify host key → register the PUBLIC key → prove the pin → configure → confirm → marker.
// What crosses to the hub is a public key and nothing else.
func TestBridge_FreshBoxRegistersPublicKeyOnly(t *testing.T) {
b, reg, pr, en, logbuf := newBridge(t, goodOffsite())
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("reconcile: %v", err)
}
if cons.calls != 1 {
t.Fatalf("consume calls = %d, want 1", cons.calls)
if reg.calls != 1 || reg.gotPub != testPub {
t.Fatalf("register calls=%d pub=%q", reg.calls, reg.gotPub)
}
if inst.calls != 1 || inst.gotPw != "the-transient-pw" || inst.gotPub == "" {
t.Fatalf("installer not called with pw+pub: %+v", inst)
if strings.Contains(reg.gotPub, "PRIVATE") {
t.Fatal("the private key was sent to the hub")
}
if inst.gotKnownHost != "[h]:23 ssh-ed25519 AAAAKEY" {
t.Fatalf("installer must receive the scanner-verified known_hosts to pin (no TOFU), got %q", inst.gotKnownHost)
if pr.calls != 2 || pr.gotKH != "[h]:23 ssh-ed25519 AAAAKEY" {
t.Fatalf("probe calls=%d kh=%q (want before+after register, pinned to the scanned host key)", pr.calls, pr.gotKH)
}
if en.calls != 1 || en.gotHost != "h" || en.gotKnownHost != "[h]:23 ssh-ed25519 AAAAKEY" || en.gotPriv != "PRIVPEM" {
t.Fatalf("enabler not called with the pinned known_hosts + key: %+v", en)
fp, _ := FingerprintOf(testPub)
if en.calls != 1 || en.gotPriv != testPriv || en.gotQuotaGB != 50 {
t.Fatalf("enabler: %+v", en)
}
if en.gotQuotaGB != 50 {
t.Fatalf("the bridge must map the descriptor's quota_gb into the target (SLICE 4), got %d", en.gotQuotaGB)
if len(reg.confirms) != 1 || reg.confirms[0] != fp {
t.Fatalf("confirm = %v, want [%s]", reg.confirms, fp)
}
if b.readMarker() != descriptorHash(b.Cfg.Offsite) {
t.Fatal("marker not persisted after a successful apply")
if _, err := os.Stat(b.MarkerPath); err != nil {
t.Fatalf("marker not persisted: %v", err)
}
if strings.Contains(logbuf.String(), "the-transient-pw") {
t.Fatal("the one-time password LEAKED into a log line")
if !strings.Contains(logbuf.String(), "append-only") {
t.Fatalf("log does not say append-only:\n%s", logbuf.String())
}
}
// SLICE 4 — a quota-only descriptor change re-applies (the hash includes QuotaGB), and with a working
// key it costs no password: key-auth-first re-pins + remaps the quota.
func TestBridge_QuotaChangeReappliesWithoutConsume(t *testing.T) {
b, cons, _, en, _ := newBridge(t, goodOffsite())
cons.panics = true
b.Prober = &fakeProber{pem: "EXISTINGPEM", ok: true}
// marker for the OLD quota (25) already applied; the descriptor now says 50
old := b.Cfg.Offsite
old.QuotaGB = 25
_ = os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700)
_ = os.WriteFile(b.MarkerPath, []byte(descriptorHash(old)), 0o600)
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("quota-change reconcile: %v", err)
// THE MIGRATION: a box upgraded from the password era has an installed (unpinned) key and an applied
// marker written under the OLD descriptor hash. The new hash ("|pinned-v1") re-applies once; the SAME
// key is registered (no new keypair) and comes back pinned.
func TestBridge_UpgradedBoxReRegistersSameKey(t *testing.T) {
b, reg, _, en, _ := newBridge(t, goodOffsite())
b.Existing = func() string { return otherPriv }
b.KeyGen = nil // must not be needed
o := goodOffsite()
if err := os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700); err != nil {
t.Fatal(err)
}
if en.calls != 1 || en.gotQuotaGB != 50 {
t.Fatalf("a quota raise must re-apply and map the NEW quota (no consume): %+v", en)
}
}
// Key-auth-first (Scenario B) — the existing key still works: NO consume, NO install; re-verify + re-pin +
// reconfigure with the EXISTING key, marker updated.
func TestBridge_KeyAuthFirstSkipsConsume(t *testing.T) {
b, cons, inst, en, _ := newBridge(t, goodOffsite())
cons.panics = true // the whole point: a working key must NEVER consume the one-time password
prober := &fakeProber{pem: "EXISTINGPEM", ok: true}
b.Prober = prober
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("key-auth-first reconcile: %v", err)
}
if prober.calls != 1 || prober.gotKH != "[h]:23 ssh-ed25519 AAAAKEY" {
t.Fatalf("probe must run once with the freshly-scanned pinned known_hosts: %+v", prober)
}
if inst.calls != 0 {
t.Fatal("installer must NOT run when the existing key authenticates")
}
if en.calls != 1 || en.gotPriv != "EXISTINGPEM" || en.gotKnownHost != "[h]:23 ssh-ed25519 AAAAKEY" {
t.Fatalf("enabler must reconfigure with the EXISTING key + fresh pin: %+v", en)
}
if b.readMarker() != descriptorHash(b.Cfg.Offsite) {
t.Fatal("marker must be updated after a key-auth-first apply")
}
}
// Scenario C — key-auth-first must NOT weaken the fresh path: probe fails → the full
// verify→consume→install path runs unchanged (with the freshly GENERATED key).
func TestBridge_FreshGuestFallsThroughToFullPath(t *testing.T) {
b, cons, inst, en, _ := newBridge(t, goodOffsite())
b.Prober = &fakeProber{ok: false} // fresh guest: no key / auth refused
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("fresh-guest reconcile: %v", err)
}
if cons.calls != 1 || inst.calls != 1 {
t.Fatalf("fresh guest must consume+install exactly once: cons=%d inst=%d", cons.calls, inst.calls)
}
if en.calls != 1 || en.gotPriv != "PRIVPEM" {
t.Fatalf("fresh guest must configure with the GENERATED key: %+v", en)
}
if b.readMarker() != descriptorHash(b.Cfg.Offsite) {
t.Fatal("marker must be persisted after a full-path apply")
}
}
// Scenario B — host-key mismatch → refuse: no consume, no install, no configure, no marker.
func TestBridge_HostKeyMismatchRefuses(t *testing.T) {
b, cons, inst, en, _ := newBridge(t, goodOffsite())
b.Scanner = &fakeScanner{fp: "SHA256:ATTACKER", line: "[h]:23 ssh-ed25519 EVIL"}
b.Prober = &fakeProber{panics: true} // the probe must NEVER run when the identity check failed
err := b.Reconcile(context.Background())
if err == nil || !strings.Contains(err.Error(), "MISMATCH") {
t.Fatalf("mismatch must refuse, got %v", err)
}
if cons.calls != 0 || inst.calls != 0 || en.calls != 0 {
t.Fatalf("nothing may proceed on a host-key mismatch: cons=%d inst=%d en=%d", cons.calls, inst.calls, en.calls)
}
if b.readMarker() != "" {
t.Fatal("no marker may be written on a mismatch")
}
}
// Scenario C — idempotent: marker already matches → no-op, consume is NOT called.
func TestBridge_IdempotentNoReconsume(t *testing.T) {
b, cons, inst, en, _ := newBridge(t, goodOffsite())
cons.panics = true // Consume must not be called
// pre-seed the marker with the current descriptor hash
_ = os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700)
if err := os.WriteFile(b.MarkerPath, []byte(descriptorHash(b.Cfg.Offsite)), 0o600); err != nil {
// The pre-v0.289.0 marker for this exact descriptor:
if err := os.WriteFile(b.MarkerPath, []byte(legacyDescriptorHash(o)), 0o600); err != nil {
t.Fatal(err)
}
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("idempotent reconcile must be a clean no-op, got %v", err)
t.Fatalf("reconcile: %v", err)
}
if cons.calls != 0 || inst.calls != 0 || en.calls != 0 {
t.Fatal("an already-applied descriptor must be a full no-op")
if reg.calls != 1 || reg.gotPub != otherPub {
t.Fatalf("the upgraded box must re-register its EXISTING key: calls=%d pub=%q", reg.calls, reg.gotPub)
}
if en.gotPriv != otherPriv {
t.Fatal("the existing key was not kept")
}
}
// Scenario D — install fails → fail-safe: marker NOT persisted, offbox NOT configured, loud log.
func TestBridge_InstallFailIsFailSafe(t *testing.T) {
b, cons, inst, en, logbuf := newBridge(t, goodOffsite())
inst.err = errors.New("ssh-copy-id refused")
err := b.Reconcile(context.Background())
if err == nil {
t.Fatal("install failure must error")
// A restart / descriptor change on a box whose key is already pinned: no hub write at all.
func TestBridge_AlreadyPinnedNeedsNoRegister(t *testing.T) {
b, reg, pr, en, _ := newBridge(t, goodOffsite())
b.Existing = func() string { return testPriv }
pr.pinnedInitially = true
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("reconcile: %v", err)
}
if reg.calls != 0 || en.calls != 1 {
t.Fatalf("register=%d enable=%d", reg.calls, en.calls)
}
}
// The registrar answered but the key still does not reach the PINNED server (e.g. an unpinned line
// won): refuse — never configure a key that can delete.
func TestBridge_RegisteredButNotPinnedRefuses(t *testing.T) {
b, reg, pr, en, _ := newBridge(t, goodOffsite())
pr.pinnedAfterRegister = false
if err := b.Reconcile(context.Background()); err == nil {
t.Fatal("a key that does not reach the pinned server must refuse")
}
if reg.calls != 1 || en.calls != 0 {
t.Fatalf("register=%d enable=%d", reg.calls, en.calls)
}
if _, err := os.Stat(b.MarkerPath); err == nil {
t.Fatal("marker persisted after a refusal")
}
}
func TestBridge_HubInstalledADifferentKeyRefuses(t *testing.T) {
b, reg, _, en, _ := newBridge(t, goodOffsite())
reg.wrongFP = true
if err := b.Reconcile(context.Background()); err == nil || en.calls != 0 {
t.Fatalf("err=%v enable=%d", err, en.calls)
}
}
// Host-key mismatch → refuse before anything touches the hub.
func TestBridge_HostKeyMismatchRefuses(t *testing.T) {
b, reg, pr, en, _ := newBridge(t, goodOffsite())
b.Scanner = &fakeScanner{fp: "SHA256:EVIL", line: "x"}
reg.panics = true
if err := b.Reconcile(context.Background()); err == nil || !strings.Contains(err.Error(), "MISMATCH") {
t.Fatalf("err = %v", err)
}
if pr.calls != 0 || en.calls != 0 {
t.Fatal("probe/configure ran after a host-key mismatch")
}
}
func TestBridge_IdempotentMarker(t *testing.T) {
b, reg, _, _, _ := newBridge(t, goodOffsite())
if err := b.Reconcile(context.Background()); err != nil {
t.Fatal(err)
}
reg.panics = true
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("second reconcile: %v", err)
}
}
func TestBridge_RegisterFailIsFailSafe(t *testing.T) {
b, reg, _, en, _ := newBridge(t, goodOffsite())
reg.err = errors.New("hub down")
if err := b.Reconcile(context.Background()); err == nil {
t.Fatal("want error")
}
if en.calls != 0 {
t.Fatal("offbox must NOT be configured when install fails")
t.Fatal("configured after a failed register")
}
if b.readMarker() != "" {
t.Fatal("marker must NOT be persisted on a failed apply (fail-safe)")
}
if cons.calls != 1 {
t.Fatal("the password was consumed (spent) before install")
}
if !strings.Contains(logbuf.String(), "password is spent") {
t.Fatal("a consumed-but-failed install must log the loud 'password is spent' signal")
if _, err := os.Stat(b.MarkerPath); err == nil {
t.Fatal("marker persisted after failure")
}
}
// A failed confirm leaves an extra PINNED line (cannot delete) — not a failure of the apply.
func TestBridge_ConfirmFailureIsNotFatal(t *testing.T) {
b, reg, _, en, logbuf := newBridge(t, goodOffsite())
reg.confirmEr = errors.New("hub blip")
if err := b.Reconcile(context.Background()); err != nil {
t.Fatalf("reconcile: %v", err)
}
if en.calls != 1 || !strings.Contains(logbuf.String(), "confirm key") {
t.Fatalf("enable=%d log=%s", en.calls, logbuf.String())
}
}
// Disabled → no-op (no consume/install/configure).
func TestBridge_DisabledNoOp(t *testing.T) {
o := goodOffsite()
o.Enabled = false
b, cons, inst, en, _ := newBridge(t, o)
if err := b.Reconcile(context.Background()); err != nil {
t.Fatal(err)
}
if cons.calls+inst.calls+en.calls != 0 {
t.Fatal("disabled offsite must be a no-op")
b, reg, _, en, _ := newBridge(t, o)
reg.panics = true
if err := b.Reconcile(context.Background()); err != nil || en.calls != 0 {
t.Fatalf("err=%v enable=%d", err, en.calls)
}
}
// legacyDescriptorHash is the pre-v0.289.0 marker key, byte for byte as v0.288.0 computed it.
func legacyDescriptorHash(o config.OffsiteConfig) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%s|%d|%s|%s|%d", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint, o.QuotaGB)))
return hex.EncodeToString(sum[:])
}
+144 -98
View File
@@ -17,16 +17,13 @@ import (
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)
// --- func adapters (convenient wiring in main.go) ---
type ConsumerFunc func(ctx context.Context) (string, error)
func (f ConsumerFunc) Consume(ctx context.Context) (string, error) { return f(ctx) }
type EnablerFunc func(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error
func (f EnablerFunc) ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string, quotaGB int) error {
@@ -40,48 +37,125 @@ type SettleFunc func() (version, floor string, updateRunning, floorKnown bool)
func (f SettleFunc) SettleState() (string, string, bool, bool) { return f() }
// --- HTTPConsumer: POST the hub consume-password endpoint with the per-customer API key ---
type HTTPConsumer struct {
// --- HubRegistrar: the box's half of the hub key registrar (decision 69, hub >= v0.127.0) ---
//
// The box sends ONLY its public key; the hub writes it into the Storage Box sub-account's
// authorized_keys pinned to `rclone serve restic --stdio --append-only <repo>`. Until controller
// v0.289.0 the box fetched the sub-account PASSWORD here (consume-password) — and that password can
// rewrite authorized_keys, i.e. remove the pin (measured 2026-10-03, R-820). No response this client
// reads can carry a password; TestHubRegistrar_NeverAsksForAPassword pins the paths it calls.
type HubRegistrar struct {
HubURL string
CustomerID string
APIKey string
HC *http.Client
}
func (c HTTPConsumer) Consume(ctx context.Context) (string, error) {
func (c HubRegistrar) post(ctx context.Context, path string, body any) ([]byte, error) {
if c.HubURL == "" || c.CustomerID == "" || c.APIKey == "" {
return "", fmt.Errorf("offsite-apply: consume: hub url/customer/apikey not configured")
return nil, fmt.Errorf("offsite-apply: hub url/customer/apikey not configured")
}
hc := c.HC
if hc == nil {
hc = &http.Client{Timeout: 20 * time.Second}
hc = &http.Client{Timeout: 3 * time.Minute} // the hub does an SSH round trip to the provider
}
url := strings.TrimRight(c.HubURL, "/") + "/api/v1/offsite/consume-password/" + c.CustomerID
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
var rd io.Reader
if body != nil {
b, err := json.Marshal(body)
if err != nil {
return nil, err
}
rd = strings.NewReader(string(b))
}
url := strings.TrimRight(c.HubURL, "/") + "/api/v1/offsite/" + path + "/" + c.CustomerID
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, rd)
if err != nil {
return "", err
return nil, err
}
req.Header.Set("Authorization", "Bearer "+c.APIKey)
req.Header.Set("Content-Type", "application/json")
resp, err := hc.Do(req)
if err != nil {
return "", err
return nil, err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
if resp.StatusCode == http.StatusNotFound {
return "", fmt.Errorf("no unconsumed offsite password (already consumed or none provisioned)")
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("consume: HTTP %d", resp.StatusCode)
return nil, fmt.Errorf("hub %s: HTTP %d: %s", path, resp.StatusCode, truncate(raw))
}
var body struct {
Password string `json:"password"`
return raw, nil
}
// Register asks the hub to install pub pinned append-only; returns the key's fingerprint as the hub saw it.
func (c HubRegistrar) Register(ctx context.Context, pub string) (string, error) {
raw, err := c.post(ctx, "register-key", map[string]string{"public_key": strings.TrimSpace(pub)})
if err != nil {
return "", err
}
if err := json.Unmarshal(raw, &body); err != nil || body.Password == "" {
return "", fmt.Errorf("consume: malformed response")
var r struct {
Installed bool `json:"installed"`
Fingerprint string `json:"fingerprint"`
}
return body.Password, nil // NEVER logged
if err := json.Unmarshal(raw, &r); err != nil || !r.Installed || r.Fingerprint == "" {
return "", fmt.Errorf("hub register-key: malformed response")
}
return r.Fingerprint, nil
}
// Confirm tells the hub the box now uses fp; the hub removes every other key line.
func (c HubRegistrar) Confirm(ctx context.Context, fp string) error {
_, err := c.post(ctx, "confirm-key", map[string]string{"fingerprint": fp})
return err
}
// MoveAside asks the hub to set the repository aside (never deletes) — the box's pinned key cannot.
func (c HubRegistrar) MoveAside(ctx context.Context) (string, error) {
raw, err := c.post(ctx, "move-aside", nil)
if err != nil {
return "", err
}
var r struct {
MovedTo string `json:"moved_to"`
}
if err := json.Unmarshal(raw, &r); err != nil || r.MovedTo == "" {
return "", fmt.Errorf("hub move-aside: malformed response")
}
return r.MovedTo, nil
}
// --- HubWindowClient: the box's half of the clean-up window (decision 68) ---
type HubWindowClient struct{ Registrar HubRegistrar }
func (c HubWindowClient) Open(ctx context.Context, countBefore int) (backup.OffsiteWindow, error) {
raw, err := c.Registrar.post(ctx, "window-open", map[string]int{"count_before": countBefore})
if err != nil {
return backup.OffsiteWindow{}, err
}
var r struct {
Granted bool `json:"granted"`
WindowID int64 `json:"window_id"`
NewestAllowed string `json:"newest_allowed"`
MaxRemove int `json:"max_remove"`
Reason string `json:"reason"`
}
if err := json.Unmarshal(raw, &r); err != nil {
return backup.OffsiteWindow{}, fmt.Errorf("hub window-open: malformed response")
}
w := backup.OffsiteWindow{Granted: r.Granted, ID: r.WindowID, MaxRemove: r.MaxRemove, Reason: r.Reason}
if r.Granted {
t, perr := time.Parse(time.RFC3339, r.NewestAllowed)
if perr != nil {
return backup.OffsiteWindow{}, fmt.Errorf("hub window-open: bad newest_allowed")
}
w.NewestAllowed = t
}
return w, nil
}
func (c HubWindowClient) Close(ctx context.Context, res backup.OffsiteWindowResult) error {
_, err := c.Registrar.post(ctx, "window-close", res)
return err
}
// --- KeyscanScanner: capture the box host key (x/crypto/ssh, no binary) → fingerprint + known_hosts line ---
@@ -145,72 +219,21 @@ func (ED25519KeyGen) Generate() (string, string, error) {
return privPEM, pubLine, nil
}
// --- SSHCopyIDInstaller: install the pubkey via the proven `sshpass -e ssh-copy-id -p N -s -f`, verify ---
type SSHCopyIDInstaller struct{}
func (SSHCopyIDInstaller) Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized, knownHosts string) error {
if strings.TrimSpace(knownHosts) == "" {
return fmt.Errorf("ssh-copy-id: empty known_hosts — refusing to install without a pinned host key")
}
// ssh-copy-id -s (SFTP mode) mktemp's its batch file under ~/.ssh and dies LOCALLY if the directory
// doesn't exist — the container image ships without /root/.ssh (live finding: the one-time password was
// consumed, then the install failed before ever connecting).
if home, err := os.UserHomeDir(); err == nil {
if err := os.MkdirAll(filepath.Join(home, ".ssh"), 0o700); err != nil {
return fmt.Errorf("ssh-copy-id: ensure ~/.ssh (needed by -s mode): %w", err)
}
}
work, err := os.MkdirTemp("", "felhom-keyinstall-")
if err != nil {
return err
}
defer os.RemoveAll(work)
pubPath := filepath.Join(work, "id.pub")
privPath := filepath.Join(work, "id")
khPath := filepath.Join(work, "known_hosts")
if err := os.WriteFile(pubPath, []byte(pubAuthorized), 0o600); err != nil {
return err
}
if err := os.WriteFile(privPath, []byte(privPEM), 0o600); err != nil {
return err
}
// Pin the scanner-VERIFIED host key: StrictHostKeyChecking=yes against this known_hosts refuses any
// other key (no accept-new/TOFU) — the ssh-copy-id + verify sessions connect ONLY to the box whose
// fingerprint the bridge already matched against the hub descriptor.
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
return err
}
// Install (SSHPASS env is read by `sshpass -e`; the password never appears on argv).
install := exec.CommandContext(ctx, "sshpass", "-e", "ssh-copy-id", "-p", strconv.Itoa(port), "-s", "-f",
"-i", pubPath, "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile="+khPath, user+"@"+host)
install.Env = append(os.Environ(), "SSHPASS="+password)
if out, err := install.CombinedOutput(); err != nil {
return fmt.Errorf("ssh-copy-id: %w: %s", err, truncate(out))
}
// Verify passwordless key auth (an SFTP no-op; the box's restricted shell only offers SFTP).
verify := exec.CommandContext(ctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
"-i", privPath, "-oBatchMode=yes", "-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
verify.Stdin = strings.NewReader("pwd\n")
if out, err := verify.CombinedOutput(); err != nil {
return fmt.Errorf("key-auth verify failed after install: %w: %s", err, truncate(out))
}
return nil
// --- PinnedProber: does this key reach the PINNED append-only server? ---
//
// Measured on the provider 2026-10-03 (audits/offsite-lock-build-2026-10-03/partA/A5): `ssh -i <key>
// host probe` with stdin closed exits 0 and prints rclone's NOTICE when the key is pinned (the forced
// rclone starts, sees EOF, exits); an UNPINNED key gets the restricted shell → "Command not found",
// exit 8. A refused key exits 255. So ok = exit 0 AND "rclone" in the output — a POSITIVE observable.
type PinnedProber struct {
Timeout time.Duration // 0 → 20 s
// Run is the exec seam (tests); nil → real ssh.
Run func(ctx context.Context, args []string) ([]byte, error)
}
// --- SFTPKeyAuthProber: does the ALREADY-INSTALLED key still authenticate? (key-auth-first) ---
// SFTPKeyAuthProber probes passwordless auth with the existing installed key (KeyPath), pinned to the
// freshly-verified knownHosts line. No key file → ok=false (fresh guest). The probe never logs secrets.
type SFTPKeyAuthProber struct {
KeyPath string // the installed key, e.g. <dataDir>/offbox/ssh_key
Timeout time.Duration // per-probe budget; 0 → 20s
}
func (p SFTPKeyAuthProber) Probe(ctx context.Context, host, user string, port int, knownHosts string) (string, bool) {
pem, err := os.ReadFile(p.KeyPath)
if err != nil {
return "", false // no existing key — a fresh guest; take the full path
func (p PinnedProber) Probe(ctx context.Context, host, user string, port int, knownHosts, privPEM string) bool {
if strings.TrimSpace(privPEM) == "" {
return false
}
timeout := p.Timeout
if timeout == 0 {
@@ -220,21 +243,44 @@ func (p SFTPKeyAuthProber) Probe(ctx context.Context, host, user string, port in
defer cancel()
work, err := os.MkdirTemp("", "felhom-keyprobe-")
if err != nil {
return "", false
return false
}
defer os.RemoveAll(work)
khPath := filepath.Join(work, "known_hosts")
if err := os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600); err != nil {
return "", false
khPath, keyPath := filepath.Join(work, "known_hosts"), filepath.Join(work, "id")
if os.WriteFile(khPath, []byte(knownHosts+"\n"), 0o600) != nil || os.WriteFile(keyPath, []byte(privPEM), 0o600) != nil {
return false
}
probe := exec.CommandContext(pctx, "sftp", "-b", "-", "-P", strconv.Itoa(port),
"-i", p.KeyPath, "-oBatchMode=yes", "-oConnectTimeout=10",
"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile="+khPath, user+"@"+host)
probe.Stdin = strings.NewReader("pwd\n")
if err := probe.Run(); err != nil {
return "", false // auth refused / unreachable — fall through to the full path
args := []string{"-p", strconv.Itoa(port), "-i", keyPath, "-oBatchMode=yes", "-oConnectTimeout=10", "-oIdentitiesOnly=yes",
"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile=" + khPath, user + "@" + host, "probe"}
run := p.Run
if run == nil {
run = func(ctx context.Context, args []string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "ssh", args...)
cmd.Stdin = strings.NewReader("")
return cmd.CombinedOutput()
}
}
return string(pem), true
out, err := run(pctx, args)
return err == nil && strings.Contains(string(out), "rclone")
}
// PublicKeyOf derives the authorized_keys line of an OpenSSH private key (the migration path: a box
// whose key predates the pin registers the SAME key, so the hub re-writes it pinned).
func PublicKeyOf(privPEM string) (string, error) {
signer, err := ssh.ParsePrivateKey([]byte(privPEM))
if err != nil {
return "", err
}
return string(ssh.MarshalAuthorizedKey(signer.PublicKey())), nil
}
// FingerprintOf returns the SHA256 fingerprint of an authorized_keys line.
func FingerprintOf(pub string) (string, error) {
pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pub))
if err != nil {
return "", err
}
return ssh.FingerprintSHA256(pk), nil
}
func truncate(b []byte) string {
@@ -68,18 +68,18 @@ func (c *fakeClock) sleep(_ context.Context, d time.Duration) {
// settleBridge builds a bridge wired for a FULL Reconcile (so a released gate consumes exactly once)
// plus the injectable settle-gate. Prober is nil → the fresh consume+install path runs on release.
func settleBridge(t *testing.T, s *fakeSettle, clk *fakeClock) (*Bridge, *fakeConsumer, *bytes.Buffer) {
func settleBridge(t *testing.T, s *fakeSettle, clk *fakeClock) (*Bridge, *fakeRegistrar, *bytes.Buffer) {
t.Helper()
cfg := &config.Config{}
cfg.Offsite = goodOffsite()
cons := &fakeConsumer{pw: "the-transient-pw"}
cons := &fakeRegistrar{}
var logbuf bytes.Buffer
b := &Bridge{
Cfg: cfg,
Consumer: cons,
Registrar: cons,
Scanner: &fakeScanner{fp: "SHA256:goodfp", line: "[h]:23 ssh-ed25519 AAAAKEY"},
KeyGen: &fakeKeyGen{priv: "PRIVPEM", pub: "ssh-ed25519 AAAAPUB felhom"},
Installer: &fakeInstaller{},
KeyGen: &realKeyGen{priv: testPriv, pub: testPub},
Prober: &fakeProber{reg: cons, pinnedAfterRegister: true},
Enabler: &fakeEnabler{},
MarkerPath: filepath.Join(t.TempDir(), "offbox", "applied_marker"),
Logger: log.New(&logbuf, "", 0),