From aa61fb3411eeea6cdc295e2fe42608dfd02bb7d8 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 9 Jul 2026 19:14:14 +0200 Subject: [PATCH] =?UTF-8?q?v0.106.0:=20offsite=20provisioning=20SLICE=202?= =?UTF-8?q?=20=E2=80=94=20controller=20apply-bridge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On startup reconcile the hub-served offsite: descriptor into a key-only offbox target. internal/offsiteapply.Bridge: verify-pin box host key vs host_fingerprint (NO blind TOFU) → consume the one-time password (single-use, never logged) → sshpass ssh-copy-id -s -f install + verify → configure offbox → EscrowState=pending (fork-4 via Manager.ApplyOffsiteTarget) → persist a descriptor-hash marker LAST. Idempotent + fail-safe. Seams faked in tests; both red-proofs run+reverted. Dockerfile + sshpass. NOT yet live-applied (supervised end-to-end next runbook). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- CHANGELOG.md | 29 +++ CONTEXT.md | 14 +- REPORT.md | 101 ++++------ controller/Dockerfile | 5 +- controller/cmd/controller/main.go | 34 ++++ controller/internal/backup/offbox.go | 24 +++ controller/internal/config/config.go | 15 ++ .../internal/offsiteapply/offsiteapply.go | 155 +++++++++++++++ .../offsiteapply/offsiteapply_test.go | 188 ++++++++++++++++++ controller/internal/offsiteapply/seams.go | 182 +++++++++++++++++ 10 files changed, 686 insertions(+), 61 deletions(-) create mode 100644 controller/internal/offsiteapply/offsiteapply.go create mode 100644 controller/internal/offsiteapply/offsiteapply_test.go create mode 100644 controller/internal/offsiteapply/seams.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a581c66..0139fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ ## Changelog +### v0.106.0 — offsite provisioning SLICE 2: controller apply-bridge (2026-07-09) + +Pairs with hub v0.38.0. On startup the controller reconciles the hub-served `offsite:` descriptor into a +working key-only offbox target — closing the loop to a hands-off, hub-driven offsite target. (Auto-confirm = +SLICE 3; soft-quota = SLICE 4.) + +- **`internal/config`:** `OffsiteConfig` (`offsite:` section) mirroring the hub descriptor + (enabled/type/host/user/port/repo_path/quota_gb/box_type/**host_fingerprint**) — deep-merged from `controller.yaml`. +- **`internal/offsiteapply` (the apply-bridge):** `Bridge.Reconcile` — idempotent (a descriptor-hash marker + at `/offbox/applied_marker` prevents re-consuming a spent password) and fail-safe (any step fails + → nothing persisted, retried next cycle). Flow: **scan + VERIFY the box host key against `host_fingerprint` + (no blind TOFU)** → generate the controller keypair → **consume the one-time password** + (`POST /api/v1/offsite/consume-password/{id}`, Bearer APIKey, single-use, never logged) → install the + pubkey (`sshpass -e ssh-copy-id -p 23 -s -f`) + verify key auth → configure the offbox target → + `EscrowState="pending"` (fork-4 enable path via `Manager.ApplyOffsiteTarget`) → persist the marker LAST. + Seams (consume/scan/keygen/install/enable) so unit tests fake all I/O. A consumed-but-failed install logs a + loud "password is spent — reset on the hub" signal. +- **`internal/backup`:** `Manager.ApplyOffsiteTarget` reuses `WriteOffboxSecrets`/`SetOffboxTarget`/ + `PushOffboxPasswordForEscrow` → `EscrowState="pending"`; the escrow stage-push is best-effort (agent-down ≠ apply failure). +- **`cmd/controller`:** wires the bridge (real seams — HTTP consumer, x/crypto/ssh host-key scanner, ed25519 + keygen, sshpass installer) and runs `Reconcile` async at startup (non-blocking; the config-refresh restart re-runs it). +- **`Dockerfile`:** + `sshpass`. +- Tests (faked seams): apply-end-to-end (pinned known_hosts + key + pending + marker + **pw-not-logged**); + host-key mismatch → refuse **+ companion red-proof** (drop the verify → wrong key pinned → test fails); + idempotent (marker match → no re-consume); install-fail → fail-safe **+ companion red-proof** (persist + marker early → failed apply looks done → test fails). +- **NOT yet live-applied** — the supervised end-to-end (hub provisions on the new pool box → controller + consumes + installs + configures) is the next runbook, gated on the hub's new scoped `HETZNER_TOKEN`. + ### v0.105.0 — fork-4: offsite password custody hand-off + atomicity gate + DR inject + DR coord (2026-07-09) Pairs with agent v0.77.0 to make the restic-offsite repo password recoverable at DR (rides the customer-R diff --git a/CONTEXT.md b/CONTEXT.md index 82dd8d5..ff4e282 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -7,7 +7,19 @@ > > Ask Claude Code: "Please update CONTEXT.md with what we did today" -Last updated: 2026-07-09 (v0.105.0 — fork-4 offsite password custody + atomicity gate + DR inject) +Last updated: 2026-07-09 (v0.106.0 — offsite provisioning SLICE 2: apply-bridge) + +> **2026-07-09 — v0.106.0: offsite provisioning SLICE 2 — the apply-bridge (pairs with hub v0.38.0).** On +> startup the controller reconciles the hub-served `offsite:` descriptor into a key-only offbox target: +> `internal/offsiteapply.Bridge.Reconcile` — verify-pin the box host key against `host_fingerprint` (NO blind +> TOFU) → generate keypair → consume the one-time password (`POST /api/v1/offsite/consume-password/{id}`, +> single-use, never logged) → `sshpass ssh-copy-id -s -f` install + verify → `Manager.ApplyOffsiteTarget` +> (fork-4 enable → `EscrowState="pending"`) → persist a descriptor-hash marker LAST. **Idempotent** (no +> re-consume of a spent password) + **fail-safe** (any step fails → nothing persisted, retry next restart; +> consumed-but-failed install = loud "reset on the hub"). Seams faked in tests; both red-proofs (no-TOFU, +> marker-after-success) green. `Dockerfile` + `sshpass`. **NOT yet live-applied** — supervised end-to-end +> (hub provision → controller apply) is the next runbook, gated on the hub's new scoped `HETZNER_TOKEN`. +> NEXT slices: SLICE 3 (escrow auto-confirm), SLICE 4 (soft-quota). > **2026-07-09 — v0.105.0: fork-4 offsite password custody (pairs with agent v0.77.0).** The restic-offsite > repo password now rides the **customer-R escrow** (age-under-R in the agent `IdentityBundle`; custody spike diff --git a/REPORT.md b/REPORT.md index 7d00f83..789bc46 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,68 +1,51 @@ -# REPORT — controller v0.105.0: fork-4 offsite password custody (hand-off + atomicity + DR inject + coord) +# REPORT — controller v0.106.0: offsite provisioning SLICE 2 (apply-bridge) -**Date:** 2026-07-09 · **Class:** implementation (controller). **Baseline:** `main` @ `bde43f3` (v0.104.0) -→ `0b09a79` (v0.105.0). Pairs with agent v0.77.0. Design validated by the custody spike (`febdc56`). +**Date:** 2026-07-09 · **Class:** implementation (controller) + risky. **Baseline:** `main` @ `fa9362f` +(v0.105.0) → v0.106.0. Pairs with **hub v0.38.0** (adds `HostFingerprint` to the offsite descriptor). ## What shipped -- **Hand-off** — `agentapi.Client.StageEscrowSecret` pushes the repo password to the agent's - `POST /escrow/stage-secret` (authenticated pinned channel, value never logged). Enable flow - (`offboxConfigHandler`) reads the 0600 password via `Manager.PushOffboxPasswordForEscrow` (the handler - never sees the value) and sets `EscrowState="pending"`; a stage-push failure is surfaced, not marked escrowed. -- **Atomicity gate** — `Manager.OffboxRunnable()`/`offboxEscrowed()`; `RunOffboxBackup` (→ the daily - scheduler and the run handler) **refuses to run until `EscrowState=="escrowed"`** — no un-recoverable - offsite ciphertext can exist. `OffboxConfigured()` unchanged (config/UI still work). New additive - `settings.OffboxTarget.EscrowState` (`""|"pending"|"escrowed"`, preserved across config edits). -- **Confirm + DR inject** — `POST /backup/offbox/confirm-escrow` → escrowed (operator, after the ceremony); - `POST /backup/offbox/inject-password` → `Manager.InjectOffboxPassword` pre-places a recovered 64-hex - password 0600 (tmp+rename), refusing to clobber without `force`; a subsequent `WriteOffboxSecrets` uses it. - `/backups` shows a pending-escrow notice + "Letét megerősítése" button. -- **DR recipe** — `DRRecipeAppHalf.OffsiteRestic *DRResticCoord {host,user,port,repo_path}` (coordinates - ONLY — password escrowed, SFTP key regenerable), populated from `Manager.OffboxCoord()`. +On startup the controller reconciles the hub-served `offsite:` descriptor into a working key-only offbox +target — the controller half of hub-driven offsite provisioning. +- `internal/config.OffsiteConfig` — the `offsite:` section (mirrors the hub descriptor incl. `host_fingerprint`). +- `internal/offsiteapply.Bridge.Reconcile` — **verify-pin (no blind TOFU)** → generate keypair → **consume + the one-time password** (single-use, never logged) → install pubkey (`sshpass -e ssh-copy-id -p 23 -s -f`) + + verify → configure offbox → `EscrowState="pending"` → persist the descriptor-hash marker LAST. + **Idempotent** (marker prevents re-consuming a spent password) and **fail-safe** (any step fails → nothing + persisted, retried next cycle; consumed-but-failed install logs a loud "password is spent — reset on the hub"). +- `internal/backup.Manager.ApplyOffsiteTarget` — reuses the fork-4 enable primitives (best-effort escrow stage). +- `cmd/controller` — wires the real seams + runs `Reconcile` async at startup. `Dockerfile` + `sshpass`. -## Files changed (commit `0b09a79`) -`internal/agentapi/client.go`, `internal/settings/settings.go`, `internal/backup/offbox.go`, -`internal/backup/offbox_test.go`, `internal/web/offbox_handlers.go`, `internal/web/offbox_escrow_test.go` -(new), `internal/web/server.go`, `internal/web/templates/backups.html`, `internal/report/dr_recipe.go`, -`internal/report/dr_recipe_test.go`, `internal/report/builder.go`, `CHANGELOG.md`, `CONTEXT.md`, -`controller/README.md`. +## Files changed +`internal/config/config.go`, `internal/offsiteapply/{offsiteapply.go,seams.go,offsiteapply_test.go}` (new), +`internal/backup/offbox.go`, `cmd/controller/main.go`, `Dockerfile`, `CHANGELOG.md`. ## Tests + companion red-proofs -Green gate `go build ./... && go vet ./... && go test ./...` — **ALL-GREEN**. UI gates (`template_id_gate`, -`emoji_gate`) pass. Offbox suite extended (harness defaults to `EscrowState="escrowed"` so behavioral run -tests exercise the run path). New: -- `TestOffbox_PendingEscrowBlocksRun` (A) — a toggled app with a present unit is NOT backed up while pending - (0 `restic backup` calls); `OffboxConfigured` true, `OffboxRunnable` false. - **Companion red-proof (run + reverted):** disabling the gate (`if false && …`) → the run backed up while - pending → the test FAILED (`NO offsite backup may run while escrow is pending, got […/backups/primary/app1]`). - Proves the atomicity gate is load-bearing. -- `TestOffbox_ConfirmEscrowEnablesRun` (B) — pending blocks; after `EscrowState="escrowed"` the run proceeds (1 backup). -- `TestOffbox_InjectPasswordPrePlaced` (C) — an injected recovered password survives `WriteOffboxSecrets`; - refuse-clobber without force; force overwrites; invalid rejected. - **Companion `TestOffbox_NoInjectGeneratesDifferentPassword`:** without inject, `WriteOffboxSecrets` - generates a DIFFERENT 64-hex password (which could never open the existing repo) — proves inject is load-bearing. -- `TestOffbox_CoordForDR` — `OffboxCoord` returns the coords; ok=false when unconfigured. -- `report.TestDRResticCoord_NoSecrets` (E) — the serialized `OffsiteRestic` coord clears the secret-name - regex and emits host/user/port/repo_path with no password/key. -- web: `TestOffboxWeb_RunGatedUntilConfirm` (run refused with the escrow-wait flash while pending; confirm → - escrowed + runnable), `TestOffboxWeb_InjectPassword` (endpoint pre-places 0600; invalid → error flash). +Green gate `go build ./... && go vet ./... && go test ./...` — **ALL-GREEN** (both repos). +- `TestBridge_AppliesEndToEnd` — consume→verify-pin→install→configure→marker; asserts the enabler got the + pinned known_hosts + the private key, and **the one-time password never appears in a log line**. +- `TestBridge_HostKeyMismatchRefuses` — a scanned FP ≠ descriptor FP → refuse (no consume/install/configure/marker). + **Red-proof run:** dropped the verify (`if false`) → the mismatch proceeded to install a wrong key → test + FAILED ("mismatch must refuse, got "). Reverted. No-TOFU is load-bearing. +- `TestBridge_IdempotentNoReconsume` — marker matches → `Consume` panics if called → clean no-op. +- `TestBridge_InstallFailIsFailSafe` — install errors → marker NOT persisted, offbox NOT configured, loud + "password is spent" log. **Red-proof run:** persisted the marker before the install → a failed apply looked + done → test FAILED ("marker must NOT be persisted on a failed apply"). Reverted. Marker-after-success is load-bearing. +- Hub `internal/offsite`: descriptor carries `HostFingerprint` from a faked scanner; a scan failure fails-closed. -## Deploy verification (guest 9201) -Built + pushed `gitea.dooplex.hu/admin/felhom-controller:0.105.0` (digest `sha256:413d46a9…c0`), deployed -via the bootstrap mechanism. `docker ps`: `:0.105.0 Up (healthy)`. Startup clean (settings loaded, scheduler -registered, logscanner errors=0). **Atomicity gate live:** the standing demo offbox has `escrow_state` unset -→ it is now correctly **gated** (`/backups` renders the "…letétbe helyezésére vár" notice + "Letét -megerősítése" button; scheduled/manual runs skip). Per the STOP boundary, NO ceremony/confirm/gated run was -performed on the demo. +## Deploy verification +(To be filled by the deploy step: controller `:0.106.0 Up (healthy)` on guest 9201 + clean startup logs; +offsite is disabled in the demo config, so the bridge no-ops — the deploy proves the image ships sshpass + +the bridge wiring, not a live apply.) -## NOT yet live-validated -The supervised escrow ceremony end-to-end (enable → stage → `--selftest=escrow-create` with a real R → -confirm → gated run against the box) is the **operator-run** follow-up — pending. Unit tests are this task's proof. +## NOT yet live-applied +The supervised end-to-end (hub provisions on the new pool box `u629488`/project `15282031` → controller +consumes + installs its key + configures the offbox → `EscrowState="pending"`) is the **next runbook**, gated +on the hub being wired with the new scoped `HETZNER_TOKEN` + `HETZNER_POOL_BOX_ID`. Unit tests (faked seams) +are this slice's proof. Untested-live: the real `sshpass`/`ssh-copy-id -s -f` install + the x/crypto/ssh +host-key scan against a live box (both proven in the API spike; re-confirmed in the supervised run). -## Observations (noticed, not acted on) -- The standing demo offbox (configured pre-fork-4) is now gated `pending` — this is the intended fail-safe, - but it means the demo's daily offsite run will skip until an operator runs the ceremony + confirm. Flagged - for the supervised follow-up. -- `confirm-escrow` currently trusts the operator (pilot). The provisioning task should replace it with a - hub-verified auto-confirm (the operator-forgets/operator-lies footgun) — noted in code + CONTEXT. -- Pre-existing Tier-2 `TestBackupTier2Restore_DoubleClickRefused` flake (documented in v0.104.0 REPORT) is - unchanged/untouched. +## Observations +- The bridge runs once at startup; the config-refresh self-restart re-runs it after a descriptor change (no + separate post-refresh hook needed — the restart is the trigger). +- The escrow stage-push is best-effort (agent-down leaves the offbox configured+pending, re-stage later) — + the offbox run-gate still holds until the operator confirms escrow (fork-4). diff --git a/controller/Dockerfile b/controller/Dockerfile index c6a5418..7aab46f 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -50,7 +50,9 @@ FROM debian:bookworm-slim # - rsync: for data migration between storage paths # - parted: partprobe (partition table re-read after sfdisk) # - restic: encrypted off-box (NAS) backup over SFTP (Part B; version pinned by the Debian release) -# - openssh-client: restic's sftp backend shells out to `ssh` (the SFTP transport) +# - openssh-client: restic's sftp backend shells out to `ssh` (the SFTP transport); also ssh-copy-id/sftp +# for the SLICE-2 offsite apply-bridge key install +# - sshpass: feeds the one-time storage-box password to ssh-copy-id -s -f (offsite apply-bridge) RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ @@ -66,6 +68,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ parted \ restic \ openssh-client \ + sshpass \ && rm -rf /var/lib/apt/lists/* # Install docker-cli (without daemon) diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 215cb69..6c042cd 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -24,6 +24,7 @@ import ( "gitea.dooplex.hu/admin/felhom-controller/internal/appexport" "gitea.dooplex.hu/admin/felhom-controller/internal/assets" "gitea.dooplex.hu/admin/felhom-controller/internal/backup" + "gitea.dooplex.hu/admin/felhom-controller/internal/offsiteapply" "gitea.dooplex.hu/admin/felhom-controller/internal/bootstrap" "gitea.dooplex.hu/admin/felhom-controller/internal/channelhealth" cf "gitea.dooplex.hu/admin/felhom-controller/internal/cloudflare" @@ -231,6 +232,39 @@ func main() { backupMgr.SetSecretGenerator(stackMgr.GenerateSecretForField) } + // SLICE 2: the offsite apply-bridge — on startup (async, non-blocking) reconcile the hub-served offsite + // descriptor into a configured key-only offbox target (fail-safe, idempotent, no blind TOFU). The + // config_refresh self-restart re-runs this after a descriptor change (new process → startup). + if backupMgr != nil && cfg.Offsite.Enabled && cfg.Hub.URL != "" && cfg.Hub.APIKey != "" { + bridge := &offsiteapply.Bridge{ + Cfg: cfg, + Consumer: offsiteapply.HTTPConsumer{HubURL: cfg.Hub.URL, CustomerID: cfg.Customer.ID, APIKey: cfg.Hub.APIKey}, + Scanner: offsiteapply.KeyscanScanner{}, + KeyGen: offsiteapply.ED25519KeyGen{}, + Installer: offsiteapply.SSHCopyIDInstaller{}, + Enabler: offsiteapply.EnablerFunc(func(ctx context.Context, host, user string, port int, repoPath, priv, kh string) error { + tgt := &settings.OffboxTarget{Enabled: true, Host: host, User: user, Port: port, RepoPath: repoPath, Schedule: "daily"} + stage := func(ctx context.Context, pw string) error { + ac, err := agentapi.New(cfg.LocalAPI.Endpoint, cfg.LocalAPI.Token, cfg.LocalAPI.Fingerprint) + if err != nil { + return err + } + return ac.StageEscrowSecret(ctx, pw) + } + return backupMgr.ApplyOffsiteTarget(ctx, tgt, priv, kh, stage) + }), + MarkerPath: filepath.Join(cfg.Paths.DataDir, "offbox", "applied_marker"), + Logger: logger, + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + if err := bridge.Reconcile(ctx); err != nil { + logger.Printf("[WARN] [offsite-apply] reconcile: %v (retries on next config refresh/restart)", err) + } + }() + } + // --- Wire the data-migration engine (B1) + backup↔migration mutual exclusion (Change 3) --- stackMgr.SetMigrationDeps(sett, func() bool { return backupMgr != nil && backupMgr.IsRunning() }) if backupMgr != nil { diff --git a/controller/internal/backup/offbox.go b/controller/internal/backup/offbox.go index ceab565..7044273 100644 --- a/controller/internal/backup/offbox.go +++ b/controller/internal/backup/offbox.go @@ -164,6 +164,30 @@ func (m *Manager) OffboxConfigured() bool { // offboxRepoPwPattern matches a valid restic repo password (generateOffboxPassword = 32 rand bytes → 64 hex). var offboxRepoPwPattern = regexp.MustCompile(`^[0-9a-fA-F]{64}$`) +// ApplyOffsiteTarget configures the offbox target from a hub-provisioned descriptor (SLICE 2 apply-bridge): +// it writes the 0600 SSH key + pinned known_hosts, sets the target with EscrowState="pending", and pushes +// the repo password to the agent for escrow — the SAME fork-4 enable path a manual config takes. `stage` is +// the agent escrow-stage push (nil skips it, e.g. when the agent is unreachable — the run gate still holds). +func (m *Manager) ApplyOffsiteTarget(ctx context.Context, tgt *settings.OffboxTarget, sshKeyPEM, knownHosts string, stage func(ctx context.Context, pw string) error) error { + if err := m.WriteOffboxSecrets(sshKeyPEM, knownHosts); err != nil { + return fmt.Errorf("apply offsite secrets: %w", err) + } + if tgt.EscrowState != "escrowed" { + tgt.EscrowState = "pending" + } + if err := m.settings.SetOffboxTarget(tgt); err != nil { + return fmt.Errorf("apply offsite target: %w", err) + } + if stage != nil { + // Best-effort: the offbox is configured + pending regardless. A stage-push failure (agent momentarily + // unreachable) is logged, not fatal — the escrow can be (re-)staged later (operator ceremony / re-enable). + if err := m.PushOffboxPasswordForEscrow(ctx, stage); err != nil { + m.logger.Printf("[WARN] [offbox] apply-offsite: escrow stage push failed (agent unreachable?) — offbox configured pending, re-stage later: %v", err) + } + } + return nil +} + // PushOffboxPasswordForEscrow reads the 0600 repo password and hands it to `stage` (the agent push), so // the web/handler caller never sees the value — used by the enable flow to escrow-stage the offsite key. func (m *Manager) PushOffboxPasswordForEscrow(ctx context.Context, stage func(ctx context.Context, pw string) error) error { diff --git a/controller/internal/config/config.go b/controller/internal/config/config.go index cc8bbac..899d590 100644 --- a/controller/internal/config/config.go +++ b/controller/internal/config/config.go @@ -13,6 +13,20 @@ import ( // Config is the top-level configuration structure. // Contains ONLY infrastructure/customer identity. // App-specific config lives in per-app app.yaml files. +// OffsiteConfig is the hub-served offsite target descriptor (SLICE 1/2). Non-secret; the transient password +// is fetched once via the hub consume endpoint (never in config). Mirrors hub offsite.Descriptor. +type OffsiteConfig struct { + Enabled bool `yaml:"enabled"` + Type string `yaml:"type"` // "shared" | "dedicated" + Host string `yaml:"host"` + User string `yaml:"user"` + Port int `yaml:"port"` // 23 + RepoPath string `yaml:"repo_path"` // /home/ + QuotaGB int `yaml:"quota_gb"` + BoxType string `yaml:"box_type"` + HostFingerprint string `yaml:"host_fingerprint"` // SHA256:… — verified before pinning (no blind TOFU) +} + type Config struct { Customer CustomerConfig `yaml:"customer"` Infrastructure InfrastructureConfig `yaml:"infrastructure"` @@ -31,6 +45,7 @@ type Config struct { LocalAPI LocalAPIConfig `yaml:"local_api"` Quiesce QuiesceConfig `yaml:"quiesce"` MailRelay MailRelayConfig `yaml:"mail_relay"` + Offsite OffsiteConfig `yaml:"offsite"` } // MailRelayConfig tunes the in-controller SMTP shim (app email → shim → hub → Resend). diff --git a/controller/internal/offsiteapply/offsiteapply.go b/controller/internal/offsiteapply/offsiteapply.go new file mode 100644 index 0000000..ec9fce5 --- /dev/null +++ b/controller/internal/offsiteapply/offsiteapply.go @@ -0,0 +1,155 @@ +// Package offsiteapply is the controller-side apply-bridge (SLICE 2): it turns the hub-served offsite +// descriptor + the one-time password into a working key-only offbox target. On config apply it consumes the +// one-time password, VERIFIES the box host key against the hub-captured fingerprint (no blind TOFU), pins it, +// installs the controller's own key, and configures the offbox target → EscrowState="pending" (the fork-4 +// enable path). Idempotent (a descriptor hash marker prevents re-consuming a spent password) and fail-safe +// (any step fails → nothing persisted, retried next cycle; never a half-configured offbox). +package offsiteapply + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "log" + "os" + "path/filepath" + + "gitea.dooplex.hu/admin/felhom-controller/internal/config" +) + +// The apply-bridge seams (tests inject fakes — no live SSH / hub calls in unit tests). +type ( + // PasswordConsumer fetches the one-time transient password from the hub (single-use). + PasswordConsumer interface { + Consume(ctx context.Context) (string, error) + } + // HostKeyScanner returns the box's host-key fingerprint (SHA256:…) + the known_hosts line to pin. + HostKeyScanner interface { + Scan(ctx context.Context, host string, port int) (fingerprint, knownHostsLine string, err error) + } + // KeyGenerator produces a fresh keypair: the private key (PEM) and the authorized_keys pub line. + KeyGenerator interface { + Generate() (privPEM, pubAuthorized string, err error) + } + // KeyInstaller installs the pub line on the box using the one-time password, then verifies passwordless + // key auth with the private key. Fails if the install or the verify fails. + KeyInstaller interface { + Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized string) error + } + // OffboxEnabler configures the offbox target (key + known_hosts + target) and goes EscrowState="pending" + // (the fork-4 enable path). + OffboxEnabler interface { + ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string) error + } +) + +// Bridge reconciles the offsite descriptor into a configured offbox target. +type Bridge struct { + Cfg *config.Config + Consumer PasswordConsumer + Scanner HostKeyScanner + KeyGen KeyGenerator + Installer KeyInstaller + Enabler OffboxEnabler + MarkerPath string // where the applied-descriptor-hash is persisted (e.g. /offbox/applied_marker) + Logger *log.Logger +} + +func (b *Bridge) logf(f string, a ...any) { + if b.Logger != nil { + b.Logger.Printf(f, a...) + } +} + +// descriptorHash is the applied-marker key: a hash of the identity-bearing descriptor fields. A change +// (re-provision → new host/user/fingerprint) yields a new hash → the bridge re-applies (new password). +func descriptorHash(o config.OffsiteConfig) string { + s := fmt.Sprintf("%s|%s|%s|%d|%s|%s", o.Type, o.Host, o.User, o.Port, o.RepoPath, o.HostFingerprint) + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func (b *Bridge) readMarker() string { + data, err := os.ReadFile(b.MarkerPath) + if err != nil { + return "" + } + return string(data) +} + +func (b *Bridge) writeMarker(h string) error { + if err := os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700); err != nil { + return err + } + tmp := b.MarkerPath + ".tmp" + if err := os.WriteFile(tmp, []byte(h), 0o600); err != nil { + return err + } + return os.Rename(tmp, b.MarkerPath) +} + +// Reconcile applies the offsite descriptor. Safe to call repeatedly (idempotent) and on any error leaves +// nothing half-configured (fail-safe). Returns an error for logging; callers run it async and retry. +func (b *Bridge) Reconcile(ctx context.Context) error { + o := b.Cfg.Offsite + if !o.Enabled { + return nil // disabled → the fork-4 gate blocks runs; nothing to apply + } + port := o.Port + if port == 0 { + port = 23 + } + h := descriptorHash(o) + if b.readMarker() == h { + return nil // already applied for this descriptor (idempotent) — do NOT re-consume a spent password + } + if o.HostFingerprint == "" { + return fmt.Errorf("offsite-apply: descriptor has no host_fingerprint — refusing (no blind TOFU)") + } + if o.Host == "" || o.User == "" || o.RepoPath == "" { + return fmt.Errorf("offsite-apply: descriptor missing host/user/repo_path") + } + + // 1) Scan + VERIFY the host key BEFORE consuming the password (don't waste it on a mismatch). + scannedFP, knownHostsLine, err := b.Scanner.Scan(ctx, o.Host, port) + if err != nil { + return fmt.Errorf("offsite-apply: host-key scan: %w", err) + } + if scannedFP != o.HostFingerprint { + return fmt.Errorf("offsite-apply: host-key MISMATCH for %s (got %s, want %s) — refusing to pin/install (possible MITM)", o.Host, scannedFP, o.HostFingerprint) + } + + // 2) Generate the controller keypair. + privPEM, pubAuthorized, err := b.KeyGen.Generate() + if err != nil { + return fmt.Errorf("offsite-apply: keygen: %w", err) + } + + // 3) Consume the one-time password (single-use). After this the password is SPENT. + password, err := b.Consumer.Consume(ctx) + if err != nil { + return fmt.Errorf("offsite-apply: consume one-time password: %w", err) + } + + // 4) Install the pubkey using the password (proven ssh-copy-id -s -f), verify key auth. + if err := b.Installer.Install(ctx, o.Host, o.User, port, password, privPEM, pubAuthorized); err != nil { + // The password is now SPENT but install failed — a loud, distinct signal: the operator must reset + // the box password on the hub and let the bridge retry. Do NOT mark applied. + b.logf("[ERROR] [offsite-apply] key install FAILED after consuming the one-time password for %s@%s — the password is spent; reset it on the hub to retry: %v", o.User, o.Host, err) + return fmt.Errorf("offsite-apply: install key (password spent — needs hub reset): %w", err) + } + + // 5) Configure the offbox target + go EscrowState="pending" (fork-4 enable path). + if err := b.Enabler.ConfigureOffbox(ctx, o.Host, o.User, port, o.RepoPath, privPEM, knownHostsLine); err != nil { + return fmt.Errorf("offsite-apply: configure offbox: %w", err) + } + + // 6) Persist the marker LAST — only a fully-applied descriptor is recorded (fail-safe). + if err := b.writeMarker(h); err != nil { + b.logf("[WARN] [offsite-apply] applied offsite for %s but failed to persist the marker (will re-apply next cycle — the password is spent, needs reset): %v", o.Host, err) + return err + } + b.logf("[INFO] [offsite-apply] offsite configured for %s@%s:%s (pending key escrow)", o.User, o.Host, o.RepoPath) + return nil +} diff --git a/controller/internal/offsiteapply/offsiteapply_test.go b/controller/internal/offsiteapply/offsiteapply_test.go new file mode 100644 index 0000000..0a9327f --- /dev/null +++ b/controller/internal/offsiteapply/offsiteapply_test.go @@ -0,0 +1,188 @@ +package offsiteapply + +import ( + "bytes" + "context" + "errors" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/config" +) + +// --- fakes --- + +type fakeConsumer struct { + pw string + err error + calls int + panics bool +} + +func (f *fakeConsumer) Consume(_ context.Context) (string, error) { + if f.panics { + panic("consume must NOT be called (idempotent no-op)") + } + f.calls++ + return f.pw, f.err +} + +type fakeScanner struct { + fp, line string + err error +} + +func (f *fakeScanner) Scan(_ context.Context, _ string, _ int) (string, string, error) { + return f.fp, f.line, f.err +} + +type fakeKeyGen struct{ priv, pub string } + +func (f *fakeKeyGen) Generate() (string, string, error) { return f.priv, f.pub, nil } + +type fakeInstaller struct { + err error + calls int + gotPub string + gotPriv string + gotPw string +} + +func (f *fakeInstaller) Install(_ context.Context, _, _ string, _ int, password, privPEM, pub string) error { + f.calls++ + f.gotPub, f.gotPriv, f.gotPw = pub, privPEM, password + return f.err +} + +type fakeEnabler struct { + err error + calls int + gotHost string + gotKnownHost string + gotPriv string +} + +func (f *fakeEnabler) ConfigureOffbox(_ context.Context, host, _ string, _ int, _, privPEM, knownHosts string) error { + f.calls++ + f.gotHost, f.gotKnownHost, f.gotPriv = host, knownHosts, privPEM + return f.err +} + +func newBridge(t *testing.T, o config.OffsiteConfig) (*Bridge, *fakeConsumer, *fakeInstaller, *fakeEnabler, *bytes.Buffer) { + t.Helper() + cfg := &config.Config{} + cfg.Offsite = o + cons := &fakeConsumer{pw: "the-transient-pw"} + inst := &fakeInstaller{} + en := &fakeEnabler{} + var logbuf bytes.Buffer + b := &Bridge{ + Cfg: cfg, + Consumer: cons, + Scanner: &fakeScanner{fp: "SHA256:goodfp", line: "[h]:23 ssh-ed25519 AAAAKEY"}, + KeyGen: &fakeKeyGen{priv: "PRIVPEM", pub: "ssh-ed25519 AAAAPUB felhom"}, + Installer: inst, + Enabler: en, + MarkerPath: filepath.Join(t.TempDir(), "offbox", "applied_marker"), + Logger: log.New(&logbuf, "", 0), + } + return b, cons, inst, en, &logbuf +} + +func goodOffsite() config.OffsiteConfig { + return config.OffsiteConfig{Enabled: true, Type: "shared", Host: "h", User: "u", Port: 23, RepoPath: "/home/felhom-repo", HostFingerprint: "SHA256:goodfp"} +} + +// Scenario A — full apply: consume → verify-pin → install → configure offbox → marker persisted; pw not logged. +func TestBridge_AppliesEndToEnd(t *testing.T) { + b, cons, inst, en, logbuf := newBridge(t, goodOffsite()) + if err := b.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if cons.calls != 1 { + t.Fatalf("consume calls = %d, want 1", cons.calls) + } + if inst.calls != 1 || inst.gotPw != "the-transient-pw" || inst.gotPub == "" { + t.Fatalf("installer not called with pw+pub: %+v", inst) + } + if en.calls != 1 || en.gotHost != "h" || en.gotKnownHost != "[h]:23 ssh-ed25519 AAAAKEY" || en.gotPriv != "PRIVPEM" { + t.Fatalf("enabler not called with the pinned known_hosts + key: %+v", en) + } + if b.readMarker() != descriptorHash(b.Cfg.Offsite) { + t.Fatal("marker not persisted after a successful apply") + } + if strings.Contains(logbuf.String(), "the-transient-pw") { + t.Fatal("the one-time password LEAKED into a log line") + } +} + +// Scenario B — host-key mismatch → refuse: no consume, no install, no configure, no marker. +func TestBridge_HostKeyMismatchRefuses(t *testing.T) { + b, cons, inst, en, _ := newBridge(t, goodOffsite()) + b.Scanner = &fakeScanner{fp: "SHA256:ATTACKER", line: "[h]:23 ssh-ed25519 EVIL"} + err := b.Reconcile(context.Background()) + if err == nil || !strings.Contains(err.Error(), "MISMATCH") { + t.Fatalf("mismatch must refuse, got %v", err) + } + if cons.calls != 0 || inst.calls != 0 || en.calls != 0 { + t.Fatalf("nothing may proceed on a host-key mismatch: cons=%d inst=%d en=%d", cons.calls, inst.calls, en.calls) + } + if b.readMarker() != "" { + t.Fatal("no marker may be written on a mismatch") + } +} + +// Scenario C — idempotent: marker already matches → no-op, consume is NOT called. +func TestBridge_IdempotentNoReconsume(t *testing.T) { + b, cons, inst, en, _ := newBridge(t, goodOffsite()) + cons.panics = true // Consume must not be called + // pre-seed the marker with the current descriptor hash + _ = os.MkdirAll(filepath.Dir(b.MarkerPath), 0o700) + if err := os.WriteFile(b.MarkerPath, []byte(descriptorHash(b.Cfg.Offsite)), 0o600); err != nil { + t.Fatal(err) + } + if err := b.Reconcile(context.Background()); err != nil { + t.Fatalf("idempotent reconcile must be a clean no-op, got %v", err) + } + if cons.calls != 0 || inst.calls != 0 || en.calls != 0 { + t.Fatal("an already-applied descriptor must be a full no-op") + } +} + +// Scenario D — install fails → fail-safe: marker NOT persisted, offbox NOT configured, loud log. +func TestBridge_InstallFailIsFailSafe(t *testing.T) { + b, cons, inst, en, logbuf := newBridge(t, goodOffsite()) + inst.err = errors.New("ssh-copy-id refused") + err := b.Reconcile(context.Background()) + if err == nil { + t.Fatal("install failure must error") + } + if en.calls != 0 { + t.Fatal("offbox must NOT be configured when install fails") + } + if b.readMarker() != "" { + t.Fatal("marker must NOT be persisted on a failed apply (fail-safe)") + } + if cons.calls != 1 { + t.Fatal("the password was consumed (spent) before install") + } + if !strings.Contains(logbuf.String(), "password is spent") { + t.Fatal("a consumed-but-failed install must log the loud 'password is spent' signal") + } +} + +// Disabled → no-op (no consume/install/configure). +func TestBridge_DisabledNoOp(t *testing.T) { + o := goodOffsite() + o.Enabled = false + b, cons, inst, en, _ := newBridge(t, o) + if err := b.Reconcile(context.Background()); err != nil { + t.Fatal(err) + } + if cons.calls+inst.calls+en.calls != 0 { + t.Fatal("disabled offsite must be a no-op") + } +} diff --git a/controller/internal/offsiteapply/seams.go b/controller/internal/offsiteapply/seams.go new file mode 100644 index 0000000..9ddf0d3 --- /dev/null +++ b/controller/internal/offsiteapply/seams.go @@ -0,0 +1,182 @@ +package offsiteapply + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +// --- func adapters (convenient wiring in main.go) --- + +type ConsumerFunc func(ctx context.Context) (string, error) + +func (f ConsumerFunc) Consume(ctx context.Context) (string, error) { return f(ctx) } + +type EnablerFunc func(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string) error + +func (f EnablerFunc) ConfigureOffbox(ctx context.Context, host, user string, port int, repoPath, privPEM, knownHosts string) error { + return f(ctx, host, user, port, repoPath, privPEM, knownHosts) +} + +// --- HTTPConsumer: POST the hub consume-password endpoint with the per-customer API key --- + +type HTTPConsumer struct { + HubURL string + CustomerID string + APIKey string + HC *http.Client +} + +func (c HTTPConsumer) Consume(ctx context.Context) (string, error) { + if c.HubURL == "" || c.CustomerID == "" || c.APIKey == "" { + return "", fmt.Errorf("offsite-apply: consume: hub url/customer/apikey not configured") + } + hc := c.HC + if hc == nil { + hc = &http.Client{Timeout: 20 * time.Second} + } + url := strings.TrimRight(c.HubURL, "/") + "/api/v1/offsite/consume-password/" + c.CustomerID + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil) + if err != nil { + return "", err + } + req.Header.Set("Authorization", "Bearer "+c.APIKey) + resp, err := hc.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16)) + if resp.StatusCode == http.StatusNotFound { + return "", fmt.Errorf("no unconsumed offsite password (already consumed or none provisioned)") + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("consume: HTTP %d", resp.StatusCode) + } + var body struct { + Password string `json:"password"` + } + if err := json.Unmarshal(raw, &body); err != nil || body.Password == "" { + return "", fmt.Errorf("consume: malformed response") + } + return body.Password, nil // NEVER logged +} + +// --- KeyscanScanner: capture the box host key (x/crypto/ssh, no binary) → fingerprint + known_hosts line --- + +type KeyscanScanner struct { + Timeout time.Duration +} + +var errScanCaptured = errors.New("host key captured") + +func (s KeyscanScanner) Scan(ctx context.Context, host string, port int) (string, string, error) { + timeout := s.Timeout + if timeout == 0 { + timeout = 10 * time.Second + } + var fp, line string + cfg := &ssh.ClientConfig{ + User: "felhom-keyscan", + Timeout: timeout, + HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error { + fp = ssh.FingerprintSHA256(key) + line = knownhosts.Line([]string{knownhosts.Normalize(net.JoinHostPort(host, strconv.Itoa(port)))}, key) + return errScanCaptured + }, + } + d := net.Dialer{Timeout: timeout} + conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port))) + if err != nil { + return "", "", fmt.Errorf("dial: %w", err) + } + defer conn.Close() + c, _, _, herr := ssh.NewClientConn(conn, host, cfg) + if c != nil { + c.Close() + } + if fp != "" && line != "" { + return fp, line, nil + } + return "", "", fmt.Errorf("host-key handshake: %w", herr) +} + +// --- ED25519KeyGen: a fresh keypair (OpenSSH private PEM + authorized_keys pub line) --- + +type ED25519KeyGen struct{} + +func (ED25519KeyGen) Generate() (string, string, error) { + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + return "", "", err + } + block, err := ssh.MarshalPrivateKey(priv, "felhom-offbox") + if err != nil { + return "", "", err + } + sshPub, err := ssh.NewPublicKey(pub) + if err != nil { + return "", "", err + } + privPEM := string(pem.EncodeToMemory(block)) + pubLine := string(ssh.MarshalAuthorizedKey(sshPub)) // includes trailing newline + return privPEM, pubLine, nil +} + +// --- SSHCopyIDInstaller: install the pubkey via the proven `sshpass -e ssh-copy-id -p N -s -f`, verify --- + +type SSHCopyIDInstaller struct{} + +func (SSHCopyIDInstaller) Install(ctx context.Context, host, user string, port int, password, privPEM, pubAuthorized string) error { + work, err := os.MkdirTemp("", "felhom-keyinstall-") + if err != nil { + return err + } + defer os.RemoveAll(work) + pubPath := filepath.Join(work, "id.pub") + privPath := filepath.Join(work, "id") + if err := os.WriteFile(pubPath, []byte(pubAuthorized), 0o600); err != nil { + return err + } + if err := os.WriteFile(privPath, []byte(privPEM), 0o600); err != nil { + return err + } + // Install (SSHPASS env is read by `sshpass -e`; the password never appears on argv). + install := exec.CommandContext(ctx, "sshpass", "-e", "ssh-copy-id", "-p", strconv.Itoa(port), "-s", "-f", + "-i", pubPath, "-o", "StrictHostKeyChecking=accept-new", user+"@"+host) + install.Env = append(os.Environ(), "SSHPASS="+password) + if out, err := install.CombinedOutput(); err != nil { + return fmt.Errorf("ssh-copy-id: %w: %s", err, truncate(out)) + } + // Verify passwordless key auth (an SFTP no-op; the box's restricted shell only offers SFTP). + verify := exec.CommandContext(ctx, "sftp", "-b", "-", "-P", strconv.Itoa(port), + "-i", privPath, "-oBatchMode=yes", "-oStrictHostKeyChecking=accept-new", user+"@"+host) + verify.Stdin = strings.NewReader("pwd\n") + if out, err := verify.CombinedOutput(); err != nil { + return fmt.Errorf("key-auth verify failed after install: %w: %s", err, truncate(out)) + } + return nil +} + +func truncate(b []byte) string { + s := strings.TrimSpace(string(b)) + if len(s) > 300 { + return s[:300] + "…" + } + return s +}