diff --git a/CHANGELOG.md b/CHANGELOG.md index 40cb9f0..4a7e68f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,35 @@ All notable changes to **felhom-agent** are recorded here. Update on every code change that gets pushed. +## v0.28.0 — backup re-target → felhom-pbs (offsite DR) + operator-signed decommission (2026-06-12) + +**Whole-guest backup now defaults to the offsite PBS tier (real DR).** `BackupConfig.BackupTarget()` +returns the configured `backup.local_backup_target` or, when empty, the new default `felhom-pbs` — a +PBS datastore on SEPARATE HARDWARE (the DooPlex box), so a host disk/hardware failure no longer takes +the backups with it. The target stays fully configurable (set `local_backup_target` to `local`/other +to override); no call site hardcodes it. All `NewBackupRunner` sites (restore-test scheduler, local-API, +`--selftest=backup`/`restore-test`) route through `BackupTarget()`. + +Proven live on demo-felhom before the re-point (PHASE 0 gate): +- snapshot-mode `vzdump → felhom-pbs` still fires the `create storage snapshot 'vzdump'` marker, so the + 8B.2 early-resume/quiesce signal survives a PBS target (the marker is mode-driven, not target-driven); +- the restore-test enumerates PBS backups through the SAME generic `StorageContent` + (`/nodes//storage/felhom-pbs/content` returns `content:"backup"` + ctime/vmid/volid), so + `PickRestoreCandidate`/`latestArchive` need NO PBS-client change; +- `pct restore` from a PBS volid round-trips cleanly (storage.cfg encryption key applied transparently); +- PBS gotchas (`ignore-verified`, node-from-UPID, privsep) touch only the verify-API path, not vzdump/restore. + +**Operator-signed `decommission` now reachable (slice 10 P3 completion).** The previously-unreachable +`IntentDecommissioned` state (no production caller) is now reached ONLY via a gate-VERIFIED operator +signature — never customer-confirmable, distinct from a safe eject. New `internal/signedjobs` +`DecommissionExecutor` (op `decommission`, classified destructive in `reconcile.Classify`) calls +`IntentStore.SetDecommissioned`, keyed by the drive's STORAGE durable-id (the watchdog's key, e.g. +`uuid:` — NOT the device-level `byid:/byuuid:` scheme `storage_wipe` uses), so the recorded +intent actually gates future remounts. New `ExecutorChain` lets the signed-jobs runner serve both +`storage_wipe` and `decommission`; the runner wiring moved below the intent-store open in `main.go`. +`felhom-opsign` builds decommission params from `-durable-id`. No controller/customer UI — the operator +path is hub jobs-queue → signed-jobs runner. + ## v0.27.0 — slice 10 P3: self-heal watchdog reconcile + 4-state intent model (2026-06-12) The storage watchdog goes from detect-only → detect-and-reconcile: the agent autonomously re-mounts an diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 7ee2610..a4635d2 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -43,7 +43,7 @@ import ( // version is the agent version. Overridable at build time with // -ldflags "-X main.version="; defaults to the in-repo CHANGELOG version. -var version = "0.27.0" +var version = "0.28.0" func main() { var ( @@ -309,18 +309,6 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { } gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger) - // Signed-jobs runner (slice 10B): the consumer of the hub's signed-jobs queue. On a heartbeat - // that flags pending signed ops, it fetches each opaque blob, runs it through the gate (the - // LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time → - // durable nonce-burn) and, only on all-pass, hands the verified op to the storage-WIPE executor - // — which re-resolves the DURABLE device id + re-inspects (8C) before mkfs. This closes the 8C - // data-bearing `pending_signature` gap. With no signers pinned the gate refuses every job - // (pending_signature) and nothing executes — correct. Wired as a second envelope observer - // alongside the desired-state syncer. - wipeExec := signedjobs.NewWipeExecutor(hostOps, logger) - jobsRunner := signedjobs.NewRunner(client, gate, wipeExec, cfg.Hub.HostID, logger) - loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner)) - // Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target // set for attached↔disconnected transitions → debounced out-of-band report; and, on a // known mount-backed target's device returning unmounted, dispatches a benign re-mount @@ -406,6 +394,28 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int { if intentStore != nil { // avoid a typed-nil interface (would defeat the nil check) intentRec = intentStore } + + // Signed-jobs runner (slice 10B + P3): the consumer of the hub's signed-jobs queue. On a + // heartbeat that flags pending signed ops, it fetches each opaque blob, runs it through the gate + // (the LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time → + // durable nonce-burn) and, only on all-pass, hands the verified op to the executor chain: + // - storage_wipe → re-resolves the DURABLE device id + re-inspects (8C) before mkfs (closes + // the 8C data-bearing `pending_signature` gap); + // - decommission → records the PERMANENT decommission intent (keyed by the watchdog's storage + // durable-id), making the previously-unreachable IntentDecommissioned state reachable ONLY + // via a verified operator signature (never customer-confirmable; distinct from a safe eject). + // With no signers pinned the gate refuses every job (pending_signature) and nothing executes — + // correct. Wired as a second envelope observer alongside the desired-state syncer. The intent + // store is opened above (line ~340), so this wiring lives here (after it) rather than earlier. + wipeExec := signedjobs.NewWipeExecutor(hostOps, logger) + var decommIntent signedjobs.IntentDecommissioner + if intentStore != nil { // typed-nil guard (a nil *IntentStore in the interface would pass != nil) + decommIntent = intentStore + } + decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger) + jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec}, cfg.Hub.HostID, logger) + loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner)) + localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, collector, intentRec, logger, &localTokens) if localTokens != nil { defer localTokens.Close() @@ -536,7 +546,8 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re } } min, max := cfg.Backup.ScratchBand() - runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom restore-test", logger) + target := cfg.Backup.BackupTarget() + runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", logger) return backup.NewScheduler(backup.SchedulerOptions{ Runner: engine, Pick: runner.PickRestoreCandidate, @@ -545,7 +556,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re RestoreStorage: cfg.Backup.RestoreStorage, ScratchMin: min, ScratchMax: max, - SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget), + SourceTier: storageTier(context.Background(), px, target), }, Cadence: cadence, Logger: logger, @@ -578,7 +589,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St return nil } logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath()) - runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger) + runner := backup.NewBackupRunner(px, cfg.Backup.BackupTarget(), "", "felhom local-api", logger) // Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown // (same fenced ExecRunner the host-storage + provision back-half use). gaMode := proxmox.RunnerMode(cfg.Privileged.Mode) @@ -838,10 +849,7 @@ func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logg fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N") return 2 } - if cfg.Backup.LocalBackupTarget == "" { - fmt.Fprintln(os.Stderr, "selftest=backup requires backup.local_backup_target in config (a content=backup storage)") - return 2 - } + target := cfg.Backup.BackupTarget() px, err := newProxmoxClient(cfg) if err != nil { fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err) @@ -850,8 +858,8 @@ func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logg ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) defer cancel() - fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, cfg.Backup.LocalBackupTarget) - runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom selftest", logger) + fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, target) + runner := backup.NewBackupRunner(px, target, "", "felhom selftest", logger) rec, err := runner.Backup(ctx, vmid) printJSON("backup record", rec) if err != nil { @@ -905,15 +913,16 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog rec := engine.Recover(ctx) fmt.Printf(" recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean) + target := cfg.Backup.BackupTarget() if archive == "" { - runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "", logger) + runner := backup.NewBackupRunner(px, target, "", "", logger) archive, err = runner.PickRestoreCandidate(ctx) if err != nil { fmt.Fprintln(os.Stderr, " [FAIL] pick backup:", err) return 1 } if archive == "" { - fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", cfg.Backup.LocalBackupTarget, "(run --selftest=backup first)") + fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", target, "(run --selftest=backup first)") return 1 } } @@ -921,7 +930,7 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage) res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{ Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage, - ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget), + ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, target), }) printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC())) if res.Skipped { diff --git a/cmd/felhom-opsign/main.go b/cmd/felhom-opsign/main.go index 61532a0..5b18dad 100644 --- a/cmd/felhom-opsign/main.go +++ b/cmd/felhom-opsign/main.go @@ -48,7 +48,7 @@ func run() error { guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)") keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)") paramsRaw = flag.String("params", "", "op params as JSON (overrides -durable-id/-fstype)") - durableID = flag.String("durable-id", "", "for storage_wipe: the DURABLE device id (byid:…|byuuid:…)") + durableID = flag.String("durable-id", "", "storage_wipe: the DURABLE device id (byid:…|byuuid:…); decommission: the drive's STORAGE durable-id (e.g. uuid:)") fstype = flag.String("fstype", "ext4", "for storage_wipe: the filesystem to mkfs after wipe") keyFile = flag.String("key", "", "operator signing key (ssh private key / sk- key handle) for ssh-keygen -Y sign") ttl = flag.Duration("ttl", 30*time.Minute, "validity window from now (issued_at..expires_at)") @@ -63,16 +63,26 @@ func run() error { return fmt.Errorf("-op, -host, -key-id and -key are required") } - // Params: explicit JSON, or built from the wipe convenience flags. + // Params: explicit JSON, or built from the convenience flags. params := strings.TrimSpace(*paramsRaw) if params == "" { - if *op == "storage_wipe" { + switch *op { + case "storage_wipe": if *durableID == "" { return fmt.Errorf("storage_wipe needs -durable-id (byid:…|byuuid:…) — a path-only binding is refused by the agent") } pj, _ := json.Marshal(map[string]string{"durable_id": *durableID, "fstype": *fstype}) params = string(pj) - } else { + case "decommission": + // Decommission binds to the drive's STORAGE durable-id (the watchdog's key, e.g. + // "uuid:"), NOT the device-level byid:/byuuid: scheme. The agent records this + // id into the intent map, so it must match what the storage observer reports. + if *durableID == "" { + return fmt.Errorf("decommission needs -durable-id (the drive's storage durable-id, e.g. uuid:)") + } + pj, _ := json.Marshal(map[string]string{"durable_id": *durableID}) + params = string(pj) + default: params = "{}" } } diff --git a/internal/config/config.go b/internal/config/config.go index 0bf78b5..b894e44 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -155,7 +155,9 @@ type EscrowConfig struct { // an agent-internal cadence (no hub policy needed — it's self-validation); the backup // schedule/retention/target-selection policy is hub-manifest-owned and unfed until slice 10. type BackupConfig struct { - // LocalBackupTarget is the vzdump storage (content=backup) backups go to, e.g. "local". + // LocalBackupTarget is the vzdump storage (content=backup) backups go to. Empty → the + // offsite PBS default (see BackupTarget); set e.g. "local" or "felhom-pbs" to override. + // (Name kept for config back-compat; the default is no longer "local".) LocalBackupTarget string `json:"local_backup_target"` // RestoreStorage is where a restore-test's restored rootfs lands, e.g. "local-lvm". RestoreStorage string `json:"restore_storage"` @@ -189,6 +191,23 @@ func (b BackupConfig) BackupCadence() time.Duration { return 24 * time.Hour } +// defaultBackupTarget is the offsite PBS storage whole-guest backups land on by default. It is +// SEPARATE HARDWARE from the guest's own disk (a PBS datastore on the DooPlex box), so a host +// disk/hardware failure doesn't take the backups with it — that's what makes it real DR. Proven +// live: snapshot-mode vzdump to PBS still fires the `create storage snapshot` marker (early-resume +// intact) and pct-restore-from-PBS round-trips cleanly via the storage.cfg encryption key. +const defaultBackupTarget = "felhom-pbs" + +// BackupTarget is the vzdump storage (content=backup) whole-guest backups go to. Defaults to the +// offsite PBS storage (see defaultBackupTarget); override via backup.local_backup_target for a +// local or other target. Kept configurable on purpose — the field is never hardcoded at a call site. +func (b BackupConfig) BackupTarget() string { + if b.LocalBackupTarget != "" { + return b.LocalBackupTarget + } + return defaultBackupTarget +} + // Default scratch VMID band + restore-test cadence. const ( defaultScratchVMIDMin = 990000 diff --git a/internal/signedjobs/chain.go b/internal/signedjobs/chain.go new file mode 100644 index 0000000..a92fbff --- /dev/null +++ b/internal/signedjobs/chain.go @@ -0,0 +1,25 @@ +package signedjobs + +import ( + "context" + "encoding/json" +) + +// ExecutorChain dispatches a verified signed op to the first sub-executor that OWNS it. A sub- +// executor returns ErrNoExecutor for an op class it does not handle; the chain then tries the next. +// If no sub-executor owns the op, the chain returns ErrNoExecutor so the runner leaves the job +// queued for a later slice (the same contract a single executor has). The chain adds no policy — the +// gate has already verified+bound the op before any Execute runs. +type ExecutorChain []Executor + +// Execute implements Executor by trying each sub-executor in order. +func (c ExecutorChain) Execute(ctx context.Context, op string, params json.RawMessage) error { + for _, e := range c { + err := e.Execute(ctx, op, params) + if errorsIs(err, ErrNoExecutor) { + continue // this sub-executor doesn't own the op — try the next + } + return err // owned (handled or failed) — done + } + return ErrNoExecutor // no sub-executor owns this op class in this build +} diff --git a/internal/signedjobs/decommission.go b/internal/signedjobs/decommission.go new file mode 100644 index 0000000..5b228bf --- /dev/null +++ b/internal/signedjobs/decommission.go @@ -0,0 +1,80 @@ +package signedjobs + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" +) + +// opDecommission is the op class this executor serves (mirrors reconcile.ClassDecommission; the +// literal avoids importing reconcile here just for the string). +const opDecommission = "decommission" + +// IntentDecommissioner records a PERMANENT drive decommission keyed by the drive's storage +// durable-id. Satisfied by *storage.IntentStore (SetDecommissioned). Decommission is the operator's +// "permanent removal" intent: once recorded, the self-heal watchdog never auto-mounts the drive +// again (it SURVIVES absent/present), cleared only by an explicit re-commission. +type IntentDecommissioner interface { + SetDecommissioned(durableID string) error +} + +// decommissionParams is the verified params of a decommission op. The drive is named by its STORAGE +// durable-id — the SAME key the storage observer / watchdog use (e.g. "uuid:"), NOT the +// device-level byid:/byuuid: scheme the storage_wipe op uses. That is deliberate: decommission +// manipulates the intent MAP the watchdog reads, so the signed id must be the watchdog's key or the +// recorded intent would never gate a remount. +type decommissionParams struct { + DurableID string `json:"durable_id"` +} + +// DecommissionExecutor is the operator "permanent removal" consumer (slice 10 P3 completion). It +// makes the previously-unreachable IntentDecommissioned state reachable — but ONLY via a gate- +// VERIFIED operator signature (decommission is classified destructive in reconcile.Classify, so the +// signed-jobs gate refuses it pending_signature without a valid operator signature). It is distinct +// from a customer-confirmable safe eject: permanent removal is never customer-authorizable. +// +// The binding IS the durable-id: the signed op authorizes decommissioning exactly that drive on +// exactly the signed host (host_scope is checked by the gate). No /dev resolution is needed — and +// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT. +type DecommissionExecutor struct { + intent IntentDecommissioner + logger *slog.Logger +} + +// NewDecommissionExecutor wires the executor to the agent's intent store. +func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor { + if logger == nil { + logger = slog.Default() + } + return &DecommissionExecutor{intent: intent, logger: logger} +} + +// Execute implements signedjobs.Executor for the decommission op class. +func (d *DecommissionExecutor) Execute(_ context.Context, op string, params json.RawMessage) error { + if op != opDecommission { + return ErrNoExecutor // not ours — the runner leaves it queued for the owning executor + } + if d.intent == nil { + // No intent store on this host (open failure) → the op can't be honored. Don't clear it as + // "done"; surface a hard error so the runner logs it and the operator can retry once fixed. + return fmt.Errorf("decommission: no intent store wired on this host — cannot record decommission") + } + var p decommissionParams + if err := json.Unmarshal(params, &p); err != nil { + return fmt.Errorf("decommission: bad params: %w", err) + } + if p.DurableID == "" { + // The durable-id IS the resource binding; a decommission with no id is unbound — refuse. + return fmt.Errorf("decommission: op has no durable_id — refusing an unbound decommission") + } + + // The gate has already verified the operator signature + burned the nonce durably BEFORE this + // point. This log line is the audit trail of an operator-authorized permanent decommission. + d.logger.Warn("decommission: executing operator-signed PERMANENT decommission", "durable_id", p.DurableID) + if err := d.intent.SetDecommissioned(p.DurableID); err != nil { + return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err) + } + d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID) + return nil +} diff --git a/internal/signedjobs/decommission_test.go b/internal/signedjobs/decommission_test.go new file mode 100644 index 0000000..1a39046 --- /dev/null +++ b/internal/signedjobs/decommission_test.go @@ -0,0 +1,91 @@ +package signedjobs + +import ( + "context" + "encoding/json" + "testing" +) + +type fakeDecommissioner struct { + got string + err error +} + +func (f *fakeDecommissioner) SetDecommissioned(durableID string) error { + f.got = durableID + return f.err +} + +func TestDecommissionExecutor_HappyPath(t *testing.T) { + fd := &fakeDecommissioner{} + ex := NewDecommissionExecutor(fd, nil) + err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":"uuid:abc-123"}`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if fd.got != "uuid:abc-123" { + t.Errorf("SetDecommissioned got %q, want uuid:abc-123", fd.got) + } +} + +func TestDecommissionExecutor_NotOurOp(t *testing.T) { + fd := &fakeDecommissioner{} + ex := NewDecommissionExecutor(fd, nil) + if err := ex.Execute(context.Background(), "storage_wipe", json.RawMessage(`{}`)); err != ErrNoExecutor { + t.Errorf("want ErrNoExecutor for a foreign op, got %v", err) + } + if fd.got != "" { + t.Errorf("SetDecommissioned must not be called for a foreign op (got %q)", fd.got) + } +} + +func TestDecommissionExecutor_RefusesUnbound(t *testing.T) { + fd := &fakeDecommissioner{} + ex := NewDecommissionExecutor(fd, nil) + if err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":""}`)); err == nil { + t.Error("want error for an empty durable_id (unbound decommission)") + } + if fd.got != "" { + t.Errorf("SetDecommissioned must not be called for an unbound op (got %q)", fd.got) + } +} + +func TestDecommissionExecutor_NoIntentStore(t *testing.T) { + ex := NewDecommissionExecutor(nil, nil) + if err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":"uuid:x"}`)); err == nil { + t.Error("want a hard error (not silent success) when no intent store is wired") + } +} + +// chainProbe records whether it was reached and what it returns. +type chainProbe struct { + owns string + called bool + ret error +} + +func (c *chainProbe) Execute(_ context.Context, op string, _ json.RawMessage) error { + c.called = true + if op == c.owns { + return c.ret + } + return ErrNoExecutor +} + +func TestExecutorChain_DispatchesToOwner(t *testing.T) { + a := &chainProbe{owns: "storage_wipe"} + b := &chainProbe{owns: "decommission"} + chain := ExecutorChain{a, b} + + if err := chain.Execute(context.Background(), "decommission", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !b.called { + t.Error("owner (decommission) executor was not reached") + } + + // An op no sub-executor owns → ErrNoExecutor (left queued). + if err := chain.Execute(context.Background(), "guest_destroy", nil); err != ErrNoExecutor { + t.Errorf("want ErrNoExecutor for an unowned op, got %v", err) + } +}