v0.28.0: backup re-target → felhom-pbs (offsite DR) + operator-signed decommission
- BackupConfig.BackupTarget() defaults whole-guest backup to felhom-pbs (separate hardware = real DR), configurable via backup.local_backup_target; all NewBackupRunner sites route through it. PBS round-trip proven live (snapshot marker + restore-test + pct-restore) before the re-point. - signedjobs DecommissionExecutor + ExecutorChain: makes IntentDecommissioned reachable ONLY via a verified operator signature (keyed by the watchdog's storage durable-id); felhom-opsign builds decommission params from -durable-id. Runner wiring moved below the intent-store open. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+34
-25
@@ -43,7 +43,7 @@ import (
|
||||
|
||||
// version is the agent version. Overridable at build time with
|
||||
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
||||
var version = "0.27.0"
|
||||
var version = "0.28.0"
|
||||
|
||||
func main() {
|
||||
var (
|
||||
@@ -309,18 +309,6 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
}
|
||||
gate := reconcile.NewGate(verifier, cfg.Hub.HostID, reconcile.SlogAudit{Logger: logger}, logger)
|
||||
|
||||
// Signed-jobs runner (slice 10B): the consumer of the hub's signed-jobs queue. On a heartbeat
|
||||
// that flags pending signed ops, it fetches each opaque blob, runs it through the gate (the
|
||||
// LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time →
|
||||
// durable nonce-burn) and, only on all-pass, hands the verified op to the storage-WIPE executor
|
||||
// — which re-resolves the DURABLE device id + re-inspects (8C) before mkfs. This closes the 8C
|
||||
// data-bearing `pending_signature` gap. With no signers pinned the gate refuses every job
|
||||
// (pending_signature) and nothing executes — correct. Wired as a second envelope observer
|
||||
// alongside the desired-state syncer.
|
||||
wipeExec := signedjobs.NewWipeExecutor(hostOps, logger)
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, wipeExec, cfg.Hub.HostID, logger)
|
||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||
|
||||
// Storage watchdog (slice 5): the third daemon goroutine. Fast-polls the known target
|
||||
// set for attached↔disconnected transitions → debounced out-of-band report; and, on a
|
||||
// known mount-backed target's device returning unmounted, dispatches a benign re-mount
|
||||
@@ -406,6 +394,28 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
if intentStore != nil { // avoid a typed-nil interface (would defeat the nil check)
|
||||
intentRec = intentStore
|
||||
}
|
||||
|
||||
// Signed-jobs runner (slice 10B + P3): the consumer of the hub's signed-jobs queue. On a
|
||||
// heartbeat that flags pending signed ops, it fetches each opaque blob, runs it through the gate
|
||||
// (the LOCKED authz pipeline: pinned-key SSHSIG → namespace → allow-list → crypto → host → time →
|
||||
// durable nonce-burn) and, only on all-pass, hands the verified op to the executor chain:
|
||||
// - storage_wipe → re-resolves the DURABLE device id + re-inspects (8C) before mkfs (closes
|
||||
// the 8C data-bearing `pending_signature` gap);
|
||||
// - decommission → records the PERMANENT decommission intent (keyed by the watchdog's storage
|
||||
// durable-id), making the previously-unreachable IntentDecommissioned state reachable ONLY
|
||||
// via a verified operator signature (never customer-confirmable; distinct from a safe eject).
|
||||
// With no signers pinned the gate refuses every job (pending_signature) and nothing executes —
|
||||
// correct. Wired as a second envelope observer alongside the desired-state syncer. The intent
|
||||
// store is opened above (line ~340), so this wiring lives here (after it) rather than earlier.
|
||||
wipeExec := signedjobs.NewWipeExecutor(hostOps, logger)
|
||||
var decommIntent signedjobs.IntentDecommissioner
|
||||
if intentStore != nil { // typed-nil guard (a nil *IntentStore in the interface would pass != nil)
|
||||
decommIntent = intentStore
|
||||
}
|
||||
decommExec := signedjobs.NewDecommissionExecutor(decommIntent, logger)
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec}, cfg.Hub.HostID, logger)
|
||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||
|
||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, collector, intentRec, logger, &localTokens)
|
||||
if localTokens != nil {
|
||||
defer localTokens.Close()
|
||||
@@ -536,7 +546,8 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
|
||||
}
|
||||
}
|
||||
min, max := cfg.Backup.ScratchBand()
|
||||
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom restore-test", logger)
|
||||
target := cfg.Backup.BackupTarget()
|
||||
runner := backup.NewBackupRunner(px, target, "", "felhom restore-test", logger)
|
||||
return backup.NewScheduler(backup.SchedulerOptions{
|
||||
Runner: engine,
|
||||
Pick: runner.PickRestoreCandidate,
|
||||
@@ -545,7 +556,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
|
||||
RestoreStorage: cfg.Backup.RestoreStorage,
|
||||
ScratchMin: min,
|
||||
ScratchMax: max,
|
||||
SourceTier: storageTier(context.Background(), px, cfg.Backup.LocalBackupTarget),
|
||||
SourceTier: storageTier(context.Background(), px, target),
|
||||
},
|
||||
Cadence: cadence,
|
||||
Logger: logger,
|
||||
@@ -578,7 +589,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
||||
return nil
|
||||
}
|
||||
logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
||||
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger)
|
||||
runner := backup.NewBackupRunner(px, cfg.Backup.BackupTarget(), "", "felhom local-api", logger)
|
||||
// Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown
|
||||
// (same fenced ExecRunner the host-storage + provision back-half use).
|
||||
gaMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||
@@ -838,10 +849,7 @@ func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logg
|
||||
fmt.Fprintln(os.Stderr, "selftest=backup requires -vmid N")
|
||||
return 2
|
||||
}
|
||||
if cfg.Backup.LocalBackupTarget == "" {
|
||||
fmt.Fprintln(os.Stderr, "selftest=backup requires backup.local_backup_target in config (a content=backup storage)")
|
||||
return 2
|
||||
}
|
||||
target := cfg.Backup.BackupTarget()
|
||||
px, err := newProxmoxClient(cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "selftest: proxmox client:", err)
|
||||
@@ -850,8 +858,8 @@ func runSelftestBackup(ctx context.Context, cfg config.Config, logger *slog.Logg
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, cfg.Backup.LocalBackupTarget)
|
||||
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom selftest", logger)
|
||||
fmt.Printf("=== felhom-agent %s selftest=backup (vmid %d → %s) ===\n", version, vmid, target)
|
||||
runner := backup.NewBackupRunner(px, target, "", "felhom selftest", logger)
|
||||
rec, err := runner.Backup(ctx, vmid)
|
||||
printJSON("backup record", rec)
|
||||
if err != nil {
|
||||
@@ -905,15 +913,16 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
|
||||
rec := engine.Recover(ctx)
|
||||
fmt.Printf(" recover: examined=%d scratch_destroyed=%d scratch_clean=%d\n", rec.Examined, rec.ScratchDestroyed, rec.ScratchClean)
|
||||
|
||||
target := cfg.Backup.BackupTarget()
|
||||
if archive == "" {
|
||||
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "", logger)
|
||||
runner := backup.NewBackupRunner(px, target, "", "", logger)
|
||||
archive, err = runner.PickRestoreCandidate(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, " [FAIL] pick backup:", err)
|
||||
return 1
|
||||
}
|
||||
if archive == "" {
|
||||
fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", cfg.Backup.LocalBackupTarget, "(run --selftest=backup first)")
|
||||
fmt.Fprintln(os.Stderr, " [FAIL] no backup available on", target, "(run --selftest=backup first)")
|
||||
return 1
|
||||
}
|
||||
}
|
||||
@@ -921,7 +930,7 @@ func runSelftestRestoreTest(ctx context.Context, cfg config.Config, logger *slog
|
||||
fmt.Printf(" restoring %s into scratch band [%d,%d] on %s …\n", archive, min, max, cfg.Backup.RestoreStorage)
|
||||
res := engine.RunRestoreTest(ctx, reconcile.RestoreTestSpec{
|
||||
Archive: archive, RestoreStorage: cfg.Backup.RestoreStorage,
|
||||
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, cfg.Backup.LocalBackupTarget),
|
||||
ScratchMin: min, ScratchMax: max, SourceTier: storageTier(ctx, px, target),
|
||||
})
|
||||
printJSON("restore-test record", backup.ToHubRestoreTest(res, time.Now().UTC()))
|
||||
if res.Skipped {
|
||||
|
||||
@@ -48,7 +48,7 @@ func run() error {
|
||||
guest = flag.String("guest", "", "target guest_id (\"\" = host-scoped op)")
|
||||
keyID = flag.String("key-id", "", "key id of the signing key (must match a pinned agent signer)")
|
||||
paramsRaw = flag.String("params", "", "op params as JSON (overrides -durable-id/-fstype)")
|
||||
durableID = flag.String("durable-id", "", "for storage_wipe: the DURABLE device id (byid:…|byuuid:…)")
|
||||
durableID = flag.String("durable-id", "", "storage_wipe: the DURABLE device id (byid:…|byuuid:…); decommission: the drive's STORAGE durable-id (e.g. uuid:<fs-uuid>)")
|
||||
fstype = flag.String("fstype", "ext4", "for storage_wipe: the filesystem to mkfs after wipe")
|
||||
keyFile = flag.String("key", "", "operator signing key (ssh private key / sk- key handle) for ssh-keygen -Y sign")
|
||||
ttl = flag.Duration("ttl", 30*time.Minute, "validity window from now (issued_at..expires_at)")
|
||||
@@ -63,16 +63,26 @@ func run() error {
|
||||
return fmt.Errorf("-op, -host, -key-id and -key are required")
|
||||
}
|
||||
|
||||
// Params: explicit JSON, or built from the wipe convenience flags.
|
||||
// Params: explicit JSON, or built from the convenience flags.
|
||||
params := strings.TrimSpace(*paramsRaw)
|
||||
if params == "" {
|
||||
if *op == "storage_wipe" {
|
||||
switch *op {
|
||||
case "storage_wipe":
|
||||
if *durableID == "" {
|
||||
return fmt.Errorf("storage_wipe needs -durable-id (byid:…|byuuid:…) — a path-only binding is refused by the agent")
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID, "fstype": *fstype})
|
||||
params = string(pj)
|
||||
} else {
|
||||
case "decommission":
|
||||
// Decommission binds to the drive's STORAGE durable-id (the watchdog's key, e.g.
|
||||
// "uuid:<fs-uuid>"), NOT the device-level byid:/byuuid: scheme. The agent records this
|
||||
// id into the intent map, so it must match what the storage observer reports.
|
||||
if *durableID == "" {
|
||||
return fmt.Errorf("decommission needs -durable-id (the drive's storage durable-id, e.g. uuid:<fs-uuid>)")
|
||||
}
|
||||
pj, _ := json.Marshal(map[string]string{"durable_id": *durableID})
|
||||
params = string(pj)
|
||||
default:
|
||||
params = "{}"
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user