v0.28.0: backup re-target → felhom-pbs (offsite DR) + operator-signed decommission
- BackupConfig.BackupTarget() defaults whole-guest backup to felhom-pbs (separate hardware = real DR), configurable via backup.local_backup_target; all NewBackupRunner sites route through it. PBS round-trip proven live (snapshot marker + restore-test + pct-restore) before the re-point. - signedjobs DecommissionExecutor + ExecutorChain: makes IntentDecommissioned reachable ONLY via a verified operator signature (keyed by the watchdog's storage durable-id); felhom-opsign builds decommission params from -durable-id. Runner wiring moved below the intent-store open. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -155,7 +155,9 @@ type EscrowConfig struct {
|
||||
// an agent-internal cadence (no hub policy needed — it's self-validation); the backup
|
||||
// schedule/retention/target-selection policy is hub-manifest-owned and unfed until slice 10.
|
||||
type BackupConfig struct {
|
||||
// LocalBackupTarget is the vzdump storage (content=backup) backups go to, e.g. "local".
|
||||
// LocalBackupTarget is the vzdump storage (content=backup) backups go to. Empty → the
|
||||
// offsite PBS default (see BackupTarget); set e.g. "local" or "felhom-pbs" to override.
|
||||
// (Name kept for config back-compat; the default is no longer "local".)
|
||||
LocalBackupTarget string `json:"local_backup_target"`
|
||||
// RestoreStorage is where a restore-test's restored rootfs lands, e.g. "local-lvm".
|
||||
RestoreStorage string `json:"restore_storage"`
|
||||
@@ -189,6 +191,23 @@ func (b BackupConfig) BackupCadence() time.Duration {
|
||||
return 24 * time.Hour
|
||||
}
|
||||
|
||||
// defaultBackupTarget is the offsite PBS storage whole-guest backups land on by default. It is
|
||||
// SEPARATE HARDWARE from the guest's own disk (a PBS datastore on the DooPlex box), so a host
|
||||
// disk/hardware failure doesn't take the backups with it — that's what makes it real DR. Proven
|
||||
// live: snapshot-mode vzdump to PBS still fires the `create storage snapshot` marker (early-resume
|
||||
// intact) and pct-restore-from-PBS round-trips cleanly via the storage.cfg encryption key.
|
||||
const defaultBackupTarget = "felhom-pbs"
|
||||
|
||||
// BackupTarget is the vzdump storage (content=backup) whole-guest backups go to. Defaults to the
|
||||
// offsite PBS storage (see defaultBackupTarget); override via backup.local_backup_target for a
|
||||
// local or other target. Kept configurable on purpose — the field is never hardcoded at a call site.
|
||||
func (b BackupConfig) BackupTarget() string {
|
||||
if b.LocalBackupTarget != "" {
|
||||
return b.LocalBackupTarget
|
||||
}
|
||||
return defaultBackupTarget
|
||||
}
|
||||
|
||||
// Default scratch VMID band + restore-test cadence.
|
||||
const (
|
||||
defaultScratchVMIDMin = 990000
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package signedjobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// ExecutorChain dispatches a verified signed op to the first sub-executor that OWNS it. A sub-
|
||||
// executor returns ErrNoExecutor for an op class it does not handle; the chain then tries the next.
|
||||
// If no sub-executor owns the op, the chain returns ErrNoExecutor so the runner leaves the job
|
||||
// queued for a later slice (the same contract a single executor has). The chain adds no policy — the
|
||||
// gate has already verified+bound the op before any Execute runs.
|
||||
type ExecutorChain []Executor
|
||||
|
||||
// Execute implements Executor by trying each sub-executor in order.
|
||||
func (c ExecutorChain) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||
for _, e := range c {
|
||||
err := e.Execute(ctx, op, params)
|
||||
if errorsIs(err, ErrNoExecutor) {
|
||||
continue // this sub-executor doesn't own the op — try the next
|
||||
}
|
||||
return err // owned (handled or failed) — done
|
||||
}
|
||||
return ErrNoExecutor // no sub-executor owns this op class in this build
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package signedjobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// opDecommission is the op class this executor serves (mirrors reconcile.ClassDecommission; the
|
||||
// literal avoids importing reconcile here just for the string).
|
||||
const opDecommission = "decommission"
|
||||
|
||||
// IntentDecommissioner records a PERMANENT drive decommission keyed by the drive's storage
|
||||
// durable-id. Satisfied by *storage.IntentStore (SetDecommissioned). Decommission is the operator's
|
||||
// "permanent removal" intent: once recorded, the self-heal watchdog never auto-mounts the drive
|
||||
// again (it SURVIVES absent/present), cleared only by an explicit re-commission.
|
||||
type IntentDecommissioner interface {
|
||||
SetDecommissioned(durableID string) error
|
||||
}
|
||||
|
||||
// decommissionParams is the verified params of a decommission op. The drive is named by its STORAGE
|
||||
// durable-id — the SAME key the storage observer / watchdog use (e.g. "uuid:<fs-uuid>"), NOT the
|
||||
// device-level byid:/byuuid: scheme the storage_wipe op uses. That is deliberate: decommission
|
||||
// manipulates the intent MAP the watchdog reads, so the signed id must be the watchdog's key or the
|
||||
// recorded intent would never gate a remount.
|
||||
type decommissionParams struct {
|
||||
DurableID string `json:"durable_id"`
|
||||
}
|
||||
|
||||
// DecommissionExecutor is the operator "permanent removal" consumer (slice 10 P3 completion). It
|
||||
// makes the previously-unreachable IntentDecommissioned state reachable — but ONLY via a gate-
|
||||
// VERIFIED operator signature (decommission is classified destructive in reconcile.Classify, so the
|
||||
// signed-jobs gate refuses it pending_signature without a valid operator signature). It is distinct
|
||||
// from a customer-confirmable safe eject: permanent removal is never customer-authorizable.
|
||||
//
|
||||
// The binding IS the durable-id: the signed op authorizes decommissioning exactly that drive on
|
||||
// exactly the signed host (host_scope is checked by the gate). No /dev resolution is needed — and
|
||||
// none is wanted, because a drive can legitimately be decommissioned while physically ABSENT.
|
||||
type DecommissionExecutor struct {
|
||||
intent IntentDecommissioner
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewDecommissionExecutor wires the executor to the agent's intent store.
|
||||
func NewDecommissionExecutor(intent IntentDecommissioner, logger *slog.Logger) *DecommissionExecutor {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &DecommissionExecutor{intent: intent, logger: logger}
|
||||
}
|
||||
|
||||
// Execute implements signedjobs.Executor for the decommission op class.
|
||||
func (d *DecommissionExecutor) Execute(_ context.Context, op string, params json.RawMessage) error {
|
||||
if op != opDecommission {
|
||||
return ErrNoExecutor // not ours — the runner leaves it queued for the owning executor
|
||||
}
|
||||
if d.intent == nil {
|
||||
// No intent store on this host (open failure) → the op can't be honored. Don't clear it as
|
||||
// "done"; surface a hard error so the runner logs it and the operator can retry once fixed.
|
||||
return fmt.Errorf("decommission: no intent store wired on this host — cannot record decommission")
|
||||
}
|
||||
var p decommissionParams
|
||||
if err := json.Unmarshal(params, &p); err != nil {
|
||||
return fmt.Errorf("decommission: bad params: %w", err)
|
||||
}
|
||||
if p.DurableID == "" {
|
||||
// The durable-id IS the resource binding; a decommission with no id is unbound — refuse.
|
||||
return fmt.Errorf("decommission: op has no durable_id — refusing an unbound decommission")
|
||||
}
|
||||
|
||||
// The gate has already verified the operator signature + burned the nonce durably BEFORE this
|
||||
// point. This log line is the audit trail of an operator-authorized permanent decommission.
|
||||
d.logger.Warn("decommission: executing operator-signed PERMANENT decommission", "durable_id", p.DurableID)
|
||||
if err := d.intent.SetDecommissioned(p.DurableID); err != nil {
|
||||
return fmt.Errorf("decommission: record intent for %q: %w", p.DurableID, err)
|
||||
}
|
||||
d.logger.Warn("decommission: operator-signed decommission complete (drive will never auto-mount)", "durable_id", p.DurableID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package signedjobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeDecommissioner struct {
|
||||
got string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeDecommissioner) SetDecommissioned(durableID string) error {
|
||||
f.got = durableID
|
||||
return f.err
|
||||
}
|
||||
|
||||
func TestDecommissionExecutor_HappyPath(t *testing.T) {
|
||||
fd := &fakeDecommissioner{}
|
||||
ex := NewDecommissionExecutor(fd, nil)
|
||||
err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":"uuid:abc-123"}`))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if fd.got != "uuid:abc-123" {
|
||||
t.Errorf("SetDecommissioned got %q, want uuid:abc-123", fd.got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecommissionExecutor_NotOurOp(t *testing.T) {
|
||||
fd := &fakeDecommissioner{}
|
||||
ex := NewDecommissionExecutor(fd, nil)
|
||||
if err := ex.Execute(context.Background(), "storage_wipe", json.RawMessage(`{}`)); err != ErrNoExecutor {
|
||||
t.Errorf("want ErrNoExecutor for a foreign op, got %v", err)
|
||||
}
|
||||
if fd.got != "" {
|
||||
t.Errorf("SetDecommissioned must not be called for a foreign op (got %q)", fd.got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecommissionExecutor_RefusesUnbound(t *testing.T) {
|
||||
fd := &fakeDecommissioner{}
|
||||
ex := NewDecommissionExecutor(fd, nil)
|
||||
if err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":""}`)); err == nil {
|
||||
t.Error("want error for an empty durable_id (unbound decommission)")
|
||||
}
|
||||
if fd.got != "" {
|
||||
t.Errorf("SetDecommissioned must not be called for an unbound op (got %q)", fd.got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecommissionExecutor_NoIntentStore(t *testing.T) {
|
||||
ex := NewDecommissionExecutor(nil, nil)
|
||||
if err := ex.Execute(context.Background(), opDecommission, json.RawMessage(`{"durable_id":"uuid:x"}`)); err == nil {
|
||||
t.Error("want a hard error (not silent success) when no intent store is wired")
|
||||
}
|
||||
}
|
||||
|
||||
// chainProbe records whether it was reached and what it returns.
|
||||
type chainProbe struct {
|
||||
owns string
|
||||
called bool
|
||||
ret error
|
||||
}
|
||||
|
||||
func (c *chainProbe) Execute(_ context.Context, op string, _ json.RawMessage) error {
|
||||
c.called = true
|
||||
if op == c.owns {
|
||||
return c.ret
|
||||
}
|
||||
return ErrNoExecutor
|
||||
}
|
||||
|
||||
func TestExecutorChain_DispatchesToOwner(t *testing.T) {
|
||||
a := &chainProbe{owns: "storage_wipe"}
|
||||
b := &chainProbe{owns: "decommission"}
|
||||
chain := ExecutorChain{a, b}
|
||||
|
||||
if err := chain.Execute(context.Background(), "decommission", nil); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !b.called {
|
||||
t.Error("owner (decommission) executor was not reached")
|
||||
}
|
||||
|
||||
// An op no sub-executor owns → ErrNoExecutor (left queued).
|
||||
if err := chain.Execute(context.Background(), "guest_destroy", nil); err != ErrNoExecutor {
|
||||
t.Errorf("want ErrNoExecutor for an unowned op, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user