A cadence cap on the local-API whole-guest backup route (R-973, unreleased)
gates / gates (push) Successful in 49s
gates / gates (push) Successful in 49s
Each successful run prunes its tier, so a compromised controller calling POST /backup in a loop pushed out every pre-compromise restore point. A tier with a cadence of at most 24 h now allows 3 successful runs per rolling 24 h (night, a press, the next night); a longer tier 1 per max(cadence - 24 h, cadence/2), 6 days for the weekly off-site tier. Only successes count. A refusal is a 429, a WARN, saved, and for 24 h a critical host-report status that the hub turns into its existing operator-only agent_capability_degraded event. The success history and the refusal are kept in the R-894 state file, so a restart does not reset the cap. No sudoers change. Tests TestBackupCap_* and the state, status and wiring tests, red-proved.
This commit is contained in:
1 parent
0963bae3c3
commit
f17e1979bd
10 files changed
+722
-8
No files matched your search
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/capability"
|
||||
)
|
||||
|
||||
// R-973: a cadence-cap refusal becomes a CRITICAL degraded status — what the hub's existing
|
||||
// agent_capability_degraded event (operator-only) keys on — and clears when the refusal ages out.
|
||||
func TestBackupCapStatus_RefusalIsCriticalDegraded(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
s := backupCapStatus(func() (string, int, time.Time, bool) { return "felhom-pbs", 9201, at, true })
|
||||
if !s.Critical || s.Status != capability.StatusDegraded || !strings.Contains(s.Reason, "9201") || !strings.Contains(s.Reason, "felhom-pbs") {
|
||||
t.Fatalf("a refusal must raise a critical degraded status naming the guest and the tier: %+v", s)
|
||||
}
|
||||
if s := backupCapStatus(func() (string, int, time.Time, bool) { return "", 0, time.Time{}, false }); s.Status != capability.StatusOK || !s.Critical {
|
||||
t.Fatalf("no refusal → ok (and still critical, so a later refusal alerts): %+v", s)
|
||||
}
|
||||
if s := backupCapStatus(nil); s.Status != capability.StatusOK {
|
||||
t.Fatalf("no local API → ok: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// Wiring (the built-but-never-wired class): runDaemon's capability probe appends backupCapStatus, and the
|
||||
// late-bound source is set from the local API server.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-18): delete `backupCapRefusal = localSrv.BackupCapRefusal` →
|
||||
// "runDaemon never binds backupCapRefusal to the local API server". Restored.
|
||||
func TestR973_CapRefusalIsWiredIntoTheReport(t *testing.T) {
|
||||
_, f := parseMain(t)
|
||||
var appended, bound bool
|
||||
for _, d := range f.Decls {
|
||||
fd, ok := d.(*ast.FuncDecl)
|
||||
if !ok || fd.Name == nil || fd.Name.Name != "runDaemon" || fd.Body == nil {
|
||||
continue
|
||||
}
|
||||
ast.Inspect(fd.Body, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
if id, ok := x.Fun.(*ast.Ident); ok && id.Name == "backupCapStatus" && len(x.Args) == 1 {
|
||||
if a, ok := x.Args[0].(*ast.Ident); ok && a.Name == "backupCapRefusal" {
|
||||
appended = true
|
||||
}
|
||||
}
|
||||
case *ast.AssignStmt:
|
||||
if len(x.Lhs) == 1 && len(x.Rhs) == 1 {
|
||||
if l, ok := x.Lhs[0].(*ast.Ident); ok && l.Name == "backupCapRefusal" {
|
||||
if sel, ok := x.Rhs[0].(*ast.SelectorExpr); ok && sel.Sel.Name == "BackupCapRefusal" {
|
||||
bound = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if !appended {
|
||||
t.Fatal("runDaemon's capability probe does not call backupCapStatus(backupCapRefusal)")
|
||||
}
|
||||
if !bound {
|
||||
t.Fatal("runDaemon never binds backupCapRefusal to the local API server")
|
||||
}
|
||||
}
|
||||
@@ -404,6 +404,31 @@ func (r *gateRemounter) Remount(ctx context.Context, t storage.KnownTarget) {
|
||||
// poolReadStatus probes the felhom-pool membership read (GET /pools/felhom) the stale-lock reaper's
|
||||
// ownership scoping depends on (v0.62.0, audit A1). Needs `Pool.Audit` at `/pool/felhom` — granted
|
||||
// by host-install v1.9.0; on an older ACL this reports degraded and the reaper fail-safes (skips).
|
||||
// backupCapStatusName is the host-report status the R-973 cadence cap raises.
|
||||
const backupCapStatusName = "localapi:backup-cadence-cap"
|
||||
|
||||
// backupCapStatus turns the local-API backup cadence cap's newest refusal (R-973) into a CRITICAL
|
||||
// capability status: degraded for 24 h after a refusal, so the hub raises its operator-only
|
||||
// agent_capability_degraded event (no hub change). The Reason names the guest, the tier and the time, and
|
||||
// says what to do. nil source (local API off) → ok.
|
||||
func backupCapStatus(refusal func() (string, int, time.Time, bool)) capability.Status {
|
||||
s := capability.Status{
|
||||
Name: backupCapStatusName,
|
||||
Feature: "whole-guest restore points: a guest asked for more whole-guest backups than its schedule allows, and the extra ones were refused (R-973)",
|
||||
Critical: true,
|
||||
Status: capability.StatusOK,
|
||||
}
|
||||
if refusal == nil {
|
||||
return s
|
||||
}
|
||||
if target, vmid, at, ok := refusal(); ok {
|
||||
s.Status = capability.StatusDegraded
|
||||
s.Reason = fmt.Sprintf("refused a whole-guest backup of guest %d on target %q at %s — more than the schedule allows; the guest's controller may be misbehaving or compromised: check the guest",
|
||||
vmid, target, at.UTC().Format(time.RFC3339))
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func poolReadStatus(ctx context.Context, px *proxmox.Client) capability.Status {
|
||||
s := capability.Status{
|
||||
Name: "pve:pool-read",
|
||||
@@ -830,8 +855,13 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
run: (&proxmox.ExecRunner{Mode: proxmox.RunnerMode(cfg.Privileged.Mode)}).Run,
|
||||
log: logger,
|
||||
}
|
||||
// R-973: the local-API backup cadence cap's refusals ride the report as a CRITICAL status, so the hub's
|
||||
// existing agent_capability_degraded event (operator-only) tells the operator. Late-bound like
|
||||
// drConfigured: the local API server is built further down, before the loop starts.
|
||||
var backupCapRefusal func() (string, int, time.Time, bool)
|
||||
probeAll := func(ctx context.Context) []capability.Status {
|
||||
out := append(capProber.Probe(ctx), poolReadStatus(ctx, px))
|
||||
out = append(out, backupCapStatus(backupCapRefusal))
|
||||
return append(out, storeGrantStatuses(ctx, px, cfg, grantRepairer)...)
|
||||
}
|
||||
// (The startup self-check log runs AFTER the pbsdr manager is wired below, so its snapshot
|
||||
@@ -1209,6 +1239,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
}
|
||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, heavyOps, observer, driveKnown, hostOps, gate, collector, client, intentRec, guestBindStore, formatJobStore, logRing, escrowCeremonyCfg, logger, &localTokens)
|
||||
if localSrv != nil {
|
||||
backupCapRefusal = localSrv.BackupCapRefusal // R-973
|
||||
internalPlaceExec.SetEnsure(localSrv.EnsureInternalPlace)
|
||||
// R-998 (doc 14): POST /host/poweroff — guest shutdown, then host power-off; guards in hostpoweroff.go.
|
||||
pwMode := proxmox.RunnerMode(cfg.Privileged.Mode)
|
||||
|
||||
Reference in new issue
Block a user