The local whole-guest tier keeps one restore point per day for 3 days: keep-daily instead of keep-last (R-1149, operator D17, unreleased)
gates / gates (push) Successful in 47s
gates / gates (push) Successful in 47s
This commit is contained in:
@@ -45,6 +45,17 @@ Committed, not released — the release and its vouch are the operator's.
|
||||
`deferUpload: false` → „json mode must defer the upload"; the `Upload:` line removed → „has no Upload". **Residual:**
|
||||
the customer's typed-back check after the claim is browser-only, so „confirmed" here means „claimed" — a customer
|
||||
who opens the code and closes the page without writing it down has replaced the package.
|
||||
- **R-1149 — the local whole-guest tier keeps one restore point per day for 3 days** (operator decision D17,
|
||||
2026-10-19). The primary (local) tier's per-run prune is now `--prune-backups keep-daily=N` (N =
|
||||
`local_backup_retention`, default 3) instead of `keep-last=N` — the same at most 3 archives, but a run of extra
|
||||
backups inside one day replaces only that day's point, so with the R-973 cap a flood cannot push out the earlier
|
||||
days' copies. Additional tiers keep their explicit `keep_last` (the weekly off-site „last 2" is two weeks).
|
||||
`config.BackupTier.PruneSpec()` is the one place the spec is built (runDaemon used to format keep-last itself); the
|
||||
„backup tier armed" log line names it. Tests `TestBackupTierPruneSpec_R1149_LocalKeepsOnePerDay` and the
|
||||
consequence `TestBackupCap_R1149_LocalFloodKeepsOlderDays` (the agent's real spec, a Proxmox-shaped prune model,
|
||||
three nights then a day-long flood → the two earlier nights survive). Red-proved: keep-last for the primary →
|
||||
„primary tier prune = "keep-last=3", want "keep-daily=3"" and „holds 0 copies made before the compromise". No
|
||||
config or sudoers change.
|
||||
- **R-973 — the local-API whole-guest backup route has a cadence cap.** Every successful run prunes its tier, so a
|
||||
compromised controller calling `POST /backup` in a loop pushed out every pre-compromise restore point (local
|
||||
keep-last 3; off-site via ep0's keep-last 2). Now (`internal/localapi/backup_cap.go`): a tier whose cadence is ≤ 24 h
|
||||
|
||||
@@ -1947,10 +1947,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
||||
var tierRunners []*backup.BackupRunner
|
||||
var tierTargets []string
|
||||
for _, t := range backupTiers {
|
||||
prune := ""
|
||||
if t.KeepLast > 0 {
|
||||
prune = fmt.Sprintf("keep-last=%d", t.KeepLast)
|
||||
}
|
||||
prune := t.PruneSpec() // primary: keep-daily=N (R-1149); an extra tier: keep-last=N; 0 → no prune
|
||||
// Pruning a PBS target is allowed ONLY for an additional tier with an explicit keep_last
|
||||
// (the primary's target AND retention both default, so it could prune the DR by accident).
|
||||
allowPBSPrune := !t.Primary && t.KeepLast > 0
|
||||
@@ -1968,7 +1965,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
||||
Service: r,
|
||||
})
|
||||
logger.Info("backup tier armed", "target", t.TargetID, "cadence", t.Cadence.String(),
|
||||
"keep_last", t.KeepLast, "wait_timeout", t.WaitTimeout.String(),
|
||||
"keep_last", t.KeepLast, "prune", prune, "wait_timeout", t.WaitTimeout.String(),
|
||||
"prune_pbs_allowed", allowPBSPrune, "primary", t.Primary)
|
||||
}
|
||||
// Guest data-drive passthrough (slice 10 P2): a root-CLI runner for the `pct set` bind + chown
|
||||
|
||||
@@ -93,7 +93,7 @@ func TestPrune_KeepLastClamp(t *testing.T) {
|
||||
t.Fatalf("keep-last must NEVER be < 1 (would prune the fresh backup), got %d for %d", b.KeepLast(), c.set)
|
||||
}
|
||||
}
|
||||
if spec := (config.BackupConfig{}).PruneBackupsSpec(); spec != "keep-last=3" {
|
||||
t.Fatalf("default PruneBackupsSpec = %q, want keep-last=3", spec)
|
||||
if spec := (config.BackupConfig{}).PruneBackupsSpec(); spec != "keep-daily=3" {
|
||||
t.Fatalf("default PruneBackupsSpec = %q, want keep-daily=3 (R-1149)", spec)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,3 +180,35 @@ func TestBackupTiers_WaitTimeoutIsPerTier(t *testing.T) {
|
||||
t.Fatalf("wait bounds are shared between tiers — the whole point is that they differ: %+v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1149 (operator decision D17, 2026-10-19): the local whole-guest tier keeps one restore point per day for N days
|
||||
// (`--prune-backups keep-daily=N`), not the newest N; an additional tier keeps its explicit count.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-19): make PruneSpec return keep-last for the primary → this fails with
|
||||
// `primary tier prune = "keep-last=3", want "keep-daily=3"`. Restored.
|
||||
func TestBackupTierPruneSpec_R1149_LocalKeepsOnePerDay(t *testing.T) {
|
||||
b := BackupConfig{
|
||||
LocalBackupTarget: "local",
|
||||
LocalBackupRetention: 3,
|
||||
ExtraTargets: []BackupTargetConfig{
|
||||
{TargetID: "felhom-pbs", CadenceSeconds: 604800, KeepLast: 2},
|
||||
{TargetID: "nas", CadenceSeconds: 86400},
|
||||
},
|
||||
}
|
||||
tiers, _ := b.BackupTiers()
|
||||
if len(tiers) != 3 {
|
||||
t.Fatalf("want 3 tiers, got %d", len(tiers))
|
||||
}
|
||||
for _, c := range []struct {
|
||||
i int
|
||||
want string
|
||||
}{{0, "keep-daily=3"}, {1, "keep-last=2"}, {2, ""}} {
|
||||
if got := tiers[c.i].PruneSpec(); got != c.want {
|
||||
t.Fatalf("%s tier prune = %q, want %q", map[bool]string{true: "primary", false: tiers[c.i].TargetID}[tiers[c.i].Primary], got, c.want)
|
||||
}
|
||||
}
|
||||
// The unset retention defaults to 3 days, never to „prune everything".
|
||||
if got := (BackupConfig{}).PruneBackupsSpec(); got != "keep-daily=3" {
|
||||
t.Fatalf("default local prune = %q, want keep-daily=3", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -391,7 +391,8 @@ type BackupConfig struct {
|
||||
// hub-served per-guest policy is slice 10; this is the agent-local cadence.
|
||||
BackupCadenceSeconds int `json:"backup_cadence_seconds"`
|
||||
|
||||
// LocalBackupRetention is keep-last=N for the per-run `--prune-backups` on a LOCAL vzdump target —
|
||||
// LocalBackupRetention is N for the per-run `--prune-backups keep-daily=N` on a LOCAL vzdump target (keep-last=N
|
||||
// before R-1149) —
|
||||
// so the agent's own local whole-guest backups can't pile up and refill root (the felhom-pve incident;
|
||||
// the host_disk + storage_fill checkers are the detectors, this is the preventive default). 0/unset →
|
||||
// default 3; ALWAYS clamped to ≥1 by KeepLast() so a mis-config can never prune the fresh backup.
|
||||
@@ -538,9 +539,26 @@ func (b BackupConfig) KeepLast() int {
|
||||
return b.LocalBackupRetention
|
||||
}
|
||||
|
||||
// PruneBackupsSpec returns the PVE `--prune-backups` value for the local vzdump (e.g. "keep-last=3").
|
||||
// PruneBackupsSpec returns the PVE `--prune-backups` value for the local vzdump: "keep-daily=N" (R-1149, operator
|
||||
// decision D17 2026-10-19 — one restore point per day for N days; `07` §6.1). keep-last=N had the same disk cost but
|
||||
// let N extra runs on one day push out every older point; keep-daily keeps the newest copy of each of the N newest
|
||||
// days that have one, so a flood inside one day replaces only that day's point. Pinned by
|
||||
// TestBackupTierPruneSpec_R1149_LocalKeepsOnePerDay and, as the consequence, TestBackupCap_R1149_LocalFloodKeepsOlderDays.
|
||||
func (b BackupConfig) PruneBackupsSpec() string {
|
||||
return fmt.Sprintf("keep-last=%d", b.KeepLast())
|
||||
return fmt.Sprintf("keep-daily=%d", b.KeepLast())
|
||||
}
|
||||
|
||||
// PruneSpec is this tier's per-run `--prune-backups` value, or "" for never prune. The PRIMARY (local) tier keeps one
|
||||
// point per day (keep-daily, R-1149, from KeepLast() — always ≥1); an additional tier keeps its explicit keep_last
|
||||
// count (a weekly off-site tier's „last 2" is two weeks, and PBS prunes stay opt-in per tier, R-82).
|
||||
func (t BackupTier) PruneSpec() string {
|
||||
if t.KeepLast <= 0 {
|
||||
return ""
|
||||
}
|
||||
if t.Primary {
|
||||
return fmt.Sprintf("keep-daily=%d", t.KeepLast)
|
||||
}
|
||||
return fmt.Sprintf("keep-last=%d", t.KeepLast)
|
||||
}
|
||||
|
||||
// BackupCadence returns the per-guest /backup/due window: positive as-is, else 24h default.
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
@@ -25,11 +26,46 @@ type keepLastTier struct {
|
||||
mu sync.Mutex
|
||||
clock *testClock
|
||||
keepLast int
|
||||
// spec, when set, is the `--prune-backups` value the agent passes (R-1149) and the tier prunes the way Proxmox
|
||||
// reads it: keep-last=N → the newest N; keep-daily=N → the newest copy of each of the N newest days with one.
|
||||
spec string
|
||||
t *testing.T
|
||||
archives []time.Time
|
||||
fail bool
|
||||
runs int
|
||||
}
|
||||
|
||||
// pruneBySpec applies a Proxmox `--prune-backups` spec to archives (oldest first) and returns what is kept.
|
||||
func pruneBySpec(t *testing.T, archives []time.Time, spec string) []time.Time {
|
||||
var n int
|
||||
var mode string
|
||||
if _, err := fmt.Sscanf(spec, "keep-last=%d", &n); err == nil {
|
||||
mode = "last"
|
||||
} else if _, err := fmt.Sscanf(spec, "keep-daily=%d", &n); err == nil {
|
||||
mode = "daily"
|
||||
} else {
|
||||
t.Fatalf("the model does not know the prune spec %q", spec)
|
||||
}
|
||||
var kept []time.Time
|
||||
days := map[string]bool{}
|
||||
for i := len(archives) - 1; i >= 0; i-- { // newest first, as Proxmox walks them
|
||||
a := archives[i]
|
||||
switch mode {
|
||||
case "last":
|
||||
if len(kept) < n {
|
||||
kept = append([]time.Time{a}, kept...)
|
||||
}
|
||||
case "daily":
|
||||
d := a.Format("2006-01-02")
|
||||
if !days[d] && len(days) < n {
|
||||
days[d] = true
|
||||
kept = append([]time.Time{a}, kept...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
func (k *keepLastTier) BackupWithSnapshotHook(_ context.Context, vmid int, _ func()) (hub.Backup, error) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
@@ -39,7 +75,9 @@ func (k *keepLastTier) BackupWithSnapshotHook(_ context.Context, vmid int, _ fun
|
||||
return hub.Backup{VMID: vmid, StartedAt: now.Format(time.RFC3339)}, fmt.Errorf("vzdump failed")
|
||||
}
|
||||
k.archives = append(k.archives, now)
|
||||
if len(k.archives) > k.keepLast {
|
||||
if k.spec != "" {
|
||||
k.archives = pruneBySpec(k.t, k.archives, k.spec)
|
||||
} else if len(k.archives) > k.keepLast {
|
||||
k.archives = k.archives[len(k.archives)-k.keepLast:]
|
||||
}
|
||||
return hub.Backup{VMID: vmid, Success: true, Archive: "x", StartedAt: now.Format(time.RFC3339)}, nil
|
||||
@@ -291,3 +329,49 @@ func TestBackupCadenceCap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R-1149 — the consequence of keep-daily on the local tier, with the agent's REAL prune spec (config.BackupTiers →
|
||||
// PruneSpec, what runDaemon hands the runner). Three honest nights, then a compromise at 01:00 that floods
|
||||
// POST /backup for the rest of the day. The cap lets 3 runs through inside the rolling 24 h (two at once, one when
|
||||
// last night's success ages out). With keep-last 3 those three were the only copies left; with keep-daily 3 the two
|
||||
// earlier nights survive the whole day.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed 2026-10-19): make BackupTier.PruneSpec return keep-last for the primary → this fails
|
||||
// with "after a day of flooding the local tier holds 0 copies made before the compromise (kept […]) — want the 2
|
||||
// earlier nights". Restored.
|
||||
func TestBackupCap_R1149_LocalFloodKeepsOlderDays(t *testing.T) {
|
||||
tiers, _ := config.BackupConfig{LocalBackupTarget: "local", LocalBackupRetention: 3}.BackupTiers()
|
||||
spec := tiers[0].PruneSpec()
|
||||
day := time.Date(2026, 6, 10, 0, 0, 0, 0, time.UTC)
|
||||
clock := &testClock{t: day}
|
||||
local := &keepLastTier{clock: clock, keepLast: 3, spec: spec, t: t}
|
||||
srv := capServer(t, filepath.Join(t.TempDir(), "s.json"), clock, local, &keepLastTier{clock: clock, keepLast: 2})
|
||||
for d := 3; d >= 1; d-- { // the nights of D-3, D-2, D-1 at 03:00
|
||||
clock.t = day.Add(-time.Duration(d)*24*time.Hour + 3*time.Hour)
|
||||
if code := runBackup(t, srv, "local"); code != http.StatusAccepted {
|
||||
t.Fatalf("seeding night D-%d: %d", d, code)
|
||||
}
|
||||
}
|
||||
compromise := day.Add(time.Hour)
|
||||
ran := 0
|
||||
for clock.t = compromise; clock.t.Before(day.Add(24 * time.Hour)); clock.Advance(10 * time.Minute) {
|
||||
if runBackup(t, srv, "local") == http.StatusAccepted {
|
||||
ran++
|
||||
}
|
||||
}
|
||||
if ran != 3 {
|
||||
t.Fatalf("the flood ran %d backups on day D — want 3 (the cap)", ran)
|
||||
}
|
||||
local.mu.Lock()
|
||||
kept := append([]time.Time(nil), local.archives...)
|
||||
local.mu.Unlock()
|
||||
pre := 0
|
||||
for _, a := range kept {
|
||||
if a.Before(compromise) {
|
||||
pre++
|
||||
}
|
||||
}
|
||||
if pre != 2 {
|
||||
t.Fatalf("after a day of flooding the local tier holds %d copies made before the compromise (kept %v) — want the 2 earlier nights", pre, kept)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user