agent: a whole-box backup that cannot fit its local target is skipped with a reason before anything starts (R-685)
gates / gates (push) Successful in 13s

Free space is read from GET /nodes/<node>/storage — GET /storage carries no usage (found live, before release).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 22:48:51 +02:00
parent 4fe2f81a32
commit 0722b2cdb0
3 changed files with 165 additions and 5 deletions
+64
View File
@@ -22,6 +22,10 @@ type BackupAPI interface {
StorageContent(ctx context.Context, store string) ([]proxmox.StorageContent, error)
// ListStorage enumerates storages (name+type) — used to scope local-only retention (never prune PBS).
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
// NodeStorage is GET /nodes/{node}/storage — the storages WITH live usage (avail/used). R-685's space
// preflight reads free space HERE: ListStorage (GET /storage) is the cluster DEFINITIONS and carries no
// usage at all — measured live 2026-09-24 on demo-hp, where reading it let a backup through.
NodeStorage(ctx context.Context) ([]proxmox.Storage, error)
// TaskLogTail reads trailing task-log lines — used to read the ACTUAL vzdump mode
// (PVE may downgrade a requested snapshot to stop for a stopped guest — spike B1).
TaskLogTail(ctx context.Context, upid string, limit int) ([]string, error)
@@ -170,6 +174,16 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
rec.UncoveredVolumes = []string{}
}
// R-685 (v0.134.0): will the new archive FIT on a local target? Asked before anything runs, so a
// target that cannot hold it is a named SKIP with the numbers, not a nightly "No space left on
// device" that only the vzdump log explains (demo-hp, every night from 2026-09-23 — R-684).
if ok, why := r.spaceFits(ctx, vmid); !ok {
rec.Error = BackupSkipNoSpacePrefix + why
rec.DurationSeconds = time.Since(start).Seconds()
r.logger.Warn("backup SKIPPED by the space preflight (R-685) — nothing was started", "vmid", vmid, "target", r.target, "reason", why)
return rec, fmt.Errorf("backup: %s", rec.Error)
}
upid, err := r.api.Vzdump(ctx, proxmox.VzdumpOptions{
VMID: vmid, Storage: r.target, Mode: r.mode, Notes: r.notes,
PruneBackups: r.localPruneSpec(ctx), // local target → keep-last=N; PBS/unknown → "" (no prune)
@@ -217,6 +231,56 @@ func (r *BackupRunner) backup(ctx context.Context, vmid int, onSnapshot func())
return rec, nil
}
// BackupSkipNoSpacePrefix starts a backup record's Error when the space preflight refused (R-685) — a
// stable prefix the controller's page and the hub can key on.
const BackupSkipNoSpacePrefix = "skipped: not enough space: "
// Space preflight margins (R-685): the new archive is predicted as the newest archive of this guest on the
// target × backupSpaceGrowth, plus backupSpaceFloorBytes of headroom for the host. MEASURED 2026-09-24:
// demo-hp 9201's archives grew 5.8 → 6.2 → 6.9 → 7.6 GB in four nights (+10 % a night at worst), so 1.25
// covers two nights' growth. PVE prunes old archives only AFTER a successful backup, so the free space
// must hold the new archive while every kept one still exists.
const (
backupSpaceGrowth = 1.25
backupSpaceFloorBytes = int64(1) << 30
)
// spaceFits answers whether a new archive of vmid fits on a LOCAL (non-PBS) target. It FAILS OPEN — a
// backup is the thing being protected, so an unreadable storage, an unknown type or a first backup (no
// previous archive to size from) proceeds and says so; only a POSITIVE "it does not fit" refuses.
func (r *BackupRunner) spaceFits(ctx context.Context, vmid int) (bool, string) {
// NodeStorage, never ListStorage: only the node view carries avail (see BackupAPI.NodeStorage).
stores, err := r.api.NodeStorage(ctx)
if err != nil {
r.logger.Warn("backup: space preflight could not read storage usage — proceeding (fail-open)", "target", r.target, "err", err)
return true, ""
}
var st *proxmox.Storage
for i := range stores {
if stores[i].Storage == r.target {
st = &stores[i]
break
}
}
if st == nil || st.Type == "pbs" || st.Avail <= 0 {
return true, "" // PBS dedups and has its own lifecycle; an unknown avail never refuses
}
_, last, err := r.latestArchive(ctx, vmid)
if err != nil || last <= 0 {
r.logger.Info("backup: space preflight has no previous archive to size from — proceeding", "vmid", vmid, "target", r.target)
return true, ""
}
need := int64(float64(last)*backupSpaceGrowth) + backupSpaceFloorBytes
if st.Avail >= need {
r.logger.Info("backup: space preflight passed", "vmid", vmid, "target", r.target, "last_archive_bytes", last, "need_bytes", need, "avail_bytes", st.Avail)
return true, ""
}
return false, fmt.Sprintf("%s has %s free; the last archive of guest %d was %s, so a new one needs about %s (old archives are removed only after a successful backup)",
r.target, humanGiB(st.Avail), vmid, humanGiB(last), humanGiB(need))
}
func humanGiB(b int64) string { return fmt.Sprintf("%.1f GiB", float64(b)/(1<<30)) }
// watchForSnapshot polls the running backup's task log until it sees the storage-snapshot marker
// (→ onSnapshot once) or the requested mode is reported as `stop` (→ downgraded; the marker will
// never come, so stop watching) or ctx is cancelled (backup finished). Best-effort: a log-read