After a power cut: remove the dead half-written backup (R-1006); a backup task from before the boot is not in flight (R-1008)
gates / gates (push) Successful in 35s

This commit is contained in:
2026-10-14 11:29:41 +02:00
parent e7d4ecd607
commit a2f3aeeeff
18 changed files with 1129 additions and 40 deletions
+27
View File
@@ -59,6 +59,7 @@ import (
"gitea.dooplex.hu/admin/felhom-agent/internal/selfupdate"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
"gitea.dooplex.hu/admin/felhom-agent/internal/vzpartial"
"gitea.dooplex.hu/admin/felhom-agent/internal/wgtunnel"
)
@@ -1532,6 +1533,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
// the dangling snapshot → start iff onboot). Runs BEFORE the backup loop starts, so a present
// backup lock is stale by definition (guarded by a no-vzdump-running check; fail-safe otherwise).
localSrv.RecoverStaleLockedGuests(ctx)
// R-1006: then remove a dead half-written backup a power cut left (the lock recovery first: same order as a human).
localSrv.SweepVzdumpPartials(ctx)
// F-REBOOT: the startup recovery above only covers a guest left LOCKED by an interrupted
// backup. A guest that simply ends up stopped-and-unlocked (a `pct reboot` whose shutdown
// half completed and whose start half never fired — Campaign 8 fault 11, 9m47s of total
@@ -1892,6 +1895,8 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
}
apiTiers := make([]localapi.BackupTier, 0, len(backupTiers))
var runner *backup.BackupRunner
var tierRunners []*backup.BackupRunner
var tierTargets []string
for _, t := range backupTiers {
prune := ""
if t.KeepLast > 0 {
@@ -1904,6 +1909,8 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
if t.Primary {
runner = r
}
tierRunners = append(tierRunners, r)
tierTargets = append(tierTargets, t.TargetID)
apiTiers = append(apiTiers, localapi.BackupTier{
TargetID: t.TargetID,
Cadence: t.Cadence,
@@ -2032,6 +2039,26 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
logger.Warn("daemon: local-api disabled (server build)", "err", err)
return nil
}
// R-1006: a dead half-written vzdump (a power cut mid-backup) is removed at start-up and before every backup of the
// guest on that target, through the root helper; the in-flight answer is R-1008's (a task from before the boot is dead).
cleaner := &vzpartial.Cleaner{
API: px,
Runner: &proxmox.ExecRunner{Mode: gaMode, SudoPath: cfg.Privileged.SudoPath},
InFlight: srv.VzdumpInFlight,
Boot: func() (int64, bool) {
b, err := strconv.ParseInt(poweroff.ProcBtime(), 10, 64)
return b, err == nil && b > 0
},
Logger: logger,
}
for _, r := range tierRunners {
r.SetBeforeBackup(func(ctx context.Context, storage string, vmid int) { cleaner.Clean(ctx, storage, vmid) })
}
srv.SetPartialSweep(func(ctx context.Context, vmid int) {
for _, t := range tierTargets {
cleaner.Clean(ctx, t, vmid)
}
})
return srv
}