v0.103.0 — R-84: an agent restart no longer triggers a redundant backup

Observed live: three redundant local backups on demo-felhom in one afternoon of
deploys. The backup Store is in-memory ('lost on restart; the cadence
re-populates'), so after every restart /backup/due said 'no successful backup
recorded yet' and the controller took another one. On the offsite tier that is a
wasted multi-hour WAN upload after every agent deploy.

- BackupRunner.NewestArchiveTime: when a backup last LANDED on this tier's
  storage, read from the storage.
- localapi.BackupArchiveLister (optional BackupService extension): the due-check
  takes whichever is newer, the in-memory record or the storage.

Asking the storage rather than persisting the store is deliberate: it is ground
truth (a pruned archive correctly stops counting, where a persisted record would
keep claiming a backup that no longer exists), needs no new on-disk state, and
answers only 'when did a backup last land' — the richer fields stay with real
records so the host-report never carries invented numbers.

Fail-safes: read error -> fall back to memory (never fake freshness, never
suppress); genuinely empty -> due; old archive -> still due; service without the
lister -> unchanged.

Red-proof observed; full suite green (29 packages).
This commit is contained in:
Claude Code
2026-07-26 18:20:59 +02:00
parent e4f22f4c4f
commit 5acf1033a2
4 changed files with 259 additions and 8 deletions
+62 -8
View File
@@ -41,6 +41,18 @@ type BackupService interface {
BackupWithSnapshotHook(ctx context.Context, vmid int, onSnapshot func()) (hub.Backup, error)
}
// BackupArchiveLister is an OPTIONAL extension to BackupService: "when did a backup last LAND on
// this tier's storage?", answered by the storage rather than by memory. *backup.BackupRunner
// satisfies it.
//
// R-84: the agent's backup Store is in-memory, so after every restart the due-check saw nothing and
// the controller took a redundant backup — a wasted multi-hour WAN upload on the offsite tier after
// every agent deploy. Consulting the storage makes the cold path truthful without persisting
// anything, and it self-corrects: a pruned archive correctly stops counting.
type BackupArchiveLister interface {
NewestArchiveTime(ctx context.Context, vmid int) (time.Time, bool, error)
}
// BackupTier (R-82) binds ONE backup tier's runner to its own policy. The agent builds one per
// resolved config tier; the local API serves each independently so "local daily + PBS weekly" is
// expressible over the wire, not just in config.
@@ -882,17 +894,32 @@ func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid in
Reason: "target storage not present yet — tier deferred until it is provisioned", Target: echo})
return
}
latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID)
if latest == nil {
// Newest backup for THIS tier: the in-memory record if this process took one, otherwise the
// storage itself (R-84 — see BackupArchiveLister). Whichever is newer wins.
var newest time.Time
var haveNewest bool
var unparseable bool
if latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID); latest != nil {
if t, ok2 := backupAge2(latest.StartedAt); ok2 {
newest, haveNewest = t, true
} else {
unparseable = true
}
}
if t, ok3 := s.newestArchiveOn(r.Context(), tier, vmid); ok3 && (!haveNewest || t.After(newest)) {
newest, haveNewest = t, true
unparseable = false // ground truth supersedes an unreadable in-memory timestamp
}
if !haveNewest {
if unparseable {
// Fail safe toward "due" so a backup still happens.
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due", Target: echo})
return
}
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet", Target: echo})
return
}
age, ok2 := backupAge(latest.StartedAt, s.now())
if !ok2 {
// Unparseable timestamp: fail safe toward "due" so a backup still happens.
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due", Target: echo})
return
}
age := s.now().Sub(newest)
ageSecs := int64(age.Seconds())
if age >= tier.Cadence {
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "older than cadence", AgeSecs: &ageSecs, Target: echo})
@@ -1044,6 +1071,33 @@ func (s *Server) pickLatestBackup(ctx context.Context, vmid int, successOnly boo
return latest
}
// newestArchiveOn asks THIS TIER's storage when a backup last landed (R-84). Errors and
// unsupported services degrade to "unknown", never to "no backup" — an unreadable storage must not
// make the tier look freshly backed up, and it must not suppress a backup either: the caller falls
// back to the in-memory record, whose absence means DUE.
func (s *Server) newestArchiveOn(ctx context.Context, tier BackupTier, vmid int) (time.Time, bool) {
lister, ok := tier.Service.(BackupArchiveLister)
if !ok {
return time.Time{}, false
}
t, found, err := lister.NewestArchiveTime(ctx, vmid)
if err != nil {
s.logger.Warn("local-api: could not read the backup storage for the due-check — falling back to the in-memory record",
"vmid", vmid, "target", tier.TargetID, "err", err)
return time.Time{}, false
}
return t, found
}
// backupAge2 parses an RFC3339 backup start time, returning it as a time.
func backupAge2(startedAt string) (time.Time, bool) {
t, err := time.Parse(time.RFC3339, startedAt)
if err != nil {
return time.Time{}, false
}
return t.UTC(), true
}
// backupAge parses an RFC3339 backup start time and returns its age relative to now.
func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
t, err := time.Parse(time.RFC3339, startedAt)