v0.103.0 — R-84: an agent restart no longer triggers a redundant backup
Observed live: three redundant local backups on demo-felhom in one afternoon of
deploys. The backup Store is in-memory ('lost on restart; the cadence
re-populates'), so after every restart /backup/due said 'no successful backup
recorded yet' and the controller took another one. On the offsite tier that is a
wasted multi-hour WAN upload after every agent deploy.
- BackupRunner.NewestArchiveTime: when a backup last LANDED on this tier's
storage, read from the storage.
- localapi.BackupArchiveLister (optional BackupService extension): the due-check
takes whichever is newer, the in-memory record or the storage.
Asking the storage rather than persisting the store is deliberate: it is ground
truth (a pruned archive correctly stops counting, where a persisted record would
keep claiming a backup that no longer exists), needs no new on-disk state, and
answers only 'when did a backup last land' — the richer fields stay with real
records so the host-report never carries invented numbers.
Fail-safes: read error -> fall back to memory (never fake freshness, never
suppress); genuinely empty -> due; old archive -> still due; service without the
lister -> unchanged.
Red-proof observed; full suite green (29 packages).
This commit is contained in:
@@ -41,6 +41,18 @@ type BackupService interface {
|
||||
BackupWithSnapshotHook(ctx context.Context, vmid int, onSnapshot func()) (hub.Backup, error)
|
||||
}
|
||||
|
||||
// BackupArchiveLister is an OPTIONAL extension to BackupService: "when did a backup last LAND on
|
||||
// this tier's storage?", answered by the storage rather than by memory. *backup.BackupRunner
|
||||
// satisfies it.
|
||||
//
|
||||
// R-84: the agent's backup Store is in-memory, so after every restart the due-check saw nothing and
|
||||
// the controller took a redundant backup — a wasted multi-hour WAN upload on the offsite tier after
|
||||
// every agent deploy. Consulting the storage makes the cold path truthful without persisting
|
||||
// anything, and it self-corrects: a pruned archive correctly stops counting.
|
||||
type BackupArchiveLister interface {
|
||||
NewestArchiveTime(ctx context.Context, vmid int) (time.Time, bool, error)
|
||||
}
|
||||
|
||||
// BackupTier (R-82) binds ONE backup tier's runner to its own policy. The agent builds one per
|
||||
// resolved config tier; the local API serves each independently so "local daily + PBS weekly" is
|
||||
// expressible over the wire, not just in config.
|
||||
@@ -882,17 +894,32 @@ func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid in
|
||||
Reason: "target storage not present yet — tier deferred until it is provisioned", Target: echo})
|
||||
return
|
||||
}
|
||||
latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID)
|
||||
if latest == nil {
|
||||
// Newest backup for THIS tier: the in-memory record if this process took one, otherwise the
|
||||
// storage itself (R-84 — see BackupArchiveLister). Whichever is newer wins.
|
||||
var newest time.Time
|
||||
var haveNewest bool
|
||||
var unparseable bool
|
||||
if latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID); latest != nil {
|
||||
if t, ok2 := backupAge2(latest.StartedAt); ok2 {
|
||||
newest, haveNewest = t, true
|
||||
} else {
|
||||
unparseable = true
|
||||
}
|
||||
}
|
||||
if t, ok3 := s.newestArchiveOn(r.Context(), tier, vmid); ok3 && (!haveNewest || t.After(newest)) {
|
||||
newest, haveNewest = t, true
|
||||
unparseable = false // ground truth supersedes an unreadable in-memory timestamp
|
||||
}
|
||||
if !haveNewest {
|
||||
if unparseable {
|
||||
// Fail safe toward "due" so a backup still happens.
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due", Target: echo})
|
||||
return
|
||||
}
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet", Target: echo})
|
||||
return
|
||||
}
|
||||
age, ok2 := backupAge(latest.StartedAt, s.now())
|
||||
if !ok2 {
|
||||
// Unparseable timestamp: fail safe toward "due" so a backup still happens.
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due", Target: echo})
|
||||
return
|
||||
}
|
||||
age := s.now().Sub(newest)
|
||||
ageSecs := int64(age.Seconds())
|
||||
if age >= tier.Cadence {
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "older than cadence", AgeSecs: &ageSecs, Target: echo})
|
||||
@@ -1044,6 +1071,33 @@ func (s *Server) pickLatestBackup(ctx context.Context, vmid int, successOnly boo
|
||||
return latest
|
||||
}
|
||||
|
||||
// newestArchiveOn asks THIS TIER's storage when a backup last landed (R-84). Errors and
|
||||
// unsupported services degrade to "unknown", never to "no backup" — an unreadable storage must not
|
||||
// make the tier look freshly backed up, and it must not suppress a backup either: the caller falls
|
||||
// back to the in-memory record, whose absence means DUE.
|
||||
func (s *Server) newestArchiveOn(ctx context.Context, tier BackupTier, vmid int) (time.Time, bool) {
|
||||
lister, ok := tier.Service.(BackupArchiveLister)
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, found, err := lister.NewestArchiveTime(ctx, vmid)
|
||||
if err != nil {
|
||||
s.logger.Warn("local-api: could not read the backup storage for the due-check — falling back to the in-memory record",
|
||||
"vmid", vmid, "target", tier.TargetID, "err", err)
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, found
|
||||
}
|
||||
|
||||
// backupAge2 parses an RFC3339 backup start time, returning it as a time.
|
||||
func backupAge2(startedAt string) (time.Time, bool) {
|
||||
t, err := time.Parse(time.RFC3339, startedAt)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t.UTC(), true
|
||||
}
|
||||
|
||||
// backupAge parses an RFC3339 backup start time and returns its age relative to now.
|
||||
func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
|
||||
t, err := time.Parse(time.RFC3339, startedAt)
|
||||
|
||||
Reference in New Issue
Block a user