v0.102.0 — R-82 Slice D: an unprovisioned tier DEFERS instead of failing
Prerequisite for the installer default (host-install 1.20.0). A fresh box now carries the offsite tier, but felhom-pbs only exists once the hub provisions the DR tier. Without this the tier would report due in that window and the controller would quiesce the apps and fire a vzdump at a missing storage every cadence. - GET /backup/due?target= defers when the target storage is absent (targetStoragePresent): due:false with a reason that says why. The tier goes live with NO restart once the storage appears. Fail-safe: a storage-view ERROR returns present and the tier stays due. 'I could not check' must never be read as 'not there' — that would silently suppress backups, the absence-is-not-failure rule relearned three times now (R-80, R-81, the R-82 wait timeout). Full suite green.
This commit is contained in:
@@ -464,3 +464,79 @@ func TestNormalizeBackupTiers(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// R-82 Slice D: a tier whose TARGET STORAGE does not exist yet is DEFERRED, not due.
|
||||
//
|
||||
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears when
|
||||
// the hub provisions the DR tier. Reporting "due" in that window would have the controller quiesce
|
||||
// the apps and fire a vzdump at a non-existent storage every cadence until provisioning happens.
|
||||
func TestBackupDue_TargetStorageMissing_Defers(t *testing.T) {
|
||||
st := &fakeStore{}
|
||||
// Storage view knows only "local" — the PBS tier's target is not provisioned yet.
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
|
||||
Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local", Type: "local"}}},
|
||||
Tokens: staticTokens{"A": 8200},
|
||||
BackupTiers: []BackupTier{
|
||||
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
||||
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
||||
},
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
srv.now = func() time.Time { return testNow }
|
||||
h := srv.Handler()
|
||||
|
||||
var pbs, local struct {
|
||||
Data BackupDueResponse `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if pbs.Data.Due {
|
||||
t.Fatalf("an unprovisioned tier must DEFER, not fire a vzdump at a storage that does not exist; got %+v", pbs.Data)
|
||||
}
|
||||
if !strings.Contains(pbs.Data.Reason, "not present") {
|
||||
t.Fatalf("the deferral must say WHY, or it is indistinguishable from a healthy tier; got %q", pbs.Data.Reason)
|
||||
}
|
||||
// The provisioned tier is unaffected — no evidence yet, so due.
|
||||
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !local.Data.Due {
|
||||
t.Fatalf("a PROVISIONED tier with no backup yet must still be due; got %+v", local.Data)
|
||||
}
|
||||
}
|
||||
|
||||
// A storage-view ERROR must NOT defer. "I could not check" is not "not there" — reading it that way
|
||||
// would silently suppress backups, the absence-is-not-failure rule this project keeps relearning.
|
||||
func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) {
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||
Storage: errStorage{}, // reused from f2_role_fallback_test.go — Observe always fails
|
||||
Tokens: staticTokens{"A": 8200},
|
||||
BackupTiers: []BackupTier{
|
||||
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
||||
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
||||
},
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
srv.now = func() time.Time { return testNow }
|
||||
|
||||
var pbs struct {
|
||||
Data BackupDueResponse `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(do(t, srv.Handler(), "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !pbs.Data.Due {
|
||||
t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -868,6 +868,20 @@ func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid in
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// A tier whose TARGET STORAGE does not exist yet is DEFERRED, not due (R-82 Slice D).
|
||||
//
|
||||
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears
|
||||
// when the hub provisions the DR tier (`felhom-pbs-apply`). Reporting "due" in that window would
|
||||
// have the controller quiesce the apps and fire a vzdump at a storage that does not exist —
|
||||
// every cadence, until provisioning happens. Deferring keeps the tier silent until it is real,
|
||||
// and it goes live with NO restart the moment the storage appears.
|
||||
//
|
||||
// Fail-safe: a storage-view ERROR does not defer. Unknown must never suppress a backup.
|
||||
if tier.TargetID != "" && !s.targetStoragePresent(r.Context(), tier.TargetID) {
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: false,
|
||||
Reason: "target storage not present yet — tier deferred until it is provisioned", Target: echo})
|
||||
return
|
||||
}
|
||||
latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID)
|
||||
if latest == nil {
|
||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet", Target: echo})
|
||||
@@ -1039,6 +1053,28 @@ func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
|
||||
return now.Sub(t), true
|
||||
}
|
||||
|
||||
// targetStoragePresent reports whether a backup target exists on this host RIGHT NOW.
|
||||
//
|
||||
// Returns TRUE on a storage-view error: "I could not check" must never be read as "not there", or a
|
||||
// transient probe failure would silently suppress backups — the absence-is-not-failure rule this
|
||||
// project keeps relearning (R-80, R-81).
|
||||
func (s *Server) targetStoragePresent(ctx context.Context, target string) bool {
|
||||
if s.storage == nil {
|
||||
return true
|
||||
}
|
||||
targets, err := s.storage.Observe(ctx)
|
||||
if err != nil {
|
||||
s.logger.Warn("local-api: storage view unavailable for the backup-target presence check — assuming present", "target", target, "err", err)
|
||||
return true
|
||||
}
|
||||
for _, t := range targets {
|
||||
if t.Name == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
|
||||
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
|
||||
func (s *Server) classByStorage(ctx context.Context) map[string]string {
|
||||
|
||||
Reference in New Issue
Block a user