v0.102.0 — R-82 Slice D: an unprovisioned tier DEFERS instead of failing
Prerequisite for the installer default (host-install 1.20.0). A fresh box now carries the offsite tier, but felhom-pbs only exists once the hub provisions the DR tier. Without this the tier would report due in that window and the controller would quiesce the apps and fire a vzdump at a missing storage every cadence. - GET /backup/due?target= defers when the target storage is absent (targetStoragePresent): due:false with a reason that says why. The tier goes live with NO restart once the storage appears. Fail-safe: a storage-view ERROR returns present and the tier stays due. 'I could not check' must never be read as 'not there' — that would silently suppress backups, the absence-is-not-failure rule relearned three times now (R-80, R-81, the R-82 wait timeout). Full suite green.
This commit is contained in:
@@ -1,3 +1,28 @@
|
|||||||
|
## v0.102.0 — R-82 Slice D: an unprovisioned tier DEFERS instead of failing (2026-07-26)
|
||||||
|
|
||||||
|
Prerequisite for putting the offsite tier into the installer defaults (host-install 1.20.0).
|
||||||
|
|
||||||
|
A fresh box now carries the offsite tier in its config, but `felhom-pbs` does not exist until the hub
|
||||||
|
provisions the DR tier (`felhom-pbs-apply`). Without this change the tier would report **due** in
|
||||||
|
that window, so the controller would quiesce the apps and fire a vzdump at a storage that is not
|
||||||
|
there — **every cadence, until provisioning happens**.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- **`GET /backup/due?target=…` DEFERS when the target storage is absent**
|
||||||
|
(`Server.targetStoragePresent`): `due:false`, reason *"target storage not present yet — tier
|
||||||
|
deferred until it is provisioned"*. The tier stays silent until it is real and goes live with **no
|
||||||
|
restart** the moment the storage appears.
|
||||||
|
|
||||||
|
**Fail-safe, and it is the point:** a storage-view ERROR returns `present` and the tier stays due.
|
||||||
|
*"I could not check"* must never be read as *"not there"* — reading it that way would silently
|
||||||
|
suppress backups, which is the absence-is-not-failure rule this project has now relearned three
|
||||||
|
times (R-80, R-81, and the R-82 wait-timeout). Pinned by
|
||||||
|
`TestBackupDue_StorageViewError_DoesNotSuppress`.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
`TestBackupDue_TargetStorageMissing_Defers` (deferral + a reason that says WHY, so it is
|
||||||
|
distinguishable from a healthy tier) and the fail-safe above. Full suite green.
|
||||||
|
|
||||||
## v0.101.0 — R-82: a leaked restore-test scratch can no longer auto-start (2026-07-26)
|
## v0.101.0 — R-82: a leaked restore-test scratch can no longer auto-start (2026-07-26)
|
||||||
|
|
||||||
**Correction first, because it matters more than the fix.** I reported earlier in this arc that the
|
**Correction first, because it matters more than the fix.** I reported earlier in this arc that the
|
||||||
|
|||||||
@@ -464,3 +464,79 @@ func TestNormalizeBackupTiers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// R-82 Slice D: a tier whose TARGET STORAGE does not exist yet is DEFERRED, not due.
|
||||||
|
//
|
||||||
|
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears when
|
||||||
|
// the hub provisions the DR tier. Reporting "due" in that window would have the controller quiesce
|
||||||
|
// the apps and fire a vzdump at a non-existent storage every cadence until provisioning happens.
|
||||||
|
func TestBackupDue_TargetStorageMissing_Defers(t *testing.T) {
|
||||||
|
st := &fakeStore{}
|
||||||
|
// Storage view knows only "local" — the PBS tier's target is not provisioned yet.
|
||||||
|
srv, err := NewServer(Options{
|
||||||
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
|
||||||
|
Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local", Type: "local"}}},
|
||||||
|
Tokens: staticTokens{"A": 8200},
|
||||||
|
BackupTiers: []BackupTier{
|
||||||
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
||||||
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
||||||
|
},
|
||||||
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv.baseCtx = context.Background()
|
||||||
|
srv.now = func() time.Time { return testNow }
|
||||||
|
h := srv.Handler()
|
||||||
|
|
||||||
|
var pbs, local struct {
|
||||||
|
Data BackupDueResponse `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if pbs.Data.Due {
|
||||||
|
t.Fatalf("an unprovisioned tier must DEFER, not fire a vzdump at a storage that does not exist; got %+v", pbs.Data)
|
||||||
|
}
|
||||||
|
if !strings.Contains(pbs.Data.Reason, "not present") {
|
||||||
|
t.Fatalf("the deferral must say WHY, or it is indistinguishable from a healthy tier; got %q", pbs.Data.Reason)
|
||||||
|
}
|
||||||
|
// The provisioned tier is unaffected — no evidence yet, so due.
|
||||||
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !local.Data.Due {
|
||||||
|
t.Fatalf("a PROVISIONED tier with no backup yet must still be due; got %+v", local.Data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A storage-view ERROR must NOT defer. "I could not check" is not "not there" — reading it that way
|
||||||
|
// would silently suppress backups, the absence-is-not-failure rule this project keeps relearning.
|
||||||
|
func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) {
|
||||||
|
srv, err := NewServer(Options{
|
||||||
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||||
|
Storage: errStorage{}, // reused from f2_role_fallback_test.go — Observe always fails
|
||||||
|
Tokens: staticTokens{"A": 8200},
|
||||||
|
BackupTiers: []BackupTier{
|
||||||
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
||||||
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
||||||
|
},
|
||||||
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv.baseCtx = context.Background()
|
||||||
|
srv.now = func() time.Time { return testNow }
|
||||||
|
|
||||||
|
var pbs struct {
|
||||||
|
Data BackupDueResponse `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(do(t, srv.Handler(), "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !pbs.Data.Due {
|
||||||
|
t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -868,6 +868,20 @@ func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid in
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// A tier whose TARGET STORAGE does not exist yet is DEFERRED, not due (R-82 Slice D).
|
||||||
|
//
|
||||||
|
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears
|
||||||
|
// when the hub provisions the DR tier (`felhom-pbs-apply`). Reporting "due" in that window would
|
||||||
|
// have the controller quiesce the apps and fire a vzdump at a storage that does not exist —
|
||||||
|
// every cadence, until provisioning happens. Deferring keeps the tier silent until it is real,
|
||||||
|
// and it goes live with NO restart the moment the storage appears.
|
||||||
|
//
|
||||||
|
// Fail-safe: a storage-view ERROR does not defer. Unknown must never suppress a backup.
|
||||||
|
if tier.TargetID != "" && !s.targetStoragePresent(r.Context(), tier.TargetID) {
|
||||||
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: false,
|
||||||
|
Reason: "target storage not present yet — tier deferred until it is provisioned", Target: echo})
|
||||||
|
return
|
||||||
|
}
|
||||||
latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID)
|
latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID)
|
||||||
if latest == nil {
|
if latest == nil {
|
||||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet", Target: echo})
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet", Target: echo})
|
||||||
@@ -1039,6 +1053,28 @@ func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
|
|||||||
return now.Sub(t), true
|
return now.Sub(t), true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// targetStoragePresent reports whether a backup target exists on this host RIGHT NOW.
|
||||||
|
//
|
||||||
|
// Returns TRUE on a storage-view error: "I could not check" must never be read as "not there", or a
|
||||||
|
// transient probe failure would silently suppress backups — the absence-is-not-failure rule this
|
||||||
|
// project keeps relearning (R-80, R-81).
|
||||||
|
func (s *Server) targetStoragePresent(ctx context.Context, target string) bool {
|
||||||
|
if s.storage == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
targets, err := s.storage.Observe(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("local-api: storage view unavailable for the backup-target presence check — assuming present", "target", target, "err", err)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, t := range targets {
|
||||||
|
if t.Name == target {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
|
// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
|
||||||
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
|
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
|
||||||
func (s *Server) classByStorage(ctx context.Context) map[string]string {
|
func (s *Server) classByStorage(ctx context.Context) map[string]string {
|
||||||
|
|||||||
Reference in New Issue
Block a user