005083b558
Prerequisite for the installer default (host-install 1.20.0). A fresh box now carries the offsite tier, but felhom-pbs only exists once the hub provisions the DR tier. Without this the tier would report due in that window and the controller would quiesce the apps and fire a vzdump at a missing storage every cadence. - GET /backup/due?target= defers when the target storage is absent (targetStoragePresent): due:false with a reason that says why. The tier goes live with NO restart once the storage appears. Fail-safe: a storage-view ERROR returns present and the tier stays due. 'I could not check' must never be read as 'not there' — that would silently suppress backups, the absence-is-not-failure rule relearned three times now (R-80, R-81, the R-82 wait timeout). Full suite green.
543 lines
22 KiB
Go
543 lines
22 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
)
|
|
|
|
// R-82 Slice A — per-target cadence, due and runner.
|
|
//
|
|
// The agent and the controller deploy INDEPENDENTLY. A new agent will serve old controllers for as
|
|
// long as it takes the fleet to catch up, so the untargeted contract is frozen, not merely
|
|
// "probably fine". These tests pin that freeze; the multi-tier behaviour is additive on top.
|
|
|
|
// tieredServer builds a two-tier server: primary "local" (24h) + "felhom-pbs" (7d), each with its
|
|
// own runner, exactly as main.go wires it.
|
|
func tieredServer(t *testing.T, st *fakeStore, localSvc, pbsSvc *fakeBackups) *Server {
|
|
t.Helper()
|
|
srv, err := NewServer(Options{
|
|
ListenAddr: "127.0.0.1:0",
|
|
Guests: &fakeGuests{},
|
|
Backups: localSvc,
|
|
Store: st,
|
|
Storage: fakeStorage{},
|
|
Tokens: staticTokens{"A": 8200, "B": 9300},
|
|
BackupTiers: []BackupTier{
|
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: localSvc},
|
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc},
|
|
},
|
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("new server: %v", err)
|
|
}
|
|
srv.baseCtx = context.Background()
|
|
srv.now = func() time.Time { return testNow }
|
|
return srv
|
|
}
|
|
|
|
// seen returns the vmids this fake runner was invoked for (mutex-guarded — the backup runs on a
|
|
// goroutine).
|
|
func (f *fakeBackups) seen() []int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]int(nil), f.vmids...)
|
|
}
|
|
|
|
// waitFor polls cond for up to 2s. POST /backup is fire-and-forget, so the assertion has to wait
|
|
// for the goroutine rather than assume it has run.
|
|
func waitFor(t *testing.T, cond func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if cond() {
|
|
return
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
t.Fatalf("condition not met within 2s")
|
|
}
|
|
|
|
func backupAt(target string, vmid int, ago time.Duration, ok bool) hub.Backup {
|
|
return hub.Backup{
|
|
TargetID: target,
|
|
VMID: vmid,
|
|
Success: ok,
|
|
StartedAt: testNow.Add(-ago).Format(time.RFC3339),
|
|
}
|
|
}
|
|
|
|
// ── RED-PROOF 1 — old controller ↔ new agent ────────────────────────────────────────────────
|
|
//
|
|
// An old controller sends `GET /backup/due` with no query string and parses the pre-R-82 response.
|
|
// The response must be BYTE-IDENTICAL — not merely semantically similar. A stray `"target":"local"`
|
|
// key is harmless to a tolerant JSON decoder and fatal to a strict one, and we do not get to choose
|
|
// which the deployed fleet has.
|
|
//
|
|
// COMPANION RED-PROOF (observed): drop the `omitempty` from BackupDueResponse.Target and have
|
|
// tierFromRequest echo the primary's id for an untargeted request → this test fails with the
|
|
// observed body carrying `"target":"local"`. Restored.
|
|
func TestBackupDue_Untargeted_ResponseBytesUnchanged(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true))
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
rr := do(t, h, "GET", "/backup/due", "A", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("status = %d, want 200", rr.Code)
|
|
}
|
|
var got map[string]any
|
|
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
|
t.Fatalf("unmarshal: %v (body %s)", err, rr.Body.String())
|
|
}
|
|
data, _ := got["data"].(map[string]any)
|
|
if data == nil {
|
|
t.Fatalf("no data object in %s", rr.Body.String())
|
|
}
|
|
if _, present := data["target"]; present {
|
|
t.Fatalf("UNTARGETED response MUST NOT carry a target key — an old controller sees a changed contract; body: %s", rr.Body.String())
|
|
}
|
|
if data["due"] != false {
|
|
t.Fatalf("2h-old local backup under a 24h cadence must not be due; body: %s", rr.Body.String())
|
|
}
|
|
if data["reason"] != "within cadence window" {
|
|
t.Fatalf("reason string changed: %v", data["reason"])
|
|
}
|
|
}
|
|
|
|
// The untargeted verdict must come from the PRIMARY tier's cadence, not from whichever tier
|
|
// happens to be freshest. With a stale local and a fresh PBS backup, untargeted must say DUE.
|
|
func TestBackupDue_Untargeted_UsesPrimaryTierOnly(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("local", 8200, 30*time.Hour, true)) // stale for 24h cadence
|
|
st.RecordBackup(backupAt("felhom-pbs", 8200, 1*time.Hour, true)) // fresh, different tier
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
var resp struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
rr := do(t, h, "GET", "/backup/due", "A", "")
|
|
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !resp.Data.Due {
|
|
t.Fatalf("a fresh backup on ANOTHER tier must not satisfy the primary's cadence; got %+v", resp.Data)
|
|
}
|
|
}
|
|
|
|
// ── Per-tier due-ness ────────────────────────────────────────────────────────────────────────
|
|
|
|
// THE POINT OF THE WHOLE SLICE: a fresh daily local backup must NOT satisfy the weekly PBS tier.
|
|
// Without the per-target filter in latestSuccessfulBackupForTarget the DR tier would never run —
|
|
// which is exactly today's "applied and empty" state, re-created in code.
|
|
func TestBackupDue_PerTier_LocalFreshDoesNotSatisfyPBS(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true)) // fresh daily
|
|
st.RecordBackup(backupAt("felhom-pbs", 8200, 8*24*time.Hour, true)) // 8d — past the 7d weekly
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
var local, pbs struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if local.Data.Due {
|
|
t.Fatalf("local tier: 2h old under 24h cadence must NOT be due; got %+v", local.Data)
|
|
}
|
|
if !pbs.Data.Due {
|
|
t.Fatalf("PBS tier: 8d old under a 7d cadence MUST be due; got %+v", pbs.Data)
|
|
}
|
|
if pbs.Data.Target != "felhom-pbs" || local.Data.Target != "local" {
|
|
t.Fatalf("a targeted response must echo its tier; got local=%q pbs=%q", local.Data.Target, pbs.Data.Target)
|
|
}
|
|
}
|
|
|
|
// A 6-day-old PBS snapshot is INSIDE the weekly window — it must not be due. The mirror of the
|
|
// hub-side threshold test in Slice C, asserted here at the source of truth.
|
|
func TestBackupDue_PerTier_PBSWithinWeeklyWindow(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("felhom-pbs", 8200, 6*24*time.Hour, true))
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
var pbs struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if pbs.Data.Due {
|
|
t.Fatalf("6d old under a 7d cadence must NOT be due; got %+v", pbs.Data)
|
|
}
|
|
}
|
|
|
|
// A failed backup must not satisfy any tier's cadence (pre-existing rule, re-asserted per-tier).
|
|
func TestBackupDue_PerTier_FailedBackupDoesNotCount(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("felhom-pbs", 8200, time.Hour, false))
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
var pbs struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !pbs.Data.Due {
|
|
t.Fatalf("a FAILED backup must not satisfy the cadence; got %+v", pbs.Data)
|
|
}
|
|
}
|
|
|
|
// The fail-safe-toward-due rule survives per-tier: an unparseable timestamp yields DUE.
|
|
// A spurious backup is cheap; a skipped one is not.
|
|
func TestBackupDue_PerTier_UnparseableTimestampIsDue(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(hub.Backup{TargetID: "felhom-pbs", VMID: 8200, Success: true, StartedAt: "not-a-time"})
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
var pbs struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !pbs.Data.Due {
|
|
t.Fatalf("unparseable timestamp must fail SAFE toward due; got %+v", pbs.Data)
|
|
}
|
|
}
|
|
|
|
// An unknown target is a 400 — never a silent fallback to the primary. A controller asking about a
|
|
// tier this agent does not serve must find out, not be handed a different tier's freshness and act
|
|
// on it.
|
|
func TestBackupDue_UnknownTarget_IsAnErrorNotAFallback(t *testing.T) {
|
|
st := &fakeStore{}
|
|
st.RecordBackup(backupAt("local", 8200, 2*time.Hour, true))
|
|
h := tieredServer(t, st, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
|
|
rr := do(t, h, "GET", "/backup/due?target=does-not-exist", "A", "")
|
|
if rr.Code != http.StatusBadRequest {
|
|
t.Fatalf("unknown target must be 400 (got %d, body %s)", rr.Code, rr.Body.String())
|
|
}
|
|
}
|
|
|
|
// ── Tier advertisement ───────────────────────────────────────────────────────────────────────
|
|
|
|
// GET /backup/tiers is the controller's capability probe. Primary must be first and flagged, so a
|
|
// controller can tell which tier the untargeted endpoints act on.
|
|
func TestBackupTiers_AdvertisesPrimaryFirst(t *testing.T) {
|
|
h := tieredServer(t, &fakeStore{}, &fakeBackups{}, &fakeBackups{}).Handler()
|
|
var resp struct {
|
|
Data BackupTiersResponse `json:"data"`
|
|
}
|
|
rr := do(t, h, "GET", "/backup/tiers", "A", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("status = %d", rr.Code)
|
|
}
|
|
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(resp.Data.Tiers) != 2 {
|
|
t.Fatalf("want 2 tiers, got %+v", resp.Data.Tiers)
|
|
}
|
|
if !resp.Data.Tiers[0].Primary || resp.Data.Tiers[0].Target != "local" {
|
|
t.Fatalf("primary must be first and flagged; got %+v", resp.Data.Tiers)
|
|
}
|
|
if resp.Data.Tiers[1].Target != "felhom-pbs" || resp.Data.Tiers[1].CadenceSeconds != int64((7 * 24 * time.Hour).Seconds()) {
|
|
t.Fatalf("PBS tier mis-advertised: %+v", resp.Data.Tiers[1])
|
|
}
|
|
}
|
|
|
|
// ── POST /backup routing + per-tier single-flight ────────────────────────────────────────────
|
|
|
|
// A targeted POST must run THAT tier's runner. Routing both tiers to one runner would silently
|
|
// write every "PBS" backup to local — a DR tier that reports success and stores nothing.
|
|
func TestBackupPost_RoutesToTheTargetsOwnRunner(t *testing.T) {
|
|
local, pbs := &fakeBackups{}, &fakeBackups{}
|
|
srv := tieredServer(t, &fakeStore{}, local, pbs)
|
|
h := srv.Handler()
|
|
|
|
if rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted {
|
|
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
|
|
}
|
|
waitFor(t, func() bool { return len(pbs.seen()) == 1 })
|
|
|
|
if got := len(local.seen()); got != 0 {
|
|
t.Fatalf("the LOCAL runner must not have run for a PBS-targeted request (ran %d times)", got)
|
|
}
|
|
}
|
|
|
|
// Untargeted POST routes to the primary — the old controller's path.
|
|
func TestBackupPost_UntargetedRoutesToPrimary(t *testing.T) {
|
|
local, pbs := &fakeBackups{}, &fakeBackups{}
|
|
h := tieredServer(t, &fakeStore{}, local, pbs).Handler()
|
|
|
|
if rr := do(t, h, "POST", "/backup", "A", ""); rr.Code != http.StatusAccepted {
|
|
t.Fatalf("status = %d", rr.Code)
|
|
}
|
|
waitFor(t, func() bool { return len(local.seen()) == 1 })
|
|
if got := len(pbs.seen()); got != 0 {
|
|
t.Fatalf("untargeted POST must not touch a non-primary tier (ran %d times)", got)
|
|
}
|
|
}
|
|
|
|
// ONE BACKUP AT A TIME PER GUEST (operator ruling 2026-07-26). A second tier's POST while another
|
|
// tier is still in flight must be REFUSED — vzdump holds the guest lock, so it could not succeed
|
|
// anyway, and attempting it records a spurious failure that leaves the tier permanently due.
|
|
//
|
|
// Crucially it must NOT be handed the busy tier's job id: that is exactly how a caller comes to
|
|
// believe its own backup ran.
|
|
func TestBackupPost_SecondTierRefusedWhileAnotherInFlight(t *testing.T) {
|
|
localGate := make(chan struct{})
|
|
local := &fakeBackups{gate: localGate}
|
|
pbs := &fakeBackups{}
|
|
h := tieredServer(t, &fakeStore{}, local, pbs).Handler()
|
|
|
|
var first BackupResponse
|
|
rr1 := do(t, h, "POST", "/backup", "A", "") // local; blocks on the gate
|
|
if err := json.Unmarshal(rr1.Body.Bytes(), &struct {
|
|
Data *BackupResponse `json:"data"`
|
|
}{Data: &first}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool { return len(local.seen()) == 1 })
|
|
|
|
rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
|
|
if rr2.Code != http.StatusConflict {
|
|
t.Fatalf("a second tier must be REFUSED while another is in flight; got %d body %s", rr2.Code, rr2.Body.String())
|
|
}
|
|
// The STRUCTURAL requirement: the refusal must not return a job the caller could mistake for
|
|
// its own. It is a 409 with ok=false and NO data object, so nothing is parseable as "my job".
|
|
// (Naming the busy job in the human-readable message is deliberate and useful for diagnosis —
|
|
// what must never happen is handing it back as BackupResponse.JobID on a 202.)
|
|
var envelope struct {
|
|
OK bool `json:"ok"`
|
|
Data *BackupResponse `json:"data"`
|
|
Error string `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(rr2.Body.Bytes(), &envelope); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if envelope.OK {
|
|
t.Fatalf("a refusal must not be ok=true; body %s", rr2.Body.String())
|
|
}
|
|
if envelope.Data != nil && envelope.Data.JobID != "" {
|
|
t.Fatalf("the refusal must NOT hand back a job id as the caller's own (got %q); body %s",
|
|
envelope.Data.JobID, rr2.Body.String())
|
|
}
|
|
if !strings.Contains(rr2.Body.String(), "local") {
|
|
t.Fatalf("the refusal must NAME the busy tier so the caller can diagnose; body %s", rr2.Body.String())
|
|
}
|
|
if got := len(pbs.seen()); got != 0 {
|
|
t.Fatalf("the refused tier must NOT have started a backup (ran %d times)", got)
|
|
}
|
|
close(localGate)
|
|
}
|
|
|
|
// Once the busy tier finishes, the other tier may start — and gets its OWN tier-scoped job id.
|
|
func TestBackupPost_SecondTierAllowedAfterFirstFinishes(t *testing.T) {
|
|
local, pbs := &fakeBackups{}, &fakeBackups{}
|
|
h := tieredServer(t, &fakeStore{}, local, pbs).Handler()
|
|
|
|
var first, second BackupResponse
|
|
if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
|
|
Data *BackupResponse `json:"data"`
|
|
}{Data: &first}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool { return len(local.seen()) == 1 })
|
|
// Wait for the local job to leave the in-flight phases.
|
|
waitFor(t, func() bool {
|
|
rr := do(t, h, "GET", "/backup/status", "A", "")
|
|
return strings.Contains(rr.Body.String(), `"phase":"done"`)
|
|
})
|
|
|
|
rr2 := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
|
|
if rr2.Code != http.StatusAccepted {
|
|
t.Fatalf("after the first tier finished the second must be allowed; got %d body %s", rr2.Code, rr2.Body.String())
|
|
}
|
|
if err := json.Unmarshal(rr2.Body.Bytes(), &struct {
|
|
Data *BackupResponse `json:"data"`
|
|
}{Data: &second}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if second.JobID == first.JobID {
|
|
t.Fatalf("job ids must stay tier-scoped: %q vs %q", first.JobID, second.JobID)
|
|
}
|
|
waitFor(t, func() bool { return len(pbs.seen()) == 1 })
|
|
}
|
|
|
|
// `snapshotted` still counts as in flight — the vzdump is uploading and still holds the guest lock.
|
|
// Checking only `running` (the pre-R-82 code) left a window where a second POST started a real
|
|
// second vzdump.
|
|
func TestBackupPost_SnapshottedCountsAsInFlight(t *testing.T) {
|
|
gate := make(chan struct{})
|
|
local := &fakeBackups{gate: gate, fireSnapshot: true} // fires onSnapshot, then blocks
|
|
pbs := &fakeBackups{}
|
|
h := tieredServer(t, &fakeStore{}, local, pbs).Handler()
|
|
|
|
do(t, h, "POST", "/backup", "A", "")
|
|
waitFor(t, func() bool {
|
|
rr := do(t, h, "GET", "/backup/status", "A", "")
|
|
return strings.Contains(rr.Body.String(), `"phase":"snapshotted"`)
|
|
})
|
|
|
|
rr := do(t, h, "POST", "/backup?target=felhom-pbs", "A", "")
|
|
if rr.Code != http.StatusConflict {
|
|
t.Fatalf("a SNAPSHOTTED backup still holds the guest — a second tier must be refused; got %d body %s", rr.Code, rr.Body.String())
|
|
}
|
|
close(gate)
|
|
}
|
|
|
|
// Same tier, still single-flight: a second POST to a running tier returns the SAME job.
|
|
func TestBackupPost_SameTierStillSingleFlight(t *testing.T) {
|
|
gate := make(chan struct{})
|
|
local := &fakeBackups{gate: gate}
|
|
h := tieredServer(t, &fakeStore{}, local, &fakeBackups{}).Handler()
|
|
|
|
var a, b BackupResponse
|
|
if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
|
|
Data *BackupResponse `json:"data"`
|
|
}{Data: &a}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "POST", "/backup", "A", "").Body.Bytes(), &struct {
|
|
Data *BackupResponse `json:"data"`
|
|
}{Data: &b}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if a.JobID != b.JobID {
|
|
t.Fatalf("same tier must single-flight: %q vs %q", a.JobID, b.JobID)
|
|
}
|
|
close(gate)
|
|
}
|
|
|
|
// ── normalizeBackupTiers — the compatibility core ────────────────────────────────────────────
|
|
|
|
func TestNormalizeBackupTiers(t *testing.T) {
|
|
svc := &fakeBackups{}
|
|
|
|
t.Run("nil tiers synthesize the legacy single tier", func(t *testing.T) {
|
|
got := normalizeBackupTiers(nil, svc, 24*time.Hour)
|
|
if len(got) != 1 || !got[0].Primary || got[0].Cadence != 24*time.Hour {
|
|
t.Fatalf("legacy synthesis broken: %+v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("primary is hoisted to the front", func(t *testing.T) {
|
|
got := normalizeBackupTiers([]BackupTier{
|
|
{TargetID: "felhom-pbs", Cadence: time.Hour, Service: svc},
|
|
{TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc},
|
|
}, svc, 24*time.Hour)
|
|
if len(got) != 2 || got[0].TargetID != "local" || !got[0].Primary || got[1].Primary {
|
|
t.Fatalf("primary not hoisted / uniqueness broken: %+v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("no primary marked → first becomes primary", func(t *testing.T) {
|
|
got := normalizeBackupTiers([]BackupTier{
|
|
{TargetID: "a", Cadence: time.Hour, Service: svc},
|
|
{TargetID: "b", Cadence: time.Hour, Service: svc},
|
|
}, svc, 24*time.Hour)
|
|
if len(got) != 2 || got[0].TargetID != "a" || !got[0].Primary {
|
|
t.Fatalf("want first-as-primary, got %+v", got)
|
|
}
|
|
})
|
|
|
|
t.Run("a tier with no runner is DROPPED, not advertised", func(t *testing.T) {
|
|
got := normalizeBackupTiers([]BackupTier{
|
|
{TargetID: "local", Cadence: time.Hour, Primary: true, Service: svc},
|
|
{TargetID: "felhom-pbs", Cadence: time.Hour, Service: nil},
|
|
}, svc, 24*time.Hour)
|
|
if len(got) != 1 || got[0].TargetID != "local" {
|
|
t.Fatalf("a serviceless tier must not be advertised (it could never run): %+v", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
// R-82 Slice D: a tier whose TARGET STORAGE does not exist yet is DEFERRED, not due.
|
|
//
|
|
// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears when
|
|
// the hub provisions the DR tier. Reporting "due" in that window would have the controller quiesce
|
|
// the apps and fire a vzdump at a non-existent storage every cadence until provisioning happens.
|
|
func TestBackupDue_TargetStorageMissing_Defers(t *testing.T) {
|
|
st := &fakeStore{}
|
|
// Storage view knows only "local" — the PBS tier's target is not provisioned yet.
|
|
srv, err := NewServer(Options{
|
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: st,
|
|
Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local", Type: "local"}}},
|
|
Tokens: staticTokens{"A": 8200},
|
|
BackupTiers: []BackupTier{
|
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
|
},
|
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srv.baseCtx = context.Background()
|
|
srv.now = func() time.Time { return testNow }
|
|
h := srv.Handler()
|
|
|
|
var pbs, local struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if pbs.Data.Due {
|
|
t.Fatalf("an unprovisioned tier must DEFER, not fire a vzdump at a storage that does not exist; got %+v", pbs.Data)
|
|
}
|
|
if !strings.Contains(pbs.Data.Reason, "not present") {
|
|
t.Fatalf("the deferral must say WHY, or it is indistinguishable from a healthy tier; got %q", pbs.Data.Reason)
|
|
}
|
|
// The provisioned tier is unaffected — no evidence yet, so due.
|
|
if err := json.Unmarshal(do(t, h, "GET", "/backup/due?target=local", "A", "").Body.Bytes(), &local); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !local.Data.Due {
|
|
t.Fatalf("a PROVISIONED tier with no backup yet must still be due; got %+v", local.Data)
|
|
}
|
|
}
|
|
|
|
// A storage-view ERROR must NOT defer. "I could not check" is not "not there" — reading it that way
|
|
// would silently suppress backups, the absence-is-not-failure rule this project keeps relearning.
|
|
func TestBackupDue_StorageViewError_DoesNotSuppress(t *testing.T) {
|
|
srv, err := NewServer(Options{
|
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
|
Storage: errStorage{}, // reused from f2_role_fallback_test.go — Observe always fails
|
|
Tokens: staticTokens{"A": 8200},
|
|
BackupTiers: []BackupTier{
|
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: &fakeBackups{}},
|
|
},
|
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srv.baseCtx = context.Background()
|
|
srv.now = func() time.Time { return testNow }
|
|
|
|
var pbs struct {
|
|
Data BackupDueResponse `json:"data"`
|
|
}
|
|
if err := json.Unmarshal(do(t, srv.Handler(), "GET", "/backup/due?target=felhom-pbs", "A", "").Body.Bytes(), &pbs); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !pbs.Data.Due {
|
|
t.Fatalf("a storage-view error must not suppress the backup (fail toward due); got %+v", pbs.Data)
|
|
}
|
|
}
|