package localapi import ( "context" "io" "log/slog" "net/http" "path/filepath" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/backup" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" ) // R-894 — after an agent restart, an UNREADABLE storage must fall back to the last success saved on // disk, not to "never". Measured 2026-10-05 on demo-hp: a restart at 04:57, the off-site storage // unreachable at 06:25, the 7-day tier (last copy 4 days old) read DUE, vzdump failed. // // Every test here builds a NEW server and a NEW BackupSuccessState from the same file — that is the // restart. The in-memory store (fakeStore) is always fresh, as after a real restart. // r894Server builds a two-tier server whose off-site tier answers the storage listing with lister. func r894Server(t *testing.T, path string, pbsSvc BackupService) *Server { t.Helper() srv, err := NewServer(Options{ ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{}, Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}}, Tokens: staticTokens{"A": 8200}, BackupTiers: []BackupTier{ {TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}}, {TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc}, }, LastKnownBackups: backup.NewBackupSuccessState(path), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) if err != nil { t.Fatal(err) } srv.baseCtx = context.Background() srv.now = func() time.Time { return testNow } return srv } // unreadable is the off-site storage as demo-hp saw it: "Can't connect to 10.77.0.1:8007". func unreadable() archiveLister { return archiveLister{fakeBackups: &fakeBackups{}, err: errStorageRead} } // THE R-894 CASE, end to end. Agent 1 takes an off-site backup through POST /backup (the fake // runner's success is 12 h before testNow). The agent restarts. The storage cannot be read. The tier // must read NOT due, from the copy saved on disk. // // COMPANION RED-PROOF (observed): delete the `lookup == archiveUnknown && s.lastKnown != nil` block in // handleBackupDue → this fails with "after a restart an unreadable storage must fall back to the saved // copy (12 h old, 7-day tier) — NOT due; got {… Due:true … AgeState:unknown …}". Restored. func TestBackupDue_R894_RestartThenUnreadableStorage_FreshSavedCopyIsNotDue(t *testing.T) { path := filepath.Join(t.TempDir(), "backup-success-state.json") // Agent 1: a real backup job through the endpoint the controller calls. first := r894Server(t, path, &fakeBackups{}) if rr := do(t, first.Handler(), "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted { t.Fatalf("POST /backup: %d %s", rr.Code, rr.Body.String()) } waitFor(t, func() bool { _, ok := backup.NewBackupSuccessState(path).LastKnownSuccess("felhom-pbs", 8200) return ok }) // Agent 2: a restart (new server, new state from the same file), and the storage is unreachable. got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs") if got.Due { t.Fatalf("after a restart an unreadable storage must fall back to the saved copy (12 h old, 7-day tier) — NOT due; got %+v", got) } if got.AgeState != AgeStateKnown || got.AgeSecs == nil || *got.AgeSecs != int64((12*time.Hour).Seconds()) { t.Fatalf("the age must come from the saved copy (12 h, known); got %+v", got) } } // The deliberate rule stays: an unreadable storage must not suppress a backup that IS due. A saved // copy older than the cadence reads DUE. // // COMPANION RED-PROOF (observed): make the fallback answer not-due whenever a saved copy exists // (`if fromDisk { …Due:false… }` before the cadence check) → this fails with "a saved copy 9 days old // under a 7-day cadence MUST read due". Restored. func TestBackupDue_R894_RestartThenUnreadableStorage_OldSavedCopyIsDue(t *testing.T) { path := filepath.Join(t.TempDir(), "backup-success-state.json") st := backup.NewBackupSuccessState(path) if err := st.RecordBackupSuccess("felhom-pbs", backupAt("felhom-pbs", 8200, 9*24*time.Hour, true)); err != nil { t.Fatal(err) } got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs") if !got.Due { t.Fatalf("a saved copy 9 days old under a 7-day cadence MUST read due; got %+v", got) } if got.AgeState != AgeStateKnown { t.Fatalf("the age is known (from disk); got %+v", got) } } // No saved copy → the pre-R-894 answer, byte for byte: DUE, age UNKNOWN (never ABSENT — the controller // fires its window-gate valve only on absent, R-88). func TestBackupDue_R894_RestartThenUnreadableStorage_NoSavedCopyIsDueUnknown(t *testing.T) { path := filepath.Join(t.TempDir(), "backup-success-state.json") got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs") if !got.Due || got.AgeState != AgeStateUnknown || got.AgeSecs != nil { t.Fatalf("no saved copy + unreadable storage must stay DUE with age unknown; got %+v", got) } } // A storage that ANSWERS is the ground truth: an archive absent there makes the tier due even when the // file remembers a fresh success (a pruned or deleted copy must be made again). // // COMPANION RED-PROOF (observed): drop `lookup == archiveUnknown &&` from the fallback condition → this // fails with "the storage answered 'no archive' — the saved copy must NOT stand in for it". Restored. func TestBackupDue_R894_SavedCopyIgnoredWhenStorageAnswers(t *testing.T) { path := filepath.Join(t.TempDir(), "backup-success-state.json") st := backup.NewBackupSuccessState(path) if err := st.RecordBackupSuccess("felhom-pbs", backupAt("felhom-pbs", 8200, time.Hour, true)); err != nil { t.Fatal(err) } absent := archiveLister{fakeBackups: &fakeBackups{}, found: false} got := dueFor(t, r894Server(t, path, absent).Handler(), "felhom-pbs") if !got.Due { t.Fatalf("the storage answered 'no archive' — the saved copy must NOT stand in for it; got %+v", got) } } // A FAILED backup is never saved: it must not make a tier look fresh after a restart. func TestBackupDue_R894_FailedBackupIsNotSaved(t *testing.T) { path := filepath.Join(t.TempDir(), "backup-success-state.json") first := r894Server(t, path, &fakeBackups{failErr: "could not activate storage 'felhom-pbs'"}) if rr := do(t, first.Handler(), "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted { t.Fatalf("POST /backup: %d %s", rr.Code, rr.Body.String()) } waitFor(t, func() bool { return len(first.store.Backups(context.Background())) == 1 }) if _, ok := backup.NewBackupSuccessState(path).LastKnownSuccess("felhom-pbs", 8200); ok { t.Fatal("a failed backup must never be saved as a success") } got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs") if !got.Due { t.Fatalf("after a failed backup and a restart the tier must still be due; got %+v", got) } }