admin
74b5eae5b0
gates / gates (push) Successful in 35s
An unreadable storage right after an agent restart read the off-site tier DUE (the in-memory record was empty). The newest success per tier is now kept on disk and read ONLY when the storage cannot be read: fresh -> not due, older than the cadence -> due, none -> due (unknown) as before. A storage that answers stays the ground truth. Ships with v0.150.0 after the 2026-10-07 read-back; nothing delivered. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
145 lines
6.8 KiB
Go
145 lines
6.8 KiB
Go
package localapi
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
)
|
|
|
|
// R-894 — after an agent restart, an UNREADABLE storage must fall back to the last success saved on
|
|
// disk, not to "never". Measured 2026-10-05 on demo-hp: a restart at 04:57, the off-site storage
|
|
// unreachable at 06:25, the 7-day tier (last copy 4 days old) read DUE, vzdump failed.
|
|
//
|
|
// Every test here builds a NEW server and a NEW BackupSuccessState from the same file — that is the
|
|
// restart. The in-memory store (fakeStore) is always fresh, as after a real restart.
|
|
|
|
// r894Server builds a two-tier server whose off-site tier answers the storage listing with lister.
|
|
func r894Server(t *testing.T, path string, pbsSvc BackupService) *Server {
|
|
t.Helper()
|
|
srv, err := NewServer(Options{
|
|
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
|
Storage: fakeStorage{targets: []hub.StorageTarget{{Name: "local"}, {Name: "felhom-pbs"}}},
|
|
Tokens: staticTokens{"A": 8200},
|
|
BackupTiers: []BackupTier{
|
|
{TargetID: "local", Cadence: 24 * time.Hour, Primary: true, Service: &fakeBackups{}},
|
|
{TargetID: "felhom-pbs", Cadence: 7 * 24 * time.Hour, Service: pbsSvc},
|
|
},
|
|
LastKnownBackups: backup.NewBackupSuccessState(path),
|
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srv.baseCtx = context.Background()
|
|
srv.now = func() time.Time { return testNow }
|
|
return srv
|
|
}
|
|
|
|
// unreadable is the off-site storage as demo-hp saw it: "Can't connect to 10.77.0.1:8007".
|
|
func unreadable() archiveLister {
|
|
return archiveLister{fakeBackups: &fakeBackups{}, err: errStorageRead}
|
|
}
|
|
|
|
// THE R-894 CASE, end to end. Agent 1 takes an off-site backup through POST /backup (the fake
|
|
// runner's success is 12 h before testNow). The agent restarts. The storage cannot be read. The tier
|
|
// must read NOT due, from the copy saved on disk.
|
|
//
|
|
// COMPANION RED-PROOF (observed): delete the `lookup == archiveUnknown && s.lastKnown != nil` block in
|
|
// handleBackupDue → this fails with "after a restart an unreadable storage must fall back to the saved
|
|
// copy (12 h old, 7-day tier) — NOT due; got {… Due:true … AgeState:unknown …}". Restored.
|
|
func TestBackupDue_R894_RestartThenUnreadableStorage_FreshSavedCopyIsNotDue(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
|
|
|
// Agent 1: a real backup job through the endpoint the controller calls.
|
|
first := r894Server(t, path, &fakeBackups{})
|
|
if rr := do(t, first.Handler(), "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted {
|
|
t.Fatalf("POST /backup: %d %s", rr.Code, rr.Body.String())
|
|
}
|
|
waitFor(t, func() bool {
|
|
_, ok := backup.NewBackupSuccessState(path).LastKnownSuccess("felhom-pbs", 8200)
|
|
return ok
|
|
})
|
|
|
|
// Agent 2: a restart (new server, new state from the same file), and the storage is unreachable.
|
|
got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs")
|
|
if got.Due {
|
|
t.Fatalf("after a restart an unreadable storage must fall back to the saved copy (12 h old, 7-day tier) — NOT due; got %+v", got)
|
|
}
|
|
if got.AgeState != AgeStateKnown || got.AgeSecs == nil || *got.AgeSecs != int64((12*time.Hour).Seconds()) {
|
|
t.Fatalf("the age must come from the saved copy (12 h, known); got %+v", got)
|
|
}
|
|
}
|
|
|
|
// The deliberate rule stays: an unreadable storage must not suppress a backup that IS due. A saved
|
|
// copy older than the cadence reads DUE.
|
|
//
|
|
// COMPANION RED-PROOF (observed): make the fallback answer not-due whenever a saved copy exists
|
|
// (`if fromDisk { …Due:false… }` before the cadence check) → this fails with "a saved copy 9 days old
|
|
// under a 7-day cadence MUST read due". Restored.
|
|
func TestBackupDue_R894_RestartThenUnreadableStorage_OldSavedCopyIsDue(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
|
st := backup.NewBackupSuccessState(path)
|
|
if err := st.RecordBackupSuccess("felhom-pbs", backupAt("felhom-pbs", 8200, 9*24*time.Hour, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs")
|
|
if !got.Due {
|
|
t.Fatalf("a saved copy 9 days old under a 7-day cadence MUST read due; got %+v", got)
|
|
}
|
|
if got.AgeState != AgeStateKnown {
|
|
t.Fatalf("the age is known (from disk); got %+v", got)
|
|
}
|
|
}
|
|
|
|
// No saved copy → the pre-R-894 answer, byte for byte: DUE, age UNKNOWN (never ABSENT — the controller
|
|
// fires its window-gate valve only on absent, R-88).
|
|
func TestBackupDue_R894_RestartThenUnreadableStorage_NoSavedCopyIsDueUnknown(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
|
got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs")
|
|
if !got.Due || got.AgeState != AgeStateUnknown || got.AgeSecs != nil {
|
|
t.Fatalf("no saved copy + unreadable storage must stay DUE with age unknown; got %+v", got)
|
|
}
|
|
}
|
|
|
|
// A storage that ANSWERS is the ground truth: an archive absent there makes the tier due even when the
|
|
// file remembers a fresh success (a pruned or deleted copy must be made again).
|
|
//
|
|
// COMPANION RED-PROOF (observed): drop `lookup == archiveUnknown &&` from the fallback condition → this
|
|
// fails with "the storage answered 'no archive' — the saved copy must NOT stand in for it". Restored.
|
|
func TestBackupDue_R894_SavedCopyIgnoredWhenStorageAnswers(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
|
st := backup.NewBackupSuccessState(path)
|
|
if err := st.RecordBackupSuccess("felhom-pbs", backupAt("felhom-pbs", 8200, time.Hour, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
absent := archiveLister{fakeBackups: &fakeBackups{}, found: false}
|
|
got := dueFor(t, r894Server(t, path, absent).Handler(), "felhom-pbs")
|
|
if !got.Due {
|
|
t.Fatalf("the storage answered 'no archive' — the saved copy must NOT stand in for it; got %+v", got)
|
|
}
|
|
}
|
|
|
|
// A FAILED backup is never saved: it must not make a tier look fresh after a restart.
|
|
func TestBackupDue_R894_FailedBackupIsNotSaved(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "backup-success-state.json")
|
|
first := r894Server(t, path, &fakeBackups{failErr: "could not activate storage 'felhom-pbs'"})
|
|
if rr := do(t, first.Handler(), "POST", "/backup?target=felhom-pbs", "A", ""); rr.Code != http.StatusAccepted {
|
|
t.Fatalf("POST /backup: %d %s", rr.Code, rr.Body.String())
|
|
}
|
|
waitFor(t, func() bool { return len(first.store.Backups(context.Background())) == 1 })
|
|
if _, ok := backup.NewBackupSuccessState(path).LastKnownSuccess("felhom-pbs", 8200); ok {
|
|
t.Fatal("a failed backup must never be saved as a success")
|
|
}
|
|
got := dueFor(t, r894Server(t, path, unreadable()).Handler(), "felhom-pbs")
|
|
if !got.Due {
|
|
t.Fatalf("after a failed backup and a restart the tier must still be due; got %+v", got)
|
|
}
|
|
}
|