package offsitekeys import ( "context" "fmt" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-895 slice 2: the hub's own listing, compared day to day. Fixtures are `ls -l` outputs in the provider's shape // (slice 1's parser) on a real store; the clock is the wall clock because the window ledger stamps rows with SQLite's // datetime('now'). const snapDir = "/home/felhom-repo/snapshots" func snapID(n int) string { return fmt.Sprintf("%08x%056x", 0xa1b2c300+n, n+1) } // lsOut renders `ls -l` lines for ids uploaded at the given times (year-less form, as the provider prints recent files). func lsOut(files map[string]time.Time) string { var b strings.Builder b.WriteString("total 8\n") for id, at := range files { fmt.Fprintf(&b, "-rw------- 1 u1-sub4 u1-sub4 324 %s %s\n", at.UTC().Format("Jan _2 15:04"), id) } return b.String() } type evt struct{ typ, sev, msg string } // watchFixture: a provisioned customer whose box key is registered AND confirmed (the pinned tier), events captured. func watchFixture(t *testing.T) (*Service, *fakeFS, *[]evt) { t.Helper() s, fs, _ := svcFixture(t) var events []evt s.Emit = func(_, typ, sev, msg, _, _ string) { events = append(events, evt{typ, sev, msg}) } pub, fp := newKey(t) if _, err := s.RegisterKey(context.Background(), "c1", pub); err != nil { t.Fatal(err) } if _, err := s.ConfirmKey(context.Background(), "c1", fp); err != nil { t.Fatal(err) } events = nil return s, fs, &events } func ofType(events []evt, typ string) []evt { var out []evt for _, e := range events { if e.typ == typ { out = append(out, e) } } return out } func reportRun(t *testing.T, st *store.Store, at time.Time) { t.Helper() if err := st.SaveReport("c1", []byte(`{"controller_version":"0.330.0","offsite":{"enabled":true,"last_run":"`+at.UTC().Format(time.RFC3339)+`"}}`)); err != nil { t.Fatal(err) } } // THE CONSEQUENCE, through the production entry point (the 07:10 AuditAll): day 1 is the baseline (no alarm), day 2 // shows three snapshots gone with no clean-up window → ONE offsite_snapshots_vanished (error), and the same listing on // day 3 does not repeat it. RED-PROOF (REPORT, 2026-10-17 night): with the `len(gone) > explains` verdict replaced by // `false`, this fails with `vanished events = 0, want 1`. func TestSnapshotWatch_VanishedOutsideAWindowAlarmsOnce(t *testing.T) { s, fs, events := watchFixture(t) now := time.Now().UTC() files := map[string]time.Time{} for i := 0; i < 10; i++ { files[snapID(i)] = now.Add(-time.Duration(48+i) * time.Hour).Truncate(time.Minute) } fs.files[snapDir] = lsOut(files) s.AuditAll(context.Background(), nil) if n := len(ofType(*events, EventSnapshotsVanished)) + len(ofType(*events, EventSnapshotsUnexplained)); n != 0 { t.Fatalf("the first listing is a baseline: %d snapshot event(s), want 0 (%+v)", n, *events) } if _, ok := s.Store.GetOffsiteSnapshotListing("c1"); !ok { t.Fatal("the first listing was not stored as the baseline") } for i := 0; i < 3; i++ { delete(files, snapID(i)) } fs.files[snapDir] = lsOut(files) s.AuditAll(context.Background(), nil) v := ofType(*events, EventSnapshotsVanished) if len(v) != 1 { t.Fatalf("vanished events = %d, want 1 (events: %+v)", len(v), *events) } if v[0].sev != "error" || !strings.Contains(v[0].msg, "3 snapshot file(s) gone") || !strings.Contains(v[0].msg, shortID(snapID(0))) { t.Fatalf("event = %+v — want severity error naming 3 files and the first id", v[0]) } s.AuditAll(context.Background(), nil) if n := len(ofType(*events, EventSnapshotsVanished)); n != 1 { t.Fatalf("the same listing again raised the alarm again: %d events, want still 1", n) } } // A clean-up window the hub opened explains up to what it removed; more than that still alarms. func TestSnapshotWatch_WindowExplainsUpToItsCredit(t *testing.T) { for _, tc := range []struct { name string gone int wantAlarm bool }{{"within the window's removal", 4, false}, {"beyond it", 7, true}} { t.Run(tc.name, func(t *testing.T) { s, _, events := watchFixture(t) now := time.Now().UTC() all := map[string]time.Time{} for i := 0; i < 20; i++ { all[snapID(i)] = now.Add(-time.Duration(48+i) * time.Hour).Truncate(time.Minute) } if err := s.Store.SetOffsiteSnapshotListing("c1", store.OffsiteSnapshotListing{ListedAt: now.Add(-24 * time.Hour), Entries: all}); err != nil { t.Fatal(err) } // The window: 20 → 15, so it explains 5 removals (min of its cap 10 and its reported drop). id, err := s.Store.OpenOffsiteWindowRow("c1", now.Add(windowLength), 20, MaxRemove(20)) if err != nil { t.Fatal(err) } if _, err := s.Store.CloseOffsiteWindowRow(id, 15, "pruned", "box"); err != nil { t.Fatal(err) } cur := map[string]time.Time{} for k, v := range all { cur[k] = v } for i := 0; i < tc.gone; i++ { delete(cur, snapID(i)) } v, _ := s.CheckSnapshotListing("c1", SnapshotListing{Files: len(cur), Entries: cur}, time.Now().UTC().Add(time.Second)) got := len(ofType(*events, EventSnapshotsVanished)) == 1 if got != tc.wantAlarm || (v > 0) != tc.wantAlarm { t.Fatalf("%d gone with a window crediting 5: alarm=%v (vanished=%d), want %v", tc.gone, got, v, tc.wantAlarm) } }) } } // A new file uploaded inside a run the box reported is explained; one uploaded at a time no run explains warns — and a // file younger than an hour is neither judged nor stored (tomorrow judges it with its run's report in hand). func TestSnapshotWatch_AppearedOutsideEveryRunWarns(t *testing.T) { s, _, events := watchFixture(t) now := time.Now().UTC() old := map[string]time.Time{snapID(0): now.Add(-72 * time.Hour).Truncate(time.Minute)} if err := s.Store.SetOffsiteSnapshotListing("c1", store.OffsiteSnapshotListing{ListedAt: now.Add(-24 * time.Hour), Entries: old}); err != nil { t.Fatal(err) } run := now.Add(-8 * time.Hour).Truncate(time.Second) reportRun(t, s.Store, run) inRun := snapID(1) // uploaded 20 minutes before the run ended fake := snapID(2) // uploaded 4 hours AFTER the run ended — no run explains it young := snapID(3) // 10 minutes old at listing time cur := map[string]time.Time{ snapID(0): old[snapID(0)], inRun: run.Add(-20 * time.Minute).Truncate(time.Minute), fake: run.Add(4 * time.Hour).Truncate(time.Minute), young: now.Add(-10 * time.Minute).Truncate(time.Minute), } _, u := s.CheckSnapshotListing("c1", SnapshotListing{Files: len(cur), Entries: cur}, now) w := ofType(*events, EventSnapshotsUnexplained) if u != 1 || len(w) != 1 { t.Fatalf("unexplained=%d events=%d, want exactly the one fake (events %+v)", u, len(w), *events) } if w[0].sev != "warning" || !strings.Contains(w[0].msg, shortID(fake)) || strings.Contains(w[0].msg, shortID(inRun)) { t.Fatalf("event = %+v — want a warning naming %s and not %s", w[0], shortID(fake), shortID(inRun)) } base, _ := s.Store.GetOffsiteSnapshotListing("c1") if _, ok := base.Entries[young]; ok { t.Fatal("a 10-minute-old file was stored in the baseline — it must be judged tomorrow, not silently accepted") } if _, ok := base.Entries[fake]; !ok { t.Fatal("the judged file is not in the new baseline — tomorrow would warn again") } } // The repository moved aside → the baseline is dropped, so the new, empty repository is a baseline and not "every // snapshot vanished". func TestSnapshotWatch_MoveAsideResetsTheBaseline(t *testing.T) { s, fs, events := watchFixture(t) now := time.Now().UTC() if err := s.Store.SetOffsiteSnapshotListing("c1", store.OffsiteSnapshotListing{ListedAt: now.Add(-24 * time.Hour), Entries: map[string]time.Time{snapID(0): now.Add(-72 * time.Hour)}}); err != nil { t.Fatal(err) } fs.dirs["/home/felhom-repo"] = true if _, err := s.MoveAside(context.Background(), "c1"); err != nil { t.Fatal(err) } if _, ok := s.Store.GetOffsiteSnapshotListing("c1"); ok { t.Fatal("the baseline survived a move-aside") } s.CheckSnapshotListing("c1", SnapshotListing{}, now) if n := len(ofType(*events, EventSnapshotsVanished)); n != 0 { t.Fatalf("an empty new repository after a move-aside raised %d vanished event(s)", n) } } // Without a confirmed append-only key the box may prune by itself: vanishing is not judged. func TestSnapshotWatch_UnpinnedTierDoesNotJudgeVanishing(t *testing.T) { s, _, _ := svcFixture(t) var events []evt s.Emit = func(_, typ, sev, msg, _, _ string) { events = append(events, evt{typ, sev, msg}) } now := time.Now().UTC() if err := s.Store.SetOffsiteSnapshotListing("c1", store.OffsiteSnapshotListing{ListedAt: now.Add(-24 * time.Hour), Entries: map[string]time.Time{snapID(0): now.Add(-72 * time.Hour), snapID(1): now.Add(-73 * time.Hour)}}); err != nil { t.Fatal(err) } s.CheckSnapshotListing("c1", SnapshotListing{Entries: map[string]time.Time{snapID(1): now.Add(-73 * time.Hour)}}, now) if n := len(ofType(events, EventSnapshotsVanished)); n != 0 { t.Fatalf("unpinned tier: %d vanished event(s), want 0", n) } } // Both events are operator-only (a missing customerMessages entry is NOT a block — the v0.78.0 defect). func TestSnapshotWatch_EventsAreOperatorOnly(t *testing.T) { for _, et := range []string{EventSnapshotsVanished, EventSnapshotsUnexplained} { if !notify.IsOperatorOnly(et) { t.Errorf("%s is not in notify.operatorOnlyEvents — it would reach a household as raw English", et) } } } func TestExplainedByRun_Bounds(t *testing.T) { r := time.Date(2026, 10, 17, 3, 0, 0, 0, time.UTC) for _, tc := range []struct { at time.Time want bool }{ {r, true}, {r.Add(snapTimeSlack), true}, {r.Add(snapTimeSlack + time.Minute), false}, {r.Add(-snapRunSpan - snapTimeSlack), true}, {r.Add(-snapRunSpan - snapTimeSlack - time.Minute), false}, {time.Time{}, false}, } { if got := explainedByRun(tc.at, []time.Time{r}); got != tc.want { t.Errorf("upload %v vs run %v: explained=%v, want %v", tc.at, r, got, tc.want) } } }