package offsitekeys import ( "fmt" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-895 SLICE 2 — THE ALARMS (hub v0.158.0; design `audits/agent-allowlist-rotation-2026-10-17/P4/design-R-895.md` // §3.2–3.3, origin `audits/night-burndown-2026-10-06/design-R-822.md` option B). // // The daily key audit (07:10) already lists the box's restic `snapshots/` over the hub's own login (slice 1). This // compares today's listing with yesterday's, by NAME, and raises two operator-only events: // // - offsite_snapshots_vanished (error): snapshot files are gone that no clean-up window explains. The box's key is // append-only (decision 69); only a hub-opened window may delete, and each window explains at most what the hub // let it remove (store.WindowCreditsBetween — the same credits R-435's count detector spends). Unlike R-431/R-435 // this does NOT trust the box's count: the names are the hub's own read. // - offsite_snapshots_unexplained (warning): snapshot files appeared whose upload time is not near any off-site run // the box reported (`offsite.last_run` in its controller reports). That is the injection signature — fakes added // through the add-only key outside a run. // // FALSE-ALARM GUARDS, each a boring reading written down: // - First listing (or a baseline that cannot be read): it becomes the baseline; no verdict. // - A set-aside or purged repository clears the baseline (MoveAside, PurgeRepos) — the next listing is a new one. // - Vanishing is judged only on a PINNED tier (a confirmed append-only key): without it the box may still prune by // itself, and the daily key check already alarms on the deleting line. // - A file younger than settleAge at listing time is not judged and not stored; tomorrow it is judged with its run's // report in hand. // - The provider shell's time zone is NOT measured (slice 1's log is the measurement): a run explains a file // uploaded in [last_run − runSpan − timeSlack, last_run + timeSlack]. timeSlack (3 h) covers UTC vs CET/CEST and // ls's minute truncation; runSpan (6 h) covers a long run (the box writes last_run at the run's END). A first // upload longer than 6 h can therefore warn once. // - A failed listing leaves the baseline unchanged; unreadable run reports leave the new files out of it (judged // again tomorrow) — never a verdict on half data. // // NOT built (design §3.1 and the "refuse the next window" half of §3.2): the window's before/after on the hub's own // count, and refusing a window while unexplained files exist. The second changes what a household's prune does on a // warning nobody has yet seen fire on a real box; it waits for the first real listings. const ( EventSnapshotsVanished = "offsite_snapshots_vanished" // error: files gone that no clean-up window explains EventSnapshotsUnexplained = "offsite_snapshots_unexplained" // warning: files no reported box run explains snapRunSpan = 6 * time.Hour snapTimeSlack = 3 * time.Hour snapSettleAge = time.Hour snapWindowSlack = 2 * time.Hour // as monitor.windowSlack: a window just before the previous listing still counts ) // shortID is a snapshot id as restic prints it (8 characters) — enough for the operator to find it, short in a mail. func shortID(id string) string { if len(id) > 8 { return id[:8] } return id } // explainedByRun reports whether a file uploaded at t falls inside some reported run's span. func explainedByRun(t time.Time, runs []time.Time) bool { if t.IsZero() { return false } for _, r := range runs { if !t.Before(r.Add(-snapRunSpan-snapTimeSlack)) && !t.After(r.Add(snapTimeSlack)) { return true } } return false } // CheckSnapshotListing compares the hub's listing l (taken at now) with the stored baseline, raises the events, and // stores the new baseline. It returns the counts it judged, for the log and the tests. func (s *Service) CheckSnapshotListing(customerID string, l SnapshotListing, now time.Time) (vanished, unexplained int) { now = now.UTC() next := store.OffsiteSnapshotListing{ListedAt: now, Entries: map[string]time.Time{}} deferred := 0 for id, at := range l.Entries { if !at.IsZero() && at.After(now.Add(-snapSettleAge)) { deferred++ // judged tomorrow, with its run's report in hand continue } next.Entries[id] = at } prev, ok := s.Store.GetOffsiteSnapshotListing(customerID) if !ok { if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil { s.logf("[WARN] offsitekeys: snapshots %s: could not store the first listing: %v", customerID, err) return 0, 0 } s.logf("[INFO] offsitekeys: snapshots %s: first listing stored as the baseline (%d file(s), %d too new to judge) — no verdict", customerID, len(next.Entries), deferred) return 0, 0 } var gone, came []string for id := range prev.Entries { if _, still := l.Entries[id]; !still { gone = append(gone, id) } } for id := range next.Entries { if _, had := prev.Entries[id]; !had { came = append(came, id) } } sort.Strings(gone) sort.Strings(came) // ── Vanished ──────────────────────────────────────────────────────────────────────────────────── if len(gone) > 0 { k, kerr := s.Store.GetOffsiteKey(customerID) switch { case kerr != nil || k == nil || k.ConfirmedAt.IsZero(): s.logf("[INFO] offsitekeys: snapshots %s: %d file(s) gone since %s — not judged: no confirmed append-only key (the box may still prune by itself)", customerID, len(gone), prev.ListedAt.Format(time.RFC3339)) default: credits, unknown := s.Store.WindowCreditsBetween(customerID, prev.ListedAt.Add(-snapWindowSlack), now) explains := 0 for _, c := range credits { explains += c.Explains } if unknown || len(gone) > explains { vanished = len(gone) why := fmt.Sprintf("the clean-up windows in that time explain at most %d", explains) if unknown { why = "the clean-up windows in that time could not be read (judged as unexplained)" } s.logf("[ERROR] offsitekeys: snapshots %s: %d snapshot file(s) VANISHED since %s, %s", customerID, len(gone), prev.ListedAt.Format(time.RFC3339), why) s.event(customerID, EventSnapshotsVanished, "error", fmt.Sprintf("Off-site: the hub's own listing of %s's repository shows %d snapshot file(s) gone since %s; %s. "+ "The box's key cannot delete outside a window. Check the clean-up window ledger and the box before any restore: %s", customerID, len(gone), prev.ListedAt.Format("2006-01-02 15:04 UTC"), why, idList(gone)), map[string]any{"vanished": shortIDs(gone), "previous_listing": prev.ListedAt, "window_credit": explains, "windows_unknown": unknown}) } else { s.logf("[INFO] offsitekeys: snapshots %s: %d file(s) gone since %s, explained by clean-up windows (up to %d)", customerID, len(gone), prev.ListedAt.Format(time.RFC3339), explains) } } } // ── Appeared, unexplained ─────────────────────────────────────────────────────────────────────── if len(came) > 0 { runs, rerr := s.Store.OffsiteRunTimesSince(customerID, prev.ListedAt.Add(-24*time.Hour)) if rerr != nil { // The vanish verdict above stands; the new files are left OUT of the stored baseline so tomorrow judges // them, instead of a verdict on half data (and without repeating the vanish alarm). for _, id := range came { delete(next.Entries, id) } s.logf("[WARN] offsitekeys: snapshots %s: could not read the box's reported runs (%v) — %d new file(s) judged again tomorrow", customerID, rerr, len(came)) if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil { s.logf("[WARN] offsitekeys: snapshots %s: could not store the listing: %v", customerID, err) } return vanished, 0 } var odd []string for _, id := range came { if !explainedByRun(next.Entries[id], runs) { odd = append(odd, id) } } if len(odd) > 0 { unexplained = len(odd) var parts []string for _, id := range odd { at := "unknown time" if t := next.Entries[id]; !t.IsZero() { at = t.Format("2006-01-02 15:04") } parts = append(parts, shortID(id)+" ("+at+")") } s.logf("[WARN] offsitekeys: snapshots %s: %d new snapshot file(s) uploaded outside every reported box run (%d run(s) reported since %s): %s", customerID, len(odd), len(runs), prev.ListedAt.Add(-24*time.Hour).Format(time.RFC3339), strings.Join(parts, ", ")) s.event(customerID, EventSnapshotsUnexplained, "warning", fmt.Sprintf("Off-site: %d new snapshot file(s) in %s's repository were uploaded when the box reported no off-site run: %s. "+ "Fakes added with the box's key would look like this; a crashed run would too. Look at the box before the next clean-up window.", len(odd), customerID, strings.Join(parts, ", ")), map[string]any{"unexplained": shortIDs(odd), "runs_reported": len(runs), "previous_listing": prev.ListedAt}) } else { s.logf("[INFO] offsitekeys: snapshots %s: %d new snapshot file(s), each inside a reported box run", customerID, len(came)) } } if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil { s.logf("[WARN] offsitekeys: snapshots %s: could not store the listing: %v", customerID, err) } return vanished, unexplained } func shortIDs(ids []string) []string { out := make([]string, 0, len(ids)) for _, id := range ids { out = append(out, shortID(id)) } return out } // idList names at most 20 ids in a message; the details carry them all. func idList(ids []string) string { s := shortIDs(ids) if len(s) > 20 { return strings.Join(s[:20], ", ") + fmt.Sprintf(" and %d more", len(s)-20) } return strings.Join(s, ", ") }