felhom.eu/hub/internal/offsitekeys/snapshot_watch.go

package offsitekeys

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-895 SLICE 2 — THE ALARMS (hub v0.158.0; design `audits/agent-allowlist-rotation-2026-10-17/P4/design-R-895.md`
// §3.2–3.3, origin `audits/night-burndown-2026-10-06/design-R-822.md` option B).
//
// The daily key audit (07:10) already lists the box's restic `snapshots/` over the hub's own login (slice 1). This
// compares today's listing with yesterday's, by NAME, and raises two operator-only events:
//
//   - offsite_snapshots_vanished (error): snapshot files are gone that no clean-up window explains. The box's key is
//     append-only (decision 69); only a hub-opened window may delete, and each window explains at most what the hub
//     let it remove (store.WindowCreditsBetween — the same credits R-435's count detector spends). Unlike R-431/R-435
//     this does NOT trust the box's count: the names are the hub's own read.
//   - offsite_snapshots_unexplained (warning): snapshot files appeared whose upload time is not near any off-site run
//     the box reported (`offsite.last_run` in its controller reports). That is the injection signature — fakes added
//     through the add-only key outside a run.
//
// FALSE-ALARM GUARDS, each a boring reading written down:
//   - First listing (or a baseline that cannot be read): it becomes the baseline; no verdict.
//   - A set-aside or purged repository clears the baseline (MoveAside, PurgeRepos) — the next listing is a new one.
//   - Vanishing is judged only on a PINNED tier (a confirmed append-only key): without it the box may still prune by
//     itself, and the daily key check already alarms on the deleting line.
//   - A file younger than settleAge at listing time is not judged and not stored; tomorrow it is judged with its run's
//     report in hand.
//   - The provider shell's time zone is NOT measured (slice 1's log is the measurement): a run explains a file
//     uploaded in [last_run − runSpan − timeSlack, last_run + timeSlack]. timeSlack (3 h) covers UTC vs CET/CEST and
//     ls's minute truncation; runSpan (6 h) covers a long run (the box writes last_run at the run's END). A first
//     upload longer than 6 h can therefore warn once.
//   - A failed listing leaves the baseline unchanged; unreadable run reports leave the new files out of it (judged
//     again tomorrow) — never a verdict on half data.
//
// NOT built (design §3.1 and the "refuse the next window" half of §3.2): the window's before/after on the hub's own
// count, and refusing a window while unexplained files exist. The second changes what a household's prune does on a
// warning nobody has yet seen fire on a real box; it waits for the first real listings.

const (
	EventSnapshotsVanished    = "offsite_snapshots_vanished"    // error: files gone that no clean-up window explains
	EventSnapshotsUnexplained = "offsite_snapshots_unexplained" // warning: files no reported box run explains

	snapRunSpan     = 6 * time.Hour
	snapTimeSlack   = 3 * time.Hour
	snapSettleAge   = time.Hour
	snapWindowSlack = 2 * time.Hour // as monitor.windowSlack: a window just before the previous listing still counts
)

// shortID is a snapshot id as restic prints it (8 characters) — enough for the operator to find it, short in a mail.
func shortID(id string) string {
	if len(id) > 8 {
		return id[:8]
	}
	return id
}

// explainedByRun reports whether a file uploaded at t falls inside some reported run's span.
func explainedByRun(t time.Time, runs []time.Time) bool {
	if t.IsZero() {
		return false
	}
	for _, r := range runs {
		if !t.Before(r.Add(-snapRunSpan-snapTimeSlack)) && !t.After(r.Add(snapTimeSlack)) {
			return true
		}
	}
	return false
}

// CheckSnapshotListing compares the hub's listing l (taken at now) with the stored baseline, raises the events, and
// stores the new baseline. It returns the counts it judged, for the log and the tests.
func (s *Service) CheckSnapshotListing(customerID string, l SnapshotListing, now time.Time) (vanished, unexplained int) {
	now = now.UTC()
	next := store.OffsiteSnapshotListing{ListedAt: now, Entries: map[string]time.Time{}}
	deferred := 0
	for id, at := range l.Entries {
		if !at.IsZero() && at.After(now.Add(-snapSettleAge)) {
			deferred++ // judged tomorrow, with its run's report in hand
			continue
		}
		next.Entries[id] = at
	}

	prev, ok := s.Store.GetOffsiteSnapshotListing(customerID)
	if !ok {
		if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil {
			s.logf("[WARN] offsitekeys: snapshots %s: could not store the first listing: %v", customerID, err)
			return 0, 0
		}
		s.logf("[INFO] offsitekeys: snapshots %s: first listing stored as the baseline (%d file(s), %d too new to judge) — no verdict", customerID, len(next.Entries), deferred)
		return 0, 0
	}

	var gone, came []string
	for id := range prev.Entries {
		if _, still := l.Entries[id]; !still {
			gone = append(gone, id)
		}
	}
	for id := range next.Entries {
		if _, had := prev.Entries[id]; !had {
			came = append(came, id)
		}
	}
	sort.Strings(gone)
	sort.Strings(came)

	// ── Vanished ────────────────────────────────────────────────────────────────────────────────────
	if len(gone) > 0 {
		k, kerr := s.Store.GetOffsiteKey(customerID)
		switch {
		case kerr != nil || k == nil || k.ConfirmedAt.IsZero():
			s.logf("[INFO] offsitekeys: snapshots %s: %d file(s) gone since %s — not judged: no confirmed append-only key (the box may still prune by itself)",
				customerID, len(gone), prev.ListedAt.Format(time.RFC3339))
		default:
			credits, unknown := s.Store.WindowCreditsBetween(customerID, prev.ListedAt.Add(-snapWindowSlack), now)
			explains := 0
			for _, c := range credits {
				explains += c.Explains
			}
			if unknown || len(gone) > explains {
				vanished = len(gone)
				why := fmt.Sprintf("the clean-up windows in that time explain at most %d", explains)
				if unknown {
					why = "the clean-up windows in that time could not be read (judged as unexplained)"
				}
				s.logf("[ERROR] offsitekeys: snapshots %s: %d snapshot file(s) VANISHED since %s, %s",
					customerID, len(gone), prev.ListedAt.Format(time.RFC3339), why)
				s.event(customerID, EventSnapshotsVanished, "error",
					fmt.Sprintf("Off-site: the hub's own listing of %s's repository shows %d snapshot file(s) gone since %s; %s. "+
						"The box's key cannot delete outside a window. Check the clean-up window ledger and the box before any restore: %s",
						customerID, len(gone), prev.ListedAt.Format("2006-01-02 15:04 UTC"), why, idList(gone)),
					map[string]any{"vanished": shortIDs(gone), "previous_listing": prev.ListedAt, "window_credit": explains, "windows_unknown": unknown})
			} else {
				s.logf("[INFO] offsitekeys: snapshots %s: %d file(s) gone since %s, explained by clean-up windows (up to %d)",
					customerID, len(gone), prev.ListedAt.Format(time.RFC3339), explains)
			}
		}
	}

	// ── Appeared, unexplained ───────────────────────────────────────────────────────────────────────
	if len(came) > 0 {
		runs, rerr := s.Store.OffsiteRunTimesSince(customerID, prev.ListedAt.Add(-24*time.Hour))
		if rerr != nil {
			// The vanish verdict above stands; the new files are left OUT of the stored baseline so tomorrow judges
			// them, instead of a verdict on half data (and without repeating the vanish alarm).
			for _, id := range came {
				delete(next.Entries, id)
			}
			s.logf("[WARN] offsitekeys: snapshots %s: could not read the box's reported runs (%v) — %d new file(s) judged again tomorrow", customerID, rerr, len(came))
			if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil {
				s.logf("[WARN] offsitekeys: snapshots %s: could not store the listing: %v", customerID, err)
			}
			return vanished, 0
		}
		var odd []string
		for _, id := range came {
			if !explainedByRun(next.Entries[id], runs) {
				odd = append(odd, id)
			}
		}
		if len(odd) > 0 {
			unexplained = len(odd)
			var parts []string
			for _, id := range odd {
				at := "unknown time"
				if t := next.Entries[id]; !t.IsZero() {
					at = t.Format("2006-01-02 15:04")
				}
				parts = append(parts, shortID(id)+" ("+at+")")
			}
			s.logf("[WARN] offsitekeys: snapshots %s: %d new snapshot file(s) uploaded outside every reported box run (%d run(s) reported since %s): %s",
				customerID, len(odd), len(runs), prev.ListedAt.Add(-24*time.Hour).Format(time.RFC3339), strings.Join(parts, ", "))
			s.event(customerID, EventSnapshotsUnexplained, "warning",
				fmt.Sprintf("Off-site: %d new snapshot file(s) in %s's repository were uploaded when the box reported no off-site run: %s. "+
					"Fakes added with the box's key would look like this; a crashed run would too. Look at the box before the next clean-up window.",
					len(odd), customerID, strings.Join(parts, ", ")),
				map[string]any{"unexplained": shortIDs(odd), "runs_reported": len(runs), "previous_listing": prev.ListedAt})
		} else {
			s.logf("[INFO] offsitekeys: snapshots %s: %d new snapshot file(s), each inside a reported box run", customerID, len(came))
		}
	}

	if err := s.Store.SetOffsiteSnapshotListing(customerID, next); err != nil {
		s.logf("[WARN] offsitekeys: snapshots %s: could not store the listing: %v", customerID, err)
	}
	return vanished, unexplained
}

func shortIDs(ids []string) []string {
	out := make([]string, 0, len(ids))
	for _, id := range ids {
		out = append(out, shortID(id))
	}
	return out
}

// idList names at most 20 ids in a message; the details carry them all.
func idList(ids []string) string {
	s := shortIDs(ids)
	if len(s) > 20 {
		return strings.Join(s[:20], ", ") + fmt.Sprintf(" and %d more", len(s)-20)
	}
	return strings.Join(s, ", ")
}