Files
felhom.eu/hub/internal/offsitekeys/ended.go
T
admin 6d2d4bd09d hub: the large request's wait is keyed on the whole app set, not its shortened display list (security review)
The key was the display string cut after ten names, so two different app sets
with the same first ten names shared one wait and one operator mail.
largeSetKey is a SHA-256 of the full sorted set; the display string is only
shown. Test seen red; 2 mutations seen red (redproof-review-s2.txt).

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-11-15 14:52:02 +01:00

375 lines
16 KiB
Go

package offsitekeys
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// D72 / D84 (`09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f) — THE HUB AGREES FIRST.
//
// A box asks, at a window's open, to remove the off-site snapshots of apps the household removed with „delete backup
// data” 30 days ago. Those removals have their own allowance, outside the window's cap — and ONLY for snapshot ids
// this file agreed to. The agreement uses nothing the box says about time:
//
// 1. THE APP. The hub's own record (store.AppPresences, written from the box's reports by the hub's clock) must say
// the app has been absent from that box's app list for EndedAppAfter. An app the newest report lists, an app the
// hub has no record of, an app absent for a shorter time and a reserved name are refused, each with its reason.
// 2. THE ID. A snapshot id is agreed only when the hub's own listing of the repository (the 07:10 audit's stored
// baseline — names and upload times, read over the hub's own login) holds it AND shows it uploaded no later than
// the moment the hub first saw the app absent. An app that is gone makes no new snapshots, so a younger file can
// not be its copy. An id the box lists under two apps is agreed only when both apps are.
//
// WHAT THIS DOES NOT PROVE, said plainly: the hub cannot read a snapshot's tags (the repository is encrypted and the
// password never leaves the box — decision 68). A box that is taken over AND has one honestly ended app can name, under
// that app, other snapshots that are older than the app's end. Rule 2 bounds that to files older than 30 days; it does
// not close it. The same box could already remove half the store per week inside the cap.
// EndedAppAfter is how long the hub must have seen an app absent before it agrees (D72: the number of a restore copy).
const EndedAppAfter = 30 * 24 * time.Hour
const (
EventEndedAppRemoved = "offsite_ended_app_removed" // info: the off-site copies of a removed app are gone (per app)
EventEndedAppRefused = "offsite_ended_app_refused" // warning: a box claimed an end the hub's records do not show
// Review of 2026-11-15 — two more operator warnings, both mailed, both hub-made (no box can POST one):
EventEndedClaimUnseen = "offsite_end_claim_unseen" // a box first names an app the hub never saw installed
EventEndedLargeAgreement = "offsite_ended_large_agreement" // one window was asked for more than half of the store
// A request for more than half of the store waits largeEndedWait after the window that first saw it; that first
// sight is remembered for largeEndedMemory (a new large request after that waits again).
largeEndedWait = 6 * 24 * time.Hour
largeEndedMemory = 30 * 24 * time.Hour
maxEndedClaims = 50 // apps in one request
maxEndedClaimIDs = 5000 // snapshot ids in one request, all apps together
)
// EndedClaim is one pending end record as the box sends it at a window's open. RemovedAt and EndsAt are the BOX's
// dates: they are stored for the operator to read and are never used to decide.
type EndedClaim struct {
App string `json:"app"`
RemovedAt string `json:"removed_at,omitempty"`
EndsAt string `json:"ends_at,omitempty"`
SnapshotIDs []string `json:"snapshot_ids"`
}
// reservedEndedName: the shares pseudo-app, the tier's marker tag, the box's own infrastructure stacks and anything
// that has not the shape of an app name are never apps a household removes (`07` §8.2f, the edge-case table). The
// same two checks decide what the hub's app record stores at all (store.NoteAppList).
func reservedEndedName(app string) bool {
return !store.ValidPresenceApp(app) || store.ReservedPresenceApp(app)
}
// agreeEnded is the PURE decision. presence is the hub's own record; listing the hub's own stored listing (ok=false
// when there is none). It returns the agreed ids (sorted), the agreed ids per agreed app, and every refusal.
func agreeEnded(now time.Time, presence map[string]store.AppPresence, listing store.OffsiteSnapshotListing, haveListing bool, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) {
if len(claims) > maxEndedClaims {
for _, c := range claims[maxEndedClaims:] {
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: fmt.Sprintf("more than %d apps in one request", maxEndedClaims)})
}
claims = claims[:maxEndedClaims]
}
agreedApp := map[string]time.Time{} // app → the hub's absent-since
refusedApp := map[string]bool{}
seen := map[string]bool{}
for _, c := range claims {
if seen[c.App] {
continue
}
seen[c.App] = true
reason := ""
p, known := presence[c.App]
switch {
case reservedEndedName(c.App):
reason = "not an app a household removes"
case !known:
reason = "the hub has no record of this app on this box"
case p.Present:
reason = "the box's newest report lists the app as installed"
case p.AbsentSince.IsZero():
reason = "the hub's record of when the app left cannot be read"
case now.Sub(p.AbsentSince) < EndedAppAfter:
reason = fmt.Sprintf("the hub has seen the app absent since %s — %d full day(s), %d needed",
p.AbsentSince.UTC().Format("2006-01-02 15:04 UTC"), int(now.Sub(p.AbsentSince).Hours()/24), int(EndedAppAfter.Hours()/24))
}
if reason != "" {
refusedApp[c.App] = true
// A claimed name that has not the shape of an app name is the box's free text: it is never written
// into the hub's ledger, an operator message or the answer (security review of 2026-11-15).
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: reason})
continue
}
agreedApp[c.App] = p.AbsentSince
}
// The ids. An id is judged against EVERY app the box listed it under.
type idState struct {
latest time.Time // the earliest absent-since among its apps: the file must be older than every one
refused bool
}
states := map[string]*idState{}
var order []string
total := 0
for _, c := range claims {
for _, id := range c.SnapshotIDs {
total++
if total > maxEndedClaimIDs {
break
}
st, ok := states[id]
if !ok {
st = &idState{}
states[id] = st
order = append(order, id)
}
since, agreed := agreedApp[c.App]
if !agreed {
st.refused = true // listed under an app the hub refused: a living app's copy is never removed
continue
}
if st.latest.IsZero() || since.Before(st.latest) {
st.latest = since
}
}
}
byApp = map[string][]string{}
idRefusals := map[string]int{}
for _, id := range order {
st := states[id]
switch {
case st.refused:
idRefusals["listed under an app the hub refused"]++
continue
case !haveListing:
idRefusals["the hub has no listing of its own for this repository yet"]++
continue
}
at, listed := listing.Entries[id]
switch {
case !snapshotID.MatchString(id) || !listed:
idRefusals["not in the hub's own listing"]++
case at.IsZero():
idRefusals["no upload time in the hub's own listing"]++
case at.After(st.latest.Add(snapTimeSlack)):
idRefusals["uploaded after the hub saw the app leave"]++
default:
ids = append(ids, id)
for _, c := range claims {
if _, ok := agreedApp[c.App]; ok {
for _, cid := range c.SnapshotIDs {
if cid == id {
byApp[c.App] = append(byApp[c.App], id)
}
}
}
}
}
}
if total > maxEndedClaimIDs {
idRefusals[fmt.Sprintf("more than %d snapshot ids in one request", maxEndedClaimIDs)] += total - maxEndedClaimIDs
}
var reasons []string
for r := range idRefusals {
reasons = append(reasons, r)
}
sort.Strings(reasons)
for _, r := range reasons {
refused = append(refused, store.EndedRefusal{App: "", Reason: fmt.Sprintf("%d snapshot id(s) refused: %s", idRefusals[r], r)})
}
sort.Strings(ids)
return ids, byApp, refused
}
// clearDisagreement: a refusal the operator must hear about. A box that asks a little early (its clock is minutes or
// hours ahead of the hub's first sight of the absence) is refused quietly and asks again next week; a box that claims
// an end for an app it reports as installed, or that the hub has seen absent for less than 29 days, is not a matter of
// minutes — its clock is wrong by a day or more (R-1384), or it is not telling the truth.
func clearDisagreement(now time.Time, presence map[string]store.AppPresence, r store.EndedRefusal) bool {
if r.App == "" {
return false
}
p, known := presence[r.App]
// An installed app has no absence date (store.AppPresences), so the zero date covers it too.
if !known || p.AbsentSince.IsZero() {
return true
}
return now.Sub(p.AbsentSince) < EndedAppAfter-24*time.Hour
}
// agreeEndedFor runs the decision for one household against the store, and tells the operator about a clear
// disagreement. It never fails a window: with a store that cannot answer, nothing is agreed.
func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) {
if len(claims) == 0 {
return nil, nil, nil
}
now := s.now()
presence, err := s.Store.AppPresences(customerID)
if err != nil {
s.logf("[WARN] offsitekeys: ended apps %s: the hub's own app record could not be read (%v) — nothing agreed", customerID, err)
for _, c := range claims {
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: "the hub's own app record could not be read"})
}
return nil, nil, refused
}
listing, have := s.Store.GetOffsiteSnapshotListing(customerID)
ids, byApp, refused = agreeEnded(now, presence, listing, have, claims)
// Review of 2026-11-15 — what bounds a box that invents an ended app, or stops listing a living one, and 30 days
// later names much of the store under it:
//
// (c) A request for more than HALF of the store's snapshots WAITS ONE WEEK: the first window that sees it agrees
// to nothing for removed apps and tells the operator (a mail); a window at least largeEndedWait later agrees.
// So the operator always has a week between the mail and the first removal. An honest household that
// removed its one big app looks the same and loses one week.
// (a) ONE window agrees to at most the cap's own size (boundEndedToCap); the rest waits for the next window.
if total := len(listing.Entries); have && len(ids)*2 > total {
// The week belongs to THESE apps: a large request for other apps waits its own week and gets its own mail.
// The first sight is the hub's own record (store.offsite_large_requests) — never a search of text a box
// could have written.
key := largeSetKey(byApp)
first := s.Store.LargeEndedRequestFirstSeen(customerID, key, now.Add(-largeEndedMemory))
if first.IsZero() || now.Sub(first) < largeEndedWait {
if first.IsZero() {
if nerr := s.Store.NoteLargeEndedRequest(customerID, key, now); nerr != nil {
// Not recorded → the next window sees it as new, tells again and waits again: the safe direction.
s.logf("[WARN] offsitekeys: ended apps %s: the first sight of a large request could not be recorded (%v) — it will wait again", customerID, nerr)
}
s.event(customerID, EventEndedLargeAgreement, "warning",
fmt.Sprintf("Off-site clean-up window %d: the box of %s asked to remove %d of %d off-site snapshot(s) as copies of removed apps (%s) — more than half of the store. "+
"NOTHING of them is removed in this window. The hub's own records agree to each app, so from %s a window removes them, at most %d per window. "+
"If the household did not remove these apps, switch the weekly windows off for it before that day and look at the box.",
windowID, customerID, len(ids), total, appNames(byApp), now.Add(largeEndedWait).UTC().Format("2006-01-02"), endedCap(maxRemove, total)),
map[string]any{"subject": "large:" + key[:16], "window_id": windowID, "asked": len(ids), "listed": total})
}
refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("the request is for %d of %d snapshots — more than half of the store: it waits one week, and the operator is told",
len(ids), total)})
ids, byApp = nil, nil
}
}
if have && len(ids) > 0 {
var waits int
ids, byApp, waits = boundEndedToCap(ids, byApp, listing, endedCap(maxRemove, len(listing.Entries)))
if waits > 0 {
refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("%d snapshot id(s) wait for the next window: one window removes at most %d snapshots of removed apps (the size of its own cap)",
waits, endedCap(maxRemove, len(listing.Entries)))})
}
}
s.logf("[INFO] offsitekeys: ended apps %s: %d app(s) claimed, %d app(s) with agreed snapshots, %d snapshot id(s) agreed, %d refusal(s)",
customerID, len(claims), len(byApp), len(ids), len(refused))
for _, r := range refused {
s.logf("[WARN] offsitekeys: ended apps %s: REFUSED %q: %s", customerID, r.App, r.Reason)
if clearDisagreement(now, presence, r) {
s.event(customerID, EventEndedAppRefused, "warning",
fmt.Sprintf("Off-site clean-up: the box of %s asked to remove the off-site copies of the removed app %q, and the hub's own records do not agree: %s. Nothing of that app was removed. Check the box's clock and its app list.",
customerID, r.App, r.Reason),
map[string]any{"subject": r.App, "app": r.App, "reason": r.Reason})
}
}
return ids, byApp, refused
}
// endedRemovedByApp groups the removed ids by the app the box listed them under, for the per-app operator record.
func endedRemovedByApp(claims map[string][]string, removed []string) map[string]int {
gone := make(map[string]bool, len(removed))
for _, id := range removed {
gone[id] = true
}
out := map[string]int{}
for app, ids := range claims {
for _, id := range ids {
if gone[id] {
out[app]++
}
}
}
return out
}
// endedCap is the most ended-app snapshots ONE window may agree to: the size of the window's own cap, and never more
// than the cap the hub's OWN count of the store would give (the window's cap comes from the box's count_before, which
// is the box's word). Decided by CC unattended on 2026-11-15, a narrowing of D84 — the operator may reverse it
// (`07-backup-architecture.md` §8.2f).
func endedCap(maxRemove, listed int) int {
c := MaxRemove(listed)
if maxRemove > 0 && maxRemove < c {
c = maxRemove
}
return c
}
// boundEndedToCap keeps at most limit of the agreed ids — the OLDEST uploads first (the newest copies of a removed
// app are the last to go) — and returns how many wait.
func boundEndedToCap(ids []string, byApp map[string][]string, listing store.OffsiteSnapshotListing, limit int) ([]string, map[string][]string, int) {
if len(ids) <= limit {
return ids, byApp, 0
}
sorted := append([]string(nil), ids...)
sort.Slice(sorted, func(i, j int) bool {
ti, tj := listing.Entries[sorted[i]], listing.Entries[sorted[j]]
if !ti.Equal(tj) {
return ti.Before(tj)
}
return sorted[i] < sorted[j]
})
keep := make(map[string]bool, limit)
for _, id := range sorted[:limit] {
keep[id] = true
}
var out []string
for _, id := range ids {
if keep[id] {
out = append(out, id)
}
}
apps := map[string][]string{}
for app, list := range byApp {
for _, id := range list {
if keep[id] {
apps[app] = append(apps[app], id)
}
}
}
return out, apps, len(ids) - limit
}
// appNames lists the agreed apps for a message (names already checked: store.ValidPresenceApp).
func appNames(byApp map[string][]string) string {
names := make([]string, 0, len(byApp))
for app := range byApp {
names = append(names, app)
}
sort.Strings(names)
if len(names) > 10 {
return strings.Join(names[:10], ", ") + fmt.Sprintf(" and %d more", len(names)-10)
}
return strings.Join(names, ", ")
}
// largeSetKey is the key of an app set in the hub's own record of large requests: a hash of the WHOLE sorted set.
// Never the display string (appNames), which is cut after ten names — two sets that differ after the tenth name
// would share one wait (security review of 2026-11-15).
func largeSetKey(byApp map[string][]string) string {
names := make([]string, 0, len(byApp))
for app := range byApp {
names = append(names, app)
}
sort.Strings(names)
sum := sha256.Sum256([]byte(strings.Join(names, "\n")))
return hex.EncodeToString(sum[:])
}
// ledgerName is a claimed name as the hub may write it down: itself when it has the shape of an app name, else a
// fixed placeholder.
func ledgerName(app string) string {
if store.ValidPresenceApp(app) {
return app
}
return notAnAppName
}
const notAnAppName = "(not-an-app-name)"