The hub agrees to remove a removed app's off-site copies only when its OWN record (offsite_end_notices) says the household was told at least 30 days ago, by the hub's clock — beside the 30 days of absence. The notice is sent from the box's report, at every report until the mail service accepts it; the box's event mails nobody. No address or a blocked household: the operator is mailed in its place and that mail starts the 30 days. A notice that cannot go out: no agreement, the window's answer names the reason, one line to the operator after 3 days. This commit also supplies the files that handler.go (c3fa43a4) already calls (store.NoteEndNotices, api.kickEndNotices) and the tests that replace TestD84_Receipt_* (their files left the tree inb8cf6c24). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
428 lines
19 KiB
Go
428 lines
19 KiB
Go
package offsitekeys
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// D72 / D84 (`09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f) — THE HUB AGREES FIRST.
|
|
//
|
|
// A box asks, at a window's open, to remove the off-site snapshots of apps the household removed with „delete backup
|
|
// data” 30 days ago. Those removals have their own allowance, outside the window's cap — and ONLY for snapshot ids
|
|
// this file agreed to. The agreement uses nothing the box says about time:
|
|
//
|
|
// 1. THE APP. The hub's own record (store.AppPresences, written from the box's reports by the hub's clock) must say
|
|
// the app has been absent from that box's app list for EndedAppAfter. An app the newest report lists, an app the
|
|
// hub has no record of, an app absent for a shorter time and a reserved name are refused, each with its reason.
|
|
// 1b. THE NOTICE (`09` §3 decision 315 — „no notice, no deletion”). The hub's own record (store.EndNotices, written
|
|
// only by the hub when its mail service accepted the mail) must say the household — or, for a household with no
|
|
// usable address, the operator — was told about THIS press EndedAppAfter ago, by the hub's clock. Beside the 30
|
|
// days of absence, not in place of them. No record, a record not told, a record told later: refused, with the
|
|
// reason. So an app a box names only in a window request (never in a report) is never agreed.
|
|
// 2. THE ID. A snapshot id is agreed only when the hub's own listing of the repository (the 07:10 audit's stored
|
|
// baseline — names and upload times, read over the hub's own login) holds it AND shows it uploaded no later than
|
|
// the moment the hub first saw the app absent. An app that is gone makes no new snapshots, so a younger file can
|
|
// not be its copy. An id the box lists under two apps is agreed only when both apps are.
|
|
//
|
|
// WHAT THIS DOES NOT PROVE, said plainly: the hub cannot read a snapshot's tags (the repository is encrypted and the
|
|
// password never leaves the box — decision 68). A box that is taken over AND has one honestly ended app can name, under
|
|
// that app, other snapshots that are older than the app's end. Rule 2 bounds that to files older than 30 days; it does
|
|
// not close it. The same box could already remove half the store per week inside the cap.
|
|
|
|
// EndedAppAfter is how long the hub must have seen an app absent before it agrees (D72: the number of a restore copy).
|
|
const EndedAppAfter = 30 * 24 * time.Hour
|
|
|
|
const (
|
|
EventEndedAppRemoved = "offsite_ended_app_removed" // info: the off-site copies of a removed app are gone (per app)
|
|
EventEndedAppRefused = "offsite_ended_app_refused" // warning: a box claimed an end the hub's records do not show
|
|
// Review of 2026-11-15 — two more operator warnings, both mailed, both hub-made (no box can POST one):
|
|
EventEndedClaimUnseen = "offsite_end_claim_unseen" // a box first names an app the hub never saw installed
|
|
EventEndedLargeAgreement = "offsite_ended_large_agreement" // one window was asked for more than half of the store
|
|
// The notice lock (`09` §3 decision 315) — two operator warnings, both mailed, both hub-made (api/end_notice.go):
|
|
EventEndNoticeOperator = "offsite_end_notice_operator" // told in place of a household with no usable address
|
|
EventEndNoticeStuck = "offsite_end_notice_stuck" // a notice has told nobody for three days
|
|
|
|
// A request for more than half of the store waits largeEndedWait after the window that first saw it; that first
|
|
// sight is remembered for largeEndedMemory (a new large request after that waits again).
|
|
largeEndedWait = 6 * 24 * time.Hour
|
|
largeEndedMemory = 30 * 24 * time.Hour
|
|
|
|
maxEndedClaims = 50 // apps in one request
|
|
maxEndedClaimIDs = 5000 // snapshot ids in one request, all apps together
|
|
)
|
|
|
|
// EndedClaim is one pending end record as the box sends it at a window's open. RemovedAt and EndsAt are the BOX's
|
|
// dates: they are stored for the operator to read and are never used to decide.
|
|
type EndedClaim struct {
|
|
App string `json:"app"`
|
|
RemovedAt string `json:"removed_at,omitempty"`
|
|
EndsAt string `json:"ends_at,omitempty"`
|
|
SnapshotIDs []string `json:"snapshot_ids"`
|
|
}
|
|
|
|
// reservedEndedName: the shares pseudo-app, the tier's marker tag, the box's own infrastructure stacks and anything
|
|
// that has not the shape of an app name are never apps a household removes (`07` §8.2f, the edge-case table). The
|
|
// same two checks decide what the hub's app record stores at all (store.NoteAppList).
|
|
func reservedEndedName(app string) bool {
|
|
return !store.ValidPresenceApp(app) || store.ReservedPresenceApp(app)
|
|
}
|
|
|
|
// agreeEnded is the PURE decision. presence is the hub's own record; listing the hub's own stored listing (ok=false
|
|
// when there is none). It returns the agreed ids (sorted), the agreed ids per agreed app, and every refusal.
|
|
func agreeEnded(now time.Time, presence map[string]store.AppPresence, notices map[string]store.EndNotice, listing store.OffsiteSnapshotListing, haveListing bool, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) {
|
|
if len(claims) > maxEndedClaims {
|
|
for _, c := range claims[maxEndedClaims:] {
|
|
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: fmt.Sprintf("more than %d apps in one request", maxEndedClaims)})
|
|
}
|
|
claims = claims[:maxEndedClaims]
|
|
}
|
|
agreedApp := map[string]time.Time{} // app → the hub's absent-since
|
|
refusedApp := map[string]bool{}
|
|
seen := map[string]bool{}
|
|
for _, c := range claims {
|
|
if seen[c.App] {
|
|
continue
|
|
}
|
|
seen[c.App] = true
|
|
reason := ""
|
|
p, known := presence[c.App]
|
|
switch {
|
|
case reservedEndedName(c.App):
|
|
reason = "not an app a household removes"
|
|
case !known:
|
|
reason = "the hub has no record of this app on this box"
|
|
case p.Present:
|
|
reason = "the box's newest report lists the app as installed"
|
|
case p.AbsentSince.IsZero():
|
|
reason = "the hub's record of when the app left cannot be read"
|
|
case now.Sub(p.AbsentSince) < EndedAppAfter:
|
|
reason = fmt.Sprintf("the hub has seen the app absent since %s — %d full day(s), %d needed",
|
|
p.AbsentSince.UTC().Format("2006-01-02 15:04 UTC"), int(now.Sub(p.AbsentSince).Hours()/24), int(EndedAppAfter.Hours()/24))
|
|
}
|
|
if reason == "" {
|
|
reason = untoldReason(now, notices, c.App)
|
|
}
|
|
if reason != "" {
|
|
refusedApp[c.App] = true
|
|
// A claimed name that has not the shape of an app name is the box's free text: it is never written
|
|
// into the hub's ledger, an operator message or the answer (security review of 2026-11-15).
|
|
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: reason})
|
|
continue
|
|
}
|
|
agreedApp[c.App] = p.AbsentSince
|
|
}
|
|
|
|
// The ids. An id is judged against EVERY app the box listed it under.
|
|
type idState struct {
|
|
latest time.Time // the earliest absent-since among its apps: the file must be older than every one
|
|
refused bool
|
|
}
|
|
states := map[string]*idState{}
|
|
var order []string
|
|
total := 0
|
|
for _, c := range claims {
|
|
for _, id := range c.SnapshotIDs {
|
|
total++
|
|
if total > maxEndedClaimIDs {
|
|
break
|
|
}
|
|
st, ok := states[id]
|
|
if !ok {
|
|
st = &idState{}
|
|
states[id] = st
|
|
order = append(order, id)
|
|
}
|
|
since, agreed := agreedApp[c.App]
|
|
if !agreed {
|
|
st.refused = true // listed under an app the hub refused: a living app's copy is never removed
|
|
continue
|
|
}
|
|
if st.latest.IsZero() || since.Before(st.latest) {
|
|
st.latest = since
|
|
}
|
|
}
|
|
}
|
|
byApp = map[string][]string{}
|
|
idRefusals := map[string]int{}
|
|
for _, id := range order {
|
|
st := states[id]
|
|
switch {
|
|
case st.refused:
|
|
idRefusals["listed under an app the hub refused"]++
|
|
continue
|
|
case !haveListing:
|
|
idRefusals["the hub has no listing of its own for this repository yet"]++
|
|
continue
|
|
}
|
|
at, listed := listing.Entries[id]
|
|
switch {
|
|
case !snapshotID.MatchString(id) || !listed:
|
|
idRefusals["not in the hub's own listing"]++
|
|
case at.IsZero():
|
|
idRefusals["no upload time in the hub's own listing"]++
|
|
case at.After(st.latest.Add(snapTimeSlack)):
|
|
idRefusals["uploaded after the hub saw the app leave"]++
|
|
default:
|
|
ids = append(ids, id)
|
|
for _, c := range claims {
|
|
if _, ok := agreedApp[c.App]; ok {
|
|
for _, cid := range c.SnapshotIDs {
|
|
if cid == id {
|
|
byApp[c.App] = append(byApp[c.App], id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if total > maxEndedClaimIDs {
|
|
idRefusals[fmt.Sprintf("more than %d snapshot ids in one request", maxEndedClaimIDs)] += total - maxEndedClaimIDs
|
|
}
|
|
var reasons []string
|
|
for r := range idRefusals {
|
|
reasons = append(reasons, r)
|
|
}
|
|
sort.Strings(reasons)
|
|
for _, r := range reasons {
|
|
refused = append(refused, store.EndedRefusal{App: "", Reason: fmt.Sprintf("%d snapshot id(s) refused: %s", idRefusals[r], r)})
|
|
}
|
|
sort.Strings(ids)
|
|
return ids, byApp, refused
|
|
}
|
|
|
|
// untoldReason is the notice lock's half of the decision: "" when the hub's own record says the household (or the
|
|
// operator in its place) was told about this app EndedAppAfter ago; else the reason, in the hub's own words.
|
|
func untoldReason(now time.Time, notices map[string]store.EndNotice, app string) string {
|
|
n, ok := notices[app]
|
|
switch {
|
|
case !ok:
|
|
return reasonNoNotice
|
|
case !n.Told():
|
|
why := n.LastReason
|
|
if why == "" {
|
|
why = "the notice has not been sent yet"
|
|
}
|
|
return "the household has not been told yet, so nothing is removed: " + why
|
|
case now.Sub(n.ToldAt) < EndedAppAfter:
|
|
who := "the household was"
|
|
if n.ToldTo == store.EndNoticeOperator {
|
|
who = "the operator (the household has no usable address) was"
|
|
}
|
|
return fmt.Sprintf("%s told on %s — %d full day(s), %d needed",
|
|
who, n.ToldAt.UTC().Format("2006-01-02 15:04 UTC"), int(now.Sub(n.ToldAt).Hours()/24), int(EndedAppAfter.Hours()/24))
|
|
}
|
|
return ""
|
|
}
|
|
|
|
const reasonNoNotice = "the hub has no record that the household was told: no report of the box names this app in a pending end record"
|
|
|
|
// clearDisagreement: a refusal the operator must hear about. A box that asks a little early (its clock is minutes or
|
|
// hours ahead of the hub's first sight of the absence) is refused quietly and asks again next week; a box that claims
|
|
// an end for an app it reports as installed, or that the hub has seen absent for less than 29 days, is not a matter of
|
|
// minutes — its clock is wrong by a day or more (R-1384), or it is not telling the truth.
|
|
//
|
|
// The notice lock adds one clear case: the box asks for an app whose absence is old enough, and NO report of it ever
|
|
// named the app in a pending end record (an honest box names it in every report). A notice that is only not sent
|
|
// yet, or not 30 days old, is quiet here — the operator's line for that is `offsite_end_notice_stuck`.
|
|
func clearDisagreement(now time.Time, presence map[string]store.AppPresence, r store.EndedRefusal) bool {
|
|
if r.App == "" {
|
|
return false
|
|
}
|
|
if r.Reason == reasonNoNotice {
|
|
return true
|
|
}
|
|
p, known := presence[r.App]
|
|
// An installed app has no absence date (store.AppPresences), so the zero date covers it too.
|
|
if !known || p.AbsentSince.IsZero() {
|
|
return true
|
|
}
|
|
return now.Sub(p.AbsentSince) < EndedAppAfter-24*time.Hour
|
|
}
|
|
|
|
// agreeEndedFor runs the decision for one household against the store, and tells the operator about a clear
|
|
// disagreement. It never fails a window: with a store that cannot answer, nothing is agreed.
|
|
func (s *Service) agreeEndedFor(customerID string, windowID int64, maxRemove int, claims []EndedClaim) (ids []string, byApp map[string][]string, refused []store.EndedRefusal) {
|
|
if len(claims) == 0 {
|
|
return nil, nil, nil
|
|
}
|
|
now := s.now()
|
|
presence, err := s.Store.AppPresences(customerID)
|
|
if err != nil {
|
|
s.logf("[WARN] offsitekeys: ended apps %s: the hub's own app record could not be read (%v) — nothing agreed", customerID, err)
|
|
for _, c := range claims {
|
|
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: "the hub's own app record could not be read"})
|
|
}
|
|
return nil, nil, refused
|
|
}
|
|
// The notice lock: a record that cannot be read is „nobody was told” — nothing agreed.
|
|
notices, nerr := s.Store.EndNotices(customerID)
|
|
if nerr != nil {
|
|
s.logf("[WARN] offsitekeys: ended apps %s: the hub's own notice record could not be read (%v) — nothing agreed", customerID, nerr)
|
|
for _, c := range claims {
|
|
refused = append(refused, store.EndedRefusal{App: ledgerName(c.App), Reason: "the hub's own notice record could not be read"})
|
|
}
|
|
return nil, nil, refused
|
|
}
|
|
listing, have := s.Store.GetOffsiteSnapshotListing(customerID)
|
|
ids, byApp, refused = agreeEnded(now, presence, notices, listing, have, claims)
|
|
// Review of 2026-11-15 — what bounds a box that invents an ended app, or stops listing a living one, and 30 days
|
|
// later names much of the store under it:
|
|
//
|
|
// (c) A request for more than HALF of the store's snapshots WAITS ONE WEEK: the first window that sees it agrees
|
|
// to nothing for removed apps and tells the operator (a mail); a window at least largeEndedWait later agrees.
|
|
// So the operator always has a week between the mail and the first removal. An honest household that
|
|
// removed its one big app looks the same and loses one week.
|
|
// (a) ONE window agrees to at most the cap's own size (boundEndedToCap); the rest waits for the next window.
|
|
if total := len(listing.Entries); have && len(ids)*2 > total {
|
|
// The week belongs to THESE apps: a large request for other apps waits its own week and gets its own mail.
|
|
// The first sight is the hub's own record (store.offsite_large_requests) — never a search of text a box
|
|
// could have written.
|
|
key := largeSetKey(byApp)
|
|
first := s.Store.LargeEndedRequestFirstSeen(customerID, key, now.Add(-largeEndedMemory))
|
|
if first.IsZero() || now.Sub(first) < largeEndedWait {
|
|
if first.IsZero() {
|
|
if nerr := s.Store.NoteLargeEndedRequest(customerID, key, now); nerr != nil {
|
|
// Not recorded → the next window sees it as new, tells again and waits again: the safe direction.
|
|
s.logf("[WARN] offsitekeys: ended apps %s: the first sight of a large request could not be recorded (%v) — it will wait again", customerID, nerr)
|
|
}
|
|
s.event(customerID, EventEndedLargeAgreement, "warning",
|
|
fmt.Sprintf("Off-site clean-up window %d: the box of %s asked to remove %d of %d off-site snapshot(s) as copies of removed apps (%s) — more than half of the store. "+
|
|
"NOTHING of them is removed in this window. The hub's own records agree to each app, so from %s a window removes them, at most %d per window. "+
|
|
"If the household did not remove these apps, switch the weekly windows off for it before that day and look at the box.",
|
|
windowID, customerID, len(ids), total, appNames(byApp), now.Add(largeEndedWait).UTC().Format("2006-01-02"), endedCap(maxRemove, total)),
|
|
map[string]any{"subject": "large:" + key[:16], "window_id": windowID, "asked": len(ids), "listed": total})
|
|
}
|
|
refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("the request is for %d of %d snapshots — more than half of the store: it waits one week, and the operator is told",
|
|
len(ids), total)})
|
|
ids, byApp = nil, nil
|
|
}
|
|
}
|
|
if have && len(ids) > 0 {
|
|
var waits int
|
|
ids, byApp, waits = boundEndedToCap(ids, byApp, listing, endedCap(maxRemove, len(listing.Entries)))
|
|
if waits > 0 {
|
|
refused = append(refused, store.EndedRefusal{Reason: fmt.Sprintf("%d snapshot id(s) wait for the next window: one window removes at most %d snapshots of removed apps (the size of its own cap)",
|
|
waits, endedCap(maxRemove, len(listing.Entries)))})
|
|
}
|
|
}
|
|
s.logf("[INFO] offsitekeys: ended apps %s: %d app(s) claimed, %d app(s) with agreed snapshots, %d snapshot id(s) agreed, %d refusal(s)",
|
|
customerID, len(claims), len(byApp), len(ids), len(refused))
|
|
for _, r := range refused {
|
|
s.logf("[WARN] offsitekeys: ended apps %s: REFUSED %q: %s", customerID, r.App, r.Reason)
|
|
if clearDisagreement(now, presence, r) {
|
|
s.event(customerID, EventEndedAppRefused, "warning",
|
|
fmt.Sprintf("Off-site clean-up: the box of %s asked to remove the off-site copies of the removed app %q, and the hub's own records do not agree: %s. Nothing of that app was removed. Check the box's clock and its app list.",
|
|
customerID, r.App, r.Reason),
|
|
map[string]any{"subject": r.App, "app": r.App, "reason": r.Reason})
|
|
}
|
|
}
|
|
return ids, byApp, refused
|
|
}
|
|
|
|
// endedRemovedByApp groups the removed ids by the app the box listed them under, for the per-app operator record.
|
|
func endedRemovedByApp(claims map[string][]string, removed []string) map[string]int {
|
|
gone := make(map[string]bool, len(removed))
|
|
for _, id := range removed {
|
|
gone[id] = true
|
|
}
|
|
out := map[string]int{}
|
|
for app, ids := range claims {
|
|
for _, id := range ids {
|
|
if gone[id] {
|
|
out[app]++
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// endedCap is the most ended-app snapshots ONE window may agree to: the size of the window's own cap, and never more
|
|
// than the cap the hub's OWN count of the store would give (the window's cap comes from the box's count_before, which
|
|
// is the box's word). Decided by CC unattended on 2026-11-15, a narrowing of D84 — the operator may reverse it
|
|
// (`07-backup-architecture.md` §8.2f).
|
|
func endedCap(maxRemove, listed int) int {
|
|
c := MaxRemove(listed)
|
|
if maxRemove > 0 && maxRemove < c {
|
|
c = maxRemove
|
|
}
|
|
return c
|
|
}
|
|
|
|
// boundEndedToCap keeps at most limit of the agreed ids — the OLDEST uploads first (the newest copies of a removed
|
|
// app are the last to go) — and returns how many wait.
|
|
func boundEndedToCap(ids []string, byApp map[string][]string, listing store.OffsiteSnapshotListing, limit int) ([]string, map[string][]string, int) {
|
|
if len(ids) <= limit {
|
|
return ids, byApp, 0
|
|
}
|
|
sorted := append([]string(nil), ids...)
|
|
sort.Slice(sorted, func(i, j int) bool {
|
|
ti, tj := listing.Entries[sorted[i]], listing.Entries[sorted[j]]
|
|
if !ti.Equal(tj) {
|
|
return ti.Before(tj)
|
|
}
|
|
return sorted[i] < sorted[j]
|
|
})
|
|
keep := make(map[string]bool, limit)
|
|
for _, id := range sorted[:limit] {
|
|
keep[id] = true
|
|
}
|
|
var out []string
|
|
for _, id := range ids {
|
|
if keep[id] {
|
|
out = append(out, id)
|
|
}
|
|
}
|
|
apps := map[string][]string{}
|
|
for app, list := range byApp {
|
|
for _, id := range list {
|
|
if keep[id] {
|
|
apps[app] = append(apps[app], id)
|
|
}
|
|
}
|
|
}
|
|
return out, apps, len(ids) - limit
|
|
}
|
|
|
|
// appNames lists the agreed apps for a message (names already checked: store.ValidPresenceApp).
|
|
func appNames(byApp map[string][]string) string {
|
|
names := make([]string, 0, len(byApp))
|
|
for app := range byApp {
|
|
names = append(names, app)
|
|
}
|
|
sort.Strings(names)
|
|
if len(names) > 10 {
|
|
return strings.Join(names[:10], ", ") + fmt.Sprintf(" and %d more", len(names)-10)
|
|
}
|
|
return strings.Join(names, ", ")
|
|
}
|
|
|
|
// largeSetKey is the key of an app set in the hub's own record of large requests: a hash of the WHOLE sorted set.
|
|
// Never the display string (appNames), which is cut after ten names — two sets that differ after the tenth name
|
|
// would share one wait (security review of 2026-11-15).
|
|
func largeSetKey(byApp map[string][]string) string {
|
|
names := make([]string, 0, len(byApp))
|
|
for app := range byApp {
|
|
names = append(names, app)
|
|
}
|
|
sort.Strings(names)
|
|
sum := sha256.Sum256([]byte(strings.Join(names, "\n")))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
// ledgerName is a claimed name as the hub may write it down: itself when it has the shape of an app name, else a
|
|
// fixed placeholder.
|
|
func ledgerName(app string) string {
|
|
if store.ValidPresenceApp(app) {
|
|
return app
|
|
}
|
|
return notAnAppName
|
|
}
|
|
|
|
const notAnAppName = "(not-an-app-name)"
|