package api import ( "encoding/json" "fmt" "sort" "sync" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/notify" "gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // THE NOTICE LOCK (`09` §3 decision 315; `07-backup-architecture.md` §8.2f „No notice, no deletion”) — the hub sends // the household's notice ITSELF, from its own state, until it has gone. // // Every controller report feeds the hub's record (store.NoteEndNotices: which apps the box names in a pending end // record). After that, endNotices walks the household's rows and, for each one whose household has not been told: // // - tries the household's mail (notify.SendEndNotice). Sent → the record says „told” at the hub's time, and the // hub agrees to remove that app's copies no earlier than 30 days after it (offsitekeys.agreeEnded). // - no address, or a blocked household → the OPERATOR is mailed in its place, once per app // (`offsite_end_notice_operator`), and that mail starts the 30 days. When an address appears later, the household // gets its own mail and the 30 days start again from it. // - the daily limit, the mail hold, a send that fails → nothing is told; the reason is written on the row (the // window's answer names it) and the next try is at a report at least endNoticeGap(tries) later (1 h, 2 h, 4 h …). // - a row that has not been told for EndNoticeStuckAfter → one line to the operator (`offsite_end_notice_stuck`). // // Nothing here reads a date a box sent, and no event of a box starts or stops anything. const ( // endNoticeRetryGap: after the FIRST try that told nobody, the next try waits this long (a report comes every 15 // minutes; a mail service that is down is not asked 96 times a day per app). It doubles with every further try // and stops at endNoticeRetryGapMax (endNoticeGap): a send that errors AFTER the mail service accepted it is a // mail each time, so the tries must thin out (review of 2026-11-15, finding 2). endNoticeRetryGap = time.Hour endNoticeRetryGapMax = 24 * time.Hour // EndNoticeStuckAfter: how long a notice may stay unsent before the operator gets one line. THREE DAYS: longer // than any outage of the mail service the hub has had and than the hold on a restored copy is meant to last, and // longer than the daily limit delays an honest household (5 notices a day: a household that removes up to 15 // apps at once is told about all of them inside it) — so the line means „this will not heal by itself”. Short // enough that a deletion the household asked for has lost three days, not a month, when the operator hears. EndNoticeStuckAfter = 3 * 24 * time.Hour ) // endNoticeGap is the wait after the tries-th try that told nobody: 1 h, 2 h, 4 h, 8 h, 16 h, then 24 h. func endNoticeGap(tries int) time.Duration { g := endNoticeRetryGap for i := 1; i < tries && g < endNoticeRetryGapMax; i++ { g *= 2 } if g > endNoticeRetryGapMax { g = endNoticeRetryGapMax } return g } // endNoticeSpawn runs the sends off the report's request (a send can wait 10 s on the mail service). A seam: tests // run it in line. var endNoticeSpawn = func(f func()) { go f() } // endNoticeBusy: one walk per household at a time — a second report that arrives while a walk is still sending does // not send the same notice again; its rows are picked up by the next report. var endNoticeBusy sync.Map // customerID → *sync.Mutex // kickEndNotices starts a walk when the household has a row that still needs one. func (h *Handler) kickEndNotices(customerID string) { ns, err := h.store.EndNotices(customerID) if err != nil { h.logger.Printf("[WARN] [end-notice] %s: the hub's own notice record could not be read: %v", customerID, err) return } work := false for _, n := range ns { if n.ToldTo != store.EndNoticeHousehold { work = true } } if work { endNoticeSpawn(func() { h.endNotices(customerID, time.Now()) }) } } // endNotices is one walk over a household's rows (see the header). start is the HUB's clock at the walk's start. // // A walk can take minutes (a send may wait 10 s on the mail service, and there can be 50 rows), so every time it // WRITES is read from `clock()` at that moment — the start plus the time the walk has run. A told time is taken // AFTER the send returned: never the walk's start, which would be before the mail was accepted (review finding 4). func (h *Handler) endNotices(customerID string, start time.Time) { wall := time.Now() clock := func() time.Time { return start.Add(time.Since(wall)) } mu, _ := endNoticeBusy.LoadOrStore(customerID, &sync.Mutex{}) if !mu.(*sync.Mutex).TryLock() { return } defer mu.(*sync.Mutex).Unlock() ns, err := h.store.EndNotices(customerID) if err != nil { h.logger.Printf("[WARN] [end-notice] %s: the hub's own notice record could not be read: %v", customerID, err) return } apps := make([]string, 0, len(ns)) for app := range ns { apps = append(apps, app) } sort.Strings(apps) for _, app := range apps { n := ns[app] if n.ToldTo == store.EndNoticeHousehold { continue } now := clock() if n.LastReason != "" && !n.LastTry.IsZero() && now.Sub(n.LastTry) < endNoticeGap(n.Tries) { h.endNoticeStuck(customerID, n, now) continue } outcome, why, addrKey := notify.EndNoticeFailed, "the hub has no mail service configured", "" if h.dispatcher != nil { // The day in the mail is the hub's own: its clock now + 30 days. Never the box's date. outcome, why, addrKey = h.dispatcher.SendEndNotice(customerID, app, now.Add(offsitekeys.EndedAppAfter)) } now = clock() // after the send returned switch outcome { case notify.EndNoticeSent: ok, merr := h.store.MarkEndNoticeTold(customerID, app, n.Press, store.EndNoticeHousehold, addrKey, now) if merr != nil || !ok { // The mail went and the record does not say so: the household is mailed again at a later report and // the 30 days start then. The safe direction (no deletion on a notice the hub cannot show). h.logger.Printf("[WARN] [end-notice] %s/%s: the household's notice was sent, but the hub's record was NOT written (matched=%v, err=%v) — it will be sent again", customerID, app, ok, merr) continue } n.ToldAt, n.ToldTo = now, store.EndNoticeHousehold h.logger.Printf("[INFO] [end-notice] %s/%s: the household was told; the hub agrees to remove this app's off-site copies no earlier than %s", customerID, app, now.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02")) case notify.EndNoticeNoAddress: if n.Told() { continue // the operator was told in its place; the household still has no address } toldAt, terr := h.endNoticeToOperator(customerID, n, why, clock) now = clock() if terr != nil { why = why + "; and the operator could not be told in its place (" + terr.Error() + ")" h.endNoticeTried(customerID, &n, why, now) } else { n.ToldAt, n.ToldTo = toldAt, store.EndNoticeOperator } default: // the daily limit, the hold, a failed send h.endNoticeTried(customerID, &n, why, now) } h.endNoticeStuck(customerID, n, now) } } func (h *Handler) endNoticeTried(customerID string, n *store.EndNotice, why string, now time.Time) { if err := h.store.MarkEndNoticeTried(customerID, n.App, n.Press, why, now); err != nil { h.logger.Printf("[WARN] [end-notice] %s/%s: the try could not be recorded: %v", customerID, n.App, err) } n.LastReason, n.LastTry = why, now h.logger.Printf("[WARN] [end-notice] %s/%s: nobody was told (%s) — nothing of this app is removed; tried again at a later report", customerID, n.App, why) } // endNoticeToOperator mails the operator in place of a household the hub cannot mail, and only when the mail service // accepted that mail does the record say „told” (to the operator): that mail starts the app's 30 days. // DECIDED BY CC UNATTENDED 2026-11-15 (the brief's A.3) — the operator may reverse it: `07` §8.2f. func (h *Handler) endNoticeToOperator(customerID string, n store.EndNotice, why string, clock func() time.Time) (time.Time, error) { if h.dispatcher == nil { return time.Time{}, fmt.Errorf("the hub has no mail service configured") } now := clock() // The limits (security review of 2026-11-15), both from the hub's own sent log, by the real clock the log is // written with. A notice that waits returns an error: the caller records the reason and nobody counts as told. wall := time.Now() if again, err := h.store.OperatorSentAboutSince(customerID, offsitekeys.EventEndNoticeOperator, operatorNoticeMarker(n.App), wall.Add(-operatorNoticeSameAppGap)); err != nil { return time.Time{}, fmt.Errorf("the hub's sent log could not be read") } else if again { return time.Time{}, fmt.Errorf("the operator was told about this app of this household less than %d days ago — the notice for this press goes later", int(operatorNoticeSameAppGap.Hours()/24)) } if sent, err := h.store.CountOperatorTriesSince(customerID, offsitekeys.EventEndNoticeOperator, wall.Add(-24*time.Hour)); err != nil { return time.Time{}, fmt.Errorf("the hub's sent log could not be read") } else if sent >= operatorNoticesPerDay { return time.Time{}, fmt.Errorf("the operator already got %d of these notices about this household in the last 24 hours (the daily limit) — this one goes later", operatorNoticesPerDay) } day := now.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02") // The text must hold operatorNoticeMarker(app): the same-app limit finds an earlier mail by it. msg := fmt.Sprintf("Off-site: the household %s "+operatorNoticeMarker(n.App)+" „delete backup data”, and the hub could not tell the household: %s. "+ "This mail is the notice in its place. The hub counts the 30 days from now: from %s a clean-up window may remove that app's off-site copies. "+ "Tell the household. If the household did not ask for this, it can take it back on its Backups page (“Keep it after all”) and must change its password. "+ "When the household gets a notification address, the hub mails the household itself and the 30 days start again from that mail.", customerID, why, day) details, _ := json.Marshal(map[string]any{"subject": n.App, "app": n.App, "agrees_from": day}) if err := h.dispatcher.SendOperatorNow(customerID, offsitekeys.EventEndNoticeOperator, "warning", msg, string(details)); err != nil { return time.Time{}, err } toldAt := clock() // after the mail service accepted the mail if _, err := h.store.SaveEvent(customerID, offsitekeys.EventEndNoticeOperator, "warning", msg, string(details), "hub"); err != nil { h.logger.Printf("[WARN] [end-notice] %s event save FAILED for %s (the mail went): %v", offsitekeys.EventEndNoticeOperator, customerID, err) } ok, err := h.store.MarkEndNoticeTold(customerID, n.App, n.Press, store.EndNoticeOperator, "", toldAt) if err != nil || !ok { return time.Time{}, fmt.Errorf("the operator's mail went, but the hub's record was not written (matched=%v, err=%v)", ok, err) } h.logger.Printf("[WARN] [end-notice] %s/%s: the household could not be told (%s) — the OPERATOR was told in its place; the hub agrees no earlier than %s", customerID, n.App, why, day) return toldAt, nil } // endNoticeStuck files the operator's one line for a notice that has told nobody for EndNoticeStuckAfter. Not while // the mail hold is on (a held hub mails nobody; the line is filed at the first report after the release). func (h *Handler) endNoticeStuck(customerID string, n store.EndNotice, now time.Time) { if n.Told() || !n.StuckTold.IsZero() || n.FirstNamed.IsZero() || now.Sub(n.FirstNamed) < EndNoticeStuckAfter { return } if h.dispatcher != nil && h.dispatcher.MailHeld() { return } // At most operatorNoticesPerDay of these lines per household in 24 hours (the hub's own event rows): a box that // keeps fifty invented presses waiting must not turn the line into fifty mails. The rest are filed later. if filed, err := h.store.GetEventsByType(customerID, offsitekeys.EventEndNoticeStuck, time.Now().Add(-24*time.Hour)); err != nil || len(filed) >= operatorNoticesPerDay { return } why := n.LastReason if why == "" { why = "no try has been recorded" } msg := fmt.Sprintf("Off-site: the household %s removed the app %q with „delete backup data” %d days ago, and the notice about it has not gone out: %s. "+ "NOTHING of that app's off-site copies is removed until the notice has gone and 30 days have passed after it. "+ "The household asked for this deletion and it is waiting: look at why the mail does not go. This line is sent once per app.", customerID, n.App, int(now.Sub(n.FirstNamed).Hours()/24), why) details, _ := json.Marshal(map[string]any{"subject": n.App, "app": n.App, "reason": why}) if _, err := h.store.SaveEvent(customerID, offsitekeys.EventEndNoticeStuck, "warning", msg, string(details), "hub"); err != nil { h.logger.Printf("[WARN] [end-notice] %s event save FAILED for %s: %v", offsitekeys.EventEndNoticeStuck, customerID, err) return // the audit row first: a mail without its event row lies } if err := h.store.MarkEndNoticeStuckTold(customerID, n.App, n.Press, now); err != nil { h.logger.Printf("[WARN] [end-notice] %s/%s: the operator's line could not be marked as filed: %v", customerID, n.App, err) } h.logger.Printf("[WARN] [end-notice] %s/%s: the notice has told nobody for %d days (%s) — operator told once", customerID, n.App, int(now.Sub(n.FirstNamed).Hours()/24), why) if h.dispatcher != nil { h.dispatcher.ProcessEvent(customerID, offsitekeys.EventEndNoticeStuck, "warning", msg, string(details), "hub") } } // The operator's in-place notice is bounded like the household's (security review of 2026-11-15): a box of a // household with no address must not be able to mail the operator without end. Both are read from the hub's own sent // log, and a notice that waits is NOT told — it starts no 30 days. const ( // operatorNoticesPerDay: the most in-place notices the operator is mailed about ONE household in 24 hours. operatorNoticesPerDay = 5 // operatorNoticeSameAppGap: the SAME app of a household is told to the operator again no sooner than this — a // press, a take-back and a press again are one matter for the operator, not a mail each. operatorNoticeSameAppGap = 7 * 24 * time.Hour ) // operatorNoticeMarker is the part of the in-place notice's text that names the app (a checked app name, quoted). func operatorNoticeMarker(app string) string { return fmt.Sprintf("removed the app %q with", app) }