257 lines
14 KiB
Go
257 lines
14 KiB
Go
package api
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys"
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// THE NOTICE LOCK (`09` §3 decision 315; `07-backup-architecture.md` §8.2f „No notice, no deletion”) — the hub sends
|
|
// the household's notice ITSELF, from its own state, until it has gone.
|
|
//
|
|
// Every controller report feeds the hub's record (store.NoteEndNotices: which apps the box names in a pending end
|
|
// record). After that, endNotices walks the household's rows and, for each one whose household has not been told:
|
|
//
|
|
// - tries the household's mail (notify.SendEndNotice). Sent → the record says „told” at the hub's time, and the
|
|
// hub agrees to remove that app's copies no earlier than 30 days after it (offsitekeys.agreeEnded).
|
|
// - no address, or a blocked household → the OPERATOR is mailed in its place, once per app
|
|
// (`offsite_end_notice_operator`), and that mail starts the 30 days. When an address appears later, the household
|
|
// gets its own mail and the 30 days start again from it.
|
|
// - the daily limit, the mail hold, a send that fails → nothing is told; the reason is written on the row (the
|
|
// window's answer names it) and the next try is at a report at least endNoticeGap(tries) later (1 h, 2 h, 4 h …).
|
|
// - a row that has not been told for EndNoticeStuckAfter → one line to the operator (`offsite_end_notice_stuck`).
|
|
//
|
|
// Nothing here reads a date a box sent, and no event of a box starts or stops anything.
|
|
|
|
const (
|
|
// endNoticeRetryGap: after the FIRST try that told nobody, the next try waits this long (a report comes every 15
|
|
// minutes; a mail service that is down is not asked 96 times a day per app). It doubles with every further try
|
|
// and stops at endNoticeRetryGapMax (endNoticeGap): a send that errors AFTER the mail service accepted it is a
|
|
// mail each time, so the tries must thin out (review of 2026-11-15, finding 2).
|
|
endNoticeRetryGap = time.Hour
|
|
endNoticeRetryGapMax = 24 * time.Hour
|
|
|
|
// EndNoticeStuckAfter: how long a notice may stay unsent before the operator gets one line. THREE DAYS: longer
|
|
// than any outage of the mail service the hub has had and than the hold on a restored copy is meant to last, and
|
|
// longer than the daily limit delays an honest household (5 notices a day: a household that removes up to 15
|
|
// apps at once is told about all of them inside it) — so the line means „this will not heal by itself”. Short
|
|
// enough that a deletion the household asked for has lost three days, not a month, when the operator hears.
|
|
EndNoticeStuckAfter = 3 * 24 * time.Hour
|
|
)
|
|
|
|
// endNoticeGap is the wait after the tries-th try that told nobody: 1 h, 2 h, 4 h, 8 h, 16 h, then 24 h.
|
|
func endNoticeGap(tries int) time.Duration {
|
|
g := endNoticeRetryGap
|
|
for i := 1; i < tries && g < endNoticeRetryGapMax; i++ {
|
|
g *= 2
|
|
}
|
|
if g > endNoticeRetryGapMax {
|
|
g = endNoticeRetryGapMax
|
|
}
|
|
return g
|
|
}
|
|
|
|
// endNoticeSpawn runs the sends off the report's request (a send can wait 10 s on the mail service). A seam: tests
|
|
// run it in line.
|
|
var endNoticeSpawn = func(f func()) { go f() }
|
|
|
|
// endNoticeBusy: one walk per household at a time — a second report that arrives while a walk is still sending does
|
|
// not send the same notice again; its rows are picked up by the next report.
|
|
var endNoticeBusy sync.Map // customerID → *sync.Mutex
|
|
|
|
// kickEndNotices starts a walk when the household has a row that still needs one.
|
|
func (h *Handler) kickEndNotices(customerID string) {
|
|
ns, err := h.store.EndNotices(customerID)
|
|
if err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s: the hub's own notice record could not be read: %v", customerID, err)
|
|
return
|
|
}
|
|
work := false
|
|
for _, n := range ns {
|
|
if n.ToldTo != store.EndNoticeHousehold {
|
|
work = true
|
|
}
|
|
}
|
|
if work {
|
|
endNoticeSpawn(func() { h.endNotices(customerID, time.Now()) })
|
|
}
|
|
}
|
|
|
|
// endNotices is one walk over a household's rows (see the header). start is the HUB's clock at the walk's start.
|
|
//
|
|
// A walk can take minutes (a send may wait 10 s on the mail service, and there can be 50 rows), so every time it
|
|
// WRITES is read from `clock()` at that moment — the start plus the time the walk has run. A told time is taken
|
|
// AFTER the send returned: never the walk's start, which would be before the mail was accepted (review finding 4).
|
|
func (h *Handler) endNotices(customerID string, start time.Time) {
|
|
wall := time.Now()
|
|
clock := func() time.Time { return start.Add(time.Since(wall)) }
|
|
mu, _ := endNoticeBusy.LoadOrStore(customerID, &sync.Mutex{})
|
|
if !mu.(*sync.Mutex).TryLock() {
|
|
return
|
|
}
|
|
defer mu.(*sync.Mutex).Unlock()
|
|
ns, err := h.store.EndNotices(customerID)
|
|
if err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s: the hub's own notice record could not be read: %v", customerID, err)
|
|
return
|
|
}
|
|
apps := make([]string, 0, len(ns))
|
|
for app := range ns {
|
|
apps = append(apps, app)
|
|
}
|
|
sort.Strings(apps)
|
|
for _, app := range apps {
|
|
n := ns[app]
|
|
if n.ToldTo == store.EndNoticeHousehold {
|
|
continue
|
|
}
|
|
now := clock()
|
|
if n.LastReason != "" && !n.LastTry.IsZero() && now.Sub(n.LastTry) < endNoticeGap(n.Tries) {
|
|
h.endNoticeStuck(customerID, n, now)
|
|
continue
|
|
}
|
|
outcome, why, addrKey := notify.EndNoticeFailed, "the hub has no mail service configured", ""
|
|
if h.dispatcher != nil {
|
|
// The day in the mail is the hub's own: its clock now + 30 days. Never the box's date.
|
|
outcome, why, addrKey = h.dispatcher.SendEndNotice(customerID, app, now.Add(offsitekeys.EndedAppAfter))
|
|
}
|
|
now = clock() // after the send returned
|
|
switch outcome {
|
|
case notify.EndNoticeSent:
|
|
ok, merr := h.store.MarkEndNoticeTold(customerID, app, n.Press, store.EndNoticeHousehold, addrKey, now)
|
|
if merr != nil || !ok {
|
|
// The mail went and the record does not say so: the household is mailed again at a later report and
|
|
// the 30 days start then. The safe direction (no deletion on a notice the hub cannot show).
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: the household's notice was sent, but the hub's record was NOT written (matched=%v, err=%v) — it will be sent again", customerID, app, ok, merr)
|
|
continue
|
|
}
|
|
n.ToldAt, n.ToldTo = now, store.EndNoticeHousehold
|
|
h.logger.Printf("[INFO] [end-notice] %s/%s: the household was told; the hub agrees to remove this app's off-site copies no earlier than %s",
|
|
customerID, app, now.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02"))
|
|
case notify.EndNoticeNoAddress:
|
|
if n.Told() {
|
|
continue // the operator was told in its place; the household still has no address
|
|
}
|
|
toldAt, terr := h.endNoticeToOperator(customerID, n, why, clock)
|
|
now = clock()
|
|
if terr != nil {
|
|
why = why + "; and the operator could not be told in its place (" + terr.Error() + ")"
|
|
h.endNoticeTried(customerID, &n, why, now)
|
|
} else {
|
|
n.ToldAt, n.ToldTo = toldAt, store.EndNoticeOperator
|
|
}
|
|
default: // the daily limit, the hold, a failed send
|
|
h.endNoticeTried(customerID, &n, why, now)
|
|
}
|
|
h.endNoticeStuck(customerID, n, now)
|
|
}
|
|
}
|
|
|
|
func (h *Handler) endNoticeTried(customerID string, n *store.EndNotice, why string, now time.Time) {
|
|
if err := h.store.MarkEndNoticeTried(customerID, n.App, n.Press, why, now); err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: the try could not be recorded: %v", customerID, n.App, err)
|
|
}
|
|
n.LastReason, n.LastTry = why, now
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: nobody was told (%s) — nothing of this app is removed; tried again at a later report", customerID, n.App, why)
|
|
}
|
|
|
|
// endNoticeToOperator mails the operator in place of a household the hub cannot mail, and only when the mail service
|
|
// accepted that mail does the record say „told” (to the operator): that mail starts the app's 30 days.
|
|
// DECIDED BY CC UNATTENDED 2026-11-15 (the brief's A.3) — the operator may reverse it: `07` §8.2f.
|
|
func (h *Handler) endNoticeToOperator(customerID string, n store.EndNotice, why string, clock func() time.Time) (time.Time, error) {
|
|
if h.dispatcher == nil {
|
|
return time.Time{}, fmt.Errorf("the hub has no mail service configured")
|
|
}
|
|
now := clock()
|
|
// The limits (security review of 2026-11-15), both from the hub's own sent log, by the real clock the log is
|
|
// written with. A notice that waits returns an error: the caller records the reason and nobody counts as told.
|
|
wall := time.Now()
|
|
if again, err := h.store.OperatorSentAboutSince(customerID, offsitekeys.EventEndNoticeOperator, operatorNoticeMarker(n.App), wall.Add(-operatorNoticeSameAppGap)); err != nil {
|
|
return time.Time{}, fmt.Errorf("the hub's sent log could not be read")
|
|
} else if again {
|
|
return time.Time{}, fmt.Errorf("the operator was told about this app of this household less than %d days ago — the notice for this press goes later", int(operatorNoticeSameAppGap.Hours()/24))
|
|
}
|
|
if sent, err := h.store.CountOperatorTriesSince(customerID, offsitekeys.EventEndNoticeOperator, wall.Add(-24*time.Hour)); err != nil {
|
|
return time.Time{}, fmt.Errorf("the hub's sent log could not be read")
|
|
} else if sent >= operatorNoticesPerDay {
|
|
return time.Time{}, fmt.Errorf("the operator already got %d of these notices about this household in the last 24 hours (the daily limit) — this one goes later", operatorNoticesPerDay)
|
|
}
|
|
day := now.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02")
|
|
// The text must hold operatorNoticeMarker(app): the same-app limit finds an earlier mail by it.
|
|
msg := fmt.Sprintf("Off-site: the household %s "+operatorNoticeMarker(n.App)+" „delete backup data”, and the hub could not tell the household: %s. "+
|
|
"This mail is the notice in its place. The hub counts the 30 days from now: from %s a clean-up window may remove that app's off-site copies. "+
|
|
"Tell the household. If the household did not ask for this, it can take it back on its Backups page (“Keep it after all”) and must change its password. "+
|
|
"When the household gets a notification address, the hub mails the household itself and the 30 days start again from that mail.",
|
|
customerID, why, day)
|
|
details, _ := json.Marshal(map[string]any{"subject": n.App, "app": n.App, "agrees_from": day})
|
|
if err := h.dispatcher.SendOperatorNow(customerID, offsitekeys.EventEndNoticeOperator, "warning", msg, string(details)); err != nil {
|
|
return time.Time{}, err
|
|
}
|
|
toldAt := clock() // after the mail service accepted the mail
|
|
if _, err := h.store.SaveEvent(customerID, offsitekeys.EventEndNoticeOperator, "warning", msg, string(details), "hub"); err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s event save FAILED for %s (the mail went): %v", offsitekeys.EventEndNoticeOperator, customerID, err)
|
|
}
|
|
ok, err := h.store.MarkEndNoticeTold(customerID, n.App, n.Press, store.EndNoticeOperator, "", toldAt)
|
|
if err != nil || !ok {
|
|
return time.Time{}, fmt.Errorf("the operator's mail went, but the hub's record was not written (matched=%v, err=%v)", ok, err)
|
|
}
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: the household could not be told (%s) — the OPERATOR was told in its place; the hub agrees no earlier than %s", customerID, n.App, why, day)
|
|
return toldAt, nil
|
|
}
|
|
|
|
// endNoticeStuck files the operator's one line for a notice that has told nobody for EndNoticeStuckAfter. Not while
|
|
// the mail hold is on (a held hub mails nobody; the line is filed at the first report after the release).
|
|
func (h *Handler) endNoticeStuck(customerID string, n store.EndNotice, now time.Time) {
|
|
if n.Told() || !n.StuckTold.IsZero() || n.FirstNamed.IsZero() || now.Sub(n.FirstNamed) < EndNoticeStuckAfter {
|
|
return
|
|
}
|
|
if h.dispatcher != nil && h.dispatcher.MailHeld() {
|
|
return
|
|
}
|
|
// At most operatorNoticesPerDay of these lines per household in 24 hours (the hub's own event rows): a box that
|
|
// keeps fifty invented presses waiting must not turn the line into fifty mails. The rest are filed later.
|
|
if filed, err := h.store.GetEventsByType(customerID, offsitekeys.EventEndNoticeStuck, time.Now().Add(-24*time.Hour)); err != nil || len(filed) >= operatorNoticesPerDay {
|
|
return
|
|
}
|
|
why := n.LastReason
|
|
if why == "" {
|
|
why = "no try has been recorded"
|
|
}
|
|
msg := fmt.Sprintf("Off-site: the household %s removed the app %q with „delete backup data” %d days ago, and the notice about it has not gone out: %s. "+
|
|
"NOTHING of that app's off-site copies is removed until the notice has gone and 30 days have passed after it. "+
|
|
"The household asked for this deletion and it is waiting: look at why the mail does not go. This line is sent once per app.",
|
|
customerID, n.App, int(now.Sub(n.FirstNamed).Hours()/24), why)
|
|
details, _ := json.Marshal(map[string]any{"subject": n.App, "app": n.App, "reason": why})
|
|
if _, err := h.store.SaveEvent(customerID, offsitekeys.EventEndNoticeStuck, "warning", msg, string(details), "hub"); err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s event save FAILED for %s: %v", offsitekeys.EventEndNoticeStuck, customerID, err)
|
|
return // the audit row first: a mail without its event row lies
|
|
}
|
|
if err := h.store.MarkEndNoticeStuckTold(customerID, n.App, n.Press, now); err != nil {
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: the operator's line could not be marked as filed: %v", customerID, n.App, err)
|
|
}
|
|
h.logger.Printf("[WARN] [end-notice] %s/%s: the notice has told nobody for %d days (%s) — operator told once", customerID, n.App, int(now.Sub(n.FirstNamed).Hours()/24), why)
|
|
if h.dispatcher != nil {
|
|
h.dispatcher.ProcessEvent(customerID, offsitekeys.EventEndNoticeStuck, "warning", msg, string(details), "hub")
|
|
}
|
|
}
|
|
|
|
// The operator's in-place notice is bounded like the household's (security review of 2026-11-15): a box of a
|
|
// household with no address must not be able to mail the operator without end. Both are read from the hub's own sent
|
|
// log, and a notice that waits is NOT told — it starts no 30 days.
|
|
const (
|
|
// operatorNoticesPerDay: the most in-place notices the operator is mailed about ONE household in 24 hours.
|
|
operatorNoticesPerDay = 5
|
|
// operatorNoticeSameAppGap: the SAME app of a household is told to the operator again no sooner than this — a
|
|
// press, a take-back and a press again are one matter for the operator, not a mail each.
|
|
operatorNoticeSameAppGap = 7 * 24 * time.Hour
|
|
)
|
|
|
|
// operatorNoticeMarker is the part of the in-place notice's text that names the app (a checked app name, quoted).
|
|
func operatorNoticeMarker(app string) string { return fmt.Sprintf("removed the app %q with", app) }
|