Files
felhom-controller/controller/internal/backup/offbox_ended.go
T
admin c5acb91a44
gates / gates (push) Successful in 32s
Review fix 5: the box sends its end records only to a hub that says it reads them (version skew)
A new box with an older hub sent `ended` in the window request; that hub reads
4 KiB of it, decided the window on a wrong count, and the weekly clean-up
stalled.

- The box sends `ended` only when the hub's last window answer said
  accepts_ended (kept in settings, re-read from every answer, so a rolled-back
  hub stops getting them after one answer). Otherwise the request is the one
  every hub always got, and the ordinary clean-up runs.
- One request carries at most 5000 snapshot ids.

Tests seen red before the fix; 6 mutations seen red (felhom.eu
documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt).

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-11-15 15:05:38 +01:00

343 lines
14 KiB
Go

package backup
import (
"errors"
"sort"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// D72 / D84 (`09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f, parts 3 and 4) — THE OFF-SITE COPIES OF
// A REMOVED APP END 30 DAYS AFTER „DELETE BACKUP DATA”, IN THE WEEKLY WINDOW AND NOWHERE ELSE.
//
// What this file adds to the clean-up window (offbox_window.go) and what it must never do:
//
// - NO RECORD, NO DELETION. Only an off-site end record (settings.OffsiteEnd, written by the household's press)
// that is DUE makes a snapshot a candidate. A removal that kept the backups, a rebuilt guest, an app removed
// before this existed: there is no record, so there is no candidate.
// - A LIVING APP'S COPY IS NEVER REMOVED BY ANOTHER APP'S END. A snapshot is a candidate only when EVERY app tag
// it carries belongs to a due record — a snapshot with two apps' tags stays until both have ended. The shares
// pseudo-app (`_shares`), the infrastructure stacks and an app that is installed right now are never due.
// - THE HUB AGREES FIRST. The candidates go to the hub with the window request; the box removes ONLY the ids the
// hub sends back (OffsiteWindow.EndedAllowed) — and of those, only ids it named itself in this same window.
// A hub that does not know this (an older hub) sends none, and then nothing of an ended app is removed.
// - ONE GUARD, NO SECOND PATH. The fake-snapshot guard runs first, on the whole repository and on the policy's own
// plan under the window's cap, exactly as before. If it refuses, the ended apps wait with everything else.
// The ended ids are OUTSIDE the cap (D84) and are named apart in the result (removed_for_ended_apps), so the
// hub's 07:10 audit can explain them.
// OffsiteEndedClaim is one due end record as the box sends it with the window request: the app, the record's own
// dates (the hub does not decide by them) and the snapshot ids the box would remove.
type OffsiteEndedClaim struct {
App string `json:"app"`
RemovedAt string `json:"removed_at"`
EndsAt string `json:"ends_at"`
SnapshotIDs []string `json:"snapshot_ids"`
}
// maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs).
const maxEndedClaimIDs = 5000
// OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged).
type OffsiteEndedRefusal struct {
App string `json:"app"`
Reason string `json:"reason"`
}
// OffsiteEndedApp is one pending end record on the hub report (`offsite.ended_apps`): it lets the hub start its own
// clock for an app it never saw installed (`07` §8.2f part 4). The dates are the box's; the hub decides by its own.
type OffsiteEndedApp struct {
App string `json:"app"`
RemovedAt string `json:"removed_at"`
EndsAt string `json:"ends_at"`
}
// SetOffsiteEndLiving wires "is this app installed on this box right now, or one of the box's own protected stacks".
// An app for which it answers true is never due. NOT WIRED (nil) = NOTHING IS DUE: a box that cannot tell what it
// runs must not delete any app's history. Wired in cmd/controller/main.go; pinned by TestD72_MainWiresTheLivingCheck.
func (m *Manager) SetOffsiteEndLiving(fn func(app string) bool) { m.offsiteEndLiving = fn }
// appTags returns a snapshot's app tags: every tag but the tier's marker.
func appTags(s guardSnap) []string {
var out []string
for _, t := range s.Tags {
if t != "" && t != OffboxMarkerTag {
out = append(out, t)
}
}
return out
}
// dueOffsiteEnds is the PURE first half: the records that may be claimed now. living == nil → none.
func dueOffsiteEnds(ends []settings.OffsiteEnd, now time.Time, living func(string) bool) map[string]settings.OffsiteEnd {
due := map[string]settings.OffsiteEnd{}
if living == nil {
return due
}
for _, e := range ends {
if !settings.ValidOffsiteEndApp(e.App) || e.App == SharesPseudoStack || !e.Due(now) || living(e.App) {
continue
}
due[e.App] = e
}
return due
}
// endedCandidates is the PURE second half: from the repository's snapshots and the due records, the claims to send
// (sorted by app; an app with no candidate is not claimed) and the set of candidate ids.
//
// A snapshot is a candidate only when it carries at least one app tag, EVERY app tag it carries is a due record,
// and it is not younger than any of those records' removal (a snapshot made under that tag AFTER the removal is not
// the removed app's copy — a later install's, or a planted one).
func endedCandidates(snaps []guardSnap, due map[string]settings.OffsiteEnd) (claims []OffsiteEndedClaim, candidate map[string]bool) {
candidate = map[string]bool{}
byApp := map[string][]string{}
for _, s := range snaps {
apps := appTags(s)
if len(apps) == 0 || s.ID == "" {
continue
}
ok := true
for _, a := range apps {
e, isDue := due[a]
if !isDue || s.Time.After(e.Removed().Add(offsiteGuardSkew)) {
ok = false
break
}
}
if !ok {
continue
}
candidate[s.ID] = true
for _, a := range apps {
byApp[a] = append(byApp[a], s.ID)
}
}
for app, ids := range byApp {
sort.Strings(ids)
e := due[app]
claims = append(claims, OffsiteEndedClaim{App: app, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt, SnapshotIDs: ids})
}
sort.Slice(claims, func(i, j int) bool { return claims[i].App < claims[j].App })
return claims, candidate
}
// endedToRemove is the PURE third half: of the ids the hub agreed to, those the box itself named as candidates in
// this window and that the policy's own plan does not already remove (those stay under the cap). Sorted.
func endedToRemove(allowed []string, candidate map[string]bool, policyIDs []string) []string {
inPolicy := make(map[string]bool, len(policyIDs))
for _, id := range policyIDs {
inPolicy[id] = true
}
seen := map[string]bool{}
var out []string
for _, id := range allowed {
if candidate[id] && !inPolicy[id] && !seen[id] {
seen[id] = true
out = append(out, id)
}
}
sort.Strings(out)
return out
}
// appStillInStore reports whether any snapshot still carries app's tag.
func appStillInStore(snaps []guardSnap, app string) bool {
for _, s := range snaps {
for _, t := range s.Tags {
if t == app {
return true
}
}
}
return false
}
// dropEndsWithNoCopies deletes the due records of apps that have NO snapshot in the repository at all: there is
// nothing to end, and a record that can never complete would be asked of the hub every week for ever. It writes no
// history line (nothing was deleted). Called only after a listing that succeeded.
func (m *Manager) dropEndsWithNoCopies(snaps []guardSnap, due map[string]settings.OffsiteEnd) {
for app, e := range due {
if appStillInStore(snaps, app) {
continue
}
if done, err := m.settings.CompleteOffsiteEnd(app, e.RemovedAt, m.now(), 0); err != nil {
m.logger.Printf("[WARN] [offbox] ended app %s: the store holds no copy of it, but its end record could not be closed: %v", app, err)
} else if done {
m.logger.Printf("[INFO] [offbox] ended app %s: the off-site store holds no copy of it — its end record is closed, nothing was deleted", app)
}
delete(due, app)
}
}
// completeOffsiteEnds closes the record of every app whose copies this window removed: at least one of its claimed
// ids is gone and no snapshot carries its tag any more. An app that still has a snapshot (one shared with a living
// app, or one the hub did not agree to) keeps its record and is asked again at the next window.
func (m *Manager) completeOffsiteEnds(claims []OffsiteEndedClaim, removed []string, after []guardSnap) {
gone := make(map[string]bool, len(removed))
for _, id := range removed {
gone[id] = true
}
for _, c := range claims {
n := 0
for _, id := range c.SnapshotIDs {
if gone[id] {
n++
}
}
if n == 0 || appStillInStore(after, c.App) {
continue
}
if _, err := m.settings.CompleteOffsiteEnd(c.App, c.RemovedAt, m.now(), n); err != nil {
m.logger.Printf("[WARN] [offbox] ended app %s: its %d off-site snapshot(s) are removed, but the end record could not be closed: %v", c.App, n, err)
}
}
}
// OffsiteEndedAppsForReport returns the pending end records for the hub report.
func (m *Manager) OffsiteEndedAppsForReport() []OffsiteEndedApp {
var out []OffsiteEndedApp
for _, e := range m.settings.GetOffsiteEnds() {
out = append(out, OffsiteEndedApp{App: e.App, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt})
}
return out
}
// ── The household's side: the press, the take-back, the date ────────────────────────────────────────────────────────
// ErrOffsiteEndNotOffered: this box has no hub-provisioned off-site tier, so there is no weekly window that could
// carry the deletion out — and no record is written. (The household's own NAS target is pruned by the box itself and
// keeps a removed app's copies, as before: `07` §8.2f.)
var ErrOffsiteEndNotOffered = errors.New("no hub-provisioned off-site tier on this box")
// ErrOffsiteEndLiving: the app is installed right now, is one of the box's own stacks, or the box cannot tell.
var ErrOffsiteEndLiving = errors.New("the app is installed or is not an app a household removes")
// OffsiteEndOffered reports whether a press can start the 30 days here: the box has the hub-provisioned tier
// (switched on or not — a tier that is off only makes the deletion wait).
func (m *Manager) OffsiteEndOffered() bool {
return m != nil && m.settings != nil && m.settings.GetOffboxTarget().Pinned()
}
// SetOffsiteEndNotify wires the household's one mail per press (D84).
func (m *Manager) SetOffsiteEndNotify(fn func(rec settings.OffsiteEnd)) { m.offsiteEndNotify = fn }
// OffsiteEndDateIfPressedNow is the date the remove dialog shows BEFORE the press.
func (m *Manager) OffsiteEndDateIfPressedNow() time.Time {
return m.now().Add(settings.OffsiteEndAfter)
}
// StartOffsiteEnd is THE press: „delete backup data” at a removal, or „delete in 30 days” on the Backups page. It
// writes the end record and tells the household once. A second press for the same app changes nothing and tells
// nobody (created == false). The ONLY writer of an end record.
func (m *Manager) StartOffsiteEnd(app string) (rec settings.OffsiteEnd, created bool, err error) {
if !m.OffsiteEndOffered() {
return settings.OffsiteEnd{}, false, ErrOffsiteEndNotOffered
}
if !settings.ValidOffsiteEndApp(app) || app == SharesPseudoStack || m.offsiteEndLiving == nil || m.offsiteEndLiving(app) {
return settings.OffsiteEnd{}, false, ErrOffsiteEndLiving
}
rec, created, err = m.settings.StartOffsiteEnd(app, m.now())
if err != nil || !created {
return rec, false, err
}
if m.offsiteEndNotify != nil {
m.offsiteEndNotify(rec)
}
return rec, true, nil
}
// KeepOffsiteCopies is „keep it after all”: it deletes the end record. Returns whether there was one TO KEEP: false
// also while a clean-up window is removing this app's copies right now (its last look at the records is behind it) —
// the page then says „not marked, or already deleted”, never „kept” over copies that are going.
func (m *Manager) KeepOffsiteCopies(app string) (bool, error) {
m.offsiteEndMu.Lock()
defer m.offsiteEndMu.Unlock()
if m.offsiteEndRemoving[app] {
m.logger.Printf("[WARN] [offbox] keep-after-all for %s came while the clean-up window is removing its off-site copies — too late, not kept", app)
return false, nil
}
return m.settings.CancelOffsiteEnd(app, "the household keeps the copies")
}
// lastLookAtEnded reads the end records AGAIN, immediately before the forget, and keeps of the agreed ids only those
// whose EVERY app still has the SAME record (app and removed_at), still due, still not installed. It returns the ids
// and the claims that are left, and marks those apps as „being removed” until endedRemovalOver. One lock with
// KeepOffsiteCopies, so a take-back is either before this look (its record is gone: its ids are taken out here) or
// after it (answered „too late”).
//
// NOT covered, said plainly: an INSTALL of the app during the forget itself. It deletes the record through settings
// (stacks.SaveAppConfig), restic is already removing, and the copies go. The window is one `forget … --prune`.
func (m *Manager) lastLookAtEnded(snaps []guardSnap, claims []OffsiteEndedClaim, endedIDs []string) ([]string, []OffsiteEndedClaim) {
m.offsiteEndMu.Lock()
defer m.offsiteEndMu.Unlock()
m.offsiteEndRemoving = nil
if len(endedIDs) == 0 {
return nil, claims
}
due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving)
claimed := make(map[string]string, len(claims))
for _, c := range claims {
claimed[c.App] = c.RemovedAt
}
for app, e := range due {
if at, ok := claimed[app]; !ok || at != e.RemovedAt {
delete(due, app) // a record this window did not claim, or one written anew since: not this window's
}
}
var left []OffsiteEndedClaim
for _, c := range claims {
if _, ok := due[c.App]; ok {
left = append(left, c)
} else {
m.logger.Printf("[WARN] [offbox] ended app %s: its end record changed while the window was being prepared (taken back, installed again, or written anew) — nothing of it is removed", c.App)
}
}
_, candidate := endedCandidates(snaps, due)
var ids []string
for _, id := range endedIDs {
if candidate[id] {
ids = append(ids, id)
}
}
if len(ids) > 0 {
m.offsiteEndRemoving = map[string]bool{}
for app := range due {
m.offsiteEndRemoving[app] = true
}
}
return ids, left
}
// endedRemovalOver ends the „being removed” mark (the forget has ended, whatever its result).
func (m *Manager) endedRemovalOver() {
m.offsiteEndMu.Lock()
m.offsiteEndRemoving = nil
m.offsiteEndMu.Unlock()
}
// PendingOffsiteEnds and OffsiteEndsHistory are what the Backups page shows.
func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings.GetOffsiteEnds() }
func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone {
return m.settings.GetOffsiteEndsDone()
}
// boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked
// again in a later window: the record stays while a copy of the app is in the store.
func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim {
left := limit
var out []OffsiteEndedClaim
for _, c := range claims {
if left <= 0 {
break
}
if len(c.SnapshotIDs) > left {
c.SnapshotIDs = c.SnapshotIDs[:left]
}
left -= len(c.SnapshotIDs)
out = append(out, c)
}
return out
}