gates / gates (push) Successful in 32s
A new box with an older hub sent `ended` in the window request; that hub reads 4 KiB of it, decided the window on a wrong count, and the weekly clean-up stalled. - The box sends `ended` only when the hub's last window answer said accepts_ended (kept in settings, re-read from every answer, so a rolled-back hub stops getting them after one answer). Otherwise the request is the one every hub always got, and the ordinary clean-up runs. - One request carries at most 5000 snapshot ids. Tests seen red before the fix; 6 mutations seen red (felhom.eu documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
343 lines
14 KiB
Go
343 lines
14 KiB
Go
package backup
|
|
|
|
import (
|
|
"errors"
|
|
"sort"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// D72 / D84 (`09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f, parts 3 and 4) — THE OFF-SITE COPIES OF
|
|
// A REMOVED APP END 30 DAYS AFTER „DELETE BACKUP DATA”, IN THE WEEKLY WINDOW AND NOWHERE ELSE.
|
|
//
|
|
// What this file adds to the clean-up window (offbox_window.go) and what it must never do:
|
|
//
|
|
// - NO RECORD, NO DELETION. Only an off-site end record (settings.OffsiteEnd, written by the household's press)
|
|
// that is DUE makes a snapshot a candidate. A removal that kept the backups, a rebuilt guest, an app removed
|
|
// before this existed: there is no record, so there is no candidate.
|
|
// - A LIVING APP'S COPY IS NEVER REMOVED BY ANOTHER APP'S END. A snapshot is a candidate only when EVERY app tag
|
|
// it carries belongs to a due record — a snapshot with two apps' tags stays until both have ended. The shares
|
|
// pseudo-app (`_shares`), the infrastructure stacks and an app that is installed right now are never due.
|
|
// - THE HUB AGREES FIRST. The candidates go to the hub with the window request; the box removes ONLY the ids the
|
|
// hub sends back (OffsiteWindow.EndedAllowed) — and of those, only ids it named itself in this same window.
|
|
// A hub that does not know this (an older hub) sends none, and then nothing of an ended app is removed.
|
|
// - ONE GUARD, NO SECOND PATH. The fake-snapshot guard runs first, on the whole repository and on the policy's own
|
|
// plan under the window's cap, exactly as before. If it refuses, the ended apps wait with everything else.
|
|
// The ended ids are OUTSIDE the cap (D84) and are named apart in the result (removed_for_ended_apps), so the
|
|
// hub's 07:10 audit can explain them.
|
|
|
|
// OffsiteEndedClaim is one due end record as the box sends it with the window request: the app, the record's own
|
|
// dates (the hub does not decide by them) and the snapshot ids the box would remove.
|
|
type OffsiteEndedClaim struct {
|
|
App string `json:"app"`
|
|
RemovedAt string `json:"removed_at"`
|
|
EndsAt string `json:"ends_at"`
|
|
SnapshotIDs []string `json:"snapshot_ids"`
|
|
}
|
|
|
|
// maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs).
|
|
const maxEndedClaimIDs = 5000
|
|
|
|
// OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged).
|
|
type OffsiteEndedRefusal struct {
|
|
App string `json:"app"`
|
|
Reason string `json:"reason"`
|
|
}
|
|
|
|
// OffsiteEndedApp is one pending end record on the hub report (`offsite.ended_apps`): it lets the hub start its own
|
|
// clock for an app it never saw installed (`07` §8.2f part 4). The dates are the box's; the hub decides by its own.
|
|
type OffsiteEndedApp struct {
|
|
App string `json:"app"`
|
|
RemovedAt string `json:"removed_at"`
|
|
EndsAt string `json:"ends_at"`
|
|
}
|
|
|
|
// SetOffsiteEndLiving wires "is this app installed on this box right now, or one of the box's own protected stacks".
|
|
// An app for which it answers true is never due. NOT WIRED (nil) = NOTHING IS DUE: a box that cannot tell what it
|
|
// runs must not delete any app's history. Wired in cmd/controller/main.go; pinned by TestD72_MainWiresTheLivingCheck.
|
|
func (m *Manager) SetOffsiteEndLiving(fn func(app string) bool) { m.offsiteEndLiving = fn }
|
|
|
|
// appTags returns a snapshot's app tags: every tag but the tier's marker.
|
|
func appTags(s guardSnap) []string {
|
|
var out []string
|
|
for _, t := range s.Tags {
|
|
if t != "" && t != OffboxMarkerTag {
|
|
out = append(out, t)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// dueOffsiteEnds is the PURE first half: the records that may be claimed now. living == nil → none.
|
|
func dueOffsiteEnds(ends []settings.OffsiteEnd, now time.Time, living func(string) bool) map[string]settings.OffsiteEnd {
|
|
due := map[string]settings.OffsiteEnd{}
|
|
if living == nil {
|
|
return due
|
|
}
|
|
for _, e := range ends {
|
|
if !settings.ValidOffsiteEndApp(e.App) || e.App == SharesPseudoStack || !e.Due(now) || living(e.App) {
|
|
continue
|
|
}
|
|
due[e.App] = e
|
|
}
|
|
return due
|
|
}
|
|
|
|
// endedCandidates is the PURE second half: from the repository's snapshots and the due records, the claims to send
|
|
// (sorted by app; an app with no candidate is not claimed) and the set of candidate ids.
|
|
//
|
|
// A snapshot is a candidate only when it carries at least one app tag, EVERY app tag it carries is a due record,
|
|
// and it is not younger than any of those records' removal (a snapshot made under that tag AFTER the removal is not
|
|
// the removed app's copy — a later install's, or a planted one).
|
|
func endedCandidates(snaps []guardSnap, due map[string]settings.OffsiteEnd) (claims []OffsiteEndedClaim, candidate map[string]bool) {
|
|
candidate = map[string]bool{}
|
|
byApp := map[string][]string{}
|
|
for _, s := range snaps {
|
|
apps := appTags(s)
|
|
if len(apps) == 0 || s.ID == "" {
|
|
continue
|
|
}
|
|
ok := true
|
|
for _, a := range apps {
|
|
e, isDue := due[a]
|
|
if !isDue || s.Time.After(e.Removed().Add(offsiteGuardSkew)) {
|
|
ok = false
|
|
break
|
|
}
|
|
}
|
|
if !ok {
|
|
continue
|
|
}
|
|
candidate[s.ID] = true
|
|
for _, a := range apps {
|
|
byApp[a] = append(byApp[a], s.ID)
|
|
}
|
|
}
|
|
for app, ids := range byApp {
|
|
sort.Strings(ids)
|
|
e := due[app]
|
|
claims = append(claims, OffsiteEndedClaim{App: app, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt, SnapshotIDs: ids})
|
|
}
|
|
sort.Slice(claims, func(i, j int) bool { return claims[i].App < claims[j].App })
|
|
return claims, candidate
|
|
}
|
|
|
|
// endedToRemove is the PURE third half: of the ids the hub agreed to, those the box itself named as candidates in
|
|
// this window and that the policy's own plan does not already remove (those stay under the cap). Sorted.
|
|
func endedToRemove(allowed []string, candidate map[string]bool, policyIDs []string) []string {
|
|
inPolicy := make(map[string]bool, len(policyIDs))
|
|
for _, id := range policyIDs {
|
|
inPolicy[id] = true
|
|
}
|
|
seen := map[string]bool{}
|
|
var out []string
|
|
for _, id := range allowed {
|
|
if candidate[id] && !inPolicy[id] && !seen[id] {
|
|
seen[id] = true
|
|
out = append(out, id)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// appStillInStore reports whether any snapshot still carries app's tag.
|
|
func appStillInStore(snaps []guardSnap, app string) bool {
|
|
for _, s := range snaps {
|
|
for _, t := range s.Tags {
|
|
if t == app {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// dropEndsWithNoCopies deletes the due records of apps that have NO snapshot in the repository at all: there is
|
|
// nothing to end, and a record that can never complete would be asked of the hub every week for ever. It writes no
|
|
// history line (nothing was deleted). Called only after a listing that succeeded.
|
|
func (m *Manager) dropEndsWithNoCopies(snaps []guardSnap, due map[string]settings.OffsiteEnd) {
|
|
for app, e := range due {
|
|
if appStillInStore(snaps, app) {
|
|
continue
|
|
}
|
|
if done, err := m.settings.CompleteOffsiteEnd(app, e.RemovedAt, m.now(), 0); err != nil {
|
|
m.logger.Printf("[WARN] [offbox] ended app %s: the store holds no copy of it, but its end record could not be closed: %v", app, err)
|
|
} else if done {
|
|
m.logger.Printf("[INFO] [offbox] ended app %s: the off-site store holds no copy of it — its end record is closed, nothing was deleted", app)
|
|
}
|
|
delete(due, app)
|
|
}
|
|
}
|
|
|
|
// completeOffsiteEnds closes the record of every app whose copies this window removed: at least one of its claimed
|
|
// ids is gone and no snapshot carries its tag any more. An app that still has a snapshot (one shared with a living
|
|
// app, or one the hub did not agree to) keeps its record and is asked again at the next window.
|
|
func (m *Manager) completeOffsiteEnds(claims []OffsiteEndedClaim, removed []string, after []guardSnap) {
|
|
gone := make(map[string]bool, len(removed))
|
|
for _, id := range removed {
|
|
gone[id] = true
|
|
}
|
|
for _, c := range claims {
|
|
n := 0
|
|
for _, id := range c.SnapshotIDs {
|
|
if gone[id] {
|
|
n++
|
|
}
|
|
}
|
|
if n == 0 || appStillInStore(after, c.App) {
|
|
continue
|
|
}
|
|
if _, err := m.settings.CompleteOffsiteEnd(c.App, c.RemovedAt, m.now(), n); err != nil {
|
|
m.logger.Printf("[WARN] [offbox] ended app %s: its %d off-site snapshot(s) are removed, but the end record could not be closed: %v", c.App, n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// OffsiteEndedAppsForReport returns the pending end records for the hub report.
|
|
func (m *Manager) OffsiteEndedAppsForReport() []OffsiteEndedApp {
|
|
var out []OffsiteEndedApp
|
|
for _, e := range m.settings.GetOffsiteEnds() {
|
|
out = append(out, OffsiteEndedApp{App: e.App, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ── The household's side: the press, the take-back, the date ────────────────────────────────────────────────────────
|
|
|
|
// ErrOffsiteEndNotOffered: this box has no hub-provisioned off-site tier, so there is no weekly window that could
|
|
// carry the deletion out — and no record is written. (The household's own NAS target is pruned by the box itself and
|
|
// keeps a removed app's copies, as before: `07` §8.2f.)
|
|
var ErrOffsiteEndNotOffered = errors.New("no hub-provisioned off-site tier on this box")
|
|
|
|
// ErrOffsiteEndLiving: the app is installed right now, is one of the box's own stacks, or the box cannot tell.
|
|
var ErrOffsiteEndLiving = errors.New("the app is installed or is not an app a household removes")
|
|
|
|
// OffsiteEndOffered reports whether a press can start the 30 days here: the box has the hub-provisioned tier
|
|
// (switched on or not — a tier that is off only makes the deletion wait).
|
|
func (m *Manager) OffsiteEndOffered() bool {
|
|
return m != nil && m.settings != nil && m.settings.GetOffboxTarget().Pinned()
|
|
}
|
|
|
|
// SetOffsiteEndNotify wires the household's one mail per press (D84).
|
|
func (m *Manager) SetOffsiteEndNotify(fn func(rec settings.OffsiteEnd)) { m.offsiteEndNotify = fn }
|
|
|
|
// OffsiteEndDateIfPressedNow is the date the remove dialog shows BEFORE the press.
|
|
func (m *Manager) OffsiteEndDateIfPressedNow() time.Time {
|
|
return m.now().Add(settings.OffsiteEndAfter)
|
|
}
|
|
|
|
// StartOffsiteEnd is THE press: „delete backup data” at a removal, or „delete in 30 days” on the Backups page. It
|
|
// writes the end record and tells the household once. A second press for the same app changes nothing and tells
|
|
// nobody (created == false). The ONLY writer of an end record.
|
|
func (m *Manager) StartOffsiteEnd(app string) (rec settings.OffsiteEnd, created bool, err error) {
|
|
if !m.OffsiteEndOffered() {
|
|
return settings.OffsiteEnd{}, false, ErrOffsiteEndNotOffered
|
|
}
|
|
if !settings.ValidOffsiteEndApp(app) || app == SharesPseudoStack || m.offsiteEndLiving == nil || m.offsiteEndLiving(app) {
|
|
return settings.OffsiteEnd{}, false, ErrOffsiteEndLiving
|
|
}
|
|
rec, created, err = m.settings.StartOffsiteEnd(app, m.now())
|
|
if err != nil || !created {
|
|
return rec, false, err
|
|
}
|
|
if m.offsiteEndNotify != nil {
|
|
m.offsiteEndNotify(rec)
|
|
}
|
|
return rec, true, nil
|
|
}
|
|
|
|
// KeepOffsiteCopies is „keep it after all”: it deletes the end record. Returns whether there was one TO KEEP: false
|
|
// also while a clean-up window is removing this app's copies right now (its last look at the records is behind it) —
|
|
// the page then says „not marked, or already deleted”, never „kept” over copies that are going.
|
|
func (m *Manager) KeepOffsiteCopies(app string) (bool, error) {
|
|
m.offsiteEndMu.Lock()
|
|
defer m.offsiteEndMu.Unlock()
|
|
if m.offsiteEndRemoving[app] {
|
|
m.logger.Printf("[WARN] [offbox] keep-after-all for %s came while the clean-up window is removing its off-site copies — too late, not kept", app)
|
|
return false, nil
|
|
}
|
|
return m.settings.CancelOffsiteEnd(app, "the household keeps the copies")
|
|
}
|
|
|
|
// lastLookAtEnded reads the end records AGAIN, immediately before the forget, and keeps of the agreed ids only those
|
|
// whose EVERY app still has the SAME record (app and removed_at), still due, still not installed. It returns the ids
|
|
// and the claims that are left, and marks those apps as „being removed” until endedRemovalOver. One lock with
|
|
// KeepOffsiteCopies, so a take-back is either before this look (its record is gone: its ids are taken out here) or
|
|
// after it (answered „too late”).
|
|
//
|
|
// NOT covered, said plainly: an INSTALL of the app during the forget itself. It deletes the record through settings
|
|
// (stacks.SaveAppConfig), restic is already removing, and the copies go. The window is one `forget … --prune`.
|
|
func (m *Manager) lastLookAtEnded(snaps []guardSnap, claims []OffsiteEndedClaim, endedIDs []string) ([]string, []OffsiteEndedClaim) {
|
|
m.offsiteEndMu.Lock()
|
|
defer m.offsiteEndMu.Unlock()
|
|
m.offsiteEndRemoving = nil
|
|
if len(endedIDs) == 0 {
|
|
return nil, claims
|
|
}
|
|
due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving)
|
|
claimed := make(map[string]string, len(claims))
|
|
for _, c := range claims {
|
|
claimed[c.App] = c.RemovedAt
|
|
}
|
|
for app, e := range due {
|
|
if at, ok := claimed[app]; !ok || at != e.RemovedAt {
|
|
delete(due, app) // a record this window did not claim, or one written anew since: not this window's
|
|
}
|
|
}
|
|
var left []OffsiteEndedClaim
|
|
for _, c := range claims {
|
|
if _, ok := due[c.App]; ok {
|
|
left = append(left, c)
|
|
} else {
|
|
m.logger.Printf("[WARN] [offbox] ended app %s: its end record changed while the window was being prepared (taken back, installed again, or written anew) — nothing of it is removed", c.App)
|
|
}
|
|
}
|
|
_, candidate := endedCandidates(snaps, due)
|
|
var ids []string
|
|
for _, id := range endedIDs {
|
|
if candidate[id] {
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
if len(ids) > 0 {
|
|
m.offsiteEndRemoving = map[string]bool{}
|
|
for app := range due {
|
|
m.offsiteEndRemoving[app] = true
|
|
}
|
|
}
|
|
return ids, left
|
|
}
|
|
|
|
// endedRemovalOver ends the „being removed” mark (the forget has ended, whatever its result).
|
|
func (m *Manager) endedRemovalOver() {
|
|
m.offsiteEndMu.Lock()
|
|
m.offsiteEndRemoving = nil
|
|
m.offsiteEndMu.Unlock()
|
|
}
|
|
|
|
// PendingOffsiteEnds and OffsiteEndsHistory are what the Backups page shows.
|
|
func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings.GetOffsiteEnds() }
|
|
func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone {
|
|
return m.settings.GetOffsiteEndsDone()
|
|
}
|
|
|
|
// boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked
|
|
// again in a later window: the record stays while a copy of the app is in the store.
|
|
func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim {
|
|
left := limit
|
|
var out []OffsiteEndedClaim
|
|
for _, c := range claims {
|
|
if left <= 0 {
|
|
break
|
|
}
|
|
if len(c.SnapshotIDs) > left {
|
|
c.SnapshotIDs = c.SnapshotIDs[:left]
|
|
}
|
|
left -= len(c.SnapshotIDs)
|
|
out = append(out, c)
|
|
}
|
|
return out
|
|
}
|