package backup import ( "errors" "sort" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // D72 / D84 (`09` §3 decisions 292 and 309; `07-backup-architecture.md` §8.2f, parts 3 and 4) — THE OFF-SITE COPIES OF // A REMOVED APP END 30 DAYS AFTER „DELETE BACKUP DATA”, IN THE WEEKLY WINDOW AND NOWHERE ELSE. // // What this file adds to the clean-up window (offbox_window.go) and what it must never do: // // - NO RECORD, NO DELETION. Only an off-site end record (settings.OffsiteEnd, written by the household's press) // that is DUE makes a snapshot a candidate. A removal that kept the backups, a rebuilt guest, an app removed // before this existed: there is no record, so there is no candidate. // - A LIVING APP'S COPY IS NEVER REMOVED BY ANOTHER APP'S END. A snapshot is a candidate only when EVERY app tag // it carries belongs to a due record — a snapshot with two apps' tags stays until both have ended. The shares // pseudo-app (`_shares`), the infrastructure stacks and an app that is installed right now are never due. // - THE HUB AGREES FIRST. The candidates go to the hub with the window request; the box removes ONLY the ids the // hub sends back (OffsiteWindow.EndedAllowed) — and of those, only ids it named itself in this same window. // A hub that does not know this (an older hub) sends none, and then nothing of an ended app is removed. // - ONE GUARD, NO SECOND PATH. The fake-snapshot guard runs first, on the whole repository and on the policy's own // plan under the window's cap, exactly as before. If it refuses, the ended apps wait with everything else. // The ended ids are OUTSIDE the cap (D84) and are named apart in the result (removed_for_ended_apps), so the // hub's 07:10 audit can explain them. // OffsiteEndedClaim is one due end record as the box sends it with the window request: the app, the record's own // dates (the hub does not decide by them) and the snapshot ids the box would remove. type OffsiteEndedClaim struct { App string `json:"app"` RemovedAt string `json:"removed_at"` EndsAt string `json:"ends_at"` SnapshotIDs []string `json:"snapshot_ids"` } // maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs). const maxEndedClaimIDs = 5000 // OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged). type OffsiteEndedRefusal struct { App string `json:"app"` Reason string `json:"reason"` } // OffsiteEndedApp is one pending end record on the hub report (`offsite.ended_apps`): it lets the hub start its own // clock for an app it never saw installed (`07` §8.2f part 4). The dates are the box's; the hub decides by its own. type OffsiteEndedApp struct { App string `json:"app"` RemovedAt string `json:"removed_at"` EndsAt string `json:"ends_at"` } // SetOffsiteEndLiving wires "is this app installed on this box right now, or one of the box's own protected stacks". // An app for which it answers true is never due. NOT WIRED (nil) = NOTHING IS DUE: a box that cannot tell what it // runs must not delete any app's history. Wired in cmd/controller/main.go; pinned by TestD72_MainWiresTheLivingCheck. func (m *Manager) SetOffsiteEndLiving(fn func(app string) bool) { m.offsiteEndLiving = fn } // appTags returns a snapshot's app tags: every tag but the tier's marker. func appTags(s guardSnap) []string { var out []string for _, t := range s.Tags { if t != "" && t != OffboxMarkerTag { out = append(out, t) } } return out } // dueOffsiteEnds is the PURE first half: the records that may be claimed now. living == nil → none. func dueOffsiteEnds(ends []settings.OffsiteEnd, now time.Time, living func(string) bool) map[string]settings.OffsiteEnd { due := map[string]settings.OffsiteEnd{} if living == nil { return due } for _, e := range ends { if !settings.ValidOffsiteEndApp(e.App) || e.App == SharesPseudoStack || !e.Due(now) || living(e.App) { continue } due[e.App] = e } return due } // endedCandidates is the PURE second half: from the repository's snapshots and the due records, the claims to send // (sorted by app; an app with no candidate is not claimed) and the set of candidate ids. // // A snapshot is a candidate only when it carries at least one app tag, EVERY app tag it carries is a due record, // and it is not younger than any of those records' removal (a snapshot made under that tag AFTER the removal is not // the removed app's copy — a later install's, or a planted one). func endedCandidates(snaps []guardSnap, due map[string]settings.OffsiteEnd) (claims []OffsiteEndedClaim, candidate map[string]bool) { candidate = map[string]bool{} byApp := map[string][]string{} for _, s := range snaps { apps := appTags(s) if len(apps) == 0 || s.ID == "" { continue } ok := true for _, a := range apps { e, isDue := due[a] if !isDue || s.Time.After(e.Removed().Add(offsiteGuardSkew)) { ok = false break } } if !ok { continue } candidate[s.ID] = true for _, a := range apps { byApp[a] = append(byApp[a], s.ID) } } for app, ids := range byApp { sort.Strings(ids) e := due[app] claims = append(claims, OffsiteEndedClaim{App: app, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt, SnapshotIDs: ids}) } sort.Slice(claims, func(i, j int) bool { return claims[i].App < claims[j].App }) return claims, candidate } // endedToRemove is the PURE third half: of the ids the hub agreed to, those the box itself named as candidates in // this window and that the policy's own plan does not already remove (those stay under the cap). Sorted. func endedToRemove(allowed []string, candidate map[string]bool, policyIDs []string) []string { inPolicy := make(map[string]bool, len(policyIDs)) for _, id := range policyIDs { inPolicy[id] = true } seen := map[string]bool{} var out []string for _, id := range allowed { if candidate[id] && !inPolicy[id] && !seen[id] { seen[id] = true out = append(out, id) } } sort.Strings(out) return out } // appStillInStore reports whether any snapshot still carries app's tag. func appStillInStore(snaps []guardSnap, app string) bool { for _, s := range snaps { for _, t := range s.Tags { if t == app { return true } } } return false } // dropEndsWithNoCopies deletes the due records of apps that have NO snapshot in the repository at all: there is // nothing to end, and a record that can never complete would be asked of the hub every week for ever. It writes no // history line (nothing was deleted). Called only after a listing that succeeded. func (m *Manager) dropEndsWithNoCopies(snaps []guardSnap, due map[string]settings.OffsiteEnd) { for app, e := range due { if appStillInStore(snaps, app) { continue } if done, err := m.settings.CompleteOffsiteEnd(app, e.RemovedAt, m.now(), 0); err != nil { m.logger.Printf("[WARN] [offbox] ended app %s: the store holds no copy of it, but its end record could not be closed: %v", app, err) } else if done { m.logger.Printf("[INFO] [offbox] ended app %s: the off-site store holds no copy of it — its end record is closed, nothing was deleted", app) } delete(due, app) } } // completeOffsiteEnds closes the record of every app whose copies this window removed: at least one of its claimed // ids is gone and no snapshot carries its tag any more. An app that still has a snapshot (one shared with a living // app, or one the hub did not agree to) keeps its record and is asked again at the next window. func (m *Manager) completeOffsiteEnds(claims []OffsiteEndedClaim, removed []string, after []guardSnap) { gone := make(map[string]bool, len(removed)) for _, id := range removed { gone[id] = true } for _, c := range claims { n := 0 for _, id := range c.SnapshotIDs { if gone[id] { n++ } } if n == 0 || appStillInStore(after, c.App) { continue } if _, err := m.settings.CompleteOffsiteEnd(c.App, c.RemovedAt, m.now(), n); err != nil { m.logger.Printf("[WARN] [offbox] ended app %s: its %d off-site snapshot(s) are removed, but the end record could not be closed: %v", c.App, n, err) } } } // OffsiteEndedAppsForReport returns the pending end records for the hub report. func (m *Manager) OffsiteEndedAppsForReport() []OffsiteEndedApp { var out []OffsiteEndedApp for _, e := range m.settings.GetOffsiteEnds() { out = append(out, OffsiteEndedApp{App: e.App, RemovedAt: e.RemovedAt, EndsAt: e.EndsAt}) } return out } // ── The household's side: the press, the take-back, the date ──────────────────────────────────────────────────────── // ErrOffsiteEndNotOffered: this box has no hub-provisioned off-site tier, so there is no weekly window that could // carry the deletion out — and no record is written. (The household's own NAS target is pruned by the box itself and // keeps a removed app's copies, as before: `07` §8.2f.) var ErrOffsiteEndNotOffered = errors.New("no hub-provisioned off-site tier on this box") // ErrOffsiteEndLiving: the app is installed right now, is one of the box's own stacks, or the box cannot tell. var ErrOffsiteEndLiving = errors.New("the app is installed or is not an app a household removes") // OffsiteEndOffered reports whether a press can start the 30 days here: the box has the hub-provisioned tier // (switched on or not — a tier that is off only makes the deletion wait). func (m *Manager) OffsiteEndOffered() bool { return m != nil && m.settings != nil && m.settings.GetOffboxTarget().Pinned() } // SetOffsiteEndNotify wires the household's one mail per press (D84). func (m *Manager) SetOffsiteEndNotify(fn func(rec settings.OffsiteEnd)) { m.offsiteEndNotify = fn } // OffsiteEndDateIfPressedNow is the date the remove dialog shows BEFORE the press. func (m *Manager) OffsiteEndDateIfPressedNow() time.Time { return m.now().Add(settings.OffsiteEndAfter) } // StartOffsiteEnd is THE press: „delete backup data” at a removal, or „delete in 30 days” on the Backups page. It // writes the end record and tells the household once. A second press for the same app changes nothing and tells // nobody (created == false). The ONLY writer of an end record. func (m *Manager) StartOffsiteEnd(app string) (rec settings.OffsiteEnd, created bool, err error) { if !m.OffsiteEndOffered() { return settings.OffsiteEnd{}, false, ErrOffsiteEndNotOffered } if !settings.ValidOffsiteEndApp(app) || app == SharesPseudoStack || m.offsiteEndLiving == nil || m.offsiteEndLiving(app) { return settings.OffsiteEnd{}, false, ErrOffsiteEndLiving } rec, created, err = m.settings.StartOffsiteEnd(app, m.now()) if err != nil || !created { return rec, false, err } if m.offsiteEndNotify != nil { m.offsiteEndNotify(rec) } return rec, true, nil } // KeepOffsiteCopies is „keep it after all”: it deletes the end record. Returns whether there was one TO KEEP: false // also while a clean-up window is removing this app's copies right now (its last look at the records is behind it) — // the page then says „not marked, or already deleted”, never „kept” over copies that are going. func (m *Manager) KeepOffsiteCopies(app string) (bool, error) { m.offsiteEndMu.Lock() defer m.offsiteEndMu.Unlock() if m.offsiteEndRemoving[app] { m.logger.Printf("[WARN] [offbox] keep-after-all for %s came while the clean-up window is removing its off-site copies — too late, not kept", app) return false, nil } return m.settings.CancelOffsiteEnd(app, "the household keeps the copies") } // lastLookAtEnded reads the end records AGAIN, immediately before the forget, and keeps of the agreed ids only those // whose EVERY app still has the SAME record (app and removed_at), still due, still not installed. It returns the ids // and the claims that are left, and marks those apps as „being removed” until endedRemovalOver. One lock with // KeepOffsiteCopies, so a take-back is either before this look (its record is gone: its ids are taken out here) or // after it (answered „too late”). // // NOT covered, said plainly: an INSTALL of the app during the forget itself. It deletes the record through settings // (stacks.SaveAppConfig), restic is already removing, and the copies go. The window is one `forget … --prune`. func (m *Manager) lastLookAtEnded(snaps []guardSnap, claims []OffsiteEndedClaim, endedIDs []string) ([]string, []OffsiteEndedClaim) { m.offsiteEndMu.Lock() defer m.offsiteEndMu.Unlock() m.offsiteEndRemoving = nil if len(endedIDs) == 0 { return nil, claims } due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving) claimed := make(map[string]string, len(claims)) for _, c := range claims { claimed[c.App] = c.RemovedAt } for app, e := range due { if at, ok := claimed[app]; !ok || at != e.RemovedAt { delete(due, app) // a record this window did not claim, or one written anew since: not this window's } } var left []OffsiteEndedClaim for _, c := range claims { if _, ok := due[c.App]; ok { left = append(left, c) } else { m.logger.Printf("[WARN] [offbox] ended app %s: its end record changed while the window was being prepared (taken back, installed again, or written anew) — nothing of it is removed", c.App) } } _, candidate := endedCandidates(snaps, due) var ids []string for _, id := range endedIDs { if candidate[id] { ids = append(ids, id) } } if len(ids) > 0 { m.offsiteEndRemoving = map[string]bool{} for app := range due { m.offsiteEndRemoving[app] = true } } return ids, left } // endedRemovalOver ends the „being removed” mark (the forget has ended, whatever its result). func (m *Manager) endedRemovalOver() { m.offsiteEndMu.Lock() m.offsiteEndRemoving = nil m.offsiteEndMu.Unlock() } // PendingOffsiteEnds and OffsiteEndsHistory are what the Backups page shows. func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings.GetOffsiteEnds() } func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone { return m.settings.GetOffsiteEndsDone() } // boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked // again in a later window: the record stays while a copy of the app is in the store. func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim { left := limit var out []OffsiteEndedClaim for _, c := range claims { if left <= 0 { break } if len(c.SnapshotIDs) > left { c.SnapshotIDs = c.SnapshotIDs[:left] } left -= len(c.SnapshotIDs) out = append(out, c) } return out }