package backup import ( "context" "errors" "os" "path/filepath" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/util" "gopkg.in/yaml.v3" ) // ── Kept data: which copy can bring it back, and the load (`09` §3 decision 36) ─────────────────── // // „Use my kept data" (at install) and „Load" (on the kept-data list) are the SAME act: the app's // recovery unit (definition + database + volumes) is restored — through RestoreFromRecoveryUnitAt, the // one body every unit restore uses (R-102) — while its files stay where they are on the drive. It is the // removed-app restore (R-487) with the unit named explicitly. // // WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a // database dump or a volume tar — a definition alone would install an empty app over the kept files), and // (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the // app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2). // // THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one // `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is // NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is // downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore, // restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three // rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before // v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only // then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body // (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as // they were. Pinned by internal/backup/r691_kept_offsite_test.go. // KeptCopy is one database copy a kept folder can be loaded from. type KeptCopy struct { UnitDir string // "" for the off-site copy (it is downloaded at load time) Tier int // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite) Time time.Time DriveLabel string // SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it. SnapshotID string UnitPath string } // KeptTierOffsite is KeptCopy.Tier for the off-site copy. const KeptTierOffsite = 3 // KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the // install choice and the kept list, so the two pages cannot name the same copy differently. func KeptCopyKey(tier int) string { switch tier { case 2: return "kept.backup.second" case KeptTierOffsite: return "kept.backup.offsite" } return "kept.backup.own" } // keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow // or unreachable repository then costs the page this long at most and the off-site copy is not offered. const keptOffsiteLookupTimeout = 20 * time.Second // unitHoldsData: a readable manifest that lists a database dump or a volume tar. func unitHoldsData(unitDir string) (*RecoveryManifest, bool) { man := readManifest(UnitManifestFile(unitDir)) if man == nil { return nil, false } return man, len(man.DBDumps)+len(man.VolumeDumps) > 0 } // unitHDDPath reads the unit's own app.yaml env HDD_PATH (a plain, non-secret field). "" when absent. func unitHDDPath(unitDir string) string { b, err := os.ReadFile(filepath.Join(unitDir, "compose", "app.yaml")) if err != nil { return "" } var doc struct { Env map[string]string `yaml:"env"` } if yaml.Unmarshal(b, &doc) != nil { return "" } return strings.TrimSpace(doc.Env["HDD_PATH"]) } // KeptCopyAt judges one unit directory for drive: usable, and when it was taken. func (m *Manager) KeptCopyAt(unitDir, drive string, tier int) (KeptCopy, bool) { if _, ok := unitHoldsData(unitDir); !ok { return KeptCopy{}, false } if h := unitHDDPath(unitDir); h != "" && filepath.Clean(h) != filepath.Clean(drive) { m.logger.Printf("[INFO] [backup] kept: unit %s was taken of %s, not %s — not offered", unitDir, h, drive) return KeptCopy{}, false } t, ok := unitNewestArtifact(unitDir) if !ok { return KeptCopy{}, false } return KeptCopy{UnitDir: unitDir, Tier: tier, Time: t}, true } // KeptDBCopy returns the NEWEST usable copy of app's data for kept files on drive: the app's own unit // (Tier 1) and every connected second-drive mirror (Tier 2). Only for an app that is NOT installed — an // installed app's unit is its live backup, never a kept-data offer. func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) { if app == "" || m.isStackDeployed(app) { return KeptCopy{}, false } var best KeptCopy found := false consider := func(c KeptCopy, ok bool) { if ok && (!found || c.Time.After(best.Time)) { best, found = c, true } } if u, ok := m.RemovedAppUnitFor(app); ok { c, ok := m.KeptCopyAt(u.UnitDir, drive, 1) c.DriveLabel = u.DriveLabel consider(c, ok) } for _, d := range m.Tier2MirrorDirsForApp(app) { consider(m.KeptCopyAt(tier2UnitDir(d), drive, 2)) } return best, found } // KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest // snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4). // Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call // for all apps (the kept list asks once per page, not once per row). Any error reading the repository // offers nothing (logged) — the local copies are still offered. func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy { out := map[string]KeptCopy{} var want []string for _, a := range apps { if a != "" && !m.isStackDeployed(a) { want = append(want, a) } } if len(want) == 0 || !m.OffboxConfigured() { return out } cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout) defer cancel() newest, _, err := m.offsiteNewestPerTag(cctx) if err != nil { m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err) return out } for _, app := range want { n, ok := newest[app] if !ok { continue } unitPath := offboxUnitPathOf(n.paths, app) if unitPath == "" { m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id) continue } out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath} } return out } // KeptOffsiteCopy is KeptOffsiteCopies for one app. func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) { c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app] return c, ok } // KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or // the only one (a tie goes local — no download). func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) { if ook && (!lok || off.Time.After(local.Time)) { return off, true } return local, lok } // KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest // usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one. func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) { local, lok := m.KeptDBCopy(app, drive) off, ook := m.KeptOffsiteCopy(ctx, app) return KeptNewer(local, lok, off, ook) } // Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything // of the app is touched. var ( ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version") ErrKeptOffsiteNotUsable = errors.New("the off-site copy is not a copy of these kept files") ) // keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY: // the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker. func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) { if m.keptUnitRestoreFn != nil { return m.keptUnitRestoreFn } return m.RestoreFromRecoveryUnitAt } // SetKeptUnitRestoreFn overrides the kept load's unit restore (tests). func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) { m.keptUnitRestoreFn = fn } // downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It // returns the unit directory and a cleanup that removes the scratch (call it on every path). func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) { noop := func() {} if err := m.acquireRunning(); err != nil { return "", noop, err } defer m.releaseRunning() scratch, nsRoot, err := m.offboxProofScratchDir(app) if err != nil { return "", noop, err } if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil { return "", noop, herr } m.removeProofScratch(app, scratch) // a copy an interrupted run left cleanup := func() { m.removeProofScratch(app, scratch) } if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil { return "", cleanup, err } unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator))) if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok { return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app) } man := readManifest(UnitManifestFile(unitDir)) if man == nil || man.Data == nil { return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app) } if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil { return "", cleanup, verr } return unitDir, cleanup, nil } // LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then // prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then // after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were. func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) { m.BeginRestoreOp("restore", app) go func() { start := time.Now() fail := func(err error) { m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err) m.EndRestoreOp(false, failMsg(err)) if after != nil { after(false) } } m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath) unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c) // The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the // outcome never finds a copy left behind. if err != nil { cleanup() fail(err) return } if prepare != nil { if perr := prepare(); perr != nil { cleanup() fail(perr) return } } res, err := m.keptUnitRestore()(app, unitDir) cleanup() if err != nil { fail(err) return } m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID, time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs) m.EndRestoreOp(true, okMsg(nil)) if after != nil { after(true) } }() } // RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when // it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise. func (m *Manager) RemovedUnitOnDrive(app, drive string) string { if m.isStackDeployed(app) { return "" } u, ok := m.RemovedAppUnitFor(app) if !ok { return "" } if !strings.HasPrefix(filepath.Clean(u.UnitDir), filepath.Clean(drive)+string(filepath.Separator)) { return "" } return u.UnitDir } // PrimaryUnitHome is where app's own unit lives on drive (backups/primary/). func (m *Manager) PrimaryUnitHome(app, drive string) string { return RecoveryUnitPath(m.namespaceRoot(drive), app) } // LoadKeptApp runs the load as a restore operation the backup pages already follow (the poll banner): // Begin → RestoreFromRecoveryUnitAt(app, unitDir) → End, then after(ok) in the same goroutine (the // after_load command, the kept folder's tidy-up). The caller has checked RestoreStatus/IsRunning. func (m *Manager) LoadKeptApp(app, unitDir string, okMsg, failMsg func(err error) string, after func(ok bool)) { m.BeginRestoreOp("restore", app) go func() { start := time.Now() res, err := m.keptUnitRestore()(app, unitDir) if err != nil { m.logger.Printf("[ERROR] [backup] kept load %s from %s FAILED after %s: %v", app, unitDir, time.Since(start).Round(time.Second), err) m.EndRestoreOp(false, failMsg(err)) if after != nil { after(false) } return } m.logger.Printf("[INFO] [backup] kept load %s from %s done in %s (volumes %d/%d, dbs %d/%d)", app, unitDir, time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs) m.EndRestoreOp(true, okMsg(nil)) if after != nil { after(true) } }() }