v0.277.0: kept data loads from the off-site copy too (R-691 (2))

Use my kept data / Load consider the off-site snapshot when it is newer than every local copy or the only one; the unit is downloaded alone, judged (drive, data, recorded data version) and only then restored. The page names the copy and its date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-28 09:57:11 +02:00
parent 0c1023e503
commit 6479933e8e
14 changed files with 696 additions and 77 deletions
@@ -1,11 +1,14 @@
 package backup

 import (
+	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"

+	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
 	"gopkg.in/yaml.v3"
 )

@@ -19,18 +22,49 @@ import (
 // WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a
 // database dump or a volume tar — a definition alone would install an empty app over the kept files), and
 // (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the
-// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2). The
-// off-site copy is NOT looked at here: its restore is a different operation (reconstitute) and it is
-// not built into this choice yet — said on the page as "none" when it is the only one.
+// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2).
+//
+// THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one
+// `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is
+// NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is
+// downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore,
+// restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three
+// rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before
+// v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only
+// then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body
+// (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as
+// they were. Pinned by internal/backup/r691_kept_offsite_test.go.

 // KeptCopy is one database copy a kept folder can be loaded from.
 type KeptCopy struct {
-	UnitDir    string
-	Tier       int // 1 own unit, 2 second drive
+	UnitDir    string // "" for the off-site copy (it is downloaded at load time)
+	Tier       int    // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite)
 	Time       time.Time
 	DriveLabel string
+	// SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it.
+	SnapshotID string
+	UnitPath   string
 }

+// KeptTierOffsite is KeptCopy.Tier for the off-site copy.
+const KeptTierOffsite = 3
+
+// KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the
+// install choice and the kept list, so the two pages cannot name the same copy differently.
+func KeptCopyKey(tier int) string {
+	switch tier {
+	case 2:
+		return "kept.backup.second"
+	case KeptTierOffsite:
+		return "kept.backup.offsite"
+	}
+	return "kept.backup.own"
+}
+
+// keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow
+// or unreachable repository then costs the page this long at most and the off-site copy is not offered.
+const keptOffsiteLookupTimeout = 20 * time.Second
+
 // unitHoldsData: a readable manifest that lists a database dump or a volume tar.
 func unitHoldsData(unitDir string) (*RecoveryManifest, bool) {
 	man := readManifest(UnitManifestFile(unitDir))
@@ -96,6 +130,167 @@ func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) {
 	return best, found
 }

+// KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest
+// snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4).
+// Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call
+// for all apps (the kept list asks once per page, not once per row). Any error reading the repository
+// offers nothing (logged) — the local copies are still offered.
+func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy {
+	out := map[string]KeptCopy{}
+	var want []string
+	for _, a := range apps {
+		if a != "" && !m.isStackDeployed(a) {
+			want = append(want, a)
+		}
+	}
+	if len(want) == 0 || !m.OffboxConfigured() {
+		return out
+	}
+	cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout)
+	defer cancel()
+	newest, _, err := m.offsiteNewestPerTag(cctx)
+	if err != nil {
+		m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err)
+		return out
+	}
+	for _, app := range want {
+		n, ok := newest[app]
+		if !ok {
+			continue
+		}
+		unitPath := offboxUnitPathOf(n.paths, app)
+		if unitPath == "" {
+			m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id)
+			continue
+		}
+		out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath}
+	}
+	return out
+}
+
+// KeptOffsiteCopy is KeptOffsiteCopies for one app.
+func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) {
+	c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app]
+	return c, ok
+}
+
+// KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or
+// the only one (a tie goes local — no download).
+func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) {
+	if ook && (!lok || off.Time.After(local.Time)) {
+		return off, true
+	}
+	return local, lok
+}
+
+// KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest
+// usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one.
+func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) {
+	local, lok := m.KeptDBCopy(app, drive)
+	off, ook := m.KeptOffsiteCopy(ctx, app)
+	return KeptNewer(local, lok, off, ook)
+}
+
+// Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything
+// of the app is touched.
+var (
+	ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version")
+	ErrKeptOffsiteNotUsable      = errors.New("the off-site copy is not a copy of these kept files")
+)
+
+// keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY:
+// the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker.
+func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) {
+	if m.keptUnitRestoreFn != nil {
+		return m.keptUnitRestoreFn
+	}
+	return m.RestoreFromRecoveryUnitAt
+}
+
+// SetKeptUnitRestoreFn overrides the kept load's unit restore (tests).
+func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) {
+	m.keptUnitRestoreFn = fn
+}
+
+// downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It
+// returns the unit directory and a cleanup that removes the scratch (call it on every path).
+func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) {
+	noop := func() {}
+	if err := m.acquireRunning(); err != nil {
+		return "", noop, err
+	}
+	defer m.releaseRunning()
+	scratch, nsRoot, err := m.offboxProofScratchDir(app)
+	if err != nil {
+		return "", noop, err
+	}
+	if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil {
+		return "", noop, herr
+	}
+	m.removeProofScratch(app, scratch) // a copy an interrupted run left
+	cleanup := func() { m.removeProofScratch(app, scratch) }
+	if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil {
+		return "", cleanup, err
+	}
+	unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator)))
+	if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok {
+		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app)
+	}
+	man := readManifest(UnitManifestFile(unitDir))
+	if man == nil || man.Data == nil {
+		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app)
+	}
+	if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil {
+		return "", cleanup, verr
+	}
+	return unitDir, cleanup, nil
+}
+
+// LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then
+// prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then
+// after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were.
+func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) {
+	m.BeginRestoreOp("restore", app)
+	go func() {
+		start := time.Now()
+		fail := func(err error) {
+			m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err)
+			m.EndRestoreOp(false, failMsg(err))
+			if after != nil {
+				after(false)
+			}
+		}
+		m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath)
+		unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c)
+		// The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the
+		// outcome never finds a copy left behind.
+		if err != nil {
+			cleanup()
+			fail(err)
+			return
+		}
+		if prepare != nil {
+			if perr := prepare(); perr != nil {
+				cleanup()
+				fail(perr)
+				return
+			}
+		}
+		res, err := m.keptUnitRestore()(app, unitDir)
+		cleanup()
+		if err != nil {
+			fail(err)
+			return
+		}
+		m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID,
+			time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
+		m.EndRestoreOp(true, okMsg(nil))
+		if after != nil {
+			after(true)
+		}
+	}()
+}
+
 // RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when
 // it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise.
 func (m *Manager) RemovedUnitOnDrive(app, drive string) string {