felhom-controller/controller/internal/backup/kept_load.go

package backup

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
	"gopkg.in/yaml.v3"
)

// ── Kept data: which copy can bring it back, and the load (`09` §3 decision 36) ───────────────────
//
// „Use my kept data" (at install) and „Load" (on the kept-data list) are the SAME act: the app's
// recovery unit (definition + database + volumes) is restored — through RestoreFromRecoveryUnitAt, the
// one body every unit restore uses (R-102) — while its files stay where they are on the drive. It is the
// removed-app restore (R-487) with the unit named explicitly.
//
// WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a
// database dump or a volume tar — a definition alone would install an empty app over the kept files), and
// (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the
// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2).
//
// THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one
// `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is
// NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is
// downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore,
// restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three
// rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before
// v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only
// then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body
// (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as
// they were. Pinned by internal/backup/r691_kept_offsite_test.go.

// KeptCopy is one database copy a kept folder can be loaded from.
type KeptCopy struct {
	UnitDir    string // "" for the off-site copy (it is downloaded at load time)
	Tier       int    // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite)
	Time       time.Time
	DriveLabel string
	// SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it.
	SnapshotID string
	UnitPath   string
}

// KeptTierOffsite is KeptCopy.Tier for the off-site copy.
const KeptTierOffsite = 3

// KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the
// install choice and the kept list, so the two pages cannot name the same copy differently.
func KeptCopyKey(tier int) string {
	switch tier {
	case 2:
		return "kept.backup.second"
	case KeptTierOffsite:
		return "kept.backup.offsite"
	}
	return "kept.backup.own"
}

// keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow
// or unreachable repository then costs the page this long at most and the off-site copy is not offered.
const keptOffsiteLookupTimeout = 20 * time.Second

// unitHoldsData: a readable manifest that lists a database dump or a volume tar.
func unitHoldsData(unitDir string) (*RecoveryManifest, bool) {
	man := readManifest(UnitManifestFile(unitDir))
	if man == nil {
		return nil, false
	}
	return man, len(man.DBDumps)+len(man.VolumeDumps) > 0
}

// unitHDDPath reads the unit's own app.yaml env HDD_PATH (a plain, non-secret field). "" when absent.
func unitHDDPath(unitDir string) string {
	b, err := os.ReadFile(filepath.Join(unitDir, "compose", "app.yaml"))
	if err != nil {
		return ""
	}
	var doc struct {
		Env map[string]string `yaml:"env"`
	}
	if yaml.Unmarshal(b, &doc) != nil {
		return ""
	}
	return strings.TrimSpace(doc.Env["HDD_PATH"])
}

// KeptCopyAt judges one unit directory for drive: usable, and when it was taken.
func (m *Manager) KeptCopyAt(unitDir, drive string, tier int) (KeptCopy, bool) {
	if _, ok := unitHoldsData(unitDir); !ok {
		return KeptCopy{}, false
	}
	if h := unitHDDPath(unitDir); h != "" && filepath.Clean(h) != filepath.Clean(drive) {
		m.logger.Printf("[INFO] [backup] kept: unit %s was taken of %s, not %s — not offered", unitDir, h, drive)
		return KeptCopy{}, false
	}
	t, ok := unitNewestArtifact(unitDir)
	if !ok {
		return KeptCopy{}, false
	}
	return KeptCopy{UnitDir: unitDir, Tier: tier, Time: t}, true
}

// KeptDBCopy returns the NEWEST usable copy of app's data for kept files on drive: the app's own unit
// (Tier 1) and every connected second-drive mirror (Tier 2). Only for an app that is NOT installed — an
// installed app's unit is its live backup, never a kept-data offer.
func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) {
	if app == "" || m.isStackDeployed(app) {
		return KeptCopy{}, false
	}
	var best KeptCopy
	found := false
	consider := func(c KeptCopy, ok bool) {
		if ok && (!found || c.Time.After(best.Time)) {
			best, found = c, true
		}
	}
	if u, ok := m.RemovedAppUnitFor(app); ok {
		c, ok := m.KeptCopyAt(u.UnitDir, drive, 1)
		c.DriveLabel = u.DriveLabel
		consider(c, ok)
	}
	for _, d := range m.Tier2MirrorDirsForApp(app) {
		consider(m.KeptCopyAt(tier2UnitDir(d), drive, 2))
	}
	return best, found
}

// KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest
// snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4).
// Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call
// for all apps (the kept list asks once per page, not once per row). Any error reading the repository
// offers nothing (logged) — the local copies are still offered.
func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy {
	out := map[string]KeptCopy{}
	var want []string
	for _, a := range apps {
		if a != "" && !m.isStackDeployed(a) {
			want = append(want, a)
		}
	}
	if len(want) == 0 || !m.OffboxConfigured() {
		return out
	}
	cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout)
	defer cancel()
	newest, _, err := m.offsiteNewestPerTag(cctx)
	if err != nil {
		m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err)
		return out
	}
	for _, app := range want {
		n, ok := newest[app]
		if !ok {
			continue
		}
		unitPath := offboxUnitPathOf(n.paths, app)
		if unitPath == "" {
			m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id)
			continue
		}
		out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath}
	}
	return out
}

// KeptOffsiteCopy is KeptOffsiteCopies for one app.
func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) {
	c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app]
	return c, ok
}

// KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or
// the only one (a tie goes local — no download).
func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) {
	if ook && (!lok || off.Time.After(local.Time)) {
		return off, true
	}
	return local, lok
}

// KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest
// usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one.
func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) {
	local, lok := m.KeptDBCopy(app, drive)
	off, ook := m.KeptOffsiteCopy(ctx, app)
	return KeptNewer(local, lok, off, ook)
}

// Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything
// of the app is touched.
var (
	ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version")
	ErrKeptOffsiteNotUsable      = errors.New("the off-site copy is not a copy of these kept files")
)

// keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY:
// the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker.
func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) {
	if m.keptUnitRestoreFn != nil {
		return m.keptUnitRestoreFn
	}
	return m.RestoreFromRecoveryUnitAt
}

// SetKeptUnitRestoreFn overrides the kept load's unit restore (tests).
func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) {
	m.keptUnitRestoreFn = fn
}

// downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It
// returns the unit directory and a cleanup that removes the scratch (call it on every path).
func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) {
	noop := func() {}
	if err := m.acquireRunning(); err != nil {
		return "", noop, err
	}
	defer m.releaseRunning()
	scratch, nsRoot, err := m.offboxProofScratchDir(app)
	if err != nil {
		return "", noop, err
	}
	if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil {
		return "", noop, herr
	}
	m.removeProofScratch(app, scratch) // a copy an interrupted run left
	cleanup := func() { m.removeProofScratch(app, scratch) }
	if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil {
		return "", cleanup, err
	}
	unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator)))
	if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok {
		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app)
	}
	man := readManifest(UnitManifestFile(unitDir))
	if man == nil || man.Data == nil {
		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app)
	}
	if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil {
		return "", cleanup, verr
	}
	return unitDir, cleanup, nil
}

// LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then
// prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then
// after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were.
func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) {
	m.BeginRestoreOp("restore", app)
	go func() {
		start := time.Now()
		fail := func(err error) {
			m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err)
			m.EndRestoreOp(false, failMsg(err))
			if after != nil {
				after(false)
			}
		}
		m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath)
		unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c)
		// The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the
		// outcome never finds a copy left behind.
		if err != nil {
			cleanup()
			fail(err)
			return
		}
		if prepare != nil {
			if perr := prepare(); perr != nil {
				cleanup()
				fail(perr)
				return
			}
		}
		res, err := m.keptUnitRestore()(app, unitDir)
		cleanup()
		if err != nil {
			fail(err)
			return
		}
		m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID,
			time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
		m.EndRestoreOp(true, okMsg(nil))
		if after != nil {
			after(true)
		}
	}()
}

// RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when
// it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise.
func (m *Manager) RemovedUnitOnDrive(app, drive string) string {
	if m.isStackDeployed(app) {
		return ""
	}
	u, ok := m.RemovedAppUnitFor(app)
	if !ok {
		return ""
	}
	if !strings.HasPrefix(filepath.Clean(u.UnitDir), filepath.Clean(drive)+string(filepath.Separator)) {
		return ""
	}
	return u.UnitDir
}

// PrimaryUnitHome is where app's own unit lives on drive (backups/primary/<app>).
func (m *Manager) PrimaryUnitHome(app, drive string) string {
	return RecoveryUnitPath(m.namespaceRoot(drive), app)
}

// LoadKeptApp runs the load as a restore operation the backup pages already follow (the poll banner):
// Begin → RestoreFromRecoveryUnitAt(app, unitDir) → End, then after(ok) in the same goroutine (the
// after_load command, the kept folder's tidy-up). The caller has checked RestoreStatus/IsRunning.
func (m *Manager) LoadKeptApp(app, unitDir string, okMsg, failMsg func(err error) string, after func(ok bool)) {
	m.BeginRestoreOp("restore", app)
	go func() {
		start := time.Now()
		res, err := m.RestoreFromRecoveryUnitAt(app, unitDir)
		if err != nil {
			m.logger.Printf("[ERROR] [backup] kept load %s from %s FAILED after %s: %v", app, unitDir, time.Since(start).Round(time.Second), err)
			m.EndRestoreOp(false, failMsg(err))
			if after != nil {
				after(false)
			}
			return
		}
		m.logger.Printf("[INFO] [backup] kept load %s from %s done in %s (volumes %d/%d, dbs %d/%d)", app, unitDir,
			time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
		m.EndRestoreOp(true, okMsg(nil))
		if after != nil {
			after(true)
		}
	}()
}