v0.277.0: kept data loads from the off-site copy too (R-691 (2))
gates / gates (push) Successful in 25s

Use my kept data / Load consider the off-site snapshot when it is newer than every local copy or
the only one; the unit is downloaded alone, judged (drive, data, recorded data version) and only
then restored. The page names the copy and its date.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-28 09:57:11 +02:00
parent 0c1023e503
commit 6479933e8e
14 changed files with 696 additions and 77 deletions
+56 -27
View File
@@ -6,6 +6,7 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
@@ -27,8 +28,9 @@ type keptChoiceData struct {
// keptDataAtInstall answers the install when the app's private drive folder already holds data:
//
// - no choice → 409 kept_data_choice with the sentences (nothing moved, nothing installed);
// - "use" → a LOAD from the newest usable copy (backup.KeptDBCopy), the removed-app restore with
// the unit named, then the app's after_load; 202. Refused 409 with use_off when no copy exists;
// - "use" → a LOAD from the newest usable copy (backup.KeptBestCopy: own unit, second drive, or —
// R-691 (2) — the off-site copy, downloaded unit-only at load time), then the app's after_load; 202.
// The page names the copy and its date. Refused 409 with use_off when no copy exists;
// - "fresh" → not handled here: DeployStack moves the old data aside and installs.
//
// Returns true when it wrote the response.
@@ -50,14 +52,11 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
}
}
var cp struct {
ok bool
time time.Time
dir string
ok bool
c backup.KeptCopy
}
if r.backupMgr != nil {
if c, ok := r.backupMgr.KeptDBCopy(name, hdd); ok {
cp.ok, cp.time, cp.dir = true, c.Time, c.UnitDir
}
cp.c, cp.ok = r.backupMgr.KeptBestCopy(req.Context(), name, hdd)
}
lang := r.langFor(req)
data := keptChoiceData{
@@ -71,7 +70,7 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
NotBacked: r.msgLang(lang, "kept.not_backed_up"),
}
if cp.ok {
data.UseDesc = r.msgLang(lang, "kept.choice.use.desc", localDay(cp.time))
data.UseDesc = r.msgLang(lang, "kept.choice.use.desc_from", r.keptCopyName(lang, cp.c))
} else {
data.UseOff = r.msgLang(lang, "kept.choice.use_off")
}
@@ -91,8 +90,12 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
return true
}
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (%s) under the kept files %v", name, cp.dir, cp.time.UTC().Format(time.RFC3339), old)
r.startKeptLoad(name, cp.dir, lang, nil)
src := cp.c.UnitDir
if cp.c.Tier == backup.KeptTierOffsite {
src = "off-site snapshot " + cp.c.SnapshotID
}
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (tier %d, %s) under the kept files %v", name, src, cp.c.Tier, cp.c.Time.UTC().Format(time.RFC3339), old)
r.startKeptLoad(name, hdd, cp.c, lang, nil)
writeJSON(w, http.StatusAccepted, apiResponse{OK: true, Message: r.msgLang(lang, "kept.load.started", display)})
return true
default:
@@ -101,34 +104,60 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
}
}
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last.
func (r *Router) startKeptLoad(name, unitDir, lang string, then func(ok bool)) {
// keptCopyName names a copy for the household: which copy and its date („saját mentés, 2026-09-28 04:15").
func (r *Router) keptCopyName(lang string, c backup.KeptCopy) string {
return r.msgLang(lang, backup.KeptCopyKey(c.Tier), localMinute(c.Time))
}
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last. The
// off-site copy goes through LoadKeptOffsite (download the unit alone, judge it, then the same restore).
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) {
display := name
if st, ok := r.stackMgr.GetStack(name); ok && st.Meta.DisplayName != "" {
display = st.Meta.DisplayName
}
r.backupMgr.LoadKeptApp(name, unitDir,
func(error) string { return r.msgLang(lang, "kept.load.done", display) },
func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) },
func(ok bool) {
if ok {
if ran, err := r.stackMgr.RunAfterLoad(name, 10*time.Minute); ran && err != nil {
r.logger.Printf("[WARN] [api] kept load %s: after_load failed: %v", name, err)
}
okMsg := func(error) string { return r.msgLang(lang, "kept.load.done", display) }
failMsg := func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) }
if c.Tier == backup.KeptTierOffsite {
r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
return
}
r.backupMgr.LoadKeptApp(name, c.UnitDir, okMsg, failMsg, r.keptAfterLoad(name, then))
}
// keptAfterLoad runs the app's after_load when the load succeeded, then then(ok).
func (r *Router) keptAfterLoad(name string, then func(ok bool)) func(ok bool) {
return func(ok bool) {
if ok {
if ran, err := r.stackMgr.RunAfterLoad(name, 10*time.Minute); ran && err != nil {
r.logger.Printf("[WARN] [api] kept load %s: after_load failed: %v", name, err)
}
if then != nil {
then(ok)
}
})
}
if then != nil {
then(ok)
}
}
}
func localDay(t time.Time) string {
if t.IsZero() {
return "?"
}
return t.In(budapest()).Format("2006-01-02")
}
// localMinute is localDay with the time of day — the kept list's format, so both pages print one date.
func localMinute(t time.Time) string {
if t.IsZero() {
return "?"
}
return t.In(budapest()).Format("2006-01-02 15:04")
}
func budapest() *time.Location {
loc, err := time.LoadLocation("Europe/Budapest")
if err != nil {
loc = time.UTC
return time.UTC
}
return t.In(loc).Format("2006-01-02")
return loc
}
@@ -1,6 +1,7 @@
package api
import (
"context"
"encoding/json"
"io"
"log"
@@ -9,8 +10,11 @@ import (
"os"
"path/filepath"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
@@ -76,3 +80,74 @@ func TestKept_InstallAPIAsksAndInstallsNothing(t *testing.T) {
t.Fatal("an install with no old data was intercepted")
}
}
// R-691 (2) — when the only database copy of kept files is OFF-SITE, the install choice offers „use" and
// names that copy and its date; before v0.277.0 it said "no backup". The repository is the restic runner
// seam; nothing reaches restic, ssh or Docker.
// COMPANION RED-PROOF: keptDataAtInstall back on KeptDBCopy (0.276.0) → use_offered=false and this fails.
func TestR691_InstallChoiceNamesTheOffsiteCopy(t *testing.T) {
dir := t.TempDir()
drive := filepath.Join(dir, "drive")
cfg := &config.Config{}
cfg.Paths.StacksDir = filepath.Join(dir, "stacks")
cfg.Paths.DataDir = filepath.Join(dir, "data")
cfg.Stacks.ComposeCommand = "docker compose"
app := filepath.Join(cfg.Paths.StacksDir, "cloudapp")
for _, d := range []string{app, filepath.Join(drive, "appdata/cloudapp"), cfg.Paths.DataDir} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
_ = os.WriteFile(filepath.Join(app, "docker-compose.yml"), []byte("services:\n cloudapp:\n image: busybox\n volumes:\n - ${HDD_PATH}/appdata/cloudapp:/data\n"), 0o644)
_ = os.WriteFile(filepath.Join(app, ".felhom.yml"), []byte("display_name: Cloud App\n"), 0o644)
_ = os.WriteFile(filepath.Join(drive, "appdata/cloudapp/old.txt"), []byte("old"), 0o644)
m, err := stacks.NewManager(cfg, log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
if err := m.ScanStacks(); err != nil {
t.Fatal(err)
}
sett, err := settings.Load(filepath.Join(cfg.Paths.DataDir, "settings.json"), log.New(io.Discard, "", 0))
if err != nil {
t.Fatal(err)
}
if err := sett.SetOffboxTarget(&settings.OffboxTarget{Enabled: true, Host: "nas.local", Port: 22, User: "felhom",
RepoPath: "/srv/repo", Schedule: "daily", EscrowState: "escrowed"}); err != nil {
t.Fatal(err)
}
bm := backup.NewManager(cfg, sett, log.New(io.Discard, "", 0))
if err := bm.WriteOffboxSecrets("KEY", "nas.local ssh-ed25519 AAAA"); err != nil {
t.Fatal(err)
}
snapAt := time.Date(2026, 9, 28, 2, 15, 0, 0, time.UTC) // 04:15 in Budapest
bm.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
for _, a := range args {
if a == "snapshots" {
return json.Marshal([]map[string]interface{}{{"short_id": "ab12cd34", "time": snapAt,
"tags": []string{"cloudapp"}, "paths": []string{drive + "/backups/primary/cloudapp", drive + "/appdata/cloudapp"}}})
}
}
return nil, nil
})
r := &Router{stackMgr: m, backupMgr: bm, logger: log.New(io.Discard, "", 0)}
for lang, want := range map[string]string{
"en": "We load the database from the off-site copy, 2026-09-28 04:15 and start the app with the old files. Changes made after that time can be missing.",
"hu": "Az adatbázist innen töltjük vissza: távoli mentés, 2026-09-28 04:15. A régi fájlokkal indítjuk az alkalmazást. Ami ezután változott, hiányozhat.",
} {
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang="+lang, nil)
if !r.keptDataAtInstall(w, req, "cloudapp", drive, "") || w.Code != http.StatusConflict {
t.Fatalf("%s: the choice was not asked (code %d)", lang, w.Code)
}
var body map[string]interface{}
_ = json.Unmarshal(w.Body.Bytes(), &body)
data, _ := body["data"].(map[string]interface{})
if data["use_offered"] != true || data["use_desc"] != want || data["use_off"] != "" {
t.Fatalf("%s: use_offered=%v use_desc=%q use_off=%q\nwant use_desc=%q", lang, data["use_offered"], data["use_desc"], data["use_off"], want)
}
}
if st, _ := m.GetStack("cloudapp"); st.Deployed || st.Deploying {
t.Fatal("something was installed")
}
}
+4
View File
@@ -156,6 +156,10 @@ type Manager struct {
// INIT/TEST ONLY. Nil in production, where offboxLatestSnapshot runs unchanged.
offboxLatestSnapFn func(ctx context.Context, stack string) (string, []string, error)
// keptUnitRestoreFn (R-691 (2)) — the kept load's unit restore; nil → RestoreFromRecoveryUnitAt.
// INIT/TEST ONLY.
keptUnitRestoreFn func(app, unitDir string) (UnitRestoreResult, error)
// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
discoverDBs func(ctx context.Context) ([]DiscoveredDB, error)
+200 -5
View File
@@ -1,11 +1,14 @@
package backup
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"gopkg.in/yaml.v3"
)
@@ -19,18 +22,49 @@ import (
// WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a
// database dump or a volume tar — a definition alone would install an empty app over the kept files), and
// (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the
// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2). The
// off-site copy is NOT looked at here: its restore is a different operation (reconstitute) and it is
// not built into this choice yet — said on the page as "none" when it is the only one.
// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2).
//
// THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one
// `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is
// NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is
// downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore,
// restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three
// rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before
// v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only
// then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body
// (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as
// they were. Pinned by internal/backup/r691_kept_offsite_test.go.
// KeptCopy is one database copy a kept folder can be loaded from.
type KeptCopy struct {
UnitDir string
Tier int // 1 own unit, 2 second drive
UnitDir string // "" for the off-site copy (it is downloaded at load time)
Tier int // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite)
Time time.Time
DriveLabel string
// SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it.
SnapshotID string
UnitPath string
}
// KeptTierOffsite is KeptCopy.Tier for the off-site copy.
const KeptTierOffsite = 3
// KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the
// install choice and the kept list, so the two pages cannot name the same copy differently.
func KeptCopyKey(tier int) string {
switch tier {
case 2:
return "kept.backup.second"
case KeptTierOffsite:
return "kept.backup.offsite"
}
return "kept.backup.own"
}
// keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow
// or unreachable repository then costs the page this long at most and the off-site copy is not offered.
const keptOffsiteLookupTimeout = 20 * time.Second
// unitHoldsData: a readable manifest that lists a database dump or a volume tar.
func unitHoldsData(unitDir string) (*RecoveryManifest, bool) {
man := readManifest(UnitManifestFile(unitDir))
@@ -96,6 +130,167 @@ func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) {
return best, found
}
// KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest
// snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4).
// Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call
// for all apps (the kept list asks once per page, not once per row). Any error reading the repository
// offers nothing (logged) — the local copies are still offered.
func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy {
out := map[string]KeptCopy{}
var want []string
for _, a := range apps {
if a != "" && !m.isStackDeployed(a) {
want = append(want, a)
}
}
if len(want) == 0 || !m.OffboxConfigured() {
return out
}
cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout)
defer cancel()
newest, _, err := m.offsiteNewestPerTag(cctx)
if err != nil {
m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err)
return out
}
for _, app := range want {
n, ok := newest[app]
if !ok {
continue
}
unitPath := offboxUnitPathOf(n.paths, app)
if unitPath == "" {
m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id)
continue
}
out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath}
}
return out
}
// KeptOffsiteCopy is KeptOffsiteCopies for one app.
func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) {
c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app]
return c, ok
}
// KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or
// the only one (a tie goes local — no download).
func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) {
if ook && (!lok || off.Time.After(local.Time)) {
return off, true
}
return local, lok
}
// KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest
// usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one.
func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) {
local, lok := m.KeptDBCopy(app, drive)
off, ook := m.KeptOffsiteCopy(ctx, app)
return KeptNewer(local, lok, off, ook)
}
// Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything
// of the app is touched.
var (
ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version")
ErrKeptOffsiteNotUsable = errors.New("the off-site copy is not a copy of these kept files")
)
// keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY:
// the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker.
func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) {
if m.keptUnitRestoreFn != nil {
return m.keptUnitRestoreFn
}
return m.RestoreFromRecoveryUnitAt
}
// SetKeptUnitRestoreFn overrides the kept load's unit restore (tests).
func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) {
m.keptUnitRestoreFn = fn
}
// downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It
// returns the unit directory and a cleanup that removes the scratch (call it on every path).
func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) {
noop := func() {}
if err := m.acquireRunning(); err != nil {
return "", noop, err
}
defer m.releaseRunning()
scratch, nsRoot, err := m.offboxProofScratchDir(app)
if err != nil {
return "", noop, err
}
if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil {
return "", noop, herr
}
m.removeProofScratch(app, scratch) // a copy an interrupted run left
cleanup := func() { m.removeProofScratch(app, scratch) }
if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil {
return "", cleanup, err
}
unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator)))
if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok {
return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app)
}
man := readManifest(UnitManifestFile(unitDir))
if man == nil || man.Data == nil {
return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app)
}
if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil {
return "", cleanup, verr
}
return unitDir, cleanup, nil
}
// LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then
// prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then
// after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were.
func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) {
m.BeginRestoreOp("restore", app)
go func() {
start := time.Now()
fail := func(err error) {
m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err)
m.EndRestoreOp(false, failMsg(err))
if after != nil {
after(false)
}
}
m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath)
unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c)
// The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the
// outcome never finds a copy left behind.
if err != nil {
cleanup()
fail(err)
return
}
if prepare != nil {
if perr := prepare(); perr != nil {
cleanup()
fail(perr)
return
}
}
res, err := m.keptUnitRestore()(app, unitDir)
cleanup()
if err != nil {
fail(err)
return
}
m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID,
time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
m.EndRestoreOp(true, okMsg(nil))
if after != nil {
after(true)
}
}()
}
// RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when
// it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise.
func (m *Manager) RemovedUnitOnDrive(app, drive string) string {
@@ -56,6 +56,9 @@ type OffsiteInventory struct {
type offsiteNewest struct {
id string
at time.Time
// paths are the snapshot's captured paths (R-691 (2): the kept-data choice asks whether the newest
// snapshot holds the app's recovery unit without a second repository call).
paths []string
}
// offsiteNewestPerTag runs ONE `snapshots --json` and returns the newest snapshot per app tag, and
@@ -81,6 +84,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
ID string `json:"id"`
Time time.Time `json:"time"`
Tags []string `json:"tags"`
Paths []string `json:"paths"`
}
if uerr := json.Unmarshal(out, &snaps); uerr != nil {
return nil, false, uerr
@@ -100,7 +104,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
continue
}
if cur, ok := newest[tag]; !ok || sn.Time.After(cur.at) {
newest[tag] = offsiteNewest{id: id, at: sn.Time}
newest[tag] = offsiteNewest{id: id, at: sn.Time, paths: sn.Paths}
}
}
}
@@ -0,0 +1,237 @@
package backup
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// R-691 (2) — „Use my kept data" / Load also look at the OFF-SITE copy (controller v0.277.0).
//
// Driven through the real KeptBestCopy / LoadKeptOffsite with the two restic seams (SetOffboxRunner answers
// `snapshots --json` and materialises the unit a `restore --include` asks for) and the unit-restore seam
// (SetKeptUnitRestoreFn records the call instead of reaching Docker). No restic, no ssh, no docker.
const r691App = "nextcloud"
type r691Harness struct {
*proofHarness
snapAt time.Time
snapPaths []string
failSnaps bool
// manifest is what the downloaded unit's manifest.json holds (raw JSON).
manifest string
unitHDD string
// restored records every unit restore call (unitDir); prepared counts prepare calls.
restored []string
prepared int
done chan bool
}
func newR691Harness(t *testing.T) *r691Harness {
t.Helper()
ph := newProofHarness(t) // nothing deployed: nextcloud is a REMOVED app with kept files
h := &r691Harness{proofHarness: ph, snapAt: time.Now().Add(-time.Hour), done: make(chan bool, 1)}
h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/backups/primary/" + r691App, "/mnt/felhom-drives/hdd_1/appdata/" + r691App}
h.unitHDD = ph.drive
h.manifest = r691Manifest(true)
ph.m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
ph.mu.Lock()
ph.argv = append(ph.argv, append([]string{}, args...))
ph.mu.Unlock()
switch {
case containsArg(args, "snapshots"):
if h.failSnaps {
return []byte("ssh: connect refused"), errors.New("exit status 1")
}
b, _ := json.Marshal([]map[string]interface{}{{
"short_id": "ab12cd34", "id": "ab12cd34ffff", "time": h.snapAt, "tags": []string{r691App}, "paths": h.snapPaths}})
return b, nil
case containsArg(args, "restore"):
if ph.failRestore {
return []byte("simulated restic failure"), os.ErrPermission
}
target, include := argValue(args, "--target"), argValue(args, "--include")
dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
for _, sub := range []string{"compose", "db-dumps"} {
_ = os.MkdirAll(filepath.Join(dest, sub), 0o755)
}
_ = os.WriteFile(filepath.Join(dest, "db-dumps", "nextcloud-mariadb.sql"), []byte("x"), 0o644)
_ = os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte("services:\n nextcloud:\n image: nextcloud:31\n db:\n image: mariadb:11.8\n"), 0o644)
_ = os.WriteFile(filepath.Join(dest, "compose", "app.yaml"), []byte("env:\n HDD_PATH: "+h.unitHDD+"\n"), 0o600)
_ = os.WriteFile(filepath.Join(dest, "manifest.json"), []byte(h.manifest), 0o644)
return nil, nil
}
return nil, nil
})
ph.m.SetKeptUnitRestoreFn(func(app, unitDir string) (UnitRestoreResult, error) {
// The unit must still be on disk while the restore reads it.
if _, err := os.Stat(UnitManifestFile(unitDir)); err != nil {
t.Errorf("the unit restore ran on %s, which no longer holds a manifest: %v", unitDir, err)
}
h.restored = append(h.restored, unitDir)
return UnitRestoreResult{ManifestDBs: 1, DBsReplayed: 1}, nil
})
return h
}
// r691Manifest is a unit manifest listing one dump; withData adds the `data` block (the data's version).
func r691Manifest(withData bool) string {
man := map[string]interface{}{"schema_version": 2, "app_name": r691App, "db_dumps": []string{"nextcloud-mariadb.sql"}}
if withData {
man["data"] = map[string]interface{}{"at": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339),
"image_pins": []string{"mariadb:11.8", "nextcloud:31"}}
}
b, _ := json.Marshal(man)
return string(b)
}
func (h *r691Harness) load(t *testing.T, c KeptCopy) (ok bool) {
t.Helper()
h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
func(error) string { return "ok" }, func(err error) string { return err.Error() },
func(ok bool) { h.done <- ok })
select {
case ok = <-h.done:
case <-time.After(10 * time.Second):
t.Fatal("the load never finished")
}
return ok
}
// The consequence, not the mechanism: which copy the household is offered.
// COMPANION RED-PROOF: KeptBestCopy returning KeptDBCopy alone (0.276.0's choice) → the first assertion fails
// with ok=false (an app whose only database copy is off-site gets „no backup").
func TestR691_OffsiteCopyIsOfferedWhenItIsTheOnlyOrNewest(t *testing.T) {
h := newR691Harness(t)
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
if !ok || c.Tier != KeptTierOffsite || c.SnapshotID != "ab12cd34" || c.UnitPath != h.snapPaths[0] {
t.Fatalf("only off-site copy: got %+v ok=%v, want the off-site snapshot ab12cd34", c, ok)
}
if !c.Time.Equal(h.snapAt) {
t.Fatalf("the copy is dated %v, want the snapshot's time %v", c.Time, h.snapAt)
}
// A local unit NEWER than the snapshot wins — no download.
unit := RecoveryUnitPath(h.m.namespaceRoot(h.drive), r691App)
writeKeptUnit(t, unit, h.drive, true, time.Now())
h.prov.hdd[r691App] = "" // removed
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != 1 || c.UnitDir != unit {
t.Fatalf("a newer local unit must win: got %+v ok=%v", c, ok)
}
// An OLDER local unit loses to the off-site copy.
old := time.Now().Add(-3 * time.Hour)
writeKeptUnit(t, unit, h.drive, true, old)
for _, f := range []string{UnitManifestFile(unit), filepath.Join(unit, "compose", "app.yaml")} {
_ = os.Chtimes(f, old, old)
}
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != KeptTierOffsite {
t.Fatalf("an older local unit must lose to the off-site copy: got %+v ok=%v", c, ok)
}
// Negative controls: a snapshot with no unit, an unreadable repository, an INSTALLED app.
_ = os.RemoveAll(unit)
h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/appdata/" + r691App}
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
t.Fatalf("a snapshot holding no recovery unit was offered: %+v", c)
}
h.snapPaths = []string{"/x/backups/primary/" + r691App}
h.failSnaps = true
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
t.Fatalf("an unreadable repository offered a copy: %+v", c)
}
h.failSnaps = false
h.prov.deployed[r691App] = true
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
t.Fatalf("an installed app's off-site copy was offered as kept data: %+v", c)
}
}
// The load: download the unit ALONE, judge it, THEN prepare, THEN the one unit restore; the scratch goes.
// COMPANION RED-PROOF: drop the `man.Data == nil` refusal in downloadKeptOffsiteUnit → the no-data-block case
// reaches the restore (restored=1, prepared=1) and fails.
func TestR691_OffsiteLoadDownloadsJudgesThenRestores(t *testing.T) {
h := newR691Harness(t)
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
if !ok {
t.Fatal("no off-site copy offered")
}
if !h.load(t, c) {
t.Fatalf("a good off-site unit did not load: %+v", h.m.RestoreStatus())
}
if h.prepared != 1 || len(h.restored) != 1 {
t.Fatalf("prepare=%d restore=%d, want 1 and 1", h.prepared, len(h.restored))
}
scratch := h.proofScratch(t, r691App)
if !strings.HasPrefix(h.restored[0], scratch+string(filepath.Separator)) || !strings.HasSuffix(h.restored[0], "/backups/primary/"+r691App) {
t.Fatalf("the restore read %s, want the downloaded unit inside %s", h.restored[0], scratch)
}
var sawUnitOnly bool
for _, a := range h.allArgs() {
if containsArg(a, "restore") && argValue(a, "--include") == h.snapPaths[0] && containsArg(a, "ab12cd34") {
sawUnitOnly = true
}
}
if !sawUnitOnly {
t.Fatalf("no unit-only restore of snapshot ab12cd34 (--include %s): %v", h.snapPaths[0], h.allArgs())
}
if _, err := os.Stat(scratch); !os.IsNotExist(err) {
t.Fatalf("the downloaded copy was left behind at %s (err=%v)", scratch, err)
}
}
// Every refusal happens BEFORE prepare: the kept files are exactly as they were, nothing is restored.
func TestR691_OffsiteLoadRefusalsLeaveTheKeptFilesAlone(t *testing.T) {
cases := []struct {
name string
set func(h *r691Harness)
want error
}{
{"data version not recorded", func(h *r691Harness) { h.manifest = r691Manifest(false) }, ErrKeptOffsiteVersionUnknown},
{"taken of another drive", func(h *r691Harness) { h.unitHDD = "/mnt/felhom-drives/other" }, ErrKeptOffsiteNotUsable},
{"definition is not the data's", func(h *r691Harness) {
h.manifest = strings.Replace(r691Manifest(true), "nextcloud:31", "nextcloud:30", 1)
}, ErrUnitVersionMismatch},
{"download failed", func(h *r691Harness) { h.failRestore = true }, nil},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
h := newR691Harness(t)
kept := filepath.Join(h.drive, "appdata", r691App, "photo.jpg")
_ = os.MkdirAll(filepath.Dir(kept), 0o755)
_ = os.WriteFile(kept, []byte("the household's photo"), 0o644)
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
if !ok {
t.Fatal("no off-site copy offered")
}
tc.set(h)
var got error
h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
func(error) string { return "ok" }, func(err error) string { got = err; return err.Error() },
func(ok bool) { h.done <- ok })
if <-h.done {
t.Fatal("the load reported success")
}
if h.prepared != 0 || len(h.restored) != 0 {
t.Fatalf("prepare=%d restore=%d after a refusal, want 0 and 0", h.prepared, len(h.restored))
}
if tc.want != nil && !errors.Is(got, tc.want) {
t.Fatalf("refused with %v, want %v", got, tc.want)
}
if b, err := os.ReadFile(kept); err != nil || string(b) != "the household's photo" {
t.Fatalf("the kept file changed: %q %v", b, err)
}
if st := h.m.RestoreStatus(); st.Running || st.Last == nil || st.Last.OK {
t.Fatalf("the restore record does not read as failed: %+v", st)
}
if _, err := os.Stat(h.proofScratch(t, r691App)); !os.IsNotExist(err) {
t.Fatalf("the downloaded copy was left behind after a refusal (err=%v)", err)
}
})
}
}
+4
View File
@@ -2441,6 +2441,10 @@
"kept.backup.second": "the second drive, %s",
"kept.backup.with": "the backup kept with it, %s",
"kept.backup.none": "no backup — the files only",
"kept.backup.offsite": "the off-site copy, %s",
"kept.choice.use.desc_from": "We load the database from %s and start the app with the old files. Changes made after that time can be missing.",
"err.kept.offsite_version_unknown": "the off-site copy of %s does not record which version wrote its data, so it is not loaded",
"err.kept.offsite_not_usable": "the off-site copy of %s does not belong to these files (it was taken of another drive, or it holds no database), so it is not loaded",
"kept.action.load": "Load",
"kept.action.look": "Look",
"kept.action.delete": "Delete",
+4
View File
@@ -2429,6 +2429,10 @@
"kept.backup.second": "második meghajtó, %s",
"kept.backup.with": "a vele megőrzött mentés, %s",
"kept.backup.none": "nincs mentés — csak a fájlok",
"kept.backup.offsite": "távoli mentés, %s",
"kept.choice.use.desc_from": "Az adatbázist innen töltjük vissza: %s. A régi fájlokkal indítjuk az alkalmazást. Ami ezután változott, hiányozhat.",
"err.kept.offsite_version_unknown": "a(z) %s távoli mentése nem rögzíti, melyik verzió írta az adatait, ezért nem töltjük be",
"err.kept.offsite_not_usable": "a(z) %s távoli mentése nem ezekhez a fájlokhoz tartozik (másik meghajtóról készült, vagy nincs benne adatbázis), ezért nem töltjük be",
"kept.action.load": "Betöltés",
"kept.action.look": "Megnézem",
"kept.action.delete": "Törlés",
+67 -39
View File
@@ -1,6 +1,7 @@
package web
import (
"context"
"net/http"
"net/url"
"os"
@@ -11,6 +12,7 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)
@@ -63,32 +65,44 @@ func KeptViewName(it stacks.KeptItem) string {
return base + filepath.Base(it.Path)
}
// keptBackupFor names the copy a Load would use for it, and whether one exists.
func (s *Server) keptBackupFor(lang string, it stacks.KeptItem) (string, string, bool) {
if s.backupMgr == nil {
return s.msgLang(lang, "kept.backup.none"), "", false
// keptBackupFor names the copy a Load would use for it, and whether one exists. off is the off-site copy
// per app (backup.KeptOffsiteCopies, asked once per page): it is used when it is newer than every local
// copy, or the only one (R-691 (2)).
func (s *Server) keptBackupFor(lang string, it stacks.KeptItem, off map[string]backup.KeptCopy) (string, backup.KeptCopy, bool) {
none := s.msgLang(lang, "kept.backup.none")
if s.backupMgr == nil || it.App == "" && it.Kind != stacks.KeptKindDated {
return none, backup.KeptCopy{}, false
}
var local backup.KeptCopy
var lok bool
if it.Kind == stacks.KeptKindDated {
if it.UnitDir == "" {
return s.msgLang(lang, "kept.backup.none"), "", false
// A dated folder's own unit is the copy taken with those files.
if it.UnitDir != "" {
if c, ok := s.backupMgr.KeptCopyAt(it.UnitDir, it.Drive, 1); ok {
return s.msgLang(lang, "kept.backup.with", c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c, true
}
}
if c, ok := s.backupMgr.KeptCopyAt(it.UnitDir, it.Drive, 1); ok {
return s.msgLang(lang, "kept.backup.with", c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c.UnitDir, true
}
return s.msgLang(lang, "kept.backup.none"), "", false
} else {
local, lok = s.backupMgr.KeptDBCopy(it.App, it.Drive)
}
if it.App == "" {
return s.msgLang(lang, "kept.backup.none"), "", false
}
c, ok := s.backupMgr.KeptDBCopy(it.App, it.Drive)
oc, ook := off[it.App]
c, ok := backup.KeptNewer(local, lok, oc, ook && it.App != "")
if !ok {
return s.msgLang(lang, "kept.backup.none"), "", false
return none, backup.KeptCopy{}, false
}
key := "kept.backup.own"
if c.Tier == 2 {
key = "kept.backup.second"
return s.msgLang(lang, backup.KeptCopyKey(c.Tier), c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c, true
}
// keptOffsite asks the off-site repository ONCE for every listed app.
func (s *Server) keptOffsite(ctx context.Context, items []stacks.KeptItem) map[string]backup.KeptCopy {
if s.backupMgr == nil {
return nil
}
return s.msgLang(lang, key, c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c.UnitDir, true
var apps []string
for _, it := range items {
apps = append(apps, it.App)
}
return s.backupMgr.KeptOffsiteCopies(ctx, apps)
}
func (s *Server) keptPageHandler(w http.ResponseWriter, r *http.Request) {
@@ -97,12 +111,14 @@ func (s *Server) keptPageHandler(w http.ResponseWriter, r *http.Request) {
data["TitleKey"] = "page.title.kept_data"
var rows []keptRow
if s.stackMgr != nil {
for _, it := range s.stackMgr.ListKept(s.keptDrives()) {
backup, _, can := s.keptBackupFor(lang, it)
items := s.stackMgr.ListKept(s.keptDrives())
off := s.keptOffsite(r.Context(), items)
for _, it := range items {
backupName, _, can := s.keptBackupFor(lang, it, off)
row := keptRow{
DisplayName: it.DisplayName, App: it.App, Path: it.Path, Drive: it.Drive,
Date: it.Date.In(getTimezone()).Format("2006-01-02 15:04"),
Size: appbackup.HumanizeBytes(it.SizeBytes), Backup: backup, CanLoad: can && it.App != "",
Size: appbackup.HumanizeBytes(it.SizeBytes), Backup: backupName, CanLoad: can && it.App != "",
LookURL: fileBrowserLink(s.cfg.Customer.Domain, s.msgLang(s.boxLang(), "kept.fb_source"), KeptViewName(it)),
}
row.DeleteText = s.msgLang(lang, "kept.delete.confirm", it.DisplayName, row.Size)
@@ -165,31 +181,43 @@ func (s *Server) keptLoadHandler(w http.ResponseWriter, r *http.Request) {
s.keptRedirect(w, r, "flash_error", msg)
return
}
_, unit, can := s.keptBackupFor(lang, it)
_, c, can := s.keptBackupFor(lang, it, s.keptOffsite(r.Context(), []stacks.KeptItem{it}))
if !can {
s.keptRedirect(w, r, "flash_error", s.msgLang(lang, "kept.choice.use_off"))
return
}
if it.Kind == stacks.KeptKindDated {
if _, err := s.stackMgr.RestoreKeptFiles(it); err != nil {
app, disp := it.App, it.DisplayName
okMsg := func(error) string { return s.msgLang(lang, "kept.load.done", disp) }
failMsg := func(err error) string { return s.msgLang(lang, "kept.load.failed", disp, err) }
after := func(ok bool) {
if ok {
if ran, err := s.stackMgr.RunAfterLoad(app, 10*time.Minute); ran && err != nil {
s.logger.Printf("[WARN] [web] kept load %s: after_load failed: %v", app, err)
}
s.stackMgr.FinishKeptLoad(it, s.backupMgr.PrimaryUnitHome(app, it.Drive))
}
s.SyncFileBrowserMounts()
}
// A dated folder's files move back BEFORE the load — for the off-site copy only after its unit is
// downloaded and judged, so a failed download or a refusal leaves the kept folder as it was.
moveBack := func() error {
if it.Kind != stacks.KeptKindDated {
return nil
}
_, err := s.stackMgr.RestoreKeptFiles(it)
return err
}
if c.Tier == backup.KeptTierOffsite {
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from the off-site snapshot %s (from %s)", it.App, it.Path, c.SnapshotID, r.RemoteAddr)
s.backupMgr.LoadKeptOffsite(app, it.Drive, c, moveBack, okMsg, failMsg, after)
} else {
if err := moveBack(); err != nil {
s.keptRedirect(w, r, "flash_error", s.errText(r, err))
return
}
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from %s (from %s)", it.App, it.Path, c.UnitDir, r.RemoteAddr)
s.backupMgr.LoadKeptApp(app, c.UnitDir, okMsg, failMsg, after)
}
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from %s (from %s)", it.App, it.Path, unit, r.RemoteAddr)
app, disp := it.App, it.DisplayName
s.backupMgr.LoadKeptApp(app, unit,
func(error) string { return s.msgLang(lang, "kept.load.done", disp) },
func(err error) string { return s.msgLang(lang, "kept.load.failed", disp, err) },
func(ok bool) {
if ok {
if ran, err := s.stackMgr.RunAfterLoad(app, 10*time.Minute); ran && err != nil {
s.logger.Printf("[WARN] [web] kept load %s: after_load failed: %v", app, err)
}
s.stackMgr.FinishKeptLoad(it, s.backupMgr.PrimaryUnitHome(app, it.Drive))
}
s.SyncFileBrowserMounts()
})
http.Redirect(w, r, "/backups/restore?"+flashQuery("flash", "flash.restore.started"), http.StatusFound)
}