v0.277.0: kept data loads from the off-site copy too (R-691 (2))
gates / gates (push) Successful in 25s
gates / gates (push) Successful in 25s
Use my kept data / Load consider the off-site snapshot when it is newer than every local copy or the only one; the unit is downloaded alone, judged (drive, data, recorded data version) and only then restored. The page names the copy and its date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -6,6 +6,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
@@ -27,8 +28,9 @@ type keptChoiceData struct {
|
||||
// keptDataAtInstall answers the install when the app's private drive folder already holds data:
|
||||
//
|
||||
// - no choice → 409 kept_data_choice with the sentences (nothing moved, nothing installed);
|
||||
// - "use" → a LOAD from the newest usable copy (backup.KeptDBCopy), the removed-app restore with
|
||||
// the unit named, then the app's after_load; 202. Refused 409 with use_off when no copy exists;
|
||||
// - "use" → a LOAD from the newest usable copy (backup.KeptBestCopy: own unit, second drive, or —
|
||||
// R-691 (2) — the off-site copy, downloaded unit-only at load time), then the app's after_load; 202.
|
||||
// The page names the copy and its date. Refused 409 with use_off when no copy exists;
|
||||
// - "fresh" → not handled here: DeployStack moves the old data aside and installs.
|
||||
//
|
||||
// Returns true when it wrote the response.
|
||||
@@ -50,14 +52,11 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
}
|
||||
}
|
||||
var cp struct {
|
||||
ok bool
|
||||
time time.Time
|
||||
dir string
|
||||
ok bool
|
||||
c backup.KeptCopy
|
||||
}
|
||||
if r.backupMgr != nil {
|
||||
if c, ok := r.backupMgr.KeptDBCopy(name, hdd); ok {
|
||||
cp.ok, cp.time, cp.dir = true, c.Time, c.UnitDir
|
||||
}
|
||||
cp.c, cp.ok = r.backupMgr.KeptBestCopy(req.Context(), name, hdd)
|
||||
}
|
||||
lang := r.langFor(req)
|
||||
data := keptChoiceData{
|
||||
@@ -71,7 +70,7 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
NotBacked: r.msgLang(lang, "kept.not_backed_up"),
|
||||
}
|
||||
if cp.ok {
|
||||
data.UseDesc = r.msgLang(lang, "kept.choice.use.desc", localDay(cp.time))
|
||||
data.UseDesc = r.msgLang(lang, "kept.choice.use.desc_from", r.keptCopyName(lang, cp.c))
|
||||
} else {
|
||||
data.UseOff = r.msgLang(lang, "kept.choice.use_off")
|
||||
}
|
||||
@@ -91,8 +90,12 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
writeJSON(w, http.StatusConflict, apiResponse{OK: false, Error: r.msgLang(lang, "api.kept.busy")})
|
||||
return true
|
||||
}
|
||||
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (%s) under the kept files %v", name, cp.dir, cp.time.UTC().Format(time.RFC3339), old)
|
||||
r.startKeptLoad(name, cp.dir, lang, nil)
|
||||
src := cp.c.UnitDir
|
||||
if cp.c.Tier == backup.KeptTierOffsite {
|
||||
src = "off-site snapshot " + cp.c.SnapshotID
|
||||
}
|
||||
r.logger.Printf("[INFO] [api] Deploy %s: USE MY KEPT DATA — loading from %s (tier %d, %s) under the kept files %v", name, src, cp.c.Tier, cp.c.Time.UTC().Format(time.RFC3339), old)
|
||||
r.startKeptLoad(name, hdd, cp.c, lang, nil)
|
||||
writeJSON(w, http.StatusAccepted, apiResponse{OK: true, Message: r.msgLang(lang, "kept.load.started", display)})
|
||||
return true
|
||||
default:
|
||||
@@ -101,34 +104,60 @@ func (r *Router) keptDataAtInstall(w http.ResponseWriter, req *http.Request, nam
|
||||
}
|
||||
}
|
||||
|
||||
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last.
|
||||
func (r *Router) startKeptLoad(name, unitDir, lang string, then func(ok bool)) {
|
||||
// keptCopyName names a copy for the household: which copy and its date („saját mentés, 2026-09-28 04:15").
|
||||
func (r *Router) keptCopyName(lang string, c backup.KeptCopy) string {
|
||||
return r.msgLang(lang, backup.KeptCopyKey(c.Tier), localMinute(c.Time))
|
||||
}
|
||||
|
||||
// startKeptLoad runs the load and, when it succeeded, the app's after_load. then(ok) runs last. The
|
||||
// off-site copy goes through LoadKeptOffsite (download the unit alone, judge it, then the same restore).
|
||||
func (r *Router) startKeptLoad(name, drive string, c backup.KeptCopy, lang string, then func(ok bool)) {
|
||||
display := name
|
||||
if st, ok := r.stackMgr.GetStack(name); ok && st.Meta.DisplayName != "" {
|
||||
display = st.Meta.DisplayName
|
||||
}
|
||||
r.backupMgr.LoadKeptApp(name, unitDir,
|
||||
func(error) string { return r.msgLang(lang, "kept.load.done", display) },
|
||||
func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) },
|
||||
func(ok bool) {
|
||||
if ok {
|
||||
if ran, err := r.stackMgr.RunAfterLoad(name, 10*time.Minute); ran && err != nil {
|
||||
r.logger.Printf("[WARN] [api] kept load %s: after_load failed: %v", name, err)
|
||||
}
|
||||
okMsg := func(error) string { return r.msgLang(lang, "kept.load.done", display) }
|
||||
failMsg := func(err error) string { return r.msgLang(lang, "kept.load.failed", display, err) }
|
||||
if c.Tier == backup.KeptTierOffsite {
|
||||
r.backupMgr.LoadKeptOffsite(name, drive, c, nil, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
return
|
||||
}
|
||||
r.backupMgr.LoadKeptApp(name, c.UnitDir, okMsg, failMsg, r.keptAfterLoad(name, then))
|
||||
}
|
||||
|
||||
// keptAfterLoad runs the app's after_load when the load succeeded, then then(ok).
|
||||
func (r *Router) keptAfterLoad(name string, then func(ok bool)) func(ok bool) {
|
||||
return func(ok bool) {
|
||||
if ok {
|
||||
if ran, err := r.stackMgr.RunAfterLoad(name, 10*time.Minute); ran && err != nil {
|
||||
r.logger.Printf("[WARN] [api] kept load %s: after_load failed: %v", name, err)
|
||||
}
|
||||
if then != nil {
|
||||
then(ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
if then != nil {
|
||||
then(ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func localDay(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "?"
|
||||
}
|
||||
return t.In(budapest()).Format("2006-01-02")
|
||||
}
|
||||
|
||||
// localMinute is localDay with the time of day — the kept list's format, so both pages print one date.
|
||||
func localMinute(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "?"
|
||||
}
|
||||
return t.In(budapest()).Format("2006-01-02 15:04")
|
||||
}
|
||||
|
||||
func budapest() *time.Location {
|
||||
loc, err := time.LoadLocation("Europe/Budapest")
|
||||
if err != nil {
|
||||
loc = time.UTC
|
||||
return time.UTC
|
||||
}
|
||||
return t.In(loc).Format("2006-01-02")
|
||||
return loc
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
@@ -9,8 +10,11 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
|
||||
@@ -76,3 +80,74 @@ func TestKept_InstallAPIAsksAndInstallsNothing(t *testing.T) {
|
||||
t.Fatal("an install with no old data was intercepted")
|
||||
}
|
||||
}
|
||||
|
||||
// R-691 (2) — when the only database copy of kept files is OFF-SITE, the install choice offers „use" and
|
||||
// names that copy and its date; before v0.277.0 it said "no backup". The repository is the restic runner
|
||||
// seam; nothing reaches restic, ssh or Docker.
|
||||
// COMPANION RED-PROOF: keptDataAtInstall back on KeptDBCopy (0.276.0) → use_offered=false and this fails.
|
||||
func TestR691_InstallChoiceNamesTheOffsiteCopy(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
drive := filepath.Join(dir, "drive")
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.StacksDir = filepath.Join(dir, "stacks")
|
||||
cfg.Paths.DataDir = filepath.Join(dir, "data")
|
||||
cfg.Stacks.ComposeCommand = "docker compose"
|
||||
app := filepath.Join(cfg.Paths.StacksDir, "cloudapp")
|
||||
for _, d := range []string{app, filepath.Join(drive, "appdata/cloudapp"), cfg.Paths.DataDir} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(app, "docker-compose.yml"), []byte("services:\n cloudapp:\n image: busybox\n volumes:\n - ${HDD_PATH}/appdata/cloudapp:/data\n"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(app, ".felhom.yml"), []byte("display_name: Cloud App\n"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(drive, "appdata/cloudapp/old.txt"), []byte("old"), 0o644)
|
||||
m, err := stacks.NewManager(cfg, log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.ScanStacks(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sett, err := settings.Load(filepath.Join(cfg.Paths.DataDir, "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sett.SetOffboxTarget(&settings.OffboxTarget{Enabled: true, Host: "nas.local", Port: 22, User: "felhom",
|
||||
RepoPath: "/srv/repo", Schedule: "daily", EscrowState: "escrowed"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bm := backup.NewManager(cfg, sett, log.New(io.Discard, "", 0))
|
||||
if err := bm.WriteOffboxSecrets("KEY", "nas.local ssh-ed25519 AAAA"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
snapAt := time.Date(2026, 9, 28, 2, 15, 0, 0, time.UTC) // 04:15 in Budapest
|
||||
bm.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
for _, a := range args {
|
||||
if a == "snapshots" {
|
||||
return json.Marshal([]map[string]interface{}{{"short_id": "ab12cd34", "time": snapAt,
|
||||
"tags": []string{"cloudapp"}, "paths": []string{drive + "/backups/primary/cloudapp", drive + "/appdata/cloudapp"}}})
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
r := &Router{stackMgr: m, backupMgr: bm, logger: log.New(io.Discard, "", 0)}
|
||||
for lang, want := range map[string]string{
|
||||
"en": "We load the database from the off-site copy, 2026-09-28 04:15 and start the app with the old files. Changes made after that time can be missing.",
|
||||
"hu": "Az adatbázist innen töltjük vissza: távoli mentés, 2026-09-28 04:15. A régi fájlokkal indítjuk az alkalmazást. Ami ezután változott, hiányozhat.",
|
||||
} {
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/stacks/cloudapp/deploy?lang="+lang, nil)
|
||||
if !r.keptDataAtInstall(w, req, "cloudapp", drive, "") || w.Code != http.StatusConflict {
|
||||
t.Fatalf("%s: the choice was not asked (code %d)", lang, w.Code)
|
||||
}
|
||||
var body map[string]interface{}
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &body)
|
||||
data, _ := body["data"].(map[string]interface{})
|
||||
if data["use_offered"] != true || data["use_desc"] != want || data["use_off"] != "" {
|
||||
t.Fatalf("%s: use_offered=%v use_desc=%q use_off=%q\nwant use_desc=%q", lang, data["use_offered"], data["use_desc"], data["use_off"], want)
|
||||
}
|
||||
}
|
||||
if st, _ := m.GetStack("cloudapp"); st.Deployed || st.Deploying {
|
||||
t.Fatal("something was installed")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,6 +156,10 @@ type Manager struct {
|
||||
// INIT/TEST ONLY. Nil in production, where offboxLatestSnapshot runs unchanged.
|
||||
offboxLatestSnapFn func(ctx context.Context, stack string) (string, []string, error)
|
||||
|
||||
// keptUnitRestoreFn (R-691 (2)) — the kept load's unit restore; nil → RestoreFromRecoveryUnitAt.
|
||||
// INIT/TEST ONLY.
|
||||
keptUnitRestoreFn func(app, unitDir string) (UnitRestoreResult, error)
|
||||
|
||||
// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
|
||||
// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
|
||||
discoverDBs func(ctx context.Context) ([]DiscoveredDB, error)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -19,18 +22,49 @@ import (
|
||||
// WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a
|
||||
// database dump or a volume tar — a definition alone would install an empty app over the kept files), and
|
||||
// (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the
|
||||
// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2). The
|
||||
// off-site copy is NOT looked at here: its restore is a different operation (reconstitute) and it is
|
||||
// not built into this choice yet — said on the page as "none" when it is the only one.
|
||||
// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2).
|
||||
//
|
||||
// THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one
|
||||
// `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is
|
||||
// NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is
|
||||
// downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore,
|
||||
// restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three
|
||||
// rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before
|
||||
// v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only
|
||||
// then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body
|
||||
// (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as
|
||||
// they were. Pinned by internal/backup/r691_kept_offsite_test.go.
|
||||
|
||||
// KeptCopy is one database copy a kept folder can be loaded from.
|
||||
type KeptCopy struct {
|
||||
UnitDir string
|
||||
Tier int // 1 own unit, 2 second drive
|
||||
UnitDir string // "" for the off-site copy (it is downloaded at load time)
|
||||
Tier int // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite)
|
||||
Time time.Time
|
||||
DriveLabel string
|
||||
// SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it.
|
||||
SnapshotID string
|
||||
UnitPath string
|
||||
}
|
||||
|
||||
// KeptTierOffsite is KeptCopy.Tier for the off-site copy.
|
||||
const KeptTierOffsite = 3
|
||||
|
||||
// KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the
|
||||
// install choice and the kept list, so the two pages cannot name the same copy differently.
|
||||
func KeptCopyKey(tier int) string {
|
||||
switch tier {
|
||||
case 2:
|
||||
return "kept.backup.second"
|
||||
case KeptTierOffsite:
|
||||
return "kept.backup.offsite"
|
||||
}
|
||||
return "kept.backup.own"
|
||||
}
|
||||
|
||||
// keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow
|
||||
// or unreachable repository then costs the page this long at most and the off-site copy is not offered.
|
||||
const keptOffsiteLookupTimeout = 20 * time.Second
|
||||
|
||||
// unitHoldsData: a readable manifest that lists a database dump or a volume tar.
|
||||
func unitHoldsData(unitDir string) (*RecoveryManifest, bool) {
|
||||
man := readManifest(UnitManifestFile(unitDir))
|
||||
@@ -96,6 +130,167 @@ func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) {
|
||||
return best, found
|
||||
}
|
||||
|
||||
// KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest
|
||||
// snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4).
|
||||
// Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call
|
||||
// for all apps (the kept list asks once per page, not once per row). Any error reading the repository
|
||||
// offers nothing (logged) — the local copies are still offered.
|
||||
func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy {
|
||||
out := map[string]KeptCopy{}
|
||||
var want []string
|
||||
for _, a := range apps {
|
||||
if a != "" && !m.isStackDeployed(a) {
|
||||
want = append(want, a)
|
||||
}
|
||||
}
|
||||
if len(want) == 0 || !m.OffboxConfigured() {
|
||||
return out
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout)
|
||||
defer cancel()
|
||||
newest, _, err := m.offsiteNewestPerTag(cctx)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err)
|
||||
return out
|
||||
}
|
||||
for _, app := range want {
|
||||
n, ok := newest[app]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
unitPath := offboxUnitPathOf(n.paths, app)
|
||||
if unitPath == "" {
|
||||
m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id)
|
||||
continue
|
||||
}
|
||||
out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// KeptOffsiteCopy is KeptOffsiteCopies for one app.
|
||||
func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) {
|
||||
c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
// KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or
|
||||
// the only one (a tie goes local — no download).
|
||||
func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) {
|
||||
if ook && (!lok || off.Time.After(local.Time)) {
|
||||
return off, true
|
||||
}
|
||||
return local, lok
|
||||
}
|
||||
|
||||
// KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest
|
||||
// usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one.
|
||||
func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) {
|
||||
local, lok := m.KeptDBCopy(app, drive)
|
||||
off, ook := m.KeptOffsiteCopy(ctx, app)
|
||||
return KeptNewer(local, lok, off, ook)
|
||||
}
|
||||
|
||||
// Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything
|
||||
// of the app is touched.
|
||||
var (
|
||||
ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version")
|
||||
ErrKeptOffsiteNotUsable = errors.New("the off-site copy is not a copy of these kept files")
|
||||
)
|
||||
|
||||
// keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY:
|
||||
// the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker.
|
||||
func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) {
|
||||
if m.keptUnitRestoreFn != nil {
|
||||
return m.keptUnitRestoreFn
|
||||
}
|
||||
return m.RestoreFromRecoveryUnitAt
|
||||
}
|
||||
|
||||
// SetKeptUnitRestoreFn overrides the kept load's unit restore (tests).
|
||||
func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) {
|
||||
m.keptUnitRestoreFn = fn
|
||||
}
|
||||
|
||||
// downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It
|
||||
// returns the unit directory and a cleanup that removes the scratch (call it on every path).
|
||||
func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) {
|
||||
noop := func() {}
|
||||
if err := m.acquireRunning(); err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
defer m.releaseRunning()
|
||||
scratch, nsRoot, err := m.offboxProofScratchDir(app)
|
||||
if err != nil {
|
||||
return "", noop, err
|
||||
}
|
||||
if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil {
|
||||
return "", noop, herr
|
||||
}
|
||||
m.removeProofScratch(app, scratch) // a copy an interrupted run left
|
||||
cleanup := func() { m.removeProofScratch(app, scratch) }
|
||||
if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil {
|
||||
return "", cleanup, err
|
||||
}
|
||||
unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator)))
|
||||
if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok {
|
||||
return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app)
|
||||
}
|
||||
man := readManifest(UnitManifestFile(unitDir))
|
||||
if man == nil || man.Data == nil {
|
||||
return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app)
|
||||
}
|
||||
if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil {
|
||||
return "", cleanup, verr
|
||||
}
|
||||
return unitDir, cleanup, nil
|
||||
}
|
||||
|
||||
// LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then
|
||||
// prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then
|
||||
// after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were.
|
||||
func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) {
|
||||
m.BeginRestoreOp("restore", app)
|
||||
go func() {
|
||||
start := time.Now()
|
||||
fail := func(err error) {
|
||||
m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err)
|
||||
m.EndRestoreOp(false, failMsg(err))
|
||||
if after != nil {
|
||||
after(false)
|
||||
}
|
||||
}
|
||||
m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath)
|
||||
unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c)
|
||||
// The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the
|
||||
// outcome never finds a copy left behind.
|
||||
if err != nil {
|
||||
cleanup()
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
if prepare != nil {
|
||||
if perr := prepare(); perr != nil {
|
||||
cleanup()
|
||||
fail(perr)
|
||||
return
|
||||
}
|
||||
}
|
||||
res, err := m.keptUnitRestore()(app, unitDir)
|
||||
cleanup()
|
||||
if err != nil {
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID,
|
||||
time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
|
||||
m.EndRestoreOp(true, okMsg(nil))
|
||||
if after != nil {
|
||||
after(true)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when
|
||||
// it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise.
|
||||
func (m *Manager) RemovedUnitOnDrive(app, drive string) string {
|
||||
|
||||
@@ -56,6 +56,9 @@ type OffsiteInventory struct {
|
||||
type offsiteNewest struct {
|
||||
id string
|
||||
at time.Time
|
||||
// paths are the snapshot's captured paths (R-691 (2): the kept-data choice asks whether the newest
|
||||
// snapshot holds the app's recovery unit without a second repository call).
|
||||
paths []string
|
||||
}
|
||||
|
||||
// offsiteNewestPerTag runs ONE `snapshots --json` and returns the newest snapshot per app tag, and
|
||||
@@ -81,6 +84,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
|
||||
ID string `json:"id"`
|
||||
Time time.Time `json:"time"`
|
||||
Tags []string `json:"tags"`
|
||||
Paths []string `json:"paths"`
|
||||
}
|
||||
if uerr := json.Unmarshal(out, &snaps); uerr != nil {
|
||||
return nil, false, uerr
|
||||
@@ -100,7 +104,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
|
||||
continue
|
||||
}
|
||||
if cur, ok := newest[tag]; !ok || sn.Time.After(cur.at) {
|
||||
newest[tag] = offsiteNewest{id: id, at: sn.Time}
|
||||
newest[tag] = offsiteNewest{id: id, at: sn.Time, paths: sn.Paths}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-691 (2) — „Use my kept data" / Load also look at the OFF-SITE copy (controller v0.277.0).
|
||||
//
|
||||
// Driven through the real KeptBestCopy / LoadKeptOffsite with the two restic seams (SetOffboxRunner answers
|
||||
// `snapshots --json` and materialises the unit a `restore --include` asks for) and the unit-restore seam
|
||||
// (SetKeptUnitRestoreFn records the call instead of reaching Docker). No restic, no ssh, no docker.
|
||||
|
||||
const r691App = "nextcloud"
|
||||
|
||||
type r691Harness struct {
|
||||
*proofHarness
|
||||
snapAt time.Time
|
||||
snapPaths []string
|
||||
failSnaps bool
|
||||
// manifest is what the downloaded unit's manifest.json holds (raw JSON).
|
||||
manifest string
|
||||
unitHDD string
|
||||
// restored records every unit restore call (unitDir); prepared counts prepare calls.
|
||||
restored []string
|
||||
prepared int
|
||||
done chan bool
|
||||
}
|
||||
|
||||
func newR691Harness(t *testing.T) *r691Harness {
|
||||
t.Helper()
|
||||
ph := newProofHarness(t) // nothing deployed: nextcloud is a REMOVED app with kept files
|
||||
h := &r691Harness{proofHarness: ph, snapAt: time.Now().Add(-time.Hour), done: make(chan bool, 1)}
|
||||
h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/backups/primary/" + r691App, "/mnt/felhom-drives/hdd_1/appdata/" + r691App}
|
||||
h.unitHDD = ph.drive
|
||||
h.manifest = r691Manifest(true)
|
||||
ph.m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
ph.mu.Lock()
|
||||
ph.argv = append(ph.argv, append([]string{}, args...))
|
||||
ph.mu.Unlock()
|
||||
switch {
|
||||
case containsArg(args, "snapshots"):
|
||||
if h.failSnaps {
|
||||
return []byte("ssh: connect refused"), errors.New("exit status 1")
|
||||
}
|
||||
b, _ := json.Marshal([]map[string]interface{}{{
|
||||
"short_id": "ab12cd34", "id": "ab12cd34ffff", "time": h.snapAt, "tags": []string{r691App}, "paths": h.snapPaths}})
|
||||
return b, nil
|
||||
case containsArg(args, "restore"):
|
||||
if ph.failRestore {
|
||||
return []byte("simulated restic failure"), os.ErrPermission
|
||||
}
|
||||
target, include := argValue(args, "--target"), argValue(args, "--include")
|
||||
dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
|
||||
for _, sub := range []string{"compose", "db-dumps"} {
|
||||
_ = os.MkdirAll(filepath.Join(dest, sub), 0o755)
|
||||
}
|
||||
_ = os.WriteFile(filepath.Join(dest, "db-dumps", "nextcloud-mariadb.sql"), []byte("x"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte("services:\n nextcloud:\n image: nextcloud:31\n db:\n image: mariadb:11.8\n"), 0o644)
|
||||
_ = os.WriteFile(filepath.Join(dest, "compose", "app.yaml"), []byte("env:\n HDD_PATH: "+h.unitHDD+"\n"), 0o600)
|
||||
_ = os.WriteFile(filepath.Join(dest, "manifest.json"), []byte(h.manifest), 0o644)
|
||||
return nil, nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
ph.m.SetKeptUnitRestoreFn(func(app, unitDir string) (UnitRestoreResult, error) {
|
||||
// The unit must still be on disk while the restore reads it.
|
||||
if _, err := os.Stat(UnitManifestFile(unitDir)); err != nil {
|
||||
t.Errorf("the unit restore ran on %s, which no longer holds a manifest: %v", unitDir, err)
|
||||
}
|
||||
h.restored = append(h.restored, unitDir)
|
||||
return UnitRestoreResult{ManifestDBs: 1, DBsReplayed: 1}, nil
|
||||
})
|
||||
return h
|
||||
}
|
||||
|
||||
// r691Manifest is a unit manifest listing one dump; withData adds the `data` block (the data's version).
|
||||
func r691Manifest(withData bool) string {
|
||||
man := map[string]interface{}{"schema_version": 2, "app_name": r691App, "db_dumps": []string{"nextcloud-mariadb.sql"}}
|
||||
if withData {
|
||||
man["data"] = map[string]interface{}{"at": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339),
|
||||
"image_pins": []string{"mariadb:11.8", "nextcloud:31"}}
|
||||
}
|
||||
b, _ := json.Marshal(man)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (h *r691Harness) load(t *testing.T, c KeptCopy) (ok bool) {
|
||||
t.Helper()
|
||||
h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
|
||||
func(error) string { return "ok" }, func(err error) string { return err.Error() },
|
||||
func(ok bool) { h.done <- ok })
|
||||
select {
|
||||
case ok = <-h.done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the load never finished")
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// The consequence, not the mechanism: which copy the household is offered.
|
||||
// COMPANION RED-PROOF: KeptBestCopy returning KeptDBCopy alone (0.276.0's choice) → the first assertion fails
|
||||
// with ok=false (an app whose only database copy is off-site gets „no backup").
|
||||
func TestR691_OffsiteCopyIsOfferedWhenItIsTheOnlyOrNewest(t *testing.T) {
|
||||
h := newR691Harness(t)
|
||||
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
|
||||
if !ok || c.Tier != KeptTierOffsite || c.SnapshotID != "ab12cd34" || c.UnitPath != h.snapPaths[0] {
|
||||
t.Fatalf("only off-site copy: got %+v ok=%v, want the off-site snapshot ab12cd34", c, ok)
|
||||
}
|
||||
if !c.Time.Equal(h.snapAt) {
|
||||
t.Fatalf("the copy is dated %v, want the snapshot's time %v", c.Time, h.snapAt)
|
||||
}
|
||||
|
||||
// A local unit NEWER than the snapshot wins — no download.
|
||||
unit := RecoveryUnitPath(h.m.namespaceRoot(h.drive), r691App)
|
||||
writeKeptUnit(t, unit, h.drive, true, time.Now())
|
||||
h.prov.hdd[r691App] = "" // removed
|
||||
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != 1 || c.UnitDir != unit {
|
||||
t.Fatalf("a newer local unit must win: got %+v ok=%v", c, ok)
|
||||
}
|
||||
// An OLDER local unit loses to the off-site copy.
|
||||
old := time.Now().Add(-3 * time.Hour)
|
||||
writeKeptUnit(t, unit, h.drive, true, old)
|
||||
for _, f := range []string{UnitManifestFile(unit), filepath.Join(unit, "compose", "app.yaml")} {
|
||||
_ = os.Chtimes(f, old, old)
|
||||
}
|
||||
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != KeptTierOffsite {
|
||||
t.Fatalf("an older local unit must lose to the off-site copy: got %+v ok=%v", c, ok)
|
||||
}
|
||||
|
||||
// Negative controls: a snapshot with no unit, an unreadable repository, an INSTALLED app.
|
||||
_ = os.RemoveAll(unit)
|
||||
h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/appdata/" + r691App}
|
||||
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
|
||||
t.Fatalf("a snapshot holding no recovery unit was offered: %+v", c)
|
||||
}
|
||||
h.snapPaths = []string{"/x/backups/primary/" + r691App}
|
||||
h.failSnaps = true
|
||||
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
|
||||
t.Fatalf("an unreadable repository offered a copy: %+v", c)
|
||||
}
|
||||
h.failSnaps = false
|
||||
h.prov.deployed[r691App] = true
|
||||
if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
|
||||
t.Fatalf("an installed app's off-site copy was offered as kept data: %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// The load: download the unit ALONE, judge it, THEN prepare, THEN the one unit restore; the scratch goes.
|
||||
// COMPANION RED-PROOF: drop the `man.Data == nil` refusal in downloadKeptOffsiteUnit → the no-data-block case
|
||||
// reaches the restore (restored=1, prepared=1) and fails.
|
||||
func TestR691_OffsiteLoadDownloadsJudgesThenRestores(t *testing.T) {
|
||||
h := newR691Harness(t)
|
||||
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
|
||||
if !ok {
|
||||
t.Fatal("no off-site copy offered")
|
||||
}
|
||||
if !h.load(t, c) {
|
||||
t.Fatalf("a good off-site unit did not load: %+v", h.m.RestoreStatus())
|
||||
}
|
||||
if h.prepared != 1 || len(h.restored) != 1 {
|
||||
t.Fatalf("prepare=%d restore=%d, want 1 and 1", h.prepared, len(h.restored))
|
||||
}
|
||||
scratch := h.proofScratch(t, r691App)
|
||||
if !strings.HasPrefix(h.restored[0], scratch+string(filepath.Separator)) || !strings.HasSuffix(h.restored[0], "/backups/primary/"+r691App) {
|
||||
t.Fatalf("the restore read %s, want the downloaded unit inside %s", h.restored[0], scratch)
|
||||
}
|
||||
var sawUnitOnly bool
|
||||
for _, a := range h.allArgs() {
|
||||
if containsArg(a, "restore") && argValue(a, "--include") == h.snapPaths[0] && containsArg(a, "ab12cd34") {
|
||||
sawUnitOnly = true
|
||||
}
|
||||
}
|
||||
if !sawUnitOnly {
|
||||
t.Fatalf("no unit-only restore of snapshot ab12cd34 (--include %s): %v", h.snapPaths[0], h.allArgs())
|
||||
}
|
||||
if _, err := os.Stat(scratch); !os.IsNotExist(err) {
|
||||
t.Fatalf("the downloaded copy was left behind at %s (err=%v)", scratch, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Every refusal happens BEFORE prepare: the kept files are exactly as they were, nothing is restored.
|
||||
func TestR691_OffsiteLoadRefusalsLeaveTheKeptFilesAlone(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
set func(h *r691Harness)
|
||||
want error
|
||||
}{
|
||||
{"data version not recorded", func(h *r691Harness) { h.manifest = r691Manifest(false) }, ErrKeptOffsiteVersionUnknown},
|
||||
{"taken of another drive", func(h *r691Harness) { h.unitHDD = "/mnt/felhom-drives/other" }, ErrKeptOffsiteNotUsable},
|
||||
{"definition is not the data's", func(h *r691Harness) {
|
||||
h.manifest = strings.Replace(r691Manifest(true), "nextcloud:31", "nextcloud:30", 1)
|
||||
}, ErrUnitVersionMismatch},
|
||||
{"download failed", func(h *r691Harness) { h.failRestore = true }, nil},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
h := newR691Harness(t)
|
||||
kept := filepath.Join(h.drive, "appdata", r691App, "photo.jpg")
|
||||
_ = os.MkdirAll(filepath.Dir(kept), 0o755)
|
||||
_ = os.WriteFile(kept, []byte("the household's photo"), 0o644)
|
||||
c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
|
||||
if !ok {
|
||||
t.Fatal("no off-site copy offered")
|
||||
}
|
||||
tc.set(h)
|
||||
var got error
|
||||
h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
|
||||
func(error) string { return "ok" }, func(err error) string { got = err; return err.Error() },
|
||||
func(ok bool) { h.done <- ok })
|
||||
if <-h.done {
|
||||
t.Fatal("the load reported success")
|
||||
}
|
||||
if h.prepared != 0 || len(h.restored) != 0 {
|
||||
t.Fatalf("prepare=%d restore=%d after a refusal, want 0 and 0", h.prepared, len(h.restored))
|
||||
}
|
||||
if tc.want != nil && !errors.Is(got, tc.want) {
|
||||
t.Fatalf("refused with %v, want %v", got, tc.want)
|
||||
}
|
||||
if b, err := os.ReadFile(kept); err != nil || string(b) != "the household's photo" {
|
||||
t.Fatalf("the kept file changed: %q %v", b, err)
|
||||
}
|
||||
if st := h.m.RestoreStatus(); st.Running || st.Last == nil || st.Last.OK {
|
||||
t.Fatalf("the restore record does not read as failed: %+v", st)
|
||||
}
|
||||
if _, err := os.Stat(h.proofScratch(t, r691App)); !os.IsNotExist(err) {
|
||||
t.Fatalf("the downloaded copy was left behind after a refusal (err=%v)", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2441,6 +2441,10 @@
|
||||
"kept.backup.second": "the second drive, %s",
|
||||
"kept.backup.with": "the backup kept with it, %s",
|
||||
"kept.backup.none": "no backup — the files only",
|
||||
"kept.backup.offsite": "the off-site copy, %s",
|
||||
"kept.choice.use.desc_from": "We load the database from %s and start the app with the old files. Changes made after that time can be missing.",
|
||||
"err.kept.offsite_version_unknown": "the off-site copy of %s does not record which version wrote its data, so it is not loaded",
|
||||
"err.kept.offsite_not_usable": "the off-site copy of %s does not belong to these files (it was taken of another drive, or it holds no database), so it is not loaded",
|
||||
"kept.action.load": "Load",
|
||||
"kept.action.look": "Look",
|
||||
"kept.action.delete": "Delete",
|
||||
|
||||
@@ -2429,6 +2429,10 @@
|
||||
"kept.backup.second": "második meghajtó, %s",
|
||||
"kept.backup.with": "a vele megőrzött mentés, %s",
|
||||
"kept.backup.none": "nincs mentés — csak a fájlok",
|
||||
"kept.backup.offsite": "távoli mentés, %s",
|
||||
"kept.choice.use.desc_from": "Az adatbázist innen töltjük vissza: %s. A régi fájlokkal indítjuk az alkalmazást. Ami ezután változott, hiányozhat.",
|
||||
"err.kept.offsite_version_unknown": "a(z) %s távoli mentése nem rögzíti, melyik verzió írta az adatait, ezért nem töltjük be",
|
||||
"err.kept.offsite_not_usable": "a(z) %s távoli mentése nem ezekhez a fájlokhoz tartozik (másik meghajtóról készült, vagy nincs benne adatbázis), ezért nem töltjük be",
|
||||
"kept.action.load": "Betöltés",
|
||||
"kept.action.look": "Megnézem",
|
||||
"kept.action.delete": "Törlés",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appbackup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/infra"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
|
||||
)
|
||||
@@ -63,32 +65,44 @@ func KeptViewName(it stacks.KeptItem) string {
|
||||
return base + filepath.Base(it.Path)
|
||||
}
|
||||
|
||||
// keptBackupFor names the copy a Load would use for it, and whether one exists.
|
||||
func (s *Server) keptBackupFor(lang string, it stacks.KeptItem) (string, string, bool) {
|
||||
if s.backupMgr == nil {
|
||||
return s.msgLang(lang, "kept.backup.none"), "", false
|
||||
// keptBackupFor names the copy a Load would use for it, and whether one exists. off is the off-site copy
|
||||
// per app (backup.KeptOffsiteCopies, asked once per page): it is used when it is newer than every local
|
||||
// copy, or the only one (R-691 (2)).
|
||||
func (s *Server) keptBackupFor(lang string, it stacks.KeptItem, off map[string]backup.KeptCopy) (string, backup.KeptCopy, bool) {
|
||||
none := s.msgLang(lang, "kept.backup.none")
|
||||
if s.backupMgr == nil || it.App == "" && it.Kind != stacks.KeptKindDated {
|
||||
return none, backup.KeptCopy{}, false
|
||||
}
|
||||
var local backup.KeptCopy
|
||||
var lok bool
|
||||
if it.Kind == stacks.KeptKindDated {
|
||||
if it.UnitDir == "" {
|
||||
return s.msgLang(lang, "kept.backup.none"), "", false
|
||||
// A dated folder's own unit is the copy taken with those files.
|
||||
if it.UnitDir != "" {
|
||||
if c, ok := s.backupMgr.KeptCopyAt(it.UnitDir, it.Drive, 1); ok {
|
||||
return s.msgLang(lang, "kept.backup.with", c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c, true
|
||||
}
|
||||
}
|
||||
if c, ok := s.backupMgr.KeptCopyAt(it.UnitDir, it.Drive, 1); ok {
|
||||
return s.msgLang(lang, "kept.backup.with", c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c.UnitDir, true
|
||||
}
|
||||
return s.msgLang(lang, "kept.backup.none"), "", false
|
||||
} else {
|
||||
local, lok = s.backupMgr.KeptDBCopy(it.App, it.Drive)
|
||||
}
|
||||
if it.App == "" {
|
||||
return s.msgLang(lang, "kept.backup.none"), "", false
|
||||
}
|
||||
c, ok := s.backupMgr.KeptDBCopy(it.App, it.Drive)
|
||||
oc, ook := off[it.App]
|
||||
c, ok := backup.KeptNewer(local, lok, oc, ook && it.App != "")
|
||||
if !ok {
|
||||
return s.msgLang(lang, "kept.backup.none"), "", false
|
||||
return none, backup.KeptCopy{}, false
|
||||
}
|
||||
key := "kept.backup.own"
|
||||
if c.Tier == 2 {
|
||||
key = "kept.backup.second"
|
||||
return s.msgLang(lang, backup.KeptCopyKey(c.Tier), c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c, true
|
||||
}
|
||||
|
||||
// keptOffsite asks the off-site repository ONCE for every listed app.
|
||||
func (s *Server) keptOffsite(ctx context.Context, items []stacks.KeptItem) map[string]backup.KeptCopy {
|
||||
if s.backupMgr == nil {
|
||||
return nil
|
||||
}
|
||||
return s.msgLang(lang, key, c.Time.In(getTimezone()).Format("2006-01-02 15:04")), c.UnitDir, true
|
||||
var apps []string
|
||||
for _, it := range items {
|
||||
apps = append(apps, it.App)
|
||||
}
|
||||
return s.backupMgr.KeptOffsiteCopies(ctx, apps)
|
||||
}
|
||||
|
||||
func (s *Server) keptPageHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -97,12 +111,14 @@ func (s *Server) keptPageHandler(w http.ResponseWriter, r *http.Request) {
|
||||
data["TitleKey"] = "page.title.kept_data"
|
||||
var rows []keptRow
|
||||
if s.stackMgr != nil {
|
||||
for _, it := range s.stackMgr.ListKept(s.keptDrives()) {
|
||||
backup, _, can := s.keptBackupFor(lang, it)
|
||||
items := s.stackMgr.ListKept(s.keptDrives())
|
||||
off := s.keptOffsite(r.Context(), items)
|
||||
for _, it := range items {
|
||||
backupName, _, can := s.keptBackupFor(lang, it, off)
|
||||
row := keptRow{
|
||||
DisplayName: it.DisplayName, App: it.App, Path: it.Path, Drive: it.Drive,
|
||||
Date: it.Date.In(getTimezone()).Format("2006-01-02 15:04"),
|
||||
Size: appbackup.HumanizeBytes(it.SizeBytes), Backup: backup, CanLoad: can && it.App != "",
|
||||
Size: appbackup.HumanizeBytes(it.SizeBytes), Backup: backupName, CanLoad: can && it.App != "",
|
||||
LookURL: fileBrowserLink(s.cfg.Customer.Domain, s.msgLang(s.boxLang(), "kept.fb_source"), KeptViewName(it)),
|
||||
}
|
||||
row.DeleteText = s.msgLang(lang, "kept.delete.confirm", it.DisplayName, row.Size)
|
||||
@@ -165,31 +181,43 @@ func (s *Server) keptLoadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
s.keptRedirect(w, r, "flash_error", msg)
|
||||
return
|
||||
}
|
||||
_, unit, can := s.keptBackupFor(lang, it)
|
||||
_, c, can := s.keptBackupFor(lang, it, s.keptOffsite(r.Context(), []stacks.KeptItem{it}))
|
||||
if !can {
|
||||
s.keptRedirect(w, r, "flash_error", s.msgLang(lang, "kept.choice.use_off"))
|
||||
return
|
||||
}
|
||||
if it.Kind == stacks.KeptKindDated {
|
||||
if _, err := s.stackMgr.RestoreKeptFiles(it); err != nil {
|
||||
app, disp := it.App, it.DisplayName
|
||||
okMsg := func(error) string { return s.msgLang(lang, "kept.load.done", disp) }
|
||||
failMsg := func(err error) string { return s.msgLang(lang, "kept.load.failed", disp, err) }
|
||||
after := func(ok bool) {
|
||||
if ok {
|
||||
if ran, err := s.stackMgr.RunAfterLoad(app, 10*time.Minute); ran && err != nil {
|
||||
s.logger.Printf("[WARN] [web] kept load %s: after_load failed: %v", app, err)
|
||||
}
|
||||
s.stackMgr.FinishKeptLoad(it, s.backupMgr.PrimaryUnitHome(app, it.Drive))
|
||||
}
|
||||
s.SyncFileBrowserMounts()
|
||||
}
|
||||
// A dated folder's files move back BEFORE the load — for the off-site copy only after its unit is
|
||||
// downloaded and judged, so a failed download or a refusal leaves the kept folder as it was.
|
||||
moveBack := func() error {
|
||||
if it.Kind != stacks.KeptKindDated {
|
||||
return nil
|
||||
}
|
||||
_, err := s.stackMgr.RestoreKeptFiles(it)
|
||||
return err
|
||||
}
|
||||
if c.Tier == backup.KeptTierOffsite {
|
||||
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from the off-site snapshot %s (from %s)", it.App, it.Path, c.SnapshotID, r.RemoteAddr)
|
||||
s.backupMgr.LoadKeptOffsite(app, it.Drive, c, moveBack, okMsg, failMsg, after)
|
||||
} else {
|
||||
if err := moveBack(); err != nil {
|
||||
s.keptRedirect(w, r, "flash_error", s.errText(r, err))
|
||||
return
|
||||
}
|
||||
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from %s (from %s)", it.App, it.Path, c.UnitDir, r.RemoteAddr)
|
||||
s.backupMgr.LoadKeptApp(app, c.UnitDir, okMsg, failMsg, after)
|
||||
}
|
||||
s.logger.Printf("[INFO] [web] kept LOAD %s: %s from %s (from %s)", it.App, it.Path, unit, r.RemoteAddr)
|
||||
app, disp := it.App, it.DisplayName
|
||||
s.backupMgr.LoadKeptApp(app, unit,
|
||||
func(error) string { return s.msgLang(lang, "kept.load.done", disp) },
|
||||
func(err error) string { return s.msgLang(lang, "kept.load.failed", disp, err) },
|
||||
func(ok bool) {
|
||||
if ok {
|
||||
if ran, err := s.stackMgr.RunAfterLoad(app, 10*time.Minute); ran && err != nil {
|
||||
s.logger.Printf("[WARN] [web] kept load %s: after_load failed: %v", app, err)
|
||||
}
|
||||
s.stackMgr.FinishKeptLoad(it, s.backupMgr.PrimaryUnitHome(app, it.Drive))
|
||||
}
|
||||
s.SyncFileBrowserMounts()
|
||||
})
|
||||
http.Redirect(w, r, "/backups/restore?"+flashQuery("flash", "flash.restore.started"), http.StatusFound)
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user