v0.277.0: kept data loads from the off-site copy too (R-691 (2))

Use my kept data / Load consider the off-site snapshot when it is newer than every local copy or
the only one; the unit is downloaded alone, judged (drive, data, recorded data version) and only
then restored. The page names the copy and its date.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/backup/backup.go

@@ -156,6 +156,10 @@ type Manager struct {
 	// INIT/TEST ONLY. Nil in production, where offboxLatestSnapshot runs unchanged.
 	offboxLatestSnapFn func(ctx context.Context, stack string) (string, []string, error)
 
+	// keptUnitRestoreFn (R-691 (2)) — the kept load's unit restore; nil → RestoreFromRecoveryUnitAt.
+	// INIT/TEST ONLY.
+	keptUnitRestoreFn func(app, unitDir string) (UnitRestoreResult, error)
+
 	// F17 restore seams — overridable in tests so the .sql re-import orchestration can be unit-tested
 	// without Docker. Default to the real DiscoverDatabases / ImportDump (lazy-init in reimportDBDumps).
 	discoverDBs  func(ctx context.Context) ([]DiscoveredDB, error)

controller/internal/backup/kept_load.go

@@ -1,11 +1,14 @@
 package backup
 
 import (
+	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
 	"gopkg.in/yaml.v3"
 )
 
@@ -19,18 +22,49 @@ import (
 // WHICH COPY. The newest unit that (1) opens (a readable manifest), (2) holds the app's data state (a
 // database dump or a volume tar — a definition alone would install an empty app over the kept files), and
 // (3) was taken of THIS drive's install: its app.yaml's HDD_PATH names this drive. Looked for in the
-// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2). The
-// off-site copy is NOT looked at here: its restore is a different operation (reconstitute) and it is
-// not built into this choice yet — said on the page as "none" when it is the only one.
+// app's own unit on the drive (Tier 1, R-487) and in every connected second-drive mirror (Tier 2).
+//
+// THE OFF-SITE COPY (Tier 3, R-691 (2), v0.277.0). KeptBestCopy also asks the off-site repository (one
+// `snapshots --json`, bounded) and offers its newest snapshot when it holds the app's recovery unit and is
+// NEWER than every local copy (a tie goes local: no download). Its unit can only be judged after it is
+// downloaded, so LoadKeptOffsite downloads the unit ALONE (the proof's unit-only restore,
+// restoreUnitReadOnly, into the proof scratch — deleted on every path), then judges it by the same three
+// rules plus `07` §6.6: a unit whose data version is not recorded (no `data` block — written before
+// v0.275.0) is REFUSED, never loaded blind; a mixed or mismatched one is refused by unitVersionCheck. Only
+// then does the caller's prepare run (a dated folder's files move back) and the one unit-restore body
+// (RestoreFromRecoveryUnitAt) load it. A failed download or a refusal leaves the kept files exactly as
+// they were. Pinned by internal/backup/r691_kept_offsite_test.go.
 
 // KeptCopy is one database copy a kept folder can be loaded from.
 type KeptCopy struct {
-	UnitDir    string
-	Tier       int // 1 own unit, 2 second drive
+	UnitDir    string // "" for the off-site copy (it is downloaded at load time)
+	Tier       int    // 1 own unit, 2 second drive, 3 off-site (KeptTierOffsite)
 	Time       time.Time
 	DriveLabel string
+	// SnapshotID and UnitPath name the off-site copy (Tier 3 only): the snapshot and the unit's path in it.
+	SnapshotID string
+	UnitPath   string
 }
 
+// KeptTierOffsite is KeptCopy.Tier for the off-site copy.
+const KeptTierOffsite = 3
+
+// KeptCopyKey is the bundle key naming a copy of this tier (its one %s is the copy's date) — shared by the
+// install choice and the kept list, so the two pages cannot name the same copy differently.
+func KeptCopyKey(tier int) string {
+	switch tier {
+	case 2:
+		return "kept.backup.second"
+	case KeptTierOffsite:
+		return "kept.backup.offsite"
+	}
+	return "kept.backup.own"
+}
+
+// keptOffsiteLookupTimeout bounds the repository question the install page and the kept list ask. A slow
+// or unreachable repository then costs the page this long at most and the off-site copy is not offered.
+const keptOffsiteLookupTimeout = 20 * time.Second
+
 // unitHoldsData: a readable manifest that lists a database dump or a volume tar.
 func unitHoldsData(unitDir string) (*RecoveryManifest, bool) {
 	man := readManifest(UnitManifestFile(unitDir))
@@ -96,6 +130,167 @@ func (m *Manager) KeptDBCopy(app, drive string) (KeptCopy, bool) {
 	return best, found
 }
 
+// KeptOffsiteCopies is the off-site copy kept files of each app could be loaded from: the app's newest
+// snapshot, when it holds the app's recovery unit. Dated by its DATA time (offsiteDataTime, `07` §6.6 A4).
+// Only for apps that are NOT installed, and only when the box has an off-site target. ONE repository call
+// for all apps (the kept list asks once per page, not once per row). Any error reading the repository
+// offers nothing (logged) — the local copies are still offered.
+func (m *Manager) KeptOffsiteCopies(ctx context.Context, apps []string) map[string]KeptCopy {
+	out := map[string]KeptCopy{}
+	var want []string
+	for _, a := range apps {
+		if a != "" && !m.isStackDeployed(a) {
+			want = append(want, a)
+		}
+	}
+	if len(want) == 0 || !m.OffboxConfigured() {
+		return out
+	}
+	cctx, cancel := context.WithTimeout(ctx, keptOffsiteLookupTimeout)
+	defer cancel()
+	newest, _, err := m.offsiteNewestPerTag(cctx)
+	if err != nil {
+		m.logger.Printf("[WARN] [backup] kept: the off-site copies of %v could not be looked up: %v — not offered", want, err)
+		return out
+	}
+	for _, app := range want {
+		n, ok := newest[app]
+		if !ok {
+			continue
+		}
+		unitPath := offboxUnitPathOf(n.paths, app)
+		if unitPath == "" {
+			m.logger.Printf("[INFO] [backup] kept: the newest off-site snapshot of %s (%s) holds no recovery unit — not offered", app, n.id)
+			continue
+		}
+		out[app] = KeptCopy{Tier: KeptTierOffsite, Time: m.offsiteDataTime(app, n.at), SnapshotID: n.id, UnitPath: unitPath}
+	}
+	return out
+}
+
+// KeptOffsiteCopy is KeptOffsiteCopies for one app.
+func (m *Manager) KeptOffsiteCopy(ctx context.Context, app string) (KeptCopy, bool) {
+	c, ok := m.KeptOffsiteCopies(ctx, []string{app})[app]
+	return c, ok
+}
+
+// KeptNewer chooses between a local copy and the off-site copy: the off-site one only when it is NEWER or
+// the only one (a tie goes local — no download).
+func KeptNewer(local KeptCopy, lok bool, off KeptCopy, ook bool) (KeptCopy, bool) {
+	if ook && (!lok || off.Time.After(local.Time)) {
+		return off, true
+	}
+	return local, lok
+}
+
+// KeptBestCopy is the copy „Use my kept data" and Load use for app's kept files on drive: the newest
+// usable local copy (KeptDBCopy), or the off-site copy when it is newer or the only one.
+func (m *Manager) KeptBestCopy(ctx context.Context, app, drive string) (KeptCopy, bool) {
+	local, lok := m.KeptDBCopy(app, drive)
+	off, ook := m.KeptOffsiteCopy(ctx, app)
+	return KeptNewer(local, lok, off, ook)
+}
+
+// Refusals of an off-site load, born as bundle keys. Each is raised BEFORE prepare and before anything
+// of the app is touched.
+var (
+	ErrKeptOffsiteVersionUnknown = errors.New("the off-site copy does not record its data version")
+	ErrKeptOffsiteNotUsable      = errors.New("the off-site copy is not a copy of these kept files")
+)
+
+// keptUnitRestore is the unit restore a kept load runs (nil → RestoreFromRecoveryUnitAt). INIT/TEST ONLY:
+// the seam lets the off-site load's order — download, judge, prepare, restore — be pinned without Docker.
+func (m *Manager) keptUnitRestore() func(app, unitDir string) (UnitRestoreResult, error) {
+	if m.keptUnitRestoreFn != nil {
+		return m.keptUnitRestoreFn
+	}
+	return m.RestoreFromRecoveryUnitAt
+}
+
+// SetKeptUnitRestoreFn overrides the kept load's unit restore (tests).
+func (m *Manager) SetKeptUnitRestoreFn(fn func(app, unitDir string) (UnitRestoreResult, error)) {
+	m.keptUnitRestoreFn = fn
+}
+
+// downloadKeptOffsiteUnit downloads c's unit alone into the proof scratch and judges it for drive. It
+// returns the unit directory and a cleanup that removes the scratch (call it on every path).
+func (m *Manager) downloadKeptOffsiteUnit(ctx context.Context, app, drive string, c KeptCopy) (string, func(), error) {
+	noop := func() {}
+	if err := m.acquireRunning(); err != nil {
+		return "", noop, err
+	}
+	defer m.releaseRunning()
+	scratch, nsRoot, err := m.offboxProofScratchDir(app)
+	if err != nil {
+		return "", noop, err
+	}
+	if herr := unitOnlyHeadroom(m.offboxFree()(nsRoot)); herr != nil {
+		return "", noop, herr
+	}
+	m.removeProofScratch(app, scratch) // a copy an interrupted run left
+	cleanup := func() { m.removeProofScratch(app, scratch) }
+	if err := m.restoreUnitReadOnly(ctx, app, c.SnapshotID, c.UnitPath, scratch); err != nil {
+		return "", cleanup, err
+	}
+	unitDir := filepath.Join(scratch, strings.TrimPrefix(c.UnitPath, string(filepath.Separator)))
+	if _, ok := m.KeptCopyAt(unitDir, drive, KeptTierOffsite); !ok {
+		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteNotUsable, "err.kept.offsite_not_usable", app)
+	}
+	man := readManifest(UnitManifestFile(unitDir))
+	if man == nil || man.Data == nil {
+		return "", cleanup, util.MsgErrorf(ErrKeptOffsiteVersionUnknown, "err.kept.offsite_version_unknown", app)
+	}
+	if _, verr := unitVersionCheck(app, man, filepath.Join(unitDir, "compose")); verr != nil {
+		return "", cleanup, verr
+	}
+	return unitDir, cleanup, nil
+}
+
+// LoadKeptOffsite is LoadKeptApp from the off-site copy c: download the unit alone, judge it, then
+// prepare (nil = nothing; a dated folder's files move back here), then the one unit restore, then
+// after(ok). A download failure or a refusal never reaches prepare, so the kept files stay as they were.
+func (m *Manager) LoadKeptOffsite(app, drive string, c KeptCopy, prepare func() error, okMsg, failMsg func(err error) string, after func(ok bool)) {
+	m.BeginRestoreOp("restore", app)
+	go func() {
+		start := time.Now()
+		fail := func(err error) {
+			m.logger.Printf("[ERROR] [backup] kept load %s from the off-site copy %s FAILED after %s: %v", app, c.SnapshotID, time.Since(start).Round(time.Second), err)
+			m.EndRestoreOp(false, failMsg(err))
+			if after != nil {
+				after(false)
+			}
+		}
+		m.logger.Printf("[INFO] [backup] kept load %s: downloading the recovery unit alone from off-site snapshot %s (%s)", app, c.SnapshotID, c.UnitPath)
+		unitDir, cleanup, err := m.downloadKeptOffsiteUnit(context.Background(), app, drive, c)
+		// The downloaded copy goes BEFORE the outcome is reported, on every path: a caller that reads the
+		// outcome never finds a copy left behind.
+		if err != nil {
+			cleanup()
+			fail(err)
+			return
+		}
+		if prepare != nil {
+			if perr := prepare(); perr != nil {
+				cleanup()
+				fail(perr)
+				return
+			}
+		}
+		res, err := m.keptUnitRestore()(app, unitDir)
+		cleanup()
+		if err != nil {
+			fail(err)
+			return
+		}
+		m.logger.Printf("[INFO] [backup] kept load %s from the off-site copy %s done in %s (volumes %d/%d, dbs %d/%d)", app, c.SnapshotID,
+			time.Since(start).Round(time.Second), res.VolumesReplayed, res.ManifestVolumes, res.DBsReplayed, res.ManifestDBs)
+		m.EndRestoreOp(true, okMsg(nil))
+		if after != nil {
+			after(true)
+		}
+	}()
+}
+
 // RemovedUnitOnDrive is the start-fresh hook (stacks.SetKeptUnitFinder): the removed app's OWN unit when
 // it sits on drive, so it moves into the kept folder with the files it belongs to. "" otherwise.
 func (m *Manager) RemovedUnitOnDrive(app, drive string) string {

controller/internal/backup/offbox_inventory.go

@@ -56,6 +56,9 @@ type OffsiteInventory struct {
 type offsiteNewest struct {
 	id string
 	at time.Time
+	// paths are the snapshot's captured paths (R-691 (2): the kept-data choice asks whether the newest
+	// snapshot holds the app's recovery unit without a second repository call).
+	paths []string
 }
 
 // offsiteNewestPerTag runs ONE `snapshots --json` and returns the newest snapshot per app tag, and
@@ -81,6 +84,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
 		ID      string    `json:"id"`
 		Time    time.Time `json:"time"`
 		Tags    []string  `json:"tags"`
+		Paths   []string  `json:"paths"`
 	}
 	if uerr := json.Unmarshal(out, &snaps); uerr != nil {
 		return nil, false, uerr
@@ -100,7 +104,7 @@ func (m *Manager) offsiteNewestPerTag(ctx context.Context) (map[string]offsiteNe
 				continue
 			}
 			if cur, ok := newest[tag]; !ok || sn.Time.After(cur.at) {
-				newest[tag] = offsiteNewest{id: id, at: sn.Time}
+				newest[tag] = offsiteNewest{id: id, at: sn.Time, paths: sn.Paths}
 			}
 		}
 	}

controller/internal/backup/r691_kept_offsite_test.go

@@ -0,0 +1,237 @@
+package backup
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// R-691 (2) — „Use my kept data" / Load also look at the OFF-SITE copy (controller v0.277.0).
+//
+// Driven through the real KeptBestCopy / LoadKeptOffsite with the two restic seams (SetOffboxRunner answers
+// `snapshots --json` and materialises the unit a `restore --include` asks for) and the unit-restore seam
+// (SetKeptUnitRestoreFn records the call instead of reaching Docker). No restic, no ssh, no docker.
+
+const r691App = "nextcloud"
+
+type r691Harness struct {
+	*proofHarness
+	snapAt    time.Time
+	snapPaths []string
+	failSnaps bool
+	// manifest is what the downloaded unit's manifest.json holds (raw JSON).
+	manifest string
+	unitHDD  string
+	// restored records every unit restore call (unitDir); prepared counts prepare calls.
+	restored []string
+	prepared int
+	done     chan bool
+}
+
+func newR691Harness(t *testing.T) *r691Harness {
+	t.Helper()
+	ph := newProofHarness(t) // nothing deployed: nextcloud is a REMOVED app with kept files
+	h := &r691Harness{proofHarness: ph, snapAt: time.Now().Add(-time.Hour), done: make(chan bool, 1)}
+	h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/backups/primary/" + r691App, "/mnt/felhom-drives/hdd_1/appdata/" + r691App}
+	h.unitHDD = ph.drive
+	h.manifest = r691Manifest(true)
+	ph.m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
+		ph.mu.Lock()
+		ph.argv = append(ph.argv, append([]string{}, args...))
+		ph.mu.Unlock()
+		switch {
+		case containsArg(args, "snapshots"):
+			if h.failSnaps {
+				return []byte("ssh: connect refused"), errors.New("exit status 1")
+			}
+			b, _ := json.Marshal([]map[string]interface{}{{
+				"short_id": "ab12cd34", "id": "ab12cd34ffff", "time": h.snapAt, "tags": []string{r691App}, "paths": h.snapPaths}})
+			return b, nil
+		case containsArg(args, "restore"):
+			if ph.failRestore {
+				return []byte("simulated restic failure"), os.ErrPermission
+			}
+			target, include := argValue(args, "--target"), argValue(args, "--include")
+			dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
+			for _, sub := range []string{"compose", "db-dumps"} {
+				_ = os.MkdirAll(filepath.Join(dest, sub), 0o755)
+			}
+			_ = os.WriteFile(filepath.Join(dest, "db-dumps", "nextcloud-mariadb.sql"), []byte("x"), 0o644)
+			_ = os.WriteFile(filepath.Join(dest, "compose", "docker-compose.yml"), []byte("services:\n  nextcloud:\n    image: nextcloud:31\n  db:\n    image: mariadb:11.8\n"), 0o644)
+			_ = os.WriteFile(filepath.Join(dest, "compose", "app.yaml"), []byte("env:\n  HDD_PATH: "+h.unitHDD+"\n"), 0o600)
+			_ = os.WriteFile(filepath.Join(dest, "manifest.json"), []byte(h.manifest), 0o644)
+			return nil, nil
+		}
+		return nil, nil
+	})
+	ph.m.SetKeptUnitRestoreFn(func(app, unitDir string) (UnitRestoreResult, error) {
+		// The unit must still be on disk while the restore reads it.
+		if _, err := os.Stat(UnitManifestFile(unitDir)); err != nil {
+			t.Errorf("the unit restore ran on %s, which no longer holds a manifest: %v", unitDir, err)
+		}
+		h.restored = append(h.restored, unitDir)
+		return UnitRestoreResult{ManifestDBs: 1, DBsReplayed: 1}, nil
+	})
+	return h
+}
+
+// r691Manifest is a unit manifest listing one dump; withData adds the `data` block (the data's version).
+func r691Manifest(withData bool) string {
+	man := map[string]interface{}{"schema_version": 2, "app_name": r691App, "db_dumps": []string{"nextcloud-mariadb.sql"}}
+	if withData {
+		man["data"] = map[string]interface{}{"at": time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339),
+			"image_pins": []string{"mariadb:11.8", "nextcloud:31"}}
+	}
+	b, _ := json.Marshal(man)
+	return string(b)
+}
+
+func (h *r691Harness) load(t *testing.T, c KeptCopy) (ok bool) {
+	t.Helper()
+	h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
+		func(error) string { return "ok" }, func(err error) string { return err.Error() },
+		func(ok bool) { h.done <- ok })
+	select {
+	case ok = <-h.done:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the load never finished")
+	}
+	return ok
+}
+
+// The consequence, not the mechanism: which copy the household is offered.
+// COMPANION RED-PROOF: KeptBestCopy returning KeptDBCopy alone (0.276.0's choice) → the first assertion fails
+// with ok=false (an app whose only database copy is off-site gets „no backup").
+func TestR691_OffsiteCopyIsOfferedWhenItIsTheOnlyOrNewest(t *testing.T) {
+	h := newR691Harness(t)
+	c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
+	if !ok || c.Tier != KeptTierOffsite || c.SnapshotID != "ab12cd34" || c.UnitPath != h.snapPaths[0] {
+		t.Fatalf("only off-site copy: got %+v ok=%v, want the off-site snapshot ab12cd34", c, ok)
+	}
+	if !c.Time.Equal(h.snapAt) {
+		t.Fatalf("the copy is dated %v, want the snapshot's time %v", c.Time, h.snapAt)
+	}
+
+	// A local unit NEWER than the snapshot wins — no download.
+	unit := RecoveryUnitPath(h.m.namespaceRoot(h.drive), r691App)
+	writeKeptUnit(t, unit, h.drive, true, time.Now())
+	h.prov.hdd[r691App] = "" // removed
+	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != 1 || c.UnitDir != unit {
+		t.Fatalf("a newer local unit must win: got %+v ok=%v", c, ok)
+	}
+	// An OLDER local unit loses to the off-site copy.
+	old := time.Now().Add(-3 * time.Hour)
+	writeKeptUnit(t, unit, h.drive, true, old)
+	for _, f := range []string{UnitManifestFile(unit), filepath.Join(unit, "compose", "app.yaml")} {
+		_ = os.Chtimes(f, old, old)
+	}
+	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); !ok || c.Tier != KeptTierOffsite {
+		t.Fatalf("an older local unit must lose to the off-site copy: got %+v ok=%v", c, ok)
+	}
+
+	// Negative controls: a snapshot with no unit, an unreadable repository, an INSTALLED app.
+	_ = os.RemoveAll(unit)
+	h.snapPaths = []string{"/mnt/felhom-drives/hdd_1/appdata/" + r691App}
+	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
+		t.Fatalf("a snapshot holding no recovery unit was offered: %+v", c)
+	}
+	h.snapPaths = []string{"/x/backups/primary/" + r691App}
+	h.failSnaps = true
+	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
+		t.Fatalf("an unreadable repository offered a copy: %+v", c)
+	}
+	h.failSnaps = false
+	h.prov.deployed[r691App] = true
+	if c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive); ok {
+		t.Fatalf("an installed app's off-site copy was offered as kept data: %+v", c)
+	}
+}
+
+// The load: download the unit ALONE, judge it, THEN prepare, THEN the one unit restore; the scratch goes.
+// COMPANION RED-PROOF: drop the `man.Data == nil` refusal in downloadKeptOffsiteUnit → the no-data-block case
+// reaches the restore (restored=1, prepared=1) and fails.
+func TestR691_OffsiteLoadDownloadsJudgesThenRestores(t *testing.T) {
+	h := newR691Harness(t)
+	c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
+	if !ok {
+		t.Fatal("no off-site copy offered")
+	}
+	if !h.load(t, c) {
+		t.Fatalf("a good off-site unit did not load: %+v", h.m.RestoreStatus())
+	}
+	if h.prepared != 1 || len(h.restored) != 1 {
+		t.Fatalf("prepare=%d restore=%d, want 1 and 1", h.prepared, len(h.restored))
+	}
+	scratch := h.proofScratch(t, r691App)
+	if !strings.HasPrefix(h.restored[0], scratch+string(filepath.Separator)) || !strings.HasSuffix(h.restored[0], "/backups/primary/"+r691App) {
+		t.Fatalf("the restore read %s, want the downloaded unit inside %s", h.restored[0], scratch)
+	}
+	var sawUnitOnly bool
+	for _, a := range h.allArgs() {
+		if containsArg(a, "restore") && argValue(a, "--include") == h.snapPaths[0] && containsArg(a, "ab12cd34") {
+			sawUnitOnly = true
+		}
+	}
+	if !sawUnitOnly {
+		t.Fatalf("no unit-only restore of snapshot ab12cd34 (--include %s): %v", h.snapPaths[0], h.allArgs())
+	}
+	if _, err := os.Stat(scratch); !os.IsNotExist(err) {
+		t.Fatalf("the downloaded copy was left behind at %s (err=%v)", scratch, err)
+	}
+}
+
+// Every refusal happens BEFORE prepare: the kept files are exactly as they were, nothing is restored.
+func TestR691_OffsiteLoadRefusalsLeaveTheKeptFilesAlone(t *testing.T) {
+	cases := []struct {
+		name string
+		set  func(h *r691Harness)
+		want error
+	}{
+		{"data version not recorded", func(h *r691Harness) { h.manifest = r691Manifest(false) }, ErrKeptOffsiteVersionUnknown},
+		{"taken of another drive", func(h *r691Harness) { h.unitHDD = "/mnt/felhom-drives/other" }, ErrKeptOffsiteNotUsable},
+		{"definition is not the data's", func(h *r691Harness) {
+			h.manifest = strings.Replace(r691Manifest(true), "nextcloud:31", "nextcloud:30", 1)
+		}, ErrUnitVersionMismatch},
+		{"download failed", func(h *r691Harness) { h.failRestore = true }, nil},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := newR691Harness(t)
+			kept := filepath.Join(h.drive, "appdata", r691App, "photo.jpg")
+			_ = os.MkdirAll(filepath.Dir(kept), 0o755)
+			_ = os.WriteFile(kept, []byte("the household's photo"), 0o644)
+			c, ok := h.m.KeptBestCopy(context.Background(), r691App, h.drive)
+			if !ok {
+				t.Fatal("no off-site copy offered")
+			}
+			tc.set(h)
+			var got error
+			h.m.LoadKeptOffsite(r691App, h.drive, c, func() error { h.prepared++; return nil },
+				func(error) string { return "ok" }, func(err error) string { got = err; return err.Error() },
+				func(ok bool) { h.done <- ok })
+			if <-h.done {
+				t.Fatal("the load reported success")
+			}
+			if h.prepared != 0 || len(h.restored) != 0 {
+				t.Fatalf("prepare=%d restore=%d after a refusal, want 0 and 0", h.prepared, len(h.restored))
+			}
+			if tc.want != nil && !errors.Is(got, tc.want) {
+				t.Fatalf("refused with %v, want %v", got, tc.want)
+			}
+			if b, err := os.ReadFile(kept); err != nil || string(b) != "the household's photo" {
+				t.Fatalf("the kept file changed: %q %v", b, err)
+			}
+			if st := h.m.RestoreStatus(); st.Running || st.Last == nil || st.Last.OK {
+				t.Fatalf("the restore record does not read as failed: %+v", st)
+			}
+			if _, err := os.Stat(h.proofScratch(t, r691App)); !os.IsNotExist(err) {
+				t.Fatalf("the downloaded copy was left behind after a refusal (err=%v)", err)
+			}
+		})
+	}
+}