package backup import ( "context" "encoding/json" "fmt" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // ── decision 68/69 (v0.289.0): the append-only tier ───────────────────────────────────────────────── func pinTarget(t *testing.T, sett *settings.Settings) { t.Helper() if err := sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.Transport = settings.TransportRclonePinned o.Port = 23 o.Host, o.User, o.RepoPath = "u1-sub4.example", "u1-sub4", "/home/felhom-repo" }); err != nil { t.Fatal(err) } } func snapJSON(s []guardSnap) []byte { b, _ := json.Marshal(s); return b } func planJSON(remove []guardSnap) []byte { b, _ := json.Marshal([]map[string]any{{"tags": []string{"app1"}, "remove": remove}}) return b } // windowRunner fakes restic for the retention step: snapshots, the dry-run plan, and records every // non-dry-run forget (the only call that deletes). type windowRunner struct { snaps []guardSnap plan []guardSnap forgets [][]string } func (w *windowRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) { switch { case contains(args, "forget") && contains(args, "--dry-run"): return planJSON(w.plan), nil case contains(args, "forget"): w.forgets = append(w.forgets, append([]string{}, args...)) return nil, nil case contains(args, "snapshots"): return snapJSON(w.snaps), nil } return nil, nil } type fakeWindow struct { grant OffsiteWindow opened int closed []OffsiteWindowResult } func (f *fakeWindow) Open(context.Context, int) (OffsiteWindow, error) { f.opened++ return f.grant, nil } func (f *fakeWindow) Close(_ context.Context, r OffsiteWindowResult) error { f.closed = append(f.closed, r) return nil } func snap(id string, at time.Time) guardSnap { return guardSnap{ID: id + "-full", ShortID: id, Time: at} } // The pinned transport: rclone over port 23, the pinned key; never sftp. func TestOffboxBaseArgs_PinnedUsesRcloneOnPort23(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.Port = 0 }) args, _ := m.offboxBaseArgs(sett.GetOffboxTarget()) j := strings.Join(args, " ") if !strings.Contains(j, "-r rclone:/home/felhom-repo") || !strings.Contains(j, "rclone.program=ssh -p 23 ") || strings.Contains(j, "sftp") { t.Fatalf("pinned args = %q", j) } if !argsContainTimeout(args) { t.Fatal("ConnectTimeout lost on the pinned transport") } // The household's NAS stays SFTP. m2, sett2 := newOffboxManager(t) if j2 := strings.Join(func() []string { a, _ := m2.offboxBaseArgs(sett2.GetOffboxTarget()); return a }(), " "); !strings.Contains(j2, "sftp:") { t.Fatalf("NAS args = %q", j2) } } // THE CONSEQUENCE: on the pinned tier, a run with no window deletes NOTHING — no forget reaches restic. // RED-PROOF: the pre-v0.289.0 retention ran `forget … --prune` unconditionally after every run. func TestRetention_PinnedWithoutWindowDeletesNothing(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) wr := &windowRunner{snaps: []guardSnap{snap("a", time.Now().Add(-40*24*time.Hour))}, plan: []guardSnap{snap("a", time.Now().Add(-40*24*time.Hour))}} m.SetOffboxRunner(wr.run) m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") // no client wired fw := &fakeWindow{grant: OffsiteWindow{Granted: false, Reason: "not due"}} m.SetOffsiteWindowClient(fw) m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") if len(wr.forgets) != 0 { t.Fatalf("a forget ran without a window: %v", wr.forgets) } if fw.opened != 1 || len(fw.closed) != 0 { t.Fatalf("opened=%d closed=%d", fw.opened, len(fw.closed)) } } // The full run path: RunOffboxBackup on a pinned target with no window never calls forget. func TestRunOffboxBackup_PinnedNeverForgetsWithoutWindow(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) var forgets int m.SetOffboxRunner(func(ctx context.Context, env []string, args ...string) ([]byte, error) { if contains(args, "forget") { forgets++ } rr := &recordingOffboxRunner{} return rr.run(ctx, env, args...) }) _ = m.RunOffboxBackup(context.Background()) if forgets != 0 { t.Fatalf("the pinned run reached forget %d time(s)", forgets) } } // R-822, the lab's shape: 13 future-dated fakes. The guard REFUSES and nothing is deleted; the hub // is told why (window closed with outcome guard-refused). func TestOffsiteGuard_LabThirteenFutureFakes_Refused(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) now := time.Now() real := []guardSnap{snap("r1", now.Add(-2*time.Hour)), snap("r2", now.Add(-26*time.Hour)), snap("r3", now.Add(-50*time.Hour))} all := append([]guardSnap{}, real...) for d := 1; d <= 7; d++ { all = append(all, snap(fmt.Sprintf("f%d", d), time.Date(2027, 1, d, 3, 0, 0, 0, time.UTC))) } for mth := 2; mth <= 7; mth++ { all = append(all, snap(fmt.Sprintf("m%d", mth), time.Date(2027, time.Month(mth), 15, 3, 0, 0, 0, time.UTC))) } wr := &windowRunner{snaps: all, plan: real} // the poisoned policy selects every REAL snapshot m.SetOffboxRunner(wr.run) fw := &fakeWindow{grant: OffsiteWindow{Granted: true, ID: 7, NewestAllowed: now, MaxRemove: 50}} m.SetOffsiteWindowClient(fw) m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") if len(wr.forgets) != 0 { t.Fatalf("the guard let a poisoned plan delete: %v", wr.forgets) } if len(fw.closed) != 1 || fw.closed[0].Outcome != "guard-refused" || !strings.Contains(fw.closed[0].Reason, "future") { t.Fatalf("window close = %+v", fw.closed) } } // Past-dated fakes that make the policy drop a RECENT real snapshot: refused too. func TestOffsiteGuard_RecentRemovalRefused(t *testing.T) { now := time.Now() all := []guardSnap{snap("old", now.Add(-60*24*time.Hour)), snap("recent", now.Add(-3*24*time.Hour))} _, why := offsiteGuard(all, []guardSnap{snap("recent", now.Add(-3*24*time.Hour))}, now, now, 50) if !strings.Contains(why, "younger than 8 days") { t.Fatalf("why = %q", why) } _, why = offsiteGuard(all, nil, now, now.Add(-10*24*time.Hour), 50) if !strings.Contains(why, "newer than the hub allows") { t.Fatalf("newest-allowed bound not enforced: %q", why) } } // Honest retention inside a window: oldest first, the forget names ids. func TestOffsiteGuard_HonestPlanPrunesOldestFirst(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) now := time.Now() plan := []guardSnap{snap("c", now.Add(-20*24*time.Hour)), snap("a", now.Add(-90*24*time.Hour)), snap("b", now.Add(-60*24*time.Hour))} all := append([]guardSnap{snap("keep", now.Add(-time.Hour))}, plan...) wr := &windowRunner{snaps: all, plan: plan} m.SetOffboxRunner(wr.run) fw := &fakeWindow{grant: OffsiteWindow{Granted: true, ID: 9, NewestAllowed: now, MaxRemove: 5}} m.SetOffsiteWindowClient(fw) m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") if len(wr.forgets) != 1 || !strings.Contains(strings.Join(wr.forgets[0], " "), "forget a-full b-full c-full --prune") { t.Fatalf("forget = %v", wr.forgets) } if len(fw.closed) != 1 || fw.closed[0].Outcome != "pruned" || fw.closed[0].ID != 9 { t.Fatalf("close = %+v", fw.closed) } } // A plan larger than one week's removal REFUSES (the 2026-10-04 brief), nothing removed. func TestOffsiteGuard_AboveWeeklyCapRefused(t *testing.T) { now := time.Now() var plan []guardSnap for i := 0; i < 6; i++ { plan = append(plan, snap(fmt.Sprintf("o%d", i), now.Add(-time.Duration(30+i)*24*time.Hour))) } ids, why := offsiteGuard(plan, plan, now, now, 5) if ids != nil || !strings.Contains(why, "more than one week") { t.Fatalf("ids=%v why=%q", ids, why) } } // R-824, THE MEASURED SHAPE (demo-hp window 1, 2026-10-03): the night run's snapshots of each app were // superseded the SAME DAY by a manual run, so the honest policy removes them while they are young. // v0.289 refused the whole window; now they are EXCLUDED (kept for later) and the old removal goes ahead. func TestOffsiteGuard_SameDaySupersededYoungExcluded(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) now := time.Now() day := now.Add(-26 * time.Hour).Truncate(24 * time.Hour) night := guardSnap{ID: "night-full", ShortID: "night", Time: day.Add(2 * time.Hour), Hostname: "demo-hp", Tags: []string{"opengist"}} manual := guardSnap{ID: "manual-full", ShortID: "manual", Time: day.Add(15 * time.Hour), Hostname: "demo-hp", Tags: []string{"opengist"}} old := guardSnap{ID: "old-full", ShortID: "old", Time: now.Add(-40 * 24 * time.Hour), Hostname: "demo-hp", Tags: []string{"opengist"}} wr := &windowRunner{snaps: []guardSnap{old, night, manual}, plan: []guardSnap{night, old}} m.SetOffboxRunner(wr.run) fw := &fakeWindow{grant: OffsiteWindow{Granted: true, ID: 2, NewestAllowed: now, MaxRemove: 5}} m.SetOffsiteWindowClient(fw) m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") if len(fw.closed) != 1 || fw.closed[0].Outcome != "pruned" { t.Fatalf("the window must run, not refuse: %+v", fw.closed) } got := strings.Join(wr.forgets[0], " ") if !strings.Contains(got, "old-full") || strings.Contains(got, "night-full") { t.Fatalf("forget = %q (the young superseded copy must be KEPT for now)", got) } // A young removal with NO same-day successor in its group is still the poisoning signature. lone := guardSnap{ID: "lone-full", ShortID: "lone", Time: now.Add(-50 * time.Hour), Hostname: "demo-hp", Tags: []string{"bookstack"}} if _, why := offsiteGuard([]guardSnap{lone, manual}, []guardSnap{lone}, now, now, 5); !strings.Contains(why, "not superseded the same day") { t.Fatalf("why = %q", why) } } // The orphan reset on the pinned tier asks the HUB; no ssh `mv` from the box. func TestResetOrphaned_PinnedAsksTheHub(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) m.SetOffboxSSH(func(context.Context, string, string, int, string, string, string) ([]byte, error) { t.Fatal("the box issued a raw ssh command on the pinned tier") return nil, nil }) called := 0 m.SetOffsiteMoveAside(func(context.Context) (string, error) { called++; return "/home/felhom-repo.orphaned-20261003", nil }) m.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) { return nil, nil }) if err := m.resetOrphanedRepo(context.Background(), nil, nil, "test"); err != nil { t.Fatal(err) } if called != 1 || sett.GetOffboxTarget().OrphanedRenamedTo != "/home/felhom-repo.orphaned-20261003" { t.Fatalf("hub move-aside called=%d recorded=%q", called, sett.GetOffboxTarget().OrphanedRenamedTo) } } // The provider's rclone notice must not reach a JSON parser — measured live on demo-felhom (v0.289.0). func TestStripRcloneNotice(t *testing.T) { in := "rclone: 2026/10/03 15:05:42 NOTICE: Config file \"/home/.config/rclone/rclone.conf\" not found - using defaults\n[{\"id\":\"s1\"}]\n" if got := string(stripRcloneNotice([]byte(in))); got != "[{\"id\":\"s1\"}]\n" { t.Fatalf("got %q", got) } keep := "rclone: 2026/10/03 15:05:42 ERROR : something real\n" if got := string(stripRcloneNotice([]byte(keep))); got != keep { t.Fatalf("an rclone ERROR line was stripped: %q", got) } } // THE CONSEQUENCE (R-331/R-431): a run whose snapshot listing cannot be read must NOT record 0 as a // measurement. Before the fix the box reported 0 snapshots with stats_known over a store holding 12. func TestRunOffbox_UnreadableCountIsNotZero(t *testing.T) { m, sett := newOffboxManager(t) sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.SnapshotCount, o.StatsKnown = 11, true }) m.SetOffboxRunner(func(ctx context.Context, env []string, args ...string) ([]byte, error) { if contains(args, "snapshots") { return []byte("rclone: 2026/10/03 15:05:42 ERROR : boom\nnot json"), nil } rr := &recordingOffboxRunner{} return rr.run(ctx, env, args...) }) _ = m.RunOffboxBackup(context.Background()) got := sett.GetOffboxTarget() if got.StatsKnown && got.SnapshotCount == 0 { t.Fatalf("an unreadable count was recorded as a measured zero: %+v", got.SnapshotCount) } } type fakeAbandon struct { requested []string state string cancels int due time.Time } func (f *fakeAbandon) Request(_ context.Context, p string) (time.Time, error) { f.requested = append(f.requested, p) f.state = "pending" return f.due, nil } func (f *fakeAbandon) Status(context.Context) (string, error) { return f.state, nil } func (f *fakeAbandon) Cancel(context.Context) error { f.cancels++; f.state = "cancelled"; return nil } // Decision 74 (R-823), the box side: a due abandonment on the pinned tier is HANDED to the hub (nothing // deleted by the box), the page keeps a dated, cancellable deletion, a recovery cancels it at the hub, and // a "deleted" from the hub completes the two-phase commit. func TestAbandon_PinnedHandsToHubAndFollows(t *testing.T) { m, sett := newOffboxManager(t) pinTarget(t, sett) setDue := func() { sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.AbandonRepoPath = "/home/felhom-repo.orphaned-20260901" o.AbandonStartedAt = time.Now().Add(-20 * 24 * time.Hour).UTC().Format(time.RFC3339) o.AbandonAt = time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) }) } setDue() m.SetOffboxSSH(func(context.Context, string, string, int, string, string, string) ([]byte, error) { t.Fatal("the box tried to delete on the pinned tier") return nil, nil }) fa := &fakeAbandon{due: time.Now().Add(7 * 24 * time.Hour)} m.SetOffsiteAbandonClient(fa) if del, err := m.AbandonSweep(context.Background()); del || err != nil || len(fa.requested) != 1 { t.Fatalf("del=%v err=%v requested=%v", del, err, fa.requested) } st := m.AbandonStatus() if !st.Active || !st.HubPending || st.DaysLeft < 6 { t.Fatalf("the page would lose the dated deletion: %+v", st) } // Still pending: nothing happens, nothing re-requested. if del, _ := m.AbandonSweep(context.Background()); del || len(fa.requested) != 1 { t.Fatal("re-requested or deleted while pending") } // The household recovers → cancelled at the hub, countdown gone. m.CancelAbandon("recovery succeeded") if fa.cancels != 1 || m.AbandonStatus().Active { t.Fatalf("cancels=%d status=%+v", fa.cancels, m.AbandonStatus()) } // Again, and this time the hub deletes. setDue() _, _ = m.AbandonSweep(context.Background()) fa.state = "deleted" var evs []string m.SetOffboxOrphanEvent(func(e, _ string) { evs = append(evs, e) }) if del, err := m.AbandonSweep(context.Background()); !del || err != nil || !m.AbandonStatus().PurgeRequested || len(evs) != 1 || evs[0] != "offbox_abandon_completed" { t.Fatalf("del=%v err=%v status=%+v evs=%v", del, err, m.AbandonStatus(), evs) } }