controller v0.239.0: any backup tier lets an app update (R-475)

Operator ruling 2026-09-13. The update precondition walks Tier 2, Tier 1 (own recovery unit, "helyi") and Tier 3 (off-site, 15 s bound; unreachable counts as absent with a WARN) and leans on the first FRESH copy; the backup_max_age rule applies to whichever tier is chosen. No copy anywhere: back up first. Refused only when nothing exists and no backup can be taken. RunAppBackupNow tolerates a Tier-2 failure (WARN) and marks the captured unit proven current. The hold names the tier (második meghajtó / saját meghajtó / távoli mentés) and the date; pre-v0.239.0 holds keep their text. A successful off-site restore now lifts an update hold. The backups page still uses Tier2UnitRestorePoint unchanged. Scenarios G-M tested; red-proofs M, L, the tail and the off-site clear in felhom.eu documentation/audits/rulings-r472-r475-2026-09-13/.
2026-09-13 17:16:24 +02:00
parent f946b0d0ca
commit b93c1543da
16 changed files with 1028 additions and 114 deletions
@@ -0,0 +1,317 @@
+package backup
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// R-475 — the backup side of "any backup tier lets an app update" (controller v0.239.0).
+
+var r475T0 = time.Date(2026, 9, 13, 12, 0, 0, 0, time.UTC)
+
+func r475Manager() (*Manager, *bytes.Buffer) {
+	var buf bytes.Buffer
+	return &Manager{logger: log.New(&buf, "", 0)}, &buf
+}
+
+func tier2At(at time.Time) func(string) (Tier2RestorePoint, error) {
+	return func(string) (Tier2RestorePoint, error) {
+		ts := at.UTC().Format(time.RFC3339)
+		return Tier2RestorePoint{Restorable: true, CopyDateProven: true, CopyLastSuccess: ts, CopyDate: ts}, nil
+	}
+}
+func noTier2(string) (Tier2RestorePoint, error) {
+	return Tier2RestorePoint{}, errors.New("no Tier-2 copy recorded for this app")
+}
+func tier1At(at time.Time) func(string) ([]RestorePoint, bool) {
+	return func(string) ([]RestorePoint, bool) {
+		return []RestorePoint{{Time: at.UTC().Format(time.RFC3339), ShortID: "helyi", Tier: 1}}, true
+	}
+}
+func noTier1(string) ([]RestorePoint, bool) { return []RestorePoint{}, true }
+func offsiteWith(app string, at time.Time) func(context.Context) (OffsiteInventory, error) {
+	return func(context.Context) (OffsiteInventory, error) {
+		return OffsiteInventory{Apps: []OffsiteInventoryApp{{App: app, LatestAt: at}}}, nil
+	}
+}
+func noOffsiteTarget(context.Context) (OffsiteInventory, error) {
+	return OffsiteInventory{}, errNoOffsiteTarget
+}
+
+func tiersOf(ps []UpdateTierPoint) []int {
+	out := make([]int, 0, len(ps))
+	for _, p := range ps {
+		out = append(out, p.Tier)
+	}
+	return out
+}
+
+// G — the order is 2, 1, 3, and the walk STOPS at the first accepted copy (so a box with a fresh
+// second-drive copy never reaches the network).
+func TestR475_G_TierOrderIsSecondDriveThenOwnUnitThenOffsite(t *testing.T) {
+	m, _ := r475Manager()
+	offsiteCalls := 0
+	m.updateTier2PointFn = tier2At(r475T0.Add(-1 * time.Hour))
+	m.updateTier1PointsFn = tier1At(r475T0.Add(-2 * time.Hour))
+	m.updateOffsiteInvFn = func(ctx context.Context) (OffsiteInventory, error) {
+		offsiteCalls++
+		return offsiteWith("gokapi", r475T0.Add(-3*time.Hour))(ctx)
+	}
+	ctx := context.Background()
+
+	p, ok, seen := m.UpdateRestorePoints(ctx, "gokapi", nil)
+	if !ok || p.Tier != UpdateTierSecondDrive || fmt.Sprint(tiersOf(seen)) != "[2]" || offsiteCalls != 0 {
+		t.Errorf("all three present: want tier 2 and a stop; got %+v ok=%v seen=%v offsite calls=%d", p, ok, tiersOf(seen), offsiteCalls)
+	}
+	p, ok, seen = m.UpdateRestorePoints(ctx, "gokapi", func(p UpdateTierPoint) bool { return p.Tier != UpdateTierSecondDrive })
+	if !ok || p.Tier != UpdateTierLocal || fmt.Sprint(tiersOf(seen)) != "[2 1]" {
+		t.Errorf("tier 2 refused: want tier 1; got %+v seen=%v", p, tiersOf(seen))
+	}
+	p, ok, seen = m.UpdateRestorePoints(ctx, "gokapi", func(p UpdateTierPoint) bool { return p.Tier == UpdateTierOffsite })
+	if !ok || p.Tier != UpdateTierOffsite || !p.At.Equal(r475T0.Add(-3*time.Hour)) || fmt.Sprint(tiersOf(seen)) != "[2 1 3]" {
+		t.Errorf("tiers 2 and 1 refused: want tier 3; got %+v seen=%v", p, tiersOf(seen))
+	}
+	if _, ok, seen = m.UpdateRestorePoints(ctx, "gokapi", func(UpdateTierPoint) bool { return false }); ok || len(seen) != 3 {
+		t.Errorf("nothing accepted: want not found with all three seen; ok=%v seen=%v", ok, tiersOf(seen))
+	}
+}
+
+func TestR475_H_OwnUnitOnly(t *testing.T) {
+	m, _ := r475Manager()
+	m.updateTier2PointFn = noTier2
+	m.updateTier1PointsFn = tier1At(r475T0.Add(-2 * time.Hour))
+	m.updateOffsiteInvFn = noOffsiteTarget
+	p, ok, _ := m.UpdateRestorePoints(context.Background(), "gokapi", nil)
+	if !ok || p.Tier != UpdateTierLocal || !p.At.Equal(r475T0.Add(-2*time.Hour)) {
+		t.Fatalf("got %+v ok=%v", p, ok)
+	}
+	// A Tier-2 record that was only ATTEMPTED is not a copy (R-101) — the own unit still wins.
+	m.updateTier2PointFn = func(string) (Tier2RestorePoint, error) {
+		return Tier2RestorePoint{Restorable: true, CopyDateProven: false}, nil
+	}
+	if p, ok, _ = m.UpdateRestorePoints(context.Background(), "gokapi", nil); !ok || p.Tier != UpdateTierLocal {
+		t.Errorf("an unproven Tier-2 record must be skipped; got %+v", p)
+	}
+	// No unit on disk (ListRestorePoints' empty list) is no copy.
+	m.updateTier1PointsFn = noTier1
+	if _, ok, _ = m.UpdateRestorePoints(context.Background(), "gokapi", nil); ok {
+		t.Error("no copy on any tier must be not found")
+	}
+}
+
+func TestR475_I_OffsiteOnly(t *testing.T) {
+	m, _ := r475Manager()
+	m.updateTier2PointFn, m.updateTier1PointsFn = noTier2, noTier1
+	m.updateOffsiteInvFn = offsiteWith("gokapi", r475T0.Add(-5*time.Hour))
+	if p, ok, _ := m.UpdateRestorePoints(context.Background(), "gokapi", nil); !ok || p.Tier != UpdateTierOffsite {
+		t.Fatalf("got %+v ok=%v", p, ok)
+	}
+	// Another app's snapshot is not this app's copy.
+	if _, ok, _ := m.UpdateRestorePoints(context.Background(), "nextcloud", nil); ok {
+		t.Error("a snapshot tagged for a different app must not count")
+	}
+}
+
+// J — an unreachable off-site repository is ABSENT with a WARN, and it is bounded in time.
+func TestR475_J_OffsiteUnreachableIsAbsentWithAWarn(t *testing.T) {
+	m, buf := r475Manager()
+	m.updateTier2PointFn, m.updateTier1PointsFn = noTier2, noTier1
+	m.updateOffsiteInvFn = func(context.Context) (OffsiteInventory, error) {
+		return OffsiteInventory{}, errors.New("ssh: connect to host: connection timed out")
+	}
+	if _, ok, _ := m.UpdateRestorePoints(context.Background(), "gokapi", nil); ok {
+		t.Error("an unreachable off-site copy must count as absent")
+	}
+	if !strings.Contains(buf.String(), "[WARN]") || !strings.Contains(buf.String(), "counted as ABSENT") {
+		t.Errorf("an unreachable off-site copy must WARN; log = %q", buf.String())
+	}
+
+	// Control: a box with NO off-site target is plainly absent — that is not a fault, so no WARN.
+	buf.Reset()
+	m.updateOffsiteInvFn = noOffsiteTarget
+	if _, ok, _ := m.UpdateRestorePoints(context.Background(), "gokapi", nil); ok || strings.Contains(buf.String(), "WARN") {
+		t.Errorf("no off-site target: want absent and silent; ok=%v log=%q", ok, buf.String())
+	}
+
+	// The bound: a repository that never answers is given up on at the timeout.
+	old := updateOffsiteCheckTimeout
+	updateOffsiteCheckTimeout = 50 * time.Millisecond
+	defer func() { updateOffsiteCheckTimeout = old }()
+	buf.Reset()
+	m.updateOffsiteInvFn = func(ctx context.Context) (OffsiteInventory, error) {
+		<-ctx.Done()
+		return OffsiteInventory{}, ctx.Err()
+	}
+	start := time.Now()
+	_, ok, _ := m.UpdateRestorePoints(context.Background(), "gokapi", nil)
+	if took := time.Since(start); ok || took > 5*time.Second || !strings.Contains(buf.String(), "counted as ABSENT") {
+		t.Errorf("a hanging off-site check must end at its bound as absent with a WARN; ok=%v took=%s log=%q", ok, took, buf.String())
+	}
+}
+
+// The hold names the tier. The labels are the ruling's exact words.
+func TestR475_HoldTextNamesTheTier(t *testing.T) {
+	at := time.Date(2026, 9, 13, 8, 0, 0, 0, time.UTC)
+	copyAt := time.Date(2026, 9, 13, 1, 30, 0, 0, time.UTC)
+	for tier, label := range map[int]string{
+		UpdateTierSecondDrive: "második meghajtó",
+		UpdateTierLocal:       "saját meghajtó",
+		UpdateTierOffsite:     "távoli mentés",
+	} {
+		sett := slice4Settings(t)
+		m := &Manager{logger: log.New(io.Discard, "", 0), settings: sett}
+		if err := m.HoldAfterFailedUpdate("gokapi", at, copyAt, tier); err != nil {
+			t.Fatal(err)
+		}
+		held, why := m.RestoreHoldFor("gokapi")
+		want := fmt.Sprintf(UpdateHoldFmt, "gokapi", "2026-09-13 10:00", label, "2026-09-13 03:30")
+		if !held || why != want {
+			t.Errorf("tier %d: hold text =\n%q\nwant\n%q", tier, why, want)
+		}
+		if !strings.HasSuffix(why, "ebből a biztonsági mentésből: "+label+", 2026-09-13 03:30.") {
+			t.Errorf("tier %d: the sentence must END naming the tier and the date, got %q", tier, why)
+		}
+	}
+}
+
+func TestR475_AHoldWrittenBeforeTheTierKeepsItsSentence(t *testing.T) {
+	sett := slice4Settings(t)
+	m := &Manager{logger: log.New(io.Discard, "", 0), settings: sett}
+	if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "uptime-kuma", At: "2026-09-13T10:18:02Z",
+		Reason: settings.HoldReasonUpdateFailed, CopyDate: "2026-09-13T10:09:51Z"}); err != nil {
+		t.Fatal(err)
+	}
+	_, why := m.RestoreHoldFor("uptime-kuma")
+	// The exact sentence quoted in audits/slice4-2026-09-13 (13-H-refusals.txt), live on v0.238.1.
+	if want := fmt.Sprintf(UpdateHoldLegacyFmt, "uptime-kuma", "2026-09-13 12:18", "2026-09-13 12:09"); why != want {
+		t.Errorf("a v0.238.1 hold must keep its sentence:\n%q\nwant\n%q", why, want)
+	}
+}
+
+// RunAppBackupNow's tail: a Tier-2 failure no longer fails "back up first", and the own unit it
+// captured reads as fresh even when the capture found nothing to rewrite.
+//
+// COMPANION RED-PROOF (REPORT.md): aim the Chtimes at a path that does not exist — this test fails on
+// the mtime: "back up first" would then leave a quiet app's own unit as old as its last definition change.
+func TestR475_PreBackupTail_Tier2FailureIsAWarnAndTheOwnUnitIsFresh(t *testing.T) {
+	nsRoot := t.TempDir()
+	mp := RecoveryUnitManifestPath(nsRoot, "gokapi")
+	if err := os.MkdirAll(filepath.Dir(mp), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(mp, []byte(`{"app_name":"gokapi"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	old := time.Now().Add(-30 * time.Hour)
+	if err := os.Chtimes(mp, old, old); err != nil {
+		t.Fatal(err)
+	}
+	m, buf := r475Manager()
+	tier2Ran := false
+	m.perAppTier2 = func(string) error { tier2Ran = true; return errors.New("no second drive with room") }
+
+	now := time.Now()
+	m.updatePreBackupTail("gokapi", nsRoot, now)
+
+	if !tier2Ran {
+		t.Fatal("the Tier-2 copy must still be attempted")
+	}
+	if !strings.Contains(buf.String(), "[WARN]") || !strings.Contains(buf.String(), "Tier 2 copy FAILED") {
+		t.Errorf("a Tier-2 failure must be logged as a WARN; log = %q", buf.String())
+	}
+	fi, err := os.Stat(mp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d := fi.ModTime().Sub(now); d < -time.Second || d > time.Second {
+		t.Errorf("the own unit must read as proven NOW (mtime %s, now %s)", fi.ModTime(), now)
+	}
+
+	// Control: the same unit through ListRestorePoints' own rule reads fresh — the newest artifact.
+	buf.Reset()
+	m.perAppTier2 = func(string) error { return nil }
+	m.updatePreBackupTail("gokapi", nsRoot, now)
+	if strings.Contains(buf.String(), "WARN") {
+		t.Errorf("a successful Tier-2 copy must not WARN; log = %q", buf.String())
+	}
+}
+
+// The tail is really what RunAppBackupNow runs — a helper tested and never called is the "seam built
+// but never wired" shape this project has shipped repeatedly.
+func TestR475_RunAppBackupNowUsesTheTolerantTail(t *testing.T) {
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "update_guard.go", nil, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var calls []string
+	found := false
+	for _, d := range f.Decls {
+		fn, ok := d.(*ast.FuncDecl)
+		if !ok || fn.Name.Name != "RunAppBackupNow" {
+			continue
+		}
+		found = true
+		ast.Inspect(fn.Body, func(n ast.Node) bool {
+			if sel, ok := n.(*ast.SelectorExpr); ok {
+				calls = append(calls, sel.Sel.Name)
+			}
+			return true
+		})
+	}
+	joined := " " + strings.Join(calls, " ") + " "
+	if !found || !strings.Contains(joined, " updatePreBackupTail ") {
+		t.Fatalf("RunAppBackupNow must end in updatePreBackupTail; selectors = %s", joined)
+	}
+	if strings.Contains(joined, " RunTier2 ") || strings.Contains(joined, " perAppTier2 ") {
+		t.Error("RunAppBackupNow must not run the Tier-2 copy itself any more — only through the tolerant tail")
+	}
+}
+
+func TestR475_CanBackUpApp(t *testing.T) {
+	var nilM *Manager
+	if ok, why := nilM.CanBackUpApp("gokapi"); ok || why == "" {
+		t.Error("no backup manager: cannot back up, with a reason")
+	}
+	m, _ := r475Manager()
+	if ok, why := m.CanBackUpApp("gokapi"); ok || !strings.Contains(why, "stack provider") {
+		t.Errorf("no stack provider: cannot back up; got ok=%v why=%q", ok, why)
+	}
+}
+
+// Tier 3's route back is the off-site restore, and it never went through RestoreFromRecoveryUnitAt —
+// so it must lift an update hold itself, or a hold naming „távoli mentés" could never be cleared.
+//
+// COMPANION RED-PROOF (REPORT.md): remove the clearUpdateHoldAfterRestore call from
+// ReconstituteFromOffsite — this test fails with the hold still in place.
+func TestR475_OffsiteRestoreClearsAnUpdateHold(t *testing.T) {
+	m, prov, _ := reconFixture(t, "run1", "2026-07-19T06:00:00Z", "")
+	prov.composePath = writeLiveCompose(t, noDBCompose)
+	m.discoverDBs = func(context.Context) ([]DiscoveredDB, error) { return nil, nil }
+	if err := m.HoldAfterFailedUpdate("immich", r475T0, r475T0.Add(-time.Hour), UpdateTierOffsite); err != nil {
+		t.Fatal(err)
+	}
+	if held, why := m.RestoreHoldFor("immich"); !held || !strings.Contains(why, "távoli mentés") {
+		t.Fatalf("fixture: the app must be held naming the off-site copy, got held=%v %q", held, why)
+	}
+	if _, err := m.ReconstituteFromOffsite(context.Background(), "immich", false); err != nil {
+		t.Fatalf("reconstitute: %v", err)
+	}
+	if held, _ := m.RestoreHoldFor("immich"); held {
+		t.Error("a successful off-site restore is the route back the hold names — the hold must be lifted")
+	}
+}