package backup import ( "bytes" "context" "fmt" "log" "strings" "sync" "sync/atomic" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // inventoryFixture wires a Manager whose repository answers a fixed snapshot list, and whose per-app // `stats` calls are observable: how many ran at once, how many in total, and which ones fail. // // The delay is what makes the concurrency assertion meaningful — with instant calls a sequential // implementation could pass a peak-of-1 check by finishing before the next one starts. func inventoryFixture(t *testing.T, tags []string, statDelay time.Duration, failFor map[string]bool) (*Manager, *int32, *int32, *bytes.Buffer) { t.Helper() m, sett := newOffboxManager(t) if err := sett.SetOffboxTarget(&settings.OffboxTarget{ Enabled: true, Host: "nas.local", Port: 22, User: "felhom", RepoPath: "/srv/repo", Schedule: "daily", EscrowState: "escrowed", }); err != nil { t.Fatal(err) } if err := m.WriteOffboxSecrets("KEYMATERIAL", "nas.local ssh-ed25519 HOSTKEY"); err != nil { t.Fatal(err) } if !m.OffboxConfigured() { t.Fatal("fixture: the target must be configured or the listing exits early") } var logbuf bytes.Buffer m.logger = log.New(&logbuf, "", 0) var snaps []string for i, tag := range tags { snaps = append(snaps, fmt.Sprintf(`{"short_id":"snap%d","time":"2026-08-0%dT06:00:00Z","tags":["%s"]}`, i, i+1, tag)) } snapshotsJSON := "[" + strings.Join(snaps, ",") + "]" var inFlight, peak, total int32 var mu sync.Mutex m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) { if contains(args, "snapshots") { return []byte(snapshotsJSON), nil } if !contains(args, "stats") { return nil, nil } atomic.AddInt32(&total, 1) cur := atomic.AddInt32(&inFlight, 1) mu.Lock() if cur > peak { peak = cur } mu.Unlock() time.Sleep(statDelay) atomic.AddInt32(&inFlight, -1) // args carries the snapshot id; map it back to its tag position. for i, tag := range tags { if contains(args, fmt.Sprintf("snap%d", i)) { if failFor[tag] { return nil, fmt.Errorf("injected failure for %s", tag) } return []byte(`{"total_size":1048576}`), nil } } return []byte(`{"total_size":0}`), nil }) return m, &peak, &total, &logbuf } // R-351 PART 4 — the per-app size calls run AT ONCE, and never more than the bound at once. // // MEASURED on demo-hp against the live off-site target before any change (2026-08-21): // snapshots --json 2605 ms once, then stats --mode restore-size 2697 ms PER APP, sequential, over // 5 app tags — 2605 + 5*2697 = ~16.1 s, which is the reported ten-to-fifteen seconds. // // THE BOUND IS THE SAFETY PROPERTY, not the speed one. The repository is a Hetzner Storage Box with // a concurrent-SSH-session cap; a size call that is refused returns SizeBytes 0, which is a SILENT // UNDER-REPORT of the customer's own data rather than a visible failure. So the peak is asserted, // not just the total. func TestOffsiteInventoryList_SizeCallsAreConcurrentButBounded(t *testing.T) { tags := []string{"immich", "nextcloud", "opengist", "paperless-ngx", "privatebin", "calibre-web", "vaultwarden"} m, peak, total, _ := inventoryFixture(t, tags, 40*time.Millisecond, nil) start := time.Now() _, err := m.OffsiteInventoryList(context.Background()) elapsed := time.Since(start) if err != nil { t.Fatalf("inventory: %v", err) } if int(*total) != len(tags) { t.Errorf("every app needs its own size call: got %d, want %d", *total, len(tags)) } if *peak < 2 { t.Errorf("the size calls must run concurrently; peak in flight was %d — that is the sequential shape that cost 16s", *peak) } if *peak > 4 { t.Errorf("peak in flight was %d, above the bound of 4 — an unbounded fan-out risks refused connections, "+ "and a refused size call silently under-reports the customer's data", *peak) } // 7 apps at 40ms: sequential would be >=280ms, four-at-a-time is two waves (~80ms). if elapsed >= time.Duration(len(tags))*40*time.Millisecond { t.Errorf("elapsed %v is the sequential cost — the fan-out is not taking effect", elapsed) } } // Nothing may be LOST or REORDERED by the fan-out. A missing app reads as "you have no backup of // this", which is the worst possible thing for this page to say by accident. func TestOffsiteInventoryList_FanOutLosesNothingAndStaysSorted(t *testing.T) { tags := []string{"vaultwarden", "immich", "calibre-web", "opengist", "nextcloud"} m, _, _, _ := inventoryFixture(t, tags, time.Millisecond, nil) inv, err := m.OffsiteInventoryList(context.Background()) if err != nil { t.Fatalf("inventory: %v", err) } if len(inv.Apps) != len(tags) { t.Fatalf("apps listed = %d, want %d — the fan-out dropped one, which reads as a missing backup", len(inv.Apps), len(tags)) } seen := map[string]bool{} for _, a := range inv.Apps { seen[a.App] = true if a.SizeBytes != 1048576 { t.Errorf("%s: size = %d, want the injected value", a.App, a.SizeBytes) } } for _, tag := range tags { if !seen[tag] { t.Errorf("%s is missing from the listing", tag) } } for i := 1; i < len(inv.Apps); i++ { if inv.Apps[i-1].App > inv.Apps[i].App { t.Fatalf("the listing must stay sorted; %q came before %q", inv.Apps[i-1].App, inv.Apps[i].App) } } if inv.Empty { t.Error("a repository with snapshots is not Empty") } } // A FAILED size call must still list the app, with size 0, AND still log the WARN. The warning is // the only thing that distinguishes "0 bytes" from "we could not tell" — and under a fan-out it is // the easiest thing to lose, because the logging moved off the goroutine that produced the error. func TestOffsiteInventoryList_FailedSizeStillListsAndStillWarns(t *testing.T) { tags := []string{"immich", "opengist"} m, _, _, logbuf := inventoryFixture(t, tags, time.Millisecond, map[string]bool{"opengist": true}) inv, err := m.OffsiteInventoryList(context.Background()) if err != nil { t.Fatalf("a per-app size failure must not fail the whole listing: %v", err) } if len(inv.Apps) != 2 { t.Fatalf("both apps must be listed; got %d", len(inv.Apps)) } for _, a := range inv.Apps { if a.App == "opengist" && a.SizeBytes != 0 { t.Errorf("an unknown size must be 0, got %d", a.SizeBytes) } if a.App == "immich" && a.SizeBytes == 0 { t.Error("the healthy app's size must survive its neighbour's failure") } } if !strings.Contains(logbuf.String(), "size of opengist's newest snapshot unknown") { t.Errorf("the WARN is what separates \"0 bytes\" from \"could not tell\"; log was: %s", logbuf.String()) } if strings.Contains(logbuf.String(), "size of immich's newest snapshot unknown") { t.Error("a healthy app must not be warned about") } }