package backup import ( "bytes" "context" "errors" "log" "strings" "testing" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // ── R-359 — the off-site store was never checked ───────────────────────────────────────────────── // // The whole-guest tier has verify jobs; the tier holding the customer's documents and photos had none. // The complete set of restic verbs this controller used contained no `check` — verified 2026-08-30. // // These drive the REAL CheckOffboxIntegrity through the EXISTING `offboxRunner` seam, which has been // injectable since the off-site tier shipped. (R-398 claimed otherwise and was my own mistake; the // seam sees every argv, including the `unlock --remove-all` escalation a `resticStepFn` would have // hidden — which is exactly what the lock-safety tests must observe.) // errFake is a plain non-nil error for seam replies; the classifier reads the OUTPUT, not the error // type, so a synthetic error is faithful here. var errFake = errors.New("restic exited non-zero") // integrityCapture records every restic invocation so both the effects and the NON-effects are // assertable. `argvs` is the whole point: a test that only checks the verdict cannot tell a check that // ran from one that did not. type integrityCapture struct { argvs [][]string reply func(args []string) ([]byte, error) logBuf *bytes.Buffer } func (c *integrityCapture) runner() offboxRunner { return func(_ context.Context, _ []string, args ...string) ([]byte, error) { c.argvs = append(c.argvs, append([]string{}, args...)) if c.reply != nil { return c.reply(args) } return nil, nil } } func (c *integrityCapture) sawVerb(verb string) bool { for _, a := range c.argvs { for _, x := range a { if x == verb { return true } } } return false } func (c *integrityCapture) checkArgv() []string { for _, a := range c.argvs { for _, x := range a { if x == "check" { return a } } } return nil } // newIntegrityManager builds a manager with a configured off-site target and a captured runner. func newIntegrityManager(t *testing.T, reply func(args []string) ([]byte, error)) (*Manager, *integrityCapture) { t.Helper() m, _ := newOffboxManager(t) cap := &integrityCapture{reply: reply, logBuf: &bytes.Buffer{}} m.logger = log.New(cap.logBuf, "", 0) m.SetOffboxRunner(cap.runner()) return m, cap } // okRepo answers `cat config` so ensureOffboxRepo passes, then defers to `then` for everything else. func okRepo(then func(args []string) ([]byte, error)) func(args []string) ([]byte, error) { return func(args []string) ([]byte, error) { for _, a := range args { if a == "config" { return []byte(`{"version":2}`), nil } } if then != nil { return then(args) } return nil, nil } } func TestR359_HealthyRepoReportsOK(t *testing.T) { m, cap := newIntegrityManager(t, okRepo(nil)) res := m.CheckOffboxIntegrity(context.Background()) if !res.OK || res.Skipped || res.Unreachable { t.Fatalf("a healthy repo did not report OK: %+v", res) } if cap.checkArgv() == nil { t.Fatal("`restic check` was never invoked — the check did not check anything") } } func TestR359_RepositoryErrorReportsFailure(t *testing.T) { // restic's own words from the 2026-08-21 damaged-pack drill. const damaged = "pack 5b1f2c3d: not found in index\nrepository contains errors" m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) { return []byte(damaged), errFake })) res := m.CheckOffboxIntegrity(context.Background()) if res.OK { t.Fatal("a repository restic said contains errors was reported as OK — this is the defect the " + "whole feature exists to prevent") } if res.Unreachable { t.Fatal("readable-and-damaged was misclassified as unreachable — those are different facts, " + "and only one of them means the customer's backups are broken") } if !strings.Contains(res.Output, "not found in index") { t.Errorf("restic's own words must reach the LOG so the operator can diagnose; got %q", res.Output) } } func TestR359_UnreachableIsNotAnIntegrityFailure(t *testing.T) { // The repo cannot even be opened. "I could not look" is not "I looked and it is broken". m, _ := newIntegrityManager(t, func(args []string) ([]byte, error) { return []byte("ssh: connect to host nas.local port 22: Connection refused"), errors.New("exit 1") }) res := m.CheckOffboxIntegrity(context.Background()) if res.OK { t.Fatal("an unreachable repository was reported as a passing check") } if !res.Unreachable { t.Fatal("an unreachable repository was reported as DAMAGE — that would alarm the customer that " + "their backups are corrupt when nothing was ever looked at, and R-339 already owns reachability") } } func TestR359_TimeoutIsNotDamage(t *testing.T) { m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) { return nil, context.DeadlineExceeded })) ctx, cancel := context.WithCancel(context.Background()) cancel() // an already-dead context: the check cannot finish res := m.CheckOffboxIntegrity(ctx) if res.OK { t.Fatal("a check that never finished reported OK") } if !res.Unreachable { t.Fatalf("a check that timed out was reported as damage: %+v — it saw nothing, so it may not "+ "claim the store is broken", res) } } func TestR359_StructureCheckPassesNoReadDataFlag(t *testing.T) { m, cap := newIntegrityManager(t, okRepo(nil)) m.CheckOffboxIntegrity(context.Background()) argv := cap.checkArgv() if argv == nil { t.Fatal("no check ran") } for _, a := range argv { if strings.HasPrefix(a, "--read-data") { t.Fatalf("the DEFAULT check downloaded pack data (%q) — that is a bandwidth cost nobody "+ "chose, and R-399 exists precisely so it is not chosen here", a) } } } func TestR359_ReadDataSubsetIsPassedWhenConfigured(t *testing.T) { m, cap := newIntegrityManager(t, okRepo(nil)) m.cfg.Monitoring.Integrity.ReadDataSubset = "5%" res := m.CheckOffboxIntegrity(context.Background()) if res.ReadDataSubset != "5%" { t.Errorf("result did not record the depth it ran at: %+v", res) } var found bool for _, a := range cap.checkArgv() { if a == "--read-data-subset=5%" { found = true } } if !found { t.Fatalf("the configured subset did not reach restic; argv=%v", cap.checkArgv()) } } func TestR359_MalformedReadDataSubsetIsTreatedAsOff(t *testing.T) { m, cap := newIntegrityManager(t, okRepo(nil)) m.cfg.Monitoring.Integrity.ReadDataSubset = "banana" res := m.CheckOffboxIntegrity(context.Background()) for _, a := range cap.checkArgv() { if strings.HasPrefix(a, "--read-data") { t.Fatalf("a malformed value was handed to restic (%q) — restic rejects it and the WHOLE "+ "check fails, so one typo silently stops the store being verified at all", a) } } if res.ReadDataSubset != "" { t.Errorf("a refused value was still recorded as the depth: %q", res.ReadDataSubset) } if !strings.Contains(cap.logBuf.String(), "WARN") { t.Error("a refused config value must say so — silence makes a typo indistinguishable from a " + "deliberate structure-only setting") } } func TestR359_MessageNeverCarriesResticOutputOrCredentials(t *testing.T) { // R-379: 615 bytes of raw database text reached a customer once. And offboxBaseArgs builds the repo // as `sftp:@:`, so a raw passthrough leaks the credential shape too. const secretish = "sftp:felhom@nas.local:/srv/repo pack 5b1f2c3d corrupt" m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) { return []byte(secretish), errFake })) res := m.CheckOffboxIntegrity(context.Background()) if res.OK { t.Fatal("fixture wrong: this should be a failure") } // The customer sentence is a CONSTANT and contains none of it. Asserted here rather than only in // the web package because this is where the output is captured. for _, bad := range []string{"sftp:", "nas.local", "5b1f2c3d", "felhom@"} { if strings.Contains(integrityFailedCustomerSentence, bad) { t.Fatalf("the customer-facing failure sentence carries %q", bad) } } // ...while the operator's log DOES get it, or the fault cannot be diagnosed without a rebuild. if !strings.Contains(res.Output, "5b1f2c3d") { t.Error("restic's output did not reach the result for the log") } } // integrityFailedCustomerSentence mirrors the constant in cmd/controller. Duplicated deliberately and // narrowly: this package cannot import main, and the property under test is that the SENTENCE carries // no machine detail — a property of the words themselves. const integrityFailedCustomerSentence = "A távoli mentés ellenőrzése hibát talált a tárolóban. A mentések egy része sérült lehet. Ne törölj semmit, és vedd fel velünk a kapcsolatot." func TestR359_NoTargetConfiguredIsASilentSkip(t *testing.T) { m, cap := newIntegrityManager(t, okRepo(nil)) if err := m.settings.SetOffboxTarget(&settings.OffboxTarget{Enabled: false}); err != nil { t.Fatal(err) } res := m.CheckOffboxIntegrity(context.Background()) if !res.Skipped { t.Fatalf("a box with no off-site tier did not skip: %+v", res) } if len(cap.argvs) != 0 { t.Fatalf("restic ran on a box with no off-site target: %v", cap.argvs) } if res.OK { t.Fatal("a skip was reported as a passing check — nothing was checked") } } // TestR359_RealResticDamageOutputIsClassifiedAsDamage uses the EXACT bytes restic produced on // `demo-hp` on 2026-08-30 against a deliberately corrupted throwaway repository (Part 5's positive // control). Invented output would only prove the classifier agrees with my guess about restic; this // closes the loop on real bytes. // // The damage was 64 zero bytes written at offset 1024 of one pack, leaving the file SIZE unchanged — // the subtlest form, and the one a structure check cannot see. See the accompanying finding: plain // `restic check` returned "no errors were found" and exit 0 over this very repository. func TestR359_RealResticDamageOutputIsClassifiedAsDamage(t *testing.T) { const realOutput = "Pack ID does not match, want 288afd3e868dc6bd210e33bd6f821e9f088a5fd71a0464c23ce82eb8217bf0cc, got 4b6847bb5eece6c56e69d7381733827330a4eb799fc26a92287a181edc496d2b\nFatal: repository contains errors" if !looksLikeRepositoryDamage([]byte(realOutput)) { t.Fatal("restic's REAL damage output was not recognised as damage — the check would report a " + "corrupted store as merely unreachable, and the customer would never be told") } m, _ := newIntegrityManager(t, okRepo(func([]string) ([]byte, error) { return []byte(realOutput), errFake })) res := m.CheckOffboxIntegrity(context.Background()) if res.OK { t.Fatal("a repository restic called corrupt was reported as passing") } if res.Unreachable { t.Fatal("readable-and-corrupt was reported as unreachable — the store WAS opened and read; " + "that misclassification would suppress the one alarm that matters") } if !strings.Contains(res.Output, "288afd3e") { t.Error("restic's own words did not reach the log") } } // TestR359_HealthyRealOutputIsNotDamage is the negative control for the classifier, from the same // live run: the healthy repository's actual output must not trip the damage predicate. func TestR359_HealthyRealOutputIsNotDamage(t *testing.T) { const realHealthy = "using temporary cache in /tmp/restic-check-cache-962728151\ncreate exclusive lock for repository\nload indexes\ncheck all packs\ncheck snapshots, trees and blobs\n\nno errors were found" if looksLikeRepositoryDamage([]byte(realHealthy)) { t.Fatalf("a HEALTHY check's real output was classified as damage — every weekly check would " + "alarm, which is how an operator learns to ignore the alarm") } }