felhom-controller/controller/internal/backup/r359_integrity_test.go

package backup

import (
	"bytes"
	"context"
	"errors"
	"log"
	"strings"
	"testing"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)

// ── R-359 — the off-site store was never checked ─────────────────────────────────────────────────
//
// The whole-guest tier has verify jobs; the tier holding the customer's documents and photos had none.
// The complete set of restic verbs this controller used contained no `check` — verified 2026-08-30.
//
// These drive the REAL CheckOffboxIntegrity through the EXISTING `offboxRunner` seam, which has been
// injectable since the off-site tier shipped. (R-398 claimed otherwise and was my own mistake; the
// seam sees every argv, including the `unlock --remove-all` escalation a `resticStepFn` would have
// hidden — which is exactly what the lock-safety tests must observe.)

// errFake is a plain non-nil error for seam replies; the classifier reads the OUTPUT, not the error
// type, so a synthetic error is faithful here.
var errFake = errors.New("restic exited non-zero")

// integrityCapture records every restic invocation so both the effects and the NON-effects are
// assertable. `argvs` is the whole point: a test that only checks the verdict cannot tell a check that
// ran from one that did not.
type integrityCapture struct {
	argvs  [][]string
	reply  func(args []string) ([]byte, error)
	logBuf *bytes.Buffer
}

func (c *integrityCapture) runner() offboxRunner {
	return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
		c.argvs = append(c.argvs, append([]string{}, args...))
		if c.reply != nil {
			return c.reply(args)
		}
		return nil, nil
	}
}

func (c *integrityCapture) sawVerb(verb string) bool {
	for _, a := range c.argvs {
		for _, x := range a {
			if x == verb {
				return true
			}
		}
	}
	return false
}

func (c *integrityCapture) checkArgv() []string {
	for _, a := range c.argvs {
		for _, x := range a {
			if x == "check" {
				return a
			}
		}
	}
	return nil
}

// newIntegrityManager builds a manager with a configured off-site target and a captured runner.
func newIntegrityManager(t *testing.T, reply func(args []string) ([]byte, error)) (*Manager, *integrityCapture) {
	t.Helper()
	m, _ := newOffboxManager(t)
	cap := &integrityCapture{reply: reply, logBuf: &bytes.Buffer{}}
	m.logger = log.New(cap.logBuf, "", 0)
	m.SetOffboxRunner(cap.runner())
	return m, cap
}

// okRepo answers `cat config` so ensureOffboxRepo passes, then defers to `then` for everything else.
func okRepo(then func(args []string) ([]byte, error)) func(args []string) ([]byte, error) {
	return func(args []string) ([]byte, error) {
		for _, a := range args {
			if a == "config" {
				return []byte(`{"version":2}`), nil
			}
		}
		if then != nil {
			return then(args)
		}
		return nil, nil
	}
}

func TestR359_HealthyRepoReportsOK(t *testing.T) {
	m, cap := newIntegrityManager(t, okRepo(nil))
	res := m.CheckOffboxIntegrity(context.Background())

	if !res.OK || res.Skipped || res.Unreachable {
		t.Fatalf("a healthy repo did not report OK: %+v", res)
	}
	if cap.checkArgv() == nil {
		t.Fatal("`restic check` was never invoked — the check did not check anything")
	}
}

func TestR359_RepositoryErrorReportsFailure(t *testing.T) {
	// restic's own words from the 2026-08-21 damaged-pack drill.
	const damaged = "pack 5b1f2c3d: not found in index\nrepository contains errors"
	m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) {
		return []byte(damaged), errFake
	}))
	res := m.CheckOffboxIntegrity(context.Background())

	if res.OK {
		t.Fatal("a repository restic said contains errors was reported as OK — this is the defect the " +
			"whole feature exists to prevent")
	}
	if res.Unreachable {
		t.Fatal("readable-and-damaged was misclassified as unreachable — those are different facts, " +
			"and only one of them means the customer's backups are broken")
	}
	if !strings.Contains(res.Output, "not found in index") {
		t.Errorf("restic's own words must reach the LOG so the operator can diagnose; got %q", res.Output)
	}
}

func TestR359_UnreachableIsNotAnIntegrityFailure(t *testing.T) {
	// The repo cannot even be opened. "I could not look" is not "I looked and it is broken".
	m, _ := newIntegrityManager(t, func(args []string) ([]byte, error) {
		return []byte("ssh: connect to host nas.local port 22: Connection refused"), errors.New("exit 1")
	})
	res := m.CheckOffboxIntegrity(context.Background())

	if res.OK {
		t.Fatal("an unreachable repository was reported as a passing check")
	}
	if !res.Unreachable {
		t.Fatal("an unreachable repository was reported as DAMAGE — that would alarm the customer that " +
			"their backups are corrupt when nothing was ever looked at, and R-339 already owns reachability")
	}
}

func TestR359_TimeoutIsNotDamage(t *testing.T) {
	m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) {
		return nil, context.DeadlineExceeded
	}))
	ctx, cancel := context.WithCancel(context.Background())
	cancel() // an already-dead context: the check cannot finish
	res := m.CheckOffboxIntegrity(ctx)

	if res.OK {
		t.Fatal("a check that never finished reported OK")
	}
	if !res.Unreachable {
		t.Fatalf("a check that timed out was reported as damage: %+v — it saw nothing, so it may not "+
			"claim the store is broken", res)
	}
}

func TestR359_StructureCheckPassesNoReadDataFlag(t *testing.T) {
	m, cap := newIntegrityManager(t, okRepo(nil))
	m.CheckOffboxIntegrity(context.Background())

	argv := cap.checkArgv()
	if argv == nil {
		t.Fatal("no check ran")
	}
	for _, a := range argv {
		if strings.HasPrefix(a, "--read-data") {
			t.Fatalf("the DEFAULT check downloaded pack data (%q) — that is a bandwidth cost nobody "+
				"chose, and R-399 exists precisely so it is not chosen here", a)
		}
	}
}

func TestR359_ReadDataSubsetIsPassedWhenConfigured(t *testing.T) {
	m, cap := newIntegrityManager(t, okRepo(nil))
	m.cfg.Monitoring.Integrity.ReadDataSubset = "5%"
	res := m.CheckOffboxIntegrity(context.Background())

	if res.ReadDataSubset != "5%" {
		t.Errorf("result did not record the depth it ran at: %+v", res)
	}
	var found bool
	for _, a := range cap.checkArgv() {
		if a == "--read-data-subset=5%" {
			found = true
		}
	}
	if !found {
		t.Fatalf("the configured subset did not reach restic; argv=%v", cap.checkArgv())
	}
}

func TestR359_MalformedReadDataSubsetIsTreatedAsOff(t *testing.T) {
	m, cap := newIntegrityManager(t, okRepo(nil))
	m.cfg.Monitoring.Integrity.ReadDataSubset = "banana"
	res := m.CheckOffboxIntegrity(context.Background())

	for _, a := range cap.checkArgv() {
		if strings.HasPrefix(a, "--read-data") {
			t.Fatalf("a malformed value was handed to restic (%q) — restic rejects it and the WHOLE "+
				"check fails, so one typo silently stops the store being verified at all", a)
		}
	}
	if res.ReadDataSubset != "" {
		t.Errorf("a refused value was still recorded as the depth: %q", res.ReadDataSubset)
	}
	if !strings.Contains(cap.logBuf.String(), "WARN") {
		t.Error("a refused config value must say so — silence makes a typo indistinguishable from a " +
			"deliberate structure-only setting")
	}
}

func TestR359_MessageNeverCarriesResticOutputOrCredentials(t *testing.T) {
	// R-379: 615 bytes of raw database text reached a customer once. And offboxBaseArgs builds the repo
	// as `sftp:<user>@<host>:<path>`, so a raw passthrough leaks the credential shape too.
	const secretish = "sftp:felhom@nas.local:/srv/repo pack 5b1f2c3d corrupt"
	m, _ := newIntegrityManager(t, okRepo(func(args []string) ([]byte, error) {
		return []byte(secretish), errFake
	}))
	res := m.CheckOffboxIntegrity(context.Background())

	if res.OK {
		t.Fatal("fixture wrong: this should be a failure")
	}
	// The customer sentence is a CONSTANT and contains none of it. Asserted here rather than only in
	// the web package because this is where the output is captured.
	for _, bad := range []string{"sftp:", "nas.local", "5b1f2c3d", "felhom@"} {
		if strings.Contains(integrityFailedCustomerSentence, bad) {
			t.Fatalf("the customer-facing failure sentence carries %q", bad)
		}
	}
	// ...while the operator's log DOES get it, or the fault cannot be diagnosed without a rebuild.
	if !strings.Contains(res.Output, "5b1f2c3d") {
		t.Error("restic's output did not reach the result for the log")
	}
}

// integrityFailedCustomerSentence mirrors the constant in cmd/controller. Duplicated deliberately and
// narrowly: this package cannot import main, and the property under test is that the SENTENCE carries
// no machine detail — a property of the words themselves.
const integrityFailedCustomerSentence = "A távoli mentés ellenőrzése hibát talált a tárolóban. A mentések egy része sérült lehet. Ne törölj semmit, és vedd fel velünk a kapcsolatot."

func TestR359_NoTargetConfiguredIsASilentSkip(t *testing.T) {
	m, cap := newIntegrityManager(t, okRepo(nil))
	if err := m.settings.SetOffboxTarget(&settings.OffboxTarget{Enabled: false}); err != nil {
		t.Fatal(err)
	}
	res := m.CheckOffboxIntegrity(context.Background())

	if !res.Skipped {
		t.Fatalf("a box with no off-site tier did not skip: %+v", res)
	}
	if len(cap.argvs) != 0 {
		t.Fatalf("restic ran on a box with no off-site target: %v", cap.argvs)
	}
	if res.OK {
		t.Fatal("a skip was reported as a passing check — nothing was checked")
	}
}

// TestR359_RealResticDamageOutputIsClassifiedAsDamage uses the EXACT bytes restic produced on
// `demo-hp` on 2026-08-30 against a deliberately corrupted throwaway repository (Part 5's positive
// control). Invented output would only prove the classifier agrees with my guess about restic; this
// closes the loop on real bytes.
//
// The damage was 64 zero bytes written at offset 1024 of one pack, leaving the file SIZE unchanged —
// the subtlest form, and the one a structure check cannot see. See the accompanying finding: plain
// `restic check` returned "no errors were found" and exit 0 over this very repository.
func TestR359_RealResticDamageOutputIsClassifiedAsDamage(t *testing.T) {
	const realOutput = "Pack ID does not match, want 288afd3e868dc6bd210e33bd6f821e9f088a5fd71a0464c23ce82eb8217bf0cc, got 4b6847bb5eece6c56e69d7381733827330a4eb799fc26a92287a181edc496d2b\nFatal: repository contains errors"

	if !looksLikeRepositoryDamage([]byte(realOutput)) {
		t.Fatal("restic's REAL damage output was not recognised as damage — the check would report a " +
			"corrupted store as merely unreachable, and the customer would never be told")
	}

	m, _ := newIntegrityManager(t, okRepo(func([]string) ([]byte, error) {
		return []byte(realOutput), errFake
	}))
	res := m.CheckOffboxIntegrity(context.Background())

	if res.OK {
		t.Fatal("a repository restic called corrupt was reported as passing")
	}
	if res.Unreachable {
		t.Fatal("readable-and-corrupt was reported as unreachable — the store WAS opened and read; " +
			"that misclassification would suppress the one alarm that matters")
	}
	if !strings.Contains(res.Output, "288afd3e") {
		t.Error("restic's own words did not reach the log")
	}
}

// TestR359_HealthyRealOutputIsNotDamage is the negative control for the classifier, from the same
// live run: the healthy repository's actual output must not trip the damage predicate.
func TestR359_HealthyRealOutputIsNotDamage(t *testing.T) {
	const realHealthy = "using temporary cache in /tmp/restic-check-cache-962728151\ncreate exclusive lock for repository\nload indexes\ncheck all packs\ncheck snapshots, trees and blobs\n\nno errors were found"

	if looksLikeRepositoryDamage([]byte(realHealthy)) {
		t.Fatalf("a HEALTHY check's real output was classified as damage — every weekly check would " +
			"alarm, which is how an operator learns to ignore the alarm")
	}
}