felhom-controller/controller/internal/backup/offsite_diag_test.go

package backup

import (
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/config"

	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"

	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)

// the real demo-hp target shape — the values the sanitiser must remove literally
func diagTarget() *settings.OffboxTarget {
	return &settings.OffboxTarget{
		Host: "u629488-sub3.your-storagebox.de", User: "u629488-sub3",
		RepoPath: "/home/felhom-repo", Port: 23,
	}
}

// F-DIAG — four causes collapsed into one string, and that string was a RAW error passthrough.
//
// Two separate defects in one line of code:
//   - an operator could not tell a full quota from a dead network without reading logs;
//   - `err.Error()` from restic/ssh carries the repo reference `sftp:<user>@<host>:<path>`, so the
//     notification carried a customer-identifying location (and potentially a credential) off the box,
//     breaking the keys-not-values rule at the one place the text leaves the machine.

func TestClassifyOffsiteFailure_EachCauseIsDistinct(t *testing.T) {
	cases := []struct {
		name string
		err  error
		want OffsiteFailureClass
	}{
		// R-553 (v0.251.0): the quota refusal is built with its KIND at the producer, exactly as the run
		// builds it — the classifier no longer recognises this sentence by its Hungarian words, and that
		// is the point (localisation slice 2 translates them).
		{"quota gate", util.KindErrorf(ErrOffsiteQuota, "A távoli mentés túllépte a tárhelykeretet (51/50 GB) — törölj régi mentéseket vagy kérj nagyobb keretet."), OffsiteFailQuota},
		{"orphaned repo", fmt.Errorf("probe: %w", ErrOffboxOrphaned), OffsiteFailOrphaned},
		{"no repo", fmt.Errorf("restic: unable to open config file: Stat: file does not exist\nIs there a repository at the following location?"), OffsiteFailNoRepo},
		{"no units", fmt.Errorf("off-box backup produced no snapshots: 3 app(s) toggled but no recovery unit was found on any connected drive (missing: a, b, c)"), OffsiteFailNoUnits},
		{"transport refused", fmt.Errorf("dial tcp 1.2.3.4:23: connect: connection refused"), OffsiteFailTransport},
		{"transport timeout", fmt.Errorf("ssh: handshake failed: i/o timeout"), OffsiteFailTransport},
		{"transport auth", fmt.Errorf("ssh: permission denied (publickey)"), OffsiteFailTransport},
		{"unclassified", fmt.Errorf("restic: some future error nobody has seen"), OffsiteFailUnknown},
	}
	seen := map[OffsiteFailureClass]bool{}
	for _, c := range cases {
		got := ClassifyOffsiteFailure(c.err)
		if got != c.want {
			t.Errorf("%s: class = %q, want %q", c.name, got, c.want)
		}
		seen[got] = true
	}
	// The whole point of F-DIAG: the causes must not collapse.
	if len(seen) < 5 {
		t.Errorf("only %d distinct classes across %d causes — the causes are still collapsing", len(seen), len(cases))
	}
}

// An unclassifiable error must say so rather than being folded into a neighbour. Inventing a precision
// the code does not have is how a confident-but-wrong diagnosis ships.
func TestClassifyOffsiteFailure_UnknownIsHonest(t *testing.T) {
	if got := ClassifyOffsiteFailure(fmt.Errorf("something entirely new")); got != OffsiteFailUnknown {
		t.Errorf("an unclassifiable error was folded into %q instead of being reported as unknown", got)
	}
	msg := offsiteFailureMessage(diagTarget(), fmt.Errorf("something entirely new"), time.Minute, "hu")
	if !strings.Contains(msg, "ismeretlen okból") {
		t.Errorf("the unknown case does not admit it is unknown: %q", msg)
	}
}

// THE SECRETS TEST. The repo reference must never survive into a message.
//
// RED-PROOF: make sanitiseOffsiteError return err.Error() unchanged → this fails with
// "the repo reference reached the message".
func TestOffsiteFailureMessage_NeverCarriesTheRepoReference(t *testing.T) {
	leaky := []error{
		fmt.Errorf(`Fatal: unable to open repository at sftp:u629488-sub3@u629488-sub3.your-storagebox.de:/home/felhom-repo: connection refused`),
		fmt.Errorf(`ssh: connect to host u629488-sub3.your-storagebox.de port 23: Connection refused`),
		fmt.Errorf(`restic: repo "sftp:u629488-sub3@u629488-sub3.your-storagebox.de:/home/felhom-repo" locked`),
	}
	for _, e := range leaky {
		msg := offsiteFailureMessage(diagTarget(), e, 42*time.Second, "hu")
		for _, forbidden := range []string{
			"sftp:",
			"your-storagebox.de",
			"u629488-sub3",
			"/home/felhom-repo",
		} {
			if strings.Contains(msg, forbidden) {
				t.Errorf("the repo reference reached the message (%q leaked):\n  %s", forbidden, msg)
			}
		}
		if !strings.Contains(msg, "<repo>") {
			t.Errorf("the redaction placeholder is absent — the detail may have been dropped silently instead of sanitised:\n  %s", msg)
		}
	}
}

// The message must still be ACTIONABLE. Sanitising must not reduce it to a shrug — an operator needs
// the cause line plus enough residual detail to act.
func TestOffsiteFailureMessage_StaysActionable(t *testing.T) {
	msg := offsiteFailureMessage(diagTarget(), fmt.Errorf("dial tcp: connect: connection refused"), 90*time.Second, "hu")
	if !strings.Contains(msg, "nem érhető el") {
		t.Errorf("the transport cause is not named: %q", msg)
	}
	if !strings.Contains(msg, "connection refused") {
		t.Errorf("all actionable detail was stripped along with the secret: %q", msg)
	}
	if !strings.Contains(msg, "1m30s") {
		t.Errorf("the duration was lost: %q", msg)
	}
}

// A very long error must be bounded — an unbounded restic dump in an email is its own problem.
func TestSanitiseOffsiteError_IsBounded(t *testing.T) {
	long := fmt.Errorf("%s", strings.Repeat("x", 5000))
	if got := sanitiseOffsiteErrorFor(diagTarget(), long); len(got) > 320 {
		t.Errorf("sanitised error is %d chars — unbounded", len(got))
	}
	if sanitiseOffsiteErrorFor(diagTarget(), nil) != "" {
		t.Error("a nil error produced text")
	}
}

// newNoteManager builds a Manager whose saved-note helpers work: release C writes a saved note in the
// BOX's language, so a Manager with no settings would render every note as its key. Hungarian here,
// because these tests assert the sentence a Hungarian household reads — which is the parity half.
func newNoteManager(t *testing.T) *Manager {
	t.Helper()
	lg := log.New(io.Discard, "", 0)
	sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), lg)
	if err != nil {
		t.Fatal(err)
	}
	cfg := &config.Config{}
	cfg.Paths.DataDir = t.TempDir()
	return NewManager(cfg, sett, lg)
}

// backupNoteHU is the Hungarian text of a saved-note key, for a test that used to compare against a
// constant. The comparison is unchanged in meaning: it still pins the sentence, now measured against
// the bundle rather than restated beside it.
func backupNoteHU(t *testing.T, key string) string {
	t.Helper()
	return newNoteManager(t).note(key)
}

// TestNoteHelpersAreNotCalledUnderTheSettingsLock — release C (R-557), and it is a REGRESSION test,
// not a precaution.
//
// `UpdateOffboxStatus` and its siblings hold the settings WRITE lock while they run their callback.
// `boxLang()` reads the language through the settings READ lock. sync.RWMutex is not reentrant, so a
// note rendered inside such a callback DEADLOCKS — and it deadlocks while holding the settings lock,
// which then wedges every other thing on that box that touches settings.json. The first draft of
// release C did exactly that, in the off-site run's final status write, and the only symptom was the
// test suite timing out at 25 minutes.
//
// The fix is to resolve the language BEFORE entering the callback. This test reads the source for the
// shape, because the failure is a hang and a hang is not something a unit test can assert on
// comfortably; the behaviour half is the suite finishing at all.
//
// RED-PROOF (REPORT): put `m.note(...)` back inside the final UpdateOffboxStatus callback → this test
// names the file and the line, in a second, instead of the suite hanging for 25 minutes.
//
// R-578: the controller-wide successor is TestR578NothingTakesTheSettingsLockInsideASettingsCallback
// (internal/settings) — it derives the lockers and helpers from source in every package.
func TestNoteHelpersAreNotCalledUnderTheSettingsLock(t *testing.T) {
	callback := regexp.MustCompile(`\.Update\w*\(func\(`)
	note := regexp.MustCompile(`\b(m|s)\.(note|noteErr|boxLang)\(`)

	files, err := filepath.Glob("*.go")
	if err != nil {
		t.Fatal(err)
	}
	checked, callbacks := 0, 0
	var bad []string
	for _, f := range files {
		if strings.HasSuffix(f, "_test.go") {
			continue
		}
		src, err := os.ReadFile(f)
		if err != nil {
			t.Fatal(err)
		}
		checked++
		depth := 0
		inside := false
		for i, line := range strings.Split(string(src), "\n") {
			if !inside && callback.MatchString(line) {
				depth = strings.Count(line, "{") - strings.Count(line, "}")
				if depth > 0 {
					inside, callbacks = true, callbacks+1
				}
				continue
			}
			if !inside {
				continue
			}
			if note.MatchString(line) {
				bad = append(bad, f+":"+strconv.Itoa(i+1)+"  "+strings.TrimSpace(line))
			}
			depth += strings.Count(line, "{") - strings.Count(line, "}")
			if depth <= 0 {
				inside = false
			}
		}
	}
	// The instrument must be shown to be looking at something.
	if checked == 0 || callbacks == 0 {
		t.Fatalf("examined %d files and found %d settings callbacks — the pattern no longer matches the code", checked, callbacks)
	}
	if len(bad) > 0 {
		t.Errorf("a saved-note helper is called inside a settings callback, which holds the WRITE lock "+
			"while boxLang() wants the READ lock — sync.RWMutex is not reentrant, so this DEADLOCKS and "+
			"wedges settings.json for everything else on the box. Resolve the language before the "+
			"callback (%d):\n  %s", len(bad), strings.Join(bad, "\n  "))
	}
	t.Logf("examined %d files, %d settings callbacks", checked, callbacks)
}