R-101 + F-DIAG: the restore dialog names the last SUCCESSFUL copy (v0.182.0)
Tier2LastRun is the attempt clock and was rendered as 'Legutóbbi másolat' in the restore confirm dialog. New LastSuccess + SuccessTracked anchor; tier2Update makes the three rebuild sites safe by construction. F-DIAG: six distinct causes, target-aware redaction.
This commit is contained in:
@@ -92,6 +92,126 @@ func classifyResticProbe(out []byte, err error) string {
|
||||
}
|
||||
}
|
||||
|
||||
// ── F-DIAG: four causes, four messages, and no secrets ──────────────────────────────────────────
|
||||
//
|
||||
// The offsite failure notification was a raw passthrough:
|
||||
//
|
||||
// "a NAS-ra mentés hibázott (<dur>): " + err.Error()
|
||||
//
|
||||
// One string for every cause, so an operator could not tell a full disk from a dead network without
|
||||
// reading logs — AND a raw restic/ssh error carries the repo URL, which is built as
|
||||
// `sftp:<user>@<host>:<path>` (offboxBaseArgs). That breaks this project's keys-not-values rule at the
|
||||
// one place the text leaves the box.
|
||||
//
|
||||
// OffsiteFailureClass names the causes that are genuinely DISTINGUISHABLE where the error is produced.
|
||||
// Nothing is invented: each maps to a signal the code already has.
|
||||
type OffsiteFailureClass string
|
||||
|
||||
const (
|
||||
OffsiteFailQuota OffsiteFailureClass = "quota" // the pre-run soft-quota gate refused (offbox.go quota state)
|
||||
OffsiteFailOrphaned OffsiteFailureClass = "orphaned" // ErrOffboxOrphaned — repo keyed under a lost passphrase
|
||||
OffsiteFailNoRepo OffsiteFailureClass = "no_repo" // classifyResticProbe "norepo" — nothing at the location
|
||||
OffsiteFailNoUnits OffsiteFailureClass = "no_units" // apps toggled but no recovery unit found on any drive
|
||||
OffsiteFailTransport OffsiteFailureClass = "transport" // network / SFTP auth / host key / timeout
|
||||
OffsiteFailUnknown OffsiteFailureClass = "unknown" // genuinely unclassified — say so rather than guess
|
||||
)
|
||||
|
||||
// offsiteRepoURLRe matches the `sftp:user@host:/path` repo reference restic echoes back in its errors.
|
||||
// It is the BACKSTOP, not the primary defence — see sanitiseOffsiteErrorFor.
|
||||
var offsiteRepoURLRe = regexp.MustCompile(`sftp:[^\s"']+`)
|
||||
|
||||
// sanitiseOffsiteErrorFor strips anything that could carry a secret or a customer-identifying location
|
||||
// out of an error before it reaches a message, an event or a report.
|
||||
//
|
||||
// IT REDACTS THE KNOWN TARGET VALUES, not a guessed pattern. The first version of this function
|
||||
// regex-matched `sftp:…` and `user@host` and looked complete; its own test caught it leaking on
|
||||
// `ssh: connect to host <host> port 23: Connection refused`, which contains a BARE hostname in neither
|
||||
// shape. Guessing at what a secret looks like fails exactly where it matters — the target's host, user
|
||||
// and repo path are known here, so they are removed literally and the regex stays only as a backstop
|
||||
// for forms built before the target is loaded.
|
||||
//
|
||||
// Whole-token replacement, not masking: a partially-masked host still identifies the customer, and
|
||||
// "it looked masked" is how a leak survives review.
|
||||
func sanitiseOffsiteErrorFor(t *settings.OffboxTarget, err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
out := offsiteRepoURLRe.ReplaceAllString(err.Error(), "<repo>")
|
||||
if t != nil {
|
||||
// Longest first, so the repo path is not half-eaten by the host replacement.
|
||||
for _, v := range []string{t.RepoPath, t.Host, t.User} {
|
||||
if len(strings.TrimSpace(v)) >= 3 {
|
||||
out = strings.ReplaceAll(out, v, "<repo>")
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) > 300 {
|
||||
out = out[:300] + "…"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ClassifyOffsiteFailure maps a run error to its cause.
|
||||
//
|
||||
// Order matters: the explicit sentinels first, then the text signatures. A cause that cannot be told
|
||||
// apart here returns OffsiteFailUnknown rather than being folded into a neighbour — inventing a
|
||||
// precision the code does not have is how a confident-but-wrong diagnosis ships.
|
||||
func ClassifyOffsiteFailure(err error) OffsiteFailureClass {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
if errors.Is(err, ErrOffboxOrphaned) {
|
||||
return OffsiteFailOrphaned
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(s, "tárhelykeretet"):
|
||||
return OffsiteFailQuota
|
||||
case strings.Contains(s, "produced no snapshots"):
|
||||
return OffsiteFailNoUnits
|
||||
case strings.Contains(s, "unable to open config file"),
|
||||
strings.Contains(s, "is there a repository at the following location"):
|
||||
return OffsiteFailNoRepo
|
||||
case strings.Contains(s, "connection refused"), strings.Contains(s, "connection reset"),
|
||||
strings.Contains(s, "no route to host"), strings.Contains(s, "i/o timeout"),
|
||||
strings.Contains(s, "timed out"), strings.Contains(s, "permission denied"),
|
||||
strings.Contains(s, "host key"), strings.Contains(s, "handshake"),
|
||||
strings.Contains(s, "could not resolve"), strings.Contains(s, "network is unreachable"):
|
||||
return OffsiteFailTransport
|
||||
default:
|
||||
return OffsiteFailUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// OffsiteFailureMessage returns the operator-facing Hungarian message for a run failure: a distinct
|
||||
// cause line plus the SANITISED detail. The detail is kept because an operator needs something to act
|
||||
// on; it is sanitised because this text leaves the box.
|
||||
//
|
||||
// A method, not a function, so it can reach the target and redact its ACTUAL host/user/path rather
|
||||
// than pattern-matching at what those might look like.
|
||||
func (m *Manager) OffsiteFailureMessage(err error, dur time.Duration) string {
|
||||
var t *settings.OffboxTarget
|
||||
if m != nil && m.settings != nil {
|
||||
t = m.settings.GetOffboxTarget()
|
||||
}
|
||||
return offsiteFailureMessage(t, err, dur)
|
||||
}
|
||||
|
||||
func offsiteFailureMessage(t *settings.OffboxTarget, err error, dur time.Duration) string {
|
||||
head := map[OffsiteFailureClass]string{
|
||||
OffsiteFailQuota: "A távoli mentés nem fért el a tárhelykereten belül",
|
||||
OffsiteFailOrphaned: "A távoli tárhely egy korábbi, már nem elérhető kulccsal készült",
|
||||
OffsiteFailNoRepo: "A távoli tárhelyen nincs mentési adattár",
|
||||
OffsiteFailNoUnits: "Nem volt mit menteni: egyetlen kijelölt alkalmazásnak sem található mentése",
|
||||
OffsiteFailTransport: "A távoli tárhely nem érhető el (hálózat vagy bejelentkezés)",
|
||||
OffsiteFailUnknown: "A távoli mentés ismeretlen okból nem sikerült",
|
||||
}[ClassifyOffsiteFailure(err)]
|
||||
if head == "" {
|
||||
head = "A távoli mentés nem sikerült"
|
||||
}
|
||||
return fmt.Sprintf("%s (%s): %s", head, dur.Round(time.Second), sanitiseOffsiteErrorFor(t, err))
|
||||
}
|
||||
|
||||
func defaultOffboxSSH(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) {
|
||||
if port == 0 {
|
||||
port = 22
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// the real demo-hp target shape — the values the sanitiser must remove literally
|
||||
func diagTarget() *settings.OffboxTarget {
|
||||
return &settings.OffboxTarget{
|
||||
Host: "u629488-sub3.your-storagebox.de", User: "u629488-sub3",
|
||||
RepoPath: "/home/felhom-repo", Port: 23,
|
||||
}
|
||||
}
|
||||
|
||||
// F-DIAG — four causes collapsed into one string, and that string was a RAW error passthrough.
|
||||
//
|
||||
// Two separate defects in one line of code:
|
||||
// - an operator could not tell a full quota from a dead network without reading logs;
|
||||
// - `err.Error()` from restic/ssh carries the repo reference `sftp:<user>@<host>:<path>`, so the
|
||||
// notification carried a customer-identifying location (and potentially a credential) off the box,
|
||||
// breaking the keys-not-values rule at the one place the text leaves the machine.
|
||||
|
||||
func TestClassifyOffsiteFailure_EachCauseIsDistinct(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want OffsiteFailureClass
|
||||
}{
|
||||
{"quota gate", fmt.Errorf("A távoli mentés túllépte a tárhelykeretet (51/50 GB) — törölj régi mentéseket vagy kérj nagyobb keretet."), OffsiteFailQuota},
|
||||
{"orphaned repo", fmt.Errorf("probe: %w", ErrOffboxOrphaned), OffsiteFailOrphaned},
|
||||
{"no repo", fmt.Errorf("restic: unable to open config file: Stat: file does not exist\nIs there a repository at the following location?"), OffsiteFailNoRepo},
|
||||
{"no units", fmt.Errorf("off-box backup produced no snapshots: 3 app(s) toggled but no recovery unit was found on any connected drive (missing: a, b, c)"), OffsiteFailNoUnits},
|
||||
{"transport refused", fmt.Errorf("dial tcp 1.2.3.4:23: connect: connection refused"), OffsiteFailTransport},
|
||||
{"transport timeout", fmt.Errorf("ssh: handshake failed: i/o timeout"), OffsiteFailTransport},
|
||||
{"transport auth", fmt.Errorf("ssh: permission denied (publickey)"), OffsiteFailTransport},
|
||||
{"unclassified", fmt.Errorf("restic: some future error nobody has seen"), OffsiteFailUnknown},
|
||||
}
|
||||
seen := map[OffsiteFailureClass]bool{}
|
||||
for _, c := range cases {
|
||||
got := ClassifyOffsiteFailure(c.err)
|
||||
if got != c.want {
|
||||
t.Errorf("%s: class = %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
seen[got] = true
|
||||
}
|
||||
// The whole point of F-DIAG: the causes must not collapse.
|
||||
if len(seen) < 5 {
|
||||
t.Errorf("only %d distinct classes across %d causes — the causes are still collapsing", len(seen), len(cases))
|
||||
}
|
||||
}
|
||||
|
||||
// An unclassifiable error must say so rather than being folded into a neighbour. Inventing a precision
|
||||
// the code does not have is how a confident-but-wrong diagnosis ships.
|
||||
func TestClassifyOffsiteFailure_UnknownIsHonest(t *testing.T) {
|
||||
if got := ClassifyOffsiteFailure(fmt.Errorf("something entirely new")); got != OffsiteFailUnknown {
|
||||
t.Errorf("an unclassifiable error was folded into %q instead of being reported as unknown", got)
|
||||
}
|
||||
msg := offsiteFailureMessage(diagTarget(), fmt.Errorf("something entirely new"), time.Minute)
|
||||
if !strings.Contains(msg, "ismeretlen okból") {
|
||||
t.Errorf("the unknown case does not admit it is unknown: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// THE SECRETS TEST. The repo reference must never survive into a message.
|
||||
//
|
||||
// RED-PROOF: make sanitiseOffsiteError return err.Error() unchanged → this fails with
|
||||
// "the repo reference reached the message".
|
||||
func TestOffsiteFailureMessage_NeverCarriesTheRepoReference(t *testing.T) {
|
||||
leaky := []error{
|
||||
fmt.Errorf(`Fatal: unable to open repository at sftp:u629488-sub3@u629488-sub3.your-storagebox.de:/home/felhom-repo: connection refused`),
|
||||
fmt.Errorf(`ssh: connect to host u629488-sub3.your-storagebox.de port 23: Connection refused`),
|
||||
fmt.Errorf(`restic: repo "sftp:u629488-sub3@u629488-sub3.your-storagebox.de:/home/felhom-repo" locked`),
|
||||
}
|
||||
for _, e := range leaky {
|
||||
msg := offsiteFailureMessage(diagTarget(), e, 42*time.Second)
|
||||
for _, forbidden := range []string{
|
||||
"sftp:",
|
||||
"your-storagebox.de",
|
||||
"u629488-sub3",
|
||||
"/home/felhom-repo",
|
||||
} {
|
||||
if strings.Contains(msg, forbidden) {
|
||||
t.Errorf("the repo reference reached the message (%q leaked):\n %s", forbidden, msg)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(msg, "<repo>") {
|
||||
t.Errorf("the redaction placeholder is absent — the detail may have been dropped silently instead of sanitised:\n %s", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The message must still be ACTIONABLE. Sanitising must not reduce it to a shrug — an operator needs
|
||||
// the cause line plus enough residual detail to act.
|
||||
func TestOffsiteFailureMessage_StaysActionable(t *testing.T) {
|
||||
msg := offsiteFailureMessage(diagTarget(), fmt.Errorf("dial tcp: connect: connection refused"), 90*time.Second)
|
||||
if !strings.Contains(msg, "nem érhető el") {
|
||||
t.Errorf("the transport cause is not named: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "connection refused") {
|
||||
t.Errorf("all actionable detail was stripped along with the secret: %q", msg)
|
||||
}
|
||||
if !strings.Contains(msg, "1m30s") {
|
||||
t.Errorf("the duration was lost: %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// A very long error must be bounded — an unbounded restic dump in an email is its own problem.
|
||||
func TestSanitiseOffsiteError_IsBounded(t *testing.T) {
|
||||
long := fmt.Errorf("%s", strings.Repeat("x", 5000))
|
||||
if got := sanitiseOffsiteErrorFor(diagTarget(), long); len(got) > 320 {
|
||||
t.Errorf("sanitised error is %d chars — unbounded", len(got))
|
||||
}
|
||||
if sanitiseOffsiteErrorFor(diagTarget(), nil) != "" {
|
||||
t.Error("a nil error produced text")
|
||||
}
|
||||
}
|
||||
@@ -530,67 +530,97 @@ func (m *Manager) Tier2Info(stackName string) Tier2Info {
|
||||
|
||||
// --- status persistence (drives the "2. mentés" UI card) ---
|
||||
|
||||
// withTier2Prefs carries the customer-preference fields (UserDisabled/PreferredTarget) from any
|
||||
// existing config into a freshly-built status struct, so a runner status write never clobbers them.
|
||||
func (m *Manager) withTier2Prefs(stackName string, cfg *settings.CrossDriveBackup) *settings.CrossDriveBackup {
|
||||
if m.settings != nil {
|
||||
if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
|
||||
cfg.UserDisabled = existing.UserDisabled
|
||||
cfg.PreferredTarget = existing.PreferredTarget
|
||||
}
|
||||
// tier2Update applies a run outcome onto a COPY OF THE EXISTING ROW, then persists it.
|
||||
//
|
||||
// R-101 Part 2 — SAFE BY CONSTRUCTION, and this replaced a real hazard rather than tidying one. The
|
||||
// three record* helpers each used to build a WHOLE `CrossDriveBackup` literal, with `withTier2Prefs`
|
||||
// re-applying exactly two fields (UserDisabled, PreferredTarget). Every other field not named in the
|
||||
// literal was silently zeroed on every status write. That is fine while the struct is stable and
|
||||
// catastrophic the moment a field is added: R-101 adds `LastSuccess`, and under the old shape
|
||||
// `recordTier2Failure` would have CLEARED it — the mirror-image of the defect being fixed, firing on
|
||||
// the first failure instead of lying dormant.
|
||||
//
|
||||
// Starting from the existing row inverts the default: a new field carries over unless a caller
|
||||
// deliberately overwrites it. The compile-safe form the R-100 review asked for; nothing is preserved
|
||||
// by a list that can fall out of date.
|
||||
//
|
||||
// Callers must therefore CLEAR explicitly what a run invalidates (a stale LastError on success, a
|
||||
// stale size on failure) — the old behaviour those clears reproduce is preserved exactly.
|
||||
func (m *Manager) tier2Update(stackName string, mutate func(*settings.CrossDriveBackup)) {
|
||||
if m.settings == nil {
|
||||
return
|
||||
}
|
||||
var cfg settings.CrossDriveBackup
|
||||
if existing := m.settings.GetCrossDriveConfig(stackName); existing != nil {
|
||||
cfg = *existing // value copy — EVERY field carries over by default
|
||||
}
|
||||
// One-time migration of a pre-anchor row. Under the old code `LastStatus=="ok"` with a LastRun
|
||||
// means that run DID succeed, so adopting it as the initial anchor is truthful — and it is what
|
||||
// keeps the deploy quiet: without it every existing row would flip to "never succeeded" at once
|
||||
// (all 7 rows on the fleet were pre-anchor). A row whose last known state was an ERROR seeds
|
||||
// nothing, because nothing in the old data evidences a success.
|
||||
if !cfg.SuccessTracked {
|
||||
if cfg.LastStatus == "ok" && cfg.LastRun != "" {
|
||||
cfg.LastSuccess = cfg.LastRun
|
||||
}
|
||||
cfg.SuccessTracked = true
|
||||
}
|
||||
mutate(&cfg)
|
||||
if err := m.settings.SetCrossDriveConfig(stackName, &cfg); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] Tier 2 status persist for %s failed: %v", stackName, err)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func (m *Manager) recordTier2Success(stackName string, target *Tier2Target, sizeBytes int64, warning string, dur time.Duration) {
|
||||
if m.settings == nil {
|
||||
return
|
||||
}
|
||||
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
||||
Enabled: true,
|
||||
Method: "rsync",
|
||||
DestinationPath: target.NamespaceRoot,
|
||||
Schedule: "daily",
|
||||
LastRun: time.Now().Format(time.RFC3339),
|
||||
LastStatus: "ok",
|
||||
LastWarning: strings.TrimSpace(warning),
|
||||
LastDuration: dur.Round(time.Second).String(),
|
||||
LastSizeHuman: humanizeBytes(sizeBytes),
|
||||
})); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] Tier 2 status persist (ok) for %s failed: %v", stackName, err)
|
||||
}
|
||||
now := time.Now().Format(time.RFC3339)
|
||||
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
||||
c.Enabled = true
|
||||
c.Method = "rsync"
|
||||
c.DestinationPath = target.NamespaceRoot
|
||||
c.Schedule = "daily"
|
||||
c.LastRun = now
|
||||
// R-101: the anchor. Only this branch advances it; no failure branch clears it.
|
||||
c.LastSuccess = now
|
||||
c.LastStatus = "ok"
|
||||
c.LastWarning = strings.TrimSpace(warning)
|
||||
c.LastDuration = dur.Round(time.Second).String()
|
||||
c.LastSizeHuman = humanizeBytes(sizeBytes)
|
||||
c.LastError = "" // a success invalidates the previous error
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Manager) recordTier2Failure(stackName string, target *Tier2Target, cause error) {
|
||||
if m.settings == nil {
|
||||
return
|
||||
}
|
||||
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
||||
Enabled: true,
|
||||
Method: "rsync",
|
||||
DestinationPath: target.NamespaceRoot,
|
||||
Schedule: "daily",
|
||||
LastRun: time.Now().Format(time.RFC3339),
|
||||
LastStatus: "error",
|
||||
LastError: cause.Error(),
|
||||
})); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] Tier 2 status persist (error) for %s failed: %v", stackName, err)
|
||||
}
|
||||
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
||||
c.Enabled = true
|
||||
c.Method = "rsync"
|
||||
c.DestinationPath = target.NamespaceRoot
|
||||
c.Schedule = "daily"
|
||||
c.LastRun = time.Now().Format(time.RFC3339) // the ATTEMPT clock — advances on failure, by design
|
||||
c.LastStatus = "error"
|
||||
c.LastError = cause.Error()
|
||||
// LastSuccess is deliberately UNTOUCHED: a failure neither advances nor clears the anchor.
|
||||
// Clearing it would make one bad night read as "no copy has ever succeeded".
|
||||
c.LastWarning = "" // a warning from the last successful run does not describe this one
|
||||
c.LastDuration = "" // preserving the old literal's clears exactly
|
||||
c.LastSizeHuman = "" // ditto — a stale size would describe a copy this run did not make
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Manager) recordTier2NoTarget(stackName, reason string) {
|
||||
if m.settings == nil {
|
||||
return
|
||||
}
|
||||
if err := m.settings.SetCrossDriveConfig(stackName, m.withTier2Prefs(stackName, &settings.CrossDriveBackup{
|
||||
Enabled: false,
|
||||
Method: "rsync",
|
||||
Schedule: "daily",
|
||||
LastStatus: "no_target",
|
||||
LastError: reason,
|
||||
})); err != nil {
|
||||
m.logger.Printf("[WARN] [backup] Tier 2 status persist (no_target) for %s failed: %v", stackName, err)
|
||||
}
|
||||
m.tier2Update(stackName, func(c *settings.CrossDriveBackup) {
|
||||
c.Enabled = false
|
||||
c.Method = "rsync"
|
||||
c.Schedule = "daily"
|
||||
c.DestinationPath = ""
|
||||
c.LastStatus = "no_target"
|
||||
c.LastError = reason
|
||||
c.LastRun = ""
|
||||
// LastSuccess survives: "there is no destination drive right now" is not evidence that the
|
||||
// last successful copy never happened. The UI gates on LastRun here, so nothing is rendered.
|
||||
c.LastWarning = ""
|
||||
c.LastDuration = ""
|
||||
c.LastSizeHuman = ""
|
||||
})
|
||||
}
|
||||
|
||||
func tier2NoTargetReason(err error) string {
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// R-101 Part 2 — THE COPY-SITE HAZARD, exercised through the REAL record* helpers.
|
||||
//
|
||||
// The three record* helpers used to build a WHOLE CrossDriveBackup literal, with a helper re-applying
|
||||
// exactly two fields. Anything not named in the literal was zeroed on every status write. Adding
|
||||
// LastSuccess to that shape would have had `recordTier2Failure` CLEAR the anchor — the mirror image of
|
||||
// the defect being fixed, and firing on the FIRST failure rather than lying dormant.
|
||||
//
|
||||
// The R-100 lesson applies: these call the production functions. A test that modelled the copy in a
|
||||
// closure would stay green through any mutation of the real code.
|
||||
|
||||
func anchorMgr(t *testing.T) *Manager {
|
||||
t.Helper()
|
||||
sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatalf("settings.Load: %v", err)
|
||||
}
|
||||
return &Manager{settings: sett, logger: log.New(io.Discard, "", 0)}
|
||||
}
|
||||
|
||||
func tgt() *Tier2Target { return &Tier2Target{NamespaceRoot: "/mnt/hdd_1/felhom-data/calibre-web"} }
|
||||
|
||||
// SCENARIO F — a FAILURE must not wipe the anchor. This is the Part 2 proof and it runs the real
|
||||
// recordTier2Success → recordTier2Failure sequence.
|
||||
//
|
||||
// RED-PROOF: make recordTier2Failure build a whole literal again (or add `c.LastSuccess = ""` to its
|
||||
// mutation) → this fails with "a FAILED run wiped the success anchor".
|
||||
func TestTier2_FailureDoesNotWipeTheAnchor(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
|
||||
// Seed a row with a DISTINCT, older anchor (a real success on the 25th) already tracked, so the
|
||||
// anchor and the attempt clock are distinguishable — a success recorded in this same test would
|
||||
// land in the same second as the failures and prove nothing about which field moved.
|
||||
const anchor = "2026-07-25T01:30:00Z"
|
||||
if err := m.settings.SetCrossDriveConfig("calibre-web", &settings.CrossDriveBackup{
|
||||
Enabled: true, Method: "rsync", Schedule: "daily",
|
||||
LastRun: anchor, LastStatus: "ok", LastSuccess: anchor, SuccessTracked: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Five consecutive failing nights, through the real helper.
|
||||
for i := 0; i < 5; i++ {
|
||||
m.recordTier2Failure("calibre-web", tgt(), errors.New("rsync: connection reset"))
|
||||
got := m.settings.GetCrossDriveConfig("calibre-web")
|
||||
if got.LastSuccess == "" {
|
||||
t.Fatalf("a FAILED run wiped the success anchor (round %d) — one bad night would read as 'no copy has ever succeeded'", i+1)
|
||||
}
|
||||
if got.LastSuccess != anchor {
|
||||
t.Fatalf("a FAILED run MOVED the anchor to %q (was %q) — that is the R-101 defect in mirror image", got.LastSuccess, anchor)
|
||||
}
|
||||
if got.LastStatus != "error" {
|
||||
t.Errorf("the failure was not recorded (status=%q)", got.LastStatus)
|
||||
}
|
||||
// the ATTEMPT clock DOES move — that is what made the old rendering wrong
|
||||
if got.LastRun == anchor {
|
||||
t.Errorf("LastRun did not advance on the attempt — it still reads the old success time %q", got.LastRun)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A successful run advances the anchor, or a tier would look permanently stale after one good night.
|
||||
func TestTier2_SuccessAdvancesTheAnchor(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
if err := m.settings.SetCrossDriveConfig("calibre-web", &settings.CrossDriveBackup{
|
||||
Enabled: true, Method: "rsync", Schedule: "daily",
|
||||
LastRun: "2026-07-25T01:30:00Z", LastStatus: "error",
|
||||
LastSuccess: "2026-07-25T01:30:00Z", SuccessTracked: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.recordTier2Success("calibre-web", tgt(), 1<<30, "", 0)
|
||||
got := m.settings.GetCrossDriveConfig("calibre-web")
|
||||
if got.LastSuccess == "2026-07-25T01:30:00Z" {
|
||||
t.Error("a successful run did not advance the anchor")
|
||||
}
|
||||
if got.LastSuccess != got.LastRun {
|
||||
t.Errorf("the anchor and the attempt clock disagree after a success (%q vs %q)", got.LastSuccess, got.LastRun)
|
||||
}
|
||||
if got.LastError != "" {
|
||||
t.Errorf("a success left the previous error in place (%q)", got.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
// A customer preference must still survive a status write — the behaviour the old helper existed to
|
||||
// provide. The new copy-the-row form should give this for free, and this pins that it does.
|
||||
//
|
||||
// RED-PROOF: make tier2Update start from a zero-value struct instead of copying the existing row →
|
||||
// this fails with "a status write wiped the customer's Tier-2 preference".
|
||||
func TestTier2_StatusWritePreservesCustomerPreference(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
if err := m.settings.SetTier2Preference("calibre-web", true, "/mnt/hdd_2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.recordTier2Failure("calibre-web", tgt(), errors.New("boom"))
|
||||
|
||||
got := m.settings.GetCrossDriveConfig("calibre-web")
|
||||
if !got.UserDisabled || got.PreferredTarget != "/mnt/hdd_2" {
|
||||
t.Errorf("a status write wiped the customer's Tier-2 preference (UserDisabled=%v PreferredTarget=%q)",
|
||||
got.UserDisabled, got.PreferredTarget)
|
||||
}
|
||||
}
|
||||
|
||||
// A no-target write must not destroy the anchor either: "there is no destination drive right now" is
|
||||
// not evidence that the last successful copy never happened.
|
||||
func TestTier2_NoTargetKeepsTheAnchor(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
m.recordTier2Success("calibre-web", tgt(), 1<<30, "", 0)
|
||||
anchor := m.settings.GetCrossDriveConfig("calibre-web").LastSuccess
|
||||
|
||||
m.recordTier2NoTarget("calibre-web", "nincs elérhető második meghajtó")
|
||||
got := m.settings.GetCrossDriveConfig("calibre-web")
|
||||
if got.LastSuccess != anchor {
|
||||
t.Errorf("a no_target write lost the anchor (%q, was %q)", got.LastSuccess, anchor)
|
||||
}
|
||||
if got.LastRun != "" {
|
||||
t.Errorf("no_target should clear the attempt clock as before, got %q", got.LastRun)
|
||||
}
|
||||
}
|
||||
|
||||
// SCENARIO E (producer half) — a LEGACY row is migrated truthfully on first touch: a row whose last
|
||||
// known state was a SUCCESS adopts that time as its anchor, so the deploy does not flip every existing
|
||||
// customer to "never succeeded".
|
||||
//
|
||||
// RED-PROOF: delete the `if !cfg.SuccessTracked` seeding block in tier2Update → this fails with
|
||||
// "a legacy OK row was not migrated".
|
||||
func TestTier2_LegacyOkRowSeedsItsAnchor(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
// A pre-R-101 row: status ok, a last_run, no anchor, not tracked.
|
||||
if err := m.settings.SetCrossDriveConfig("calibre-web", &settings.CrossDriveBackup{
|
||||
Enabled: true, Method: "rsync", Schedule: "daily",
|
||||
LastRun: "2026-07-28T01:30:00Z", LastStatus: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The next run fails — the first new-code touch of this row.
|
||||
m.recordTier2Failure("calibre-web", tgt(), errors.New("boom"))
|
||||
|
||||
got := m.settings.GetCrossDriveConfig("calibre-web")
|
||||
if !got.SuccessTracked {
|
||||
t.Fatal("the row was not marked as tracked")
|
||||
}
|
||||
if got.LastSuccess != "2026-07-28T01:30:00Z" {
|
||||
t.Errorf("a legacy OK row was not migrated — its known-good run should have become the anchor (got %q)", got.LastSuccess)
|
||||
}
|
||||
}
|
||||
|
||||
// ...but a legacy row whose last known state was an ERROR seeds NOTHING: the old data contains no
|
||||
// evidence of a success, and inventing one would be the original defect.
|
||||
func TestTier2_LegacyErrorRowSeedsNothing(t *testing.T) {
|
||||
m := anchorMgr(t)
|
||||
if err := m.settings.SetCrossDriveConfig("calibre-web", &settings.CrossDriveBackup{
|
||||
Enabled: true, Method: "rsync", Schedule: "daily",
|
||||
LastRun: "2026-07-28T01:30:00Z", LastStatus: "error",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.recordTier2Failure("calibre-web", tgt(), errors.New("boom"))
|
||||
|
||||
if got := m.settings.GetCrossDriveConfig("calibre-web"); got.LastSuccess != "" {
|
||||
t.Errorf("a legacy ERROR row invented an anchor (%q) — that is the defect, not the fix", got.LastSuccess)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user