596505ed64
- Part A: classify restic cat-config failure (wrong-password=orphaned vs no-repo vs other); ORPHANED state + Hungarian card + offbox_repo_orphaned/reset events (once, not nightly); reset = move-aside (never delete) + init, unclaimed auto / claimed confirm. Red-proofs TestOffbox_OrphanDetection_* + ConfirmedReset. - Part C: GET /backup/offbox/status + poll on backups_remote → flips Fut→Rendben/Hiba without manual reload.
173 lines
6.2 KiB
Go
173 lines
6.2 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// classifyResticProbe maps the exact restic stderr to a repo class (the 2026-07-17 diagnosis
|
|
// signatures). ORPHANED only on the definitive wrong-password line; ambiguous errors are NOT orphaned.
|
|
func TestClassifyResticProbe(t *testing.T) {
|
|
cases := []struct {
|
|
out string
|
|
err error
|
|
want string
|
|
}{
|
|
{"", nil, ""}, // success
|
|
{"Fatal: wrong password or no key found", fmt.Errorf("exit status 1"), "orphaned"},
|
|
{"Fatal: unable to open config file: <sftp:...> does not exist\nIs there a repository at the following location?", fmt.Errorf("exit status 1"), "norepo"},
|
|
{"ssh: connect to host nas.local port 22: Connection timed out", fmt.Errorf("exit status 255"), "other"},
|
|
{"Load(<lock/...>): permission denied", fmt.Errorf("exit status 1"), "other"},
|
|
}
|
|
for _, c := range cases {
|
|
if got := classifyResticProbe([]byte(c.out), c.err); got != c.want {
|
|
t.Errorf("classify(%q) = %q, want %q", c.out, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// wrongPwRunner: `cat config` returns the wrong-password signature; other restic steps succeed (so a
|
|
// post-reset run can proceed). Records the subcommands seen.
|
|
func wrongPwRunner(seen *[]string) offboxRunner {
|
|
return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
|
sub := ""
|
|
for i, a := range args {
|
|
if a == "cat" && i+1 < len(args) && args[i+1] == "config" {
|
|
sub = "cat-config"
|
|
} else if a == "init" {
|
|
sub = "init"
|
|
}
|
|
}
|
|
if sub == "" && len(args) > 0 {
|
|
sub = args[len(args)-1]
|
|
}
|
|
if seen != nil {
|
|
*seen = append(*seen, sub)
|
|
}
|
|
if sub == "cat-config" {
|
|
return []byte("Fatal: wrong password or no key found"), fmt.Errorf("exit status 1")
|
|
}
|
|
return nil, nil // init / unlock / backup / stats succeed
|
|
}
|
|
}
|
|
|
|
// Scenario A (RED-PROOF = the incident): a CLAIMED box whose repo is wrong-keyed enters the explicit
|
|
// ORPHANED state — the run skips cleanly (no raw restic banner, ONE event, no nightly re-fire) instead
|
|
// of erroring nightly with "exit status 1". Pre-fix (no classification) surfaced the raw error and set
|
|
// no state → these assertions FAIL.
|
|
func TestOffbox_OrphanDetection_Claimed(t *testing.T) {
|
|
m, sett := newOffboxManager(t)
|
|
if err := sett.SetClaimed(); err != nil { // claimed → orphan card, NEVER auto-reset
|
|
t.Fatal(err)
|
|
}
|
|
var events []string
|
|
m.SetOffboxOrphanEvent(func(evt, _ string) { events = append(events, evt) })
|
|
m.SetOffboxRunner(wrongPwRunner(nil))
|
|
|
|
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
|
t.Fatalf("run should skip cleanly on an orphaned repo, got %v", err)
|
|
}
|
|
if !m.OffboxOrphaned() {
|
|
t.Fatal("repo was not classified/persisted as ORPHANED")
|
|
}
|
|
if len(events) != 1 || events[0] != "offbox_repo_orphaned" {
|
|
t.Fatalf("expected exactly one offbox_repo_orphaned event, got %v", events)
|
|
}
|
|
got := sett.GetOffboxTarget()
|
|
if got.RepoState != "orphaned" || got.OrphanedAt == "" {
|
|
t.Fatalf("RepoState=%q OrphanedAt=%q, want orphaned + a stamp", got.RepoState, got.OrphanedAt)
|
|
}
|
|
// The raw restic error must NOT be surfaced as the last-error banner (the card explains instead).
|
|
if strings.Contains(got.LastError, "wrong password") || strings.Contains(got.LastError, "exit status") {
|
|
t.Fatalf("raw restic error leaked into LastError: %q", got.LastError)
|
|
}
|
|
// A second scheduled run SKIPS (no nightly spam) — no new event.
|
|
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
|
t.Fatalf("second run: %v", err)
|
|
}
|
|
if len(events) != 1 {
|
|
t.Fatalf("nightly re-fire — events=%v, want the single transition event only", events)
|
|
}
|
|
}
|
|
|
|
// Scenario B: an UNCLAIMED box auto-resets on detection — move-aside (never delete) + re-init; both
|
|
// events fire and the box ends un-orphaned (next run green).
|
|
func TestOffbox_OrphanDetection_UnclaimedAutoReset(t *testing.T) {
|
|
m, sett := newOffboxManager(t) // unclaimed by default
|
|
var events []string
|
|
m.SetOffboxOrphanEvent(func(evt, _ string) { events = append(events, evt) })
|
|
var sshCmds []string
|
|
m.SetOffboxSSH(func(_ context.Context, _, _ string, _ int, _, _, remoteCmd string) ([]byte, error) {
|
|
sshCmds = append(sshCmds, remoteCmd)
|
|
if strings.HasPrefix(remoteCmd, "test -e") {
|
|
return nil, fmt.Errorf("exit status 1") // absent → free name
|
|
}
|
|
return nil, nil // mv OK
|
|
})
|
|
m.SetOffboxRunner(wrongPwRunner(nil))
|
|
|
|
if err := m.RunOffboxBackup(context.Background()); err != nil {
|
|
t.Fatalf("unclaimed run should auto-reset + succeed, got %v", err)
|
|
}
|
|
if m.OffboxOrphaned() {
|
|
t.Fatal("unclaimed box stayed orphaned — auto-reset did not clear the state")
|
|
}
|
|
got := sett.GetOffboxTarget()
|
|
if got.OrphanedRenamedTo == "" || !strings.Contains(got.OrphanedRenamedTo, ".orphaned-") {
|
|
t.Fatalf("move-aside path not recorded: %q", got.OrphanedRenamedTo)
|
|
}
|
|
var mvSeen bool
|
|
for _, c := range sshCmds {
|
|
if strings.HasPrefix(c, "mv ") {
|
|
mvSeen = true
|
|
}
|
|
}
|
|
if !mvSeen {
|
|
t.Fatalf("no move-aside mv issued: %v", sshCmds)
|
|
}
|
|
// Both transition events fired (orphaned → reset). No delete anywhere.
|
|
if len(events) != 2 || events[0] != "offbox_repo_orphaned" || events[1] != "offbox_repo_reset" {
|
|
t.Fatalf("events = %v, want [orphaned reset]", events)
|
|
}
|
|
}
|
|
|
|
// Scenario C: the claimed confirmed reset (ResetOrphanedRepo) refuses unless orphaned, then move-aside +
|
|
// re-init + clear state.
|
|
func TestOffbox_ConfirmedReset(t *testing.T) {
|
|
m, sett := newOffboxManager(t)
|
|
if err := sett.SetClaimed(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// refuse when not orphaned
|
|
if err := m.ResetOrphanedRepo(context.Background()); err == nil {
|
|
t.Fatal("reset must refuse when the repo is not orphaned")
|
|
}
|
|
// mark orphaned, then confirm reset
|
|
m.SetOffboxRunner(wrongPwRunner(nil))
|
|
_ = m.RunOffboxBackup(context.Background())
|
|
if !m.OffboxOrphaned() {
|
|
t.Fatal("precondition: not orphaned")
|
|
}
|
|
var mv bool
|
|
m.SetOffboxSSH(func(_ context.Context, _, _ string, _ int, _, _, cmd string) ([]byte, error) {
|
|
if strings.HasPrefix(cmd, "test -e") {
|
|
return nil, fmt.Errorf("exit 1")
|
|
}
|
|
if strings.HasPrefix(cmd, "mv ") {
|
|
mv = true
|
|
}
|
|
return nil, nil
|
|
})
|
|
if err := m.ResetOrphanedRepo(context.Background()); err != nil {
|
|
t.Fatalf("confirmed reset: %v", err)
|
|
}
|
|
if !mv {
|
|
t.Fatal("confirmed reset did not move the old repo aside")
|
|
}
|
|
if m.OffboxOrphaned() {
|
|
t.Fatal("state not cleared after confirmed reset")
|
|
}
|
|
}
|