253 lines
10 KiB
Go
253 lines
10 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// R-1105: the off-site code was measured against restic 0.14.0; v0.323.0 ships 0.18.1. The fixtures in
|
|
// testdata/restic/<version>/ are REAL outputs of both official binaries, captured 2026-10-16 on one shared scratch
|
|
// repository (made by 0.14.0, so both versions read the same snapshot set; paths rewritten to /data/x). Each
|
|
// assumption the code makes about restic's output is asserted for BOTH versions through the code's own functions —
|
|
// so a future restic that rewords one of them fails here, by name, instead of moving a failure to `unknown` on a
|
|
// customer's box (07-backup-architecture.md §6.7, §6.8).
|
|
var r1105Versions = []string{"0.14.0", "0.18.1"}
|
|
|
|
func r1105Fixture(t *testing.T, version, name string) []byte {
|
|
t.Helper()
|
|
b, err := os.ReadFile(filepath.Join("testdata", "restic", version, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// fixtureRunner answers each restic subcommand with a captured output; err != nil when restic exited non-zero.
|
|
func r1105Runner(t *testing.T, version string, answers map[string]string) offboxRunner {
|
|
return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
|
joined := " " + strings.Join(args, " ") + " "
|
|
for _, key := range sortedKeys(answers) {
|
|
if strings.Contains(joined, " "+key+" ") {
|
|
name := answers[key]
|
|
var err error
|
|
if strings.HasPrefix(name, "!") { // restic exited non-zero for this one
|
|
name, err = name[1:], errors.New("exit status 1")
|
|
}
|
|
return r1105Fixture(t, version, name), err
|
|
}
|
|
}
|
|
t.Fatalf("restic %s: no fixture for %q", version, joined)
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
func sortedKeys(m map[string]string) []string {
|
|
k := make([]string, 0, len(m))
|
|
for s := range m {
|
|
k = append(k, s)
|
|
}
|
|
sort.Slice(k, func(i, j int) bool { return len(k[i]) > len(k[j]) }) // the longest (most specific) match first
|
|
return k
|
|
}
|
|
|
|
// The probe and the failure classifier: each captured failure lands in the class it has always landed in.
|
|
func TestR1105_EachResticFailureKeepsItsClass(t *testing.T) {
|
|
cases := []struct {
|
|
fixture string
|
|
probe string // classifyResticProbe on `cat config`
|
|
runClass OffsiteFailureClass // ClassifyOffsiteFailure on the error ensureOffboxRepo returns ("" = not reached)
|
|
}{
|
|
{"wrong-password.txt", "orphaned", ""},
|
|
{"no-repo.txt", "norepo", ""},
|
|
{"sftp-refused-port.txt", "other", OffsiteFailTransport},
|
|
{"sftp-refused-key.txt", "other", OffsiteFailTransport},
|
|
{"sftp-unreachable.txt", "other", OffsiteFailTransport},
|
|
{"rclone-dead-ssh.txt", "other", OffsiteFailTransport},
|
|
}
|
|
for _, v := range r1105Versions {
|
|
for _, c := range cases {
|
|
out := r1105Fixture(t, v, c.fixture)
|
|
if got := classifyResticProbe(out, errors.New("exit status 1")); got != c.probe {
|
|
t.Errorf("restic %s %s: probe class %q, want %q", v, c.fixture, got, c.probe)
|
|
}
|
|
if c.runClass == "" {
|
|
continue
|
|
}
|
|
m, _ := newOffboxManager(t)
|
|
m.SetOffboxRunner(r1105Runner(t, v, map[string]string{"cat": "!" + c.fixture}))
|
|
err := m.ensureOffboxRepo(context.Background(), []string{"-r", "x"}, nil)
|
|
if got := ClassifyOffsiteFailure(err); got != c.runClass {
|
|
t.Errorf("restic %s %s: run class %q, want %q (err: %v)", v, c.fixture, got, c.runClass, err)
|
|
}
|
|
}
|
|
// A lock left by a crashed run on another hostname (the C2 shape), both forms.
|
|
for _, f := range []string{"locked-shared.txt", "locked-exclusive.txt"} {
|
|
out := r1105Fixture(t, v, f)
|
|
if !offboxLockRe.Match(out) {
|
|
t.Errorf("restic %s %s: the lock text is no longer recognised — resticStep would not self-heal", v, f)
|
|
}
|
|
if got := ClassifyOffsiteFailure(errors.New(string(out))); got != OffsiteFailLocked {
|
|
t.Errorf("restic %s %s: class %q, want locked", v, f, got)
|
|
}
|
|
}
|
|
// The integrity check: a damaged store reads as damage, a healthy one does not.
|
|
if !looksLikeRepositoryDamage(r1105Fixture(t, v, "check-damaged.txt")) {
|
|
t.Errorf("restic %s: a damaged pack is no longer recognised as damage", v)
|
|
}
|
|
if looksLikeRepositoryDamage(r1105Fixture(t, v, "check-ok.txt")) {
|
|
t.Errorf("restic %s: a healthy check reads as damage", v)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A first run that races another init: „already exists" is still read as success, so the run proceeds.
|
|
func TestR1105_InitRaceIsStillSuccess(t *testing.T) {
|
|
for _, v := range r1105Versions {
|
|
m, _ := newOffboxManager(t)
|
|
m.SetOffboxRunner(r1105Runner(t, v, map[string]string{"cat": "!no-repo.txt", "init": "!init-exists.txt"}))
|
|
if err := m.ensureOffboxRepo(context.Background(), []string{"-r", "x"}, nil); err != nil {
|
|
t.Errorf("restic %s: an init that finds the repository already made must be success, got %v", v, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The JSON the code parses: snapshot lists, the newest snapshot of a tag, the sizes, the progress lines.
|
|
func TestR1105_TheJSONTheCodeReadsStillParses(t *testing.T) {
|
|
counts := map[string]int{}
|
|
for _, v := range r1105Versions {
|
|
m, sett := newOffboxManager(t)
|
|
m.SetOffboxRunner(r1105Runner(t, v, map[string]string{
|
|
"snapshots": "snapshots.json", "latest": "snapshots-latest-tag.json",
|
|
"stats": "stats-id.json", "raw-data": "stats-raw.json",
|
|
}))
|
|
snaps, err := m.listGuardSnaps(context.Background(), nil, nil)
|
|
if err != nil || len(snaps) == 0 {
|
|
t.Fatalf("restic %s: snapshot list: %d, %v", v, len(snaps), err)
|
|
}
|
|
for _, s := range snaps {
|
|
if s.ID == "" || s.ShortID == "" || s.Time.IsZero() || s.Hostname == "" || len(s.Tags) == 0 {
|
|
t.Fatalf("restic %s: a snapshot lost a field the guard reads: %+v", v, s)
|
|
}
|
|
}
|
|
counts[v] = len(snaps)
|
|
id, paths, err := m.offboxLatestSnapshot(context.Background(), "a")
|
|
if err != nil || id == "" || len(paths) == 0 {
|
|
t.Errorf("restic %s: newest snapshot of a tag: %q %v %v", v, id, paths, err)
|
|
}
|
|
if size, err := m.offboxSnapshotSize(context.Background(), id); err != nil || size <= 0 {
|
|
t.Errorf("restic %s: snapshot size: %d %v", v, size, err)
|
|
}
|
|
n, ok := m.offboxRecordStats(context.Background(), nil, nil)
|
|
if !ok || n != counts[v] || sett.GetOffboxTarget().RepoSizeBytes <= 0 {
|
|
t.Errorf("restic %s: repo stats: count %d ok %v size %d", v, n, ok, sett.GetOffboxTarget().RepoSizeBytes)
|
|
}
|
|
moved := false
|
|
for _, line := range strings.Split(strings.TrimSpace(string(r1105Fixture(t, v, "backup-status.jsonl"))), "\n") {
|
|
p, ok := parseResticStatus(line)
|
|
if !ok {
|
|
t.Errorf("restic %s: a status line no longer parses: %s", v, line)
|
|
}
|
|
if p.TotalBytes > 0 && p.TotalFiles > 0 {
|
|
moved = true
|
|
}
|
|
}
|
|
if !moved {
|
|
t.Errorf("restic %s: no status line carried the totals the progress bar shows", v)
|
|
}
|
|
}
|
|
if counts["0.14.0"] != counts["0.18.1"] {
|
|
t.Errorf("the two versions list a different number of snapshots of ONE repository: %v", counts)
|
|
}
|
|
}
|
|
|
|
// The clean-up plan — the ONE assumption that MOVED. On the same snapshot set, 0.18.1 removes a SUBSET of what 0.14.0
|
|
// removes: since restic 0.17, while a --keep-monthly window has unused slots, restic also keeps the group's OLDEST
|
|
// snapshot (reason „oldest monthly snapshot"). Measured: a 41-day history → 0.14.0 removes 62, 0.18.1 removes 60 (one
|
|
// per group); a 230-day history (every monthly slot used) → identical plans (474 of 502, audits/debian13-restic-
|
|
// 2026-10-16/). It only ever KEEPS more, never removes more, and what it keeps extra is old — so the fake-snapshot
|
|
// guard (R-822/R-867), which refuses a plan that removes a YOUNG snapshot, passes the 0.18.1 plan whenever it passes
|
|
// the 0.14.0 one. Pinned here: subset, the difference is exactly the „oldest" keeps, and the guard's verdict holds.
|
|
func TestR1105_TheForgetPlanOnlyEverKeepsMore(t *testing.T) {
|
|
plans := map[string]map[string]bool{}
|
|
var all []guardSnap
|
|
for _, v := range r1105Versions {
|
|
m, _ := newOffboxManager(t)
|
|
m.SetOffboxRunner(r1105Runner(t, v, map[string]string{"forget": "forget-dryrun.json", "snapshots": "snapshots.json"}))
|
|
plan, err := m.planRemovals(context.Background(), nil, nil)
|
|
if err != nil {
|
|
t.Fatalf("restic %s: %v", v, err)
|
|
}
|
|
plans[v] = map[string]bool{}
|
|
for _, s := range plan {
|
|
plans[v][s.ID] = true
|
|
}
|
|
// Independent read of the same file: the parser drops nothing restic planned.
|
|
var groups []struct {
|
|
Remove []json.RawMessage `json:"remove"`
|
|
}
|
|
if err := json.Unmarshal(r1105Fixture(t, v, "forget-dryrun.json"), &groups); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := 0
|
|
for _, g := range groups {
|
|
want += len(g.Remove)
|
|
}
|
|
if len(plan) != want || want == 0 {
|
|
t.Errorf("restic %s: the parsed plan holds %d removals, restic planned %d", v, len(plan), want)
|
|
}
|
|
// The guard, at the minute the plans were captured: both plans pass it.
|
|
if all == nil {
|
|
if all, err = m.listGuardSnaps(context.Background(), nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
captured := time.Date(2026, 10, 16, 18, 20, 0, 0, time.UTC)
|
|
if _, refuse := offsiteGuard(all, plan, captured, time.Time{}, -1); refuse != "" {
|
|
t.Errorf("restic %s: the guard refuses the honest plan: %s", v, refuse)
|
|
}
|
|
}
|
|
for id := range plans["0.18.1"] {
|
|
if !plans["0.14.0"][id] {
|
|
t.Errorf("0.18.1 removes snapshot %s, which 0.14.0 keeps — the new restic removes MORE", id)
|
|
}
|
|
}
|
|
// Every snapshot only 0.14.0 removes is one 0.18.1 keeps as the „oldest" of a window with unused slots.
|
|
var groups []struct {
|
|
Reasons []struct {
|
|
Snapshot struct {
|
|
ID string `json:"id"`
|
|
} `json:"snapshot"`
|
|
Matches []string `json:"matches"`
|
|
} `json:"reasons"`
|
|
}
|
|
if err := json.Unmarshal(r1105Fixture(t, "0.18.1", "forget-dryrun.json"), &groups); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
why := map[string]string{}
|
|
for _, g := range groups {
|
|
for _, r := range g.Reasons {
|
|
why[r.Snapshot.ID] = strings.Join(r.Matches, ",")
|
|
}
|
|
}
|
|
extra := 0
|
|
for id := range plans["0.14.0"] {
|
|
if plans["0.18.1"][id] {
|
|
continue
|
|
}
|
|
extra++
|
|
if !strings.Contains(why[id], "oldest") {
|
|
t.Errorf("0.18.1 keeps snapshot %s that 0.14.0 removes, for a reason other than “oldest”: %q", id, why[id])
|
|
}
|
|
}
|
|
if extra == 0 {
|
|
t.Error("the fixture no longer shows the 0.17+ “oldest” keep — it no longer tests the difference it documents")
|
|
}
|
|
}
|