v0.311.0: after a power cut, an app repairs the files the cut left empty; the household sees lost photos (R-1007)

New template block after_unclean_boot (decision 236): once per controller start that followed an UNCLEAN host boot (crashboot's own verdict, after the 15-minute crash grace; a normal or unknown boot never runs it), each app with the block rebuilds ONLY its zero-byte files (scan: regular files, no symlinks, its own filesystem; ids appended to the app's own command in batches; one retry, then reported) and its own lost-file check is read. The household sees lost files (name, date taken) on the app page until dismissed; the operator gets app_repaired_after_crash with counts only. Skips, with the reason, an unproven image, an unknown key, an app not running. MinAgent 0.155.0 unchanged. Tests red-proofed (14 guards). One new parity fixture (the new card only). Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
2026-10-14 16:47:13 +02:00
parent acbbf4b305
commit 8d862e4e69
24 changed files with 1865 additions and 3 deletions
@@ -0,0 +1,269 @@
+package stacks
+
+import (
+	"bytes"
+	"errors"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+	"time"
+
+	"gopkg.in/yaml.v3"
+)
+
+// R-1007 (decision 236) — after_unclean_boot: the app rebuilds ONLY the files a power cut left empty, and its own
+// lost-file check is read. The exec is the seam; nothing reaches Docker. Each guard's red-proof is named on its test.
+
+const (
+	idA = "11111111-1111-4111-8111-111111111111"
+	idB = "22222222-2222-4222-8222-222222222222"
+	idC = "33333333-3333-4333-8333-333333333333"
+	idD = "44444444-4444-4444-8444-444444444444"
+)
+
+var aubRe = regexp.MustCompile(`^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})_`)
+
+func aubWrite(t *testing.T, p string, data string) {
+	t.Helper()
+	must(t, os.MkdirAll(filepath.Dir(p), 0o755))
+	must(t, os.WriteFile(p, []byte(data), 0o644))
+}
+
+// RED-PROOF: drop the `info.Size() != 0` test in ScanEmptyFiles → idB (non-empty) is sent → fails.
+// RED-PROOF: drop the ModeSymlink skip → the link to an empty file OUTSIDE the folder is counted → fails.
+func TestAUB_ScanFindsOnlyEmptyRegularFilesInsideTheFolder(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "thumbs")
+	aubWrite(t, filepath.Join(root, "u1/11/11", idA+"_thumbnail.webp"), "")     // empty: found
+	aubWrite(t, filepath.Join(root, "u1/11/11", idA+"_preview.jpeg"), "")       // the same id twice: once
+	aubWrite(t, filepath.Join(root, "u1/22/22", idB+"_thumbnail.webp"), "data") // the non-empty CONTROL
+	aubWrite(t, filepath.Join(root, "u2/33/33", idC+"_preview.jpeg"), "")       // another user: found
+	aubWrite(t, filepath.Join(root, "u2/notes.txt"), "")                        // an empty file with no id
+	aubWrite(t, filepath.Join(root, "u2", "-rf_thumbnail.webp"), "")            // a name that is no plain id
+	aubWrite(t, filepath.Join(dir, "outside", idD+"_thumbnail.webp"), "")       // empty, but OUTSIDE the folder
+	must(t, os.Symlink(filepath.Join(dir, "outside", idD+"_thumbnail.webp"), filepath.Join(root, "u2", idD+"_thumbnail.webp")))
+	ids, err := ScanEmptyFiles(root, aubRe)
+	must(t, err)
+	if strings.Join(ids, ",") != idA+","+idC {
+		t.Fatalf("ids = %v, want exactly [%s %s]", ids, idA, idC)
+	}
+	if ids, err := ScanEmptyFiles(filepath.Join(dir, "nope"), aubRe); err != nil || len(ids) != 0 {
+		t.Fatalf("a missing folder: %v %v", ids, err)
+	}
+}
+
+// aubManager: the kept-test manager with an Immich-shaped block on "cloudapp", running, its image proven.
+func aubManager(t *testing.T) (*Manager, string, *AfterUncleanBootSpec, *bytes.Buffer) {
+	t.Helper()
+	m, drive := keptManager(t)
+	var logBuf bytes.Buffer
+	m.logger = log.New(&logBuf, "", 0)
+	dir := filepath.Dir(m.stacks["cloudapp"].ComposePath)
+	must(t, SaveAppConfig(dir, &AppConfig{Deployed: true, Env: map[string]string{"HDD_PATH": drive},
+		InstalledImages: map[string]InstalledImage{"cloudapp": {Ref: "example/app:v1"}}}, m.encKey, nil))
+	spec := &AfterUncleanBootSpec{
+		Service: "cloudapp", Images: []string{"example/app:v1"},
+		EmptyFiles: AUBEmptyFiles{Under: "${HDD_PATH}/appdata/cloudapp/thumbs", IDFromName: aubRe.String()},
+		Repair:     AUBCommand{Command: []string{"node", "-e", "repair"}, Success: "FELHOM_REPAIR_OK", Batch: 2},
+		ReportLost: &AUBCommand{Command: []string{"node", "-e", "lost"}, Success: "FELHOM_LOST_OK"},
+	}
+	m.mu.Lock()
+	m.stacks["cloudapp"].Meta.AfterUncleanBoot = spec
+	m.stacks["cloudapp"].Deployed = true
+	m.mu.Unlock()
+	m.aubRunningFn = func(string) bool { return true }
+	m.aubSleepFn = func(time.Duration) {}
+	return m, drive, spec, &logBuf
+}
+
+// fakeApp plays the app: the repair fills the files of the ids it is given (unless told to leave one), the lost
+// check prints its report. It records every command.
+type fakeApp struct {
+	root      string
+	leave     map[string]bool // ids whose files the app fails to rebuild
+	calls     [][]string
+	lostOut   string
+	repairErr error
+}
+
+func (f *fakeApp) exec(_ string, args ...string) (string, error) {
+	f.calls = append(f.calls, append([]string(nil), args...))
+	if len(args) >= 6 && args[5] == "repair" {
+		for _, id := range args[6:] {
+			if f.leave[id] {
+				continue
+			}
+			_ = filepath.Walk(f.root, func(p string, info os.FileInfo, err error) error {
+				if err == nil && !info.IsDir() && strings.HasPrefix(info.Name(), id+"_") {
+					_ = os.WriteFile(p, []byte("rebuilt"), 0o644)
+				}
+				return nil
+			})
+		}
+		if f.repairErr != nil {
+			return "boom", f.repairErr
+		}
+		return "FELHOM_REPAIR_OK", nil
+	}
+	return f.lostOut, nil
+}
+
+func (f *fakeApp) repairCalls() [][]string {
+	var out [][]string
+	for _, c := range f.calls {
+		if len(c) >= 6 && c[5] == "repair" {
+			out = append(out, c[6:])
+		}
+	}
+	return out
+}
+
+// The consequence: only the empty files' ids are sent; the non-empty control is not; batches hold; one still-empty
+// file is retried ONCE and then reported; the lost list is exactly the app's report; the log carries no file name.
+// RED-PROOF: let the retry loop run 3 times (`attempt <= 3`) → the still-broken id is sent 3 times → fails.
+// RED-PROOF: drop the intersect() (re-scan) → StillBroken stays 0 for the file the app never rebuilt → fails.
+func TestAUB_RepairsOnlyTheEmptyFilesRetriesOnceAndReadsTheLostList(t *testing.T) {
+	m, drive, _, logBuf := aubManager(t)
+	root := filepath.Join(drive, "appdata/cloudapp/thumbs")
+	aubWrite(t, filepath.Join(root, "u1", idA+"_thumbnail.webp"), "")
+	aubWrite(t, filepath.Join(root, "u1", idB+"_preview.jpeg"), "real") // control
+	aubWrite(t, filepath.Join(root, "u2", idC+"_preview.jpeg"), "")
+	aubWrite(t, filepath.Join(root, "u2", idD+"_thumbnail.webp"), "")
+	ctlInfo, _ := os.Stat(filepath.Join(root, "u1", idB+"_preview.jpeg"))
+	app := &fakeApp{root: root, leave: map[string]bool{idD: true},
+		lostOut: "LOST {\"name\":\"IMG_0042.jpg\",\"taken\":\"2026-10-14 10:41\"}\nnoise\nFELHOM_LOST_OK lost=1\n"}
+	m.afterLoadFn = app.exec
+	boot := time.Date(2026, 10, 14, 12, 7, 26, 0, time.UTC)
+
+	res := m.RunAfterUncleanBoot("cloudapp", boot)
+
+	if res.Skipped != "" || res.Problem != "" {
+		t.Fatalf("skipped=%q problem=%q", res.Skipped, res.Problem)
+	}
+	if res.Found != 3 || res.Repaired != 2 || res.StillBroken != 1 || res.Lost != 1 || !res.LostChecked {
+		t.Fatalf("counts %+v, want found 3, repaired 2, still 1, lost 1", res)
+	}
+	sent := map[string]int{}
+	for _, batch := range app.repairCalls() {
+		if len(batch) > 2 {
+			t.Fatalf("a batch of %d, the template says 2", len(batch))
+		}
+		for _, id := range batch {
+			sent[id]++
+		}
+	}
+	if sent[idB] != 0 {
+		t.Fatal("the non-empty control was sent for repair")
+	}
+	if sent[idA] != 1 || sent[idC] != 1 || sent[idD] != 2 {
+		t.Fatalf("sent %v: the repaired ones once, the still-broken one twice (one retry)", sent)
+	}
+	if after, _ := os.Stat(filepath.Join(root, "u1", idB+"_preview.jpeg")); !after.ModTime().Equal(ctlInfo.ModTime()) {
+		t.Fatal("the control file was rewritten")
+	}
+	c := LoadAppConfig(filepath.Dir(m.stacks["cloudapp"].ComposePath))
+	r := c.AfterUncleanBoot
+	if r == nil || r.Found != 3 || r.StillBroken != 1 || r.BootAt != "2026-10-14T12:07:26Z" {
+		t.Fatalf("record %+v", r)
+	}
+	if len(r.Lost) != 1 || r.Lost[0].Name != "IMG_0042.jpg" || r.Lost[0].Taken != "2026-10-14 10:41" {
+		t.Fatalf("the household's lost list %+v, want exactly the app's report", r.Lost)
+	}
+	if strings.Contains(logBuf.String(), "IMG_0042") {
+		t.Fatal("a lost file's name reached the controller log")
+	}
+}
+
+// A run never guesses: an image the template has not proven, a key this box does not know, a stopped app — each is
+// skipped with its reason and NO command runs. RED-PROOF: drop the containsString(spec.Images, …) check → the
+// unproven image's run sends commands → fails.
+func TestAUB_SkipsAnUnprovenImageAnUnknownKeyAndAStoppedApp(t *testing.T) {
+	m, drive, spec, _ := aubManager(t)
+	aubWrite(t, filepath.Join(drive, "appdata/cloudapp/thumbs/u1", idA+"_thumbnail.webp"), "")
+	app := &fakeApp{root: drive}
+	m.afterLoadFn = app.exec
+
+	spec.Images = []string{"example/app:v2"}
+	if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "has not proven") || len(app.calls) != 0 {
+		t.Fatalf("unproven image: %+v, calls %v", res, app.calls)
+	}
+	spec.Images = []string{"example/app:v1"}
+	spec.Extra = map[string]any{"verify_after": true}
+	if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "verify_after") || len(app.calls) != 0 {
+		t.Fatalf("unknown key: %+v, calls %v", res, app.calls)
+	}
+	spec.Extra = nil
+	m.aubRunningFn = func(string) bool { return false }
+	aubHealthyWait = 0
+	t.Cleanup(func() { aubHealthyWait = 10 * time.Minute })
+	if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "not running") || len(app.calls) != 0 {
+		t.Fatalf("stopped app: %+v, calls %v", res, app.calls)
+	}
+}
+
+// A failing repair is a reported problem, never a stopped app or an endless loop.
+func TestAUB_AFailingRepairIsReportedNotRetriedForever(t *testing.T) {
+	m, drive, _, _ := aubManager(t)
+	root := filepath.Join(drive, "appdata/cloudapp/thumbs")
+	aubWrite(t, filepath.Join(root, "u1", idA+"_thumbnail.webp"), "")
+	app := &fakeApp{root: root, leave: map[string]bool{idA: true}, repairErr: errors.New("exit 1"), lostOut: "FELHOM_LOST_OK lost=0"}
+	m.afterLoadFn = app.exec
+	res := m.RunAfterUncleanBoot("cloudapp", time.Time{})
+	if res.Problem == "" || res.StillBroken != 1 || len(app.repairCalls()) != 2 {
+		t.Fatalf("%+v, repair calls %d (want the run + one retry)", res, len(app.repairCalls()))
+	}
+}
+
+// The catalog block parses as written, and a newer key lands in Extra (so the box skips, never half-runs).
+func TestAUB_TheCatalogShapeParses(t *testing.T) {
+	src := `
+after_unclean_boot:
+  service: immich-server
+  images: ["ghcr.io/immich-app/immich-server:v3.2.4"]
+  empty_files:
+    under: "${HDD_PATH}/appdata/immich/thumbs"
+    id_from_name: "^([0-9a-f-]{36})_"
+  repair:
+    command: ["node", "-e", "x"]
+    success: FELHOM_REPAIR_OK
+    batch: 200
+  report_lost:
+    command: ["node", "-e", "y"]
+    success: FELHOM_LOST_OK
+  some_future_key: 1
+`
+	var meta Metadata
+	must(t, yaml.Unmarshal([]byte(src), &meta))
+	s := meta.AfterUncleanBoot
+	if s == nil || s.Service != "immich-server" || s.Repair.Batch != 200 || s.ReportLost == nil || s.EmptyFiles.IDFromName == "" {
+		t.Fatalf("parsed %+v", s)
+	}
+	if _, ok := s.Extra["some_future_key"]; !ok || aubUnsupported(s) == "" {
+		t.Fatalf("a newer key must be seen and refused: extra=%v", s.Extra)
+	}
+	delete(s.Extra, "some_future_key")
+	if why := aubUnsupported(s); why != "" {
+		t.Fatalf("the shipped shape must be supported: %s", why)
+	}
+}
+
+// The dismiss hides the household's note and keeps the counts.
+func TestAUB_DismissHidesTheNoteKeepsTheRecord(t *testing.T) {
+	m, _, _, _ := aubManager(t)
+	dir := filepath.Dir(m.stacks["cloudapp"].ComposePath)
+	if err := m.DismissLostFiles("cloudapp"); err == nil {
+		t.Fatal("nothing to dismiss must refuse")
+	}
+	m.mutateAppConfig("cloudapp", dir, "t", func(c *AppConfig) bool {
+		c.AfterUncleanBoot = &AfterUncleanBootRecord{Found: 3, Lost: []LostFile{{Name: "a.jpg"}}}
+		return true
+	})
+	must(t, m.DismissLostFiles("cloudapp"))
+	r := LoadAppConfig(dir).AfterUncleanBoot
+	if r == nil || !r.Dismissed || r.Found != 3 || len(r.Lost) != 1 {
+		t.Fatalf("%+v", r)
+	}
+}