admin
8d862e4e69
gates / gates (push) Successful in 52s
New template block after_unclean_boot (decision 236): once per controller start that followed an UNCLEAN host boot (crashboot's own verdict, after the 15-minute crash grace; a normal or unknown boot never runs it), each app with the block rebuilds ONLY its zero-byte files (scan: regular files, no symlinks, its own filesystem; ids appended to the app's own command in batches; one retry, then reported) and its own lost-file check is read. The household sees lost files (name, date taken) on the app page until dismissed; the operator gets app_repaired_after_crash with counts only. Skips, with the reason, an unproven image, an unknown key, an app not running. MinAgent 0.155.0 unchanged. Tests red-proofed (14 guards). One new parity fixture (the new card only). Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
270 lines
11 KiB
Go
270 lines
11 KiB
Go
package stacks
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// R-1007 (decision 236) — after_unclean_boot: the app rebuilds ONLY the files a power cut left empty, and its own
|
|
// lost-file check is read. The exec is the seam; nothing reaches Docker. Each guard's red-proof is named on its test.
|
|
|
|
const (
|
|
idA = "11111111-1111-4111-8111-111111111111"
|
|
idB = "22222222-2222-4222-8222-222222222222"
|
|
idC = "33333333-3333-4333-8333-333333333333"
|
|
idD = "44444444-4444-4444-8444-444444444444"
|
|
)
|
|
|
|
var aubRe = regexp.MustCompile(`^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})_`)
|
|
|
|
func aubWrite(t *testing.T, p string, data string) {
|
|
t.Helper()
|
|
must(t, os.MkdirAll(filepath.Dir(p), 0o755))
|
|
must(t, os.WriteFile(p, []byte(data), 0o644))
|
|
}
|
|
|
|
// RED-PROOF: drop the `info.Size() != 0` test in ScanEmptyFiles → idB (non-empty) is sent → fails.
|
|
// RED-PROOF: drop the ModeSymlink skip → the link to an empty file OUTSIDE the folder is counted → fails.
|
|
func TestAUB_ScanFindsOnlyEmptyRegularFilesInsideTheFolder(t *testing.T) {
|
|
dir := t.TempDir()
|
|
root := filepath.Join(dir, "thumbs")
|
|
aubWrite(t, filepath.Join(root, "u1/11/11", idA+"_thumbnail.webp"), "") // empty: found
|
|
aubWrite(t, filepath.Join(root, "u1/11/11", idA+"_preview.jpeg"), "") // the same id twice: once
|
|
aubWrite(t, filepath.Join(root, "u1/22/22", idB+"_thumbnail.webp"), "data") // the non-empty CONTROL
|
|
aubWrite(t, filepath.Join(root, "u2/33/33", idC+"_preview.jpeg"), "") // another user: found
|
|
aubWrite(t, filepath.Join(root, "u2/notes.txt"), "") // an empty file with no id
|
|
aubWrite(t, filepath.Join(root, "u2", "-rf_thumbnail.webp"), "") // a name that is no plain id
|
|
aubWrite(t, filepath.Join(dir, "outside", idD+"_thumbnail.webp"), "") // empty, but OUTSIDE the folder
|
|
must(t, os.Symlink(filepath.Join(dir, "outside", idD+"_thumbnail.webp"), filepath.Join(root, "u2", idD+"_thumbnail.webp")))
|
|
ids, err := ScanEmptyFiles(root, aubRe)
|
|
must(t, err)
|
|
if strings.Join(ids, ",") != idA+","+idC {
|
|
t.Fatalf("ids = %v, want exactly [%s %s]", ids, idA, idC)
|
|
}
|
|
if ids, err := ScanEmptyFiles(filepath.Join(dir, "nope"), aubRe); err != nil || len(ids) != 0 {
|
|
t.Fatalf("a missing folder: %v %v", ids, err)
|
|
}
|
|
}
|
|
|
|
// aubManager: the kept-test manager with an Immich-shaped block on "cloudapp", running, its image proven.
|
|
func aubManager(t *testing.T) (*Manager, string, *AfterUncleanBootSpec, *bytes.Buffer) {
|
|
t.Helper()
|
|
m, drive := keptManager(t)
|
|
var logBuf bytes.Buffer
|
|
m.logger = log.New(&logBuf, "", 0)
|
|
dir := filepath.Dir(m.stacks["cloudapp"].ComposePath)
|
|
must(t, SaveAppConfig(dir, &AppConfig{Deployed: true, Env: map[string]string{"HDD_PATH": drive},
|
|
InstalledImages: map[string]InstalledImage{"cloudapp": {Ref: "example/app:v1"}}}, m.encKey, nil))
|
|
spec := &AfterUncleanBootSpec{
|
|
Service: "cloudapp", Images: []string{"example/app:v1"},
|
|
EmptyFiles: AUBEmptyFiles{Under: "${HDD_PATH}/appdata/cloudapp/thumbs", IDFromName: aubRe.String()},
|
|
Repair: AUBCommand{Command: []string{"node", "-e", "repair"}, Success: "FELHOM_REPAIR_OK", Batch: 2},
|
|
ReportLost: &AUBCommand{Command: []string{"node", "-e", "lost"}, Success: "FELHOM_LOST_OK"},
|
|
}
|
|
m.mu.Lock()
|
|
m.stacks["cloudapp"].Meta.AfterUncleanBoot = spec
|
|
m.stacks["cloudapp"].Deployed = true
|
|
m.mu.Unlock()
|
|
m.aubRunningFn = func(string) bool { return true }
|
|
m.aubSleepFn = func(time.Duration) {}
|
|
return m, drive, spec, &logBuf
|
|
}
|
|
|
|
// fakeApp plays the app: the repair fills the files of the ids it is given (unless told to leave one), the lost
|
|
// check prints its report. It records every command.
|
|
type fakeApp struct {
|
|
root string
|
|
leave map[string]bool // ids whose files the app fails to rebuild
|
|
calls [][]string
|
|
lostOut string
|
|
repairErr error
|
|
}
|
|
|
|
func (f *fakeApp) exec(_ string, args ...string) (string, error) {
|
|
f.calls = append(f.calls, append([]string(nil), args...))
|
|
if len(args) >= 6 && args[5] == "repair" {
|
|
for _, id := range args[6:] {
|
|
if f.leave[id] {
|
|
continue
|
|
}
|
|
_ = filepath.Walk(f.root, func(p string, info os.FileInfo, err error) error {
|
|
if err == nil && !info.IsDir() && strings.HasPrefix(info.Name(), id+"_") {
|
|
_ = os.WriteFile(p, []byte("rebuilt"), 0o644)
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
if f.repairErr != nil {
|
|
return "boom", f.repairErr
|
|
}
|
|
return "FELHOM_REPAIR_OK", nil
|
|
}
|
|
return f.lostOut, nil
|
|
}
|
|
|
|
func (f *fakeApp) repairCalls() [][]string {
|
|
var out [][]string
|
|
for _, c := range f.calls {
|
|
if len(c) >= 6 && c[5] == "repair" {
|
|
out = append(out, c[6:])
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The consequence: only the empty files' ids are sent; the non-empty control is not; batches hold; one still-empty
|
|
// file is retried ONCE and then reported; the lost list is exactly the app's report; the log carries no file name.
|
|
// RED-PROOF: let the retry loop run 3 times (`attempt <= 3`) → the still-broken id is sent 3 times → fails.
|
|
// RED-PROOF: drop the intersect() (re-scan) → StillBroken stays 0 for the file the app never rebuilt → fails.
|
|
func TestAUB_RepairsOnlyTheEmptyFilesRetriesOnceAndReadsTheLostList(t *testing.T) {
|
|
m, drive, _, logBuf := aubManager(t)
|
|
root := filepath.Join(drive, "appdata/cloudapp/thumbs")
|
|
aubWrite(t, filepath.Join(root, "u1", idA+"_thumbnail.webp"), "")
|
|
aubWrite(t, filepath.Join(root, "u1", idB+"_preview.jpeg"), "real") // control
|
|
aubWrite(t, filepath.Join(root, "u2", idC+"_preview.jpeg"), "")
|
|
aubWrite(t, filepath.Join(root, "u2", idD+"_thumbnail.webp"), "")
|
|
ctlInfo, _ := os.Stat(filepath.Join(root, "u1", idB+"_preview.jpeg"))
|
|
app := &fakeApp{root: root, leave: map[string]bool{idD: true},
|
|
lostOut: "LOST {\"name\":\"IMG_0042.jpg\",\"taken\":\"2026-10-14 10:41\"}\nnoise\nFELHOM_LOST_OK lost=1\n"}
|
|
m.afterLoadFn = app.exec
|
|
boot := time.Date(2026, 10, 14, 12, 7, 26, 0, time.UTC)
|
|
|
|
res := m.RunAfterUncleanBoot("cloudapp", boot)
|
|
|
|
if res.Skipped != "" || res.Problem != "" {
|
|
t.Fatalf("skipped=%q problem=%q", res.Skipped, res.Problem)
|
|
}
|
|
if res.Found != 3 || res.Repaired != 2 || res.StillBroken != 1 || res.Lost != 1 || !res.LostChecked {
|
|
t.Fatalf("counts %+v, want found 3, repaired 2, still 1, lost 1", res)
|
|
}
|
|
sent := map[string]int{}
|
|
for _, batch := range app.repairCalls() {
|
|
if len(batch) > 2 {
|
|
t.Fatalf("a batch of %d, the template says 2", len(batch))
|
|
}
|
|
for _, id := range batch {
|
|
sent[id]++
|
|
}
|
|
}
|
|
if sent[idB] != 0 {
|
|
t.Fatal("the non-empty control was sent for repair")
|
|
}
|
|
if sent[idA] != 1 || sent[idC] != 1 || sent[idD] != 2 {
|
|
t.Fatalf("sent %v: the repaired ones once, the still-broken one twice (one retry)", sent)
|
|
}
|
|
if after, _ := os.Stat(filepath.Join(root, "u1", idB+"_preview.jpeg")); !after.ModTime().Equal(ctlInfo.ModTime()) {
|
|
t.Fatal("the control file was rewritten")
|
|
}
|
|
c := LoadAppConfig(filepath.Dir(m.stacks["cloudapp"].ComposePath))
|
|
r := c.AfterUncleanBoot
|
|
if r == nil || r.Found != 3 || r.StillBroken != 1 || r.BootAt != "2026-10-14T12:07:26Z" {
|
|
t.Fatalf("record %+v", r)
|
|
}
|
|
if len(r.Lost) != 1 || r.Lost[0].Name != "IMG_0042.jpg" || r.Lost[0].Taken != "2026-10-14 10:41" {
|
|
t.Fatalf("the household's lost list %+v, want exactly the app's report", r.Lost)
|
|
}
|
|
if strings.Contains(logBuf.String(), "IMG_0042") {
|
|
t.Fatal("a lost file's name reached the controller log")
|
|
}
|
|
}
|
|
|
|
// A run never guesses: an image the template has not proven, a key this box does not know, a stopped app — each is
|
|
// skipped with its reason and NO command runs. RED-PROOF: drop the containsString(spec.Images, …) check → the
|
|
// unproven image's run sends commands → fails.
|
|
func TestAUB_SkipsAnUnprovenImageAnUnknownKeyAndAStoppedApp(t *testing.T) {
|
|
m, drive, spec, _ := aubManager(t)
|
|
aubWrite(t, filepath.Join(drive, "appdata/cloudapp/thumbs/u1", idA+"_thumbnail.webp"), "")
|
|
app := &fakeApp{root: drive}
|
|
m.afterLoadFn = app.exec
|
|
|
|
spec.Images = []string{"example/app:v2"}
|
|
if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "has not proven") || len(app.calls) != 0 {
|
|
t.Fatalf("unproven image: %+v, calls %v", res, app.calls)
|
|
}
|
|
spec.Images = []string{"example/app:v1"}
|
|
spec.Extra = map[string]any{"verify_after": true}
|
|
if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "verify_after") || len(app.calls) != 0 {
|
|
t.Fatalf("unknown key: %+v, calls %v", res, app.calls)
|
|
}
|
|
spec.Extra = nil
|
|
m.aubRunningFn = func(string) bool { return false }
|
|
aubHealthyWait = 0
|
|
t.Cleanup(func() { aubHealthyWait = 10 * time.Minute })
|
|
if res := m.RunAfterUncleanBoot("cloudapp", time.Time{}); !strings.Contains(res.Skipped, "not running") || len(app.calls) != 0 {
|
|
t.Fatalf("stopped app: %+v, calls %v", res, app.calls)
|
|
}
|
|
}
|
|
|
|
// A failing repair is a reported problem, never a stopped app or an endless loop.
|
|
func TestAUB_AFailingRepairIsReportedNotRetriedForever(t *testing.T) {
|
|
m, drive, _, _ := aubManager(t)
|
|
root := filepath.Join(drive, "appdata/cloudapp/thumbs")
|
|
aubWrite(t, filepath.Join(root, "u1", idA+"_thumbnail.webp"), "")
|
|
app := &fakeApp{root: root, leave: map[string]bool{idA: true}, repairErr: errors.New("exit 1"), lostOut: "FELHOM_LOST_OK lost=0"}
|
|
m.afterLoadFn = app.exec
|
|
res := m.RunAfterUncleanBoot("cloudapp", time.Time{})
|
|
if res.Problem == "" || res.StillBroken != 1 || len(app.repairCalls()) != 2 {
|
|
t.Fatalf("%+v, repair calls %d (want the run + one retry)", res, len(app.repairCalls()))
|
|
}
|
|
}
|
|
|
|
// The catalog block parses as written, and a newer key lands in Extra (so the box skips, never half-runs).
|
|
func TestAUB_TheCatalogShapeParses(t *testing.T) {
|
|
src := `
|
|
after_unclean_boot:
|
|
service: immich-server
|
|
images: ["ghcr.io/immich-app/immich-server:v3.2.4"]
|
|
empty_files:
|
|
under: "${HDD_PATH}/appdata/immich/thumbs"
|
|
id_from_name: "^([0-9a-f-]{36})_"
|
|
repair:
|
|
command: ["node", "-e", "x"]
|
|
success: FELHOM_REPAIR_OK
|
|
batch: 200
|
|
report_lost:
|
|
command: ["node", "-e", "y"]
|
|
success: FELHOM_LOST_OK
|
|
some_future_key: 1
|
|
`
|
|
var meta Metadata
|
|
must(t, yaml.Unmarshal([]byte(src), &meta))
|
|
s := meta.AfterUncleanBoot
|
|
if s == nil || s.Service != "immich-server" || s.Repair.Batch != 200 || s.ReportLost == nil || s.EmptyFiles.IDFromName == "" {
|
|
t.Fatalf("parsed %+v", s)
|
|
}
|
|
if _, ok := s.Extra["some_future_key"]; !ok || aubUnsupported(s) == "" {
|
|
t.Fatalf("a newer key must be seen and refused: extra=%v", s.Extra)
|
|
}
|
|
delete(s.Extra, "some_future_key")
|
|
if why := aubUnsupported(s); why != "" {
|
|
t.Fatalf("the shipped shape must be supported: %s", why)
|
|
}
|
|
}
|
|
|
|
// The dismiss hides the household's note and keeps the counts.
|
|
func TestAUB_DismissHidesTheNoteKeepsTheRecord(t *testing.T) {
|
|
m, _, _, _ := aubManager(t)
|
|
dir := filepath.Dir(m.stacks["cloudapp"].ComposePath)
|
|
if err := m.DismissLostFiles("cloudapp"); err == nil {
|
|
t.Fatal("nothing to dismiss must refuse")
|
|
}
|
|
m.mutateAppConfig("cloudapp", dir, "t", func(c *AppConfig) bool {
|
|
c.AfterUncleanBoot = &AfterUncleanBootRecord{Found: 3, Lost: []LostFile{{Name: "a.jpg"}}}
|
|
return true
|
|
})
|
|
must(t, m.DismissLostFiles("cloudapp"))
|
|
r := LoadAppConfig(dir).AfterUncleanBoot
|
|
if r == nil || !r.Dismissed || r.Found != 3 || len(r.Lost) != 1 {
|
|
t.Fatalf("%+v", r)
|
|
}
|
|
}
|