package stacks import ( "bytes" "encoding/json" "errors" "fmt" "io" "io/fs" "os" "os/exec" "path/filepath" "regexp" "sort" "strings" "syscall" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" ) // ── after_unclean_boot (R-1007, `09` §3 decision 236): an app repairs the files a power cut left empty ── // // A power cut while photos upload leaves some of Immich's thumbnails and previews ZERO BYTES (ext4 had the name, not // the data). Immich still lists the photos as done and never redoes them; rarely an original is gone too. The // template declares, for its own app, where such files are, how the app is asked to rebuild THOSE (and only those), // and how the app's own lost-file check is read: // // after_unclean_boot: // service: immich-server # the commands run in this compose service // images: ["ghcr.io/immich-app/immich-server:v3.2.4"] # the images it was PROVEN on; any other: skip // empty_files: // under: "${HDD_PATH}/appdata/immich/thumbs" # only ${HDD_PATH} is filled in // id_from_name: "^([0-9a-f-]{36})_" # one capture group: the app's id for that file // repair: // command: ["node", "-e", "…"] # the ids are appended as arguments // success: FELHOM_REPAIR_OK // batch: 200 // report_lost: # optional // command: ["node", "-e", "…"] # prints `LOST {"name":…,"taken":…}` lines // success: FELHOM_LOST_OK // // The controller holds NO app knowledge: the catalog block is all of it. Ran ONLY once per controller start that // followed an unclean host boot (internal/aftercrash decides when). Never deletes or rewrites anything itself; the // app rebuilds its own files. A run that fails never stops the app. File names never reach a log line or an event: // the lost list lives only in app.yaml on this box, for the household's note. Pinned by after_unclean_boot_test.go. // AfterUncleanBootSpec is `.felhom.yml`'s `after_unclean_boot:`. type AfterUncleanBootSpec struct { Service string `yaml:"service" json:"service"` Images []string `yaml:"images" json:"images"` EmptyFiles AUBEmptyFiles `yaml:"empty_files" json:"empty_files"` Repair AUBCommand `yaml:"repair" json:"repair"` ReportLost *AUBCommand `yaml:"report_lost,omitempty" json:"report_lost,omitempty"` Extra map[string]any `yaml:",inline" json:"-"` // a newer key this box does not know → skip } // AUBEmptyFiles says where the empty files are and how a file name maps to the app's id. type AUBEmptyFiles struct { Under string `yaml:"under" json:"under"` IDFromName string `yaml:"id_from_name" json:"id_from_name"` } // AUBCommand is one command run in the service's container. type AUBCommand struct { Command []string `yaml:"command" json:"command"` Success string `yaml:"success" json:"success"` Batch int `yaml:"batch,omitempty" json:"batch,omitempty"` } // LostFile is one file the app's own check reports lost: the household's note shows it, nothing else does. type LostFile struct { Name string `yaml:"name" json:"name"` Taken string `yaml:"taken,omitempty" json:"taken,omitempty"` } // AfterUncleanBootRecord is app.yaml's `after_unclean_boot:` — what the last run found and did. type AfterUncleanBootRecord struct { At string `yaml:"at" json:"at"` BootAt string `yaml:"boot_at,omitempty" json:"boot_at,omitempty"` Found int `yaml:"found" json:"found"` Repaired int `yaml:"repaired" json:"repaired"` StillBroken int `yaml:"still_broken" json:"still_broken"` Lost []LostFile `yaml:"lost,omitempty" json:"lost,omitempty"` LostChecked bool `yaml:"lost_checked" json:"lost_checked"` DurationMS int64 `yaml:"duration_ms" json:"duration_ms"` Skipped string `yaml:"skipped,omitempty" json:"skipped,omitempty"` Problem string `yaml:"problem,omitempty" json:"problem,omitempty"` Dismissed bool `yaml:"dismissed,omitempty" json:"dismissed,omitempty"` } // AfterUncleanBootResult is a run's outcome for the log and the operator event (counts only). type AfterUncleanBootResult struct { Found, Repaired, StillBroken, Lost int LostChecked bool Duration time.Duration Skipped string // why nothing ran ("" = it ran) Problem string // a step that failed (the counts say what was done) } var ( aubIDSafe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$`) aubHealthyWait = 10 * time.Minute aubDefaultBatch = 200 aubMaxLostListed = 200 ) // aubUnsupported names why this box cannot run the block, or "" when it can. func aubUnsupported(s *AfterUncleanBootSpec) string { if len(s.Extra) > 0 { keys := make([]string, 0, len(s.Extra)) for k := range s.Extra { keys = append(keys, k) } sort.Strings(keys) return "the template has keys this controller does not know: " + strings.Join(keys, ", ") } switch { case s.Service == "" || len(s.Images) == 0: return "the template names no service or no proven image" case s.EmptyFiles.Under == "" || s.EmptyFiles.IDFromName == "": return "the template names no folder or no id pattern" case len(s.Repair.Command) == 0 || s.Repair.Success == "": return "the template names no repair command" case s.ReportLost != nil && (len(s.ReportLost.Command) == 0 || s.ReportLost.Success == ""): return "the template's lost-file check is incomplete" } re, err := regexp.Compile(s.EmptyFiles.IDFromName) if err != nil || re.NumSubexp() != 1 { return "the template's id pattern is not one capture group" } return "" } // aubUnder fills ${HDD_PATH} (and nothing else) into the folder; an empty or relative result refuses. func aubUnder(tmpl string, env map[string]string) (string, error) { var bad []string out := os.Expand(tmpl, func(k string) string { if k != "HDD_PATH" || strings.TrimSpace(env["HDD_PATH"]) == "" { bad = append(bad, k) return "" } return strings.TrimSpace(env["HDD_PATH"]) }) if len(bad) > 0 { return "", fmt.Errorf("the folder needs %v, which this app does not have", bad) } out = filepath.Clean(out) if !filepath.IsAbs(out) || out == "/" { return "", fmt.Errorf("the folder %q is not an absolute app folder", out) } return out, nil } // ScanEmptyFiles walks root and returns the ids (de-duplicated, sorted) of the REGULAR, zero-byte files whose names // match re. Symlinks are never followed nor counted; the walk stays on root's own filesystem; a name whose id is not // plain id text is skipped. A missing root is no error (nothing to repair). func ScanEmptyFiles(root string, re *regexp.Regexp) ([]string, error) { rootInfo, err := os.Lstat(root) if errors.Is(err, fs.ErrNotExist) { return nil, nil } if err != nil { return nil, err } if !rootInfo.IsDir() { return nil, fmt.Errorf("%s is not a folder", root) } rootDev, haveDev := devOf(rootInfo) seen := map[string]bool{} err = filepath.WalkDir(root, func(p string, d fs.DirEntry, werr error) error { if werr != nil { if p == root { return werr } return nil // an unreadable corner is skipped, not fatal } if d.Type()&fs.ModeSymlink != 0 { return nil } info, ierr := d.Info() if ierr != nil { return nil } if d.IsDir() { if haveDev && p != root { if dev, ok := devOf(info); ok && dev != rootDev { return filepath.SkipDir // another filesystem mounted inside: not the app's folder } } return nil } if !info.Mode().IsRegular() || info.Size() != 0 { return nil } m := re.FindStringSubmatch(d.Name()) if len(m) != 2 || !aubIDSafe.MatchString(m[1]) { return nil } seen[m[1]] = true return nil }) ids := make([]string, 0, len(seen)) for id := range seen { ids = append(ids, id) } sort.Strings(ids) return ids, err } func devOf(info fs.FileInfo) (uint64, bool) { if st, ok := info.Sys().(*syscall.Stat_t); ok { return uint64(st.Dev), true } return 0, false } // parseLost reads the lost-file check's `LOST {json}` lines. func parseLost(out string) []LostFile { var lost []LostFile for _, line := range strings.Split(out, "\n") { rest, ok := strings.CutPrefix(strings.TrimSpace(line), "LOST ") if !ok { continue } var f LostFile if json.Unmarshal([]byte(rest), &f) == nil && strings.TrimSpace(f.Name) != "" { f.Name = truncateStr(f.Name, 200) f.Taken = truncateStr(f.Taken, 40) lost = append(lost, f) } } return lost } // AfterUncleanBootApps lists the deployed apps whose template carries the block. func (m *Manager) AfterUncleanBootApps() []string { var out []string for _, st := range m.GetStacks() { if st.Deployed && st.Meta.AfterUncleanBoot != nil { out = append(out, st.Name) } } sort.Strings(out) return out } // RunAfterUncleanBoot runs one app's block once (internal/aftercrash calls it after an unclean boot). It records // the outcome in app.yaml and returns the counts. It never stops, restarts or changes the app. func (m *Manager) RunAfterUncleanBoot(name string, bootAt time.Time) AfterUncleanBootResult { t0 := time.Now() res := AfterUncleanBootResult{} st, ok := m.GetStack(name) if !ok || st.Meta.AfterUncleanBoot == nil { res.Skipped = "the app has no after_unclean_boot block" return res } spec := st.Meta.AfterUncleanBoot dir := filepath.Dir(st.ComposePath) var lost []LostFile defer func() { res.Duration = time.Since(t0) rec := &AfterUncleanBootRecord{ At: m.now().UTC().Format(time.RFC3339), Found: res.Found, Repaired: res.Repaired, StillBroken: res.StillBroken, Lost: lost, LostChecked: res.LostChecked, DurationMS: res.Duration.Milliseconds(), Skipped: res.Skipped, Problem: truncateStr(res.Problem, 300), } if !bootAt.IsZero() { rec.BootAt = bootAt.UTC().Format(time.RFC3339) } m.mutateAppConfig(name, dir, "after_unclean_boot", func(cfg *AppConfig) bool { cfg.AfterUncleanBoot = rec; return true }) }() if why := aubUnsupported(spec); why != "" { res.Skipped = why return res } cfg := LoadAppConfigDecrypted(dir, m.encKey) if cfg == nil { res.Skipped = "the app's settings could not be read" return res } if img := installedRef(cfg, spec.Service); !containsString(spec.Images, img) { if img == "" { img = "unknown" } res.Skipped = fmt.Sprintf("%s runs %s, which the template has not proven (%s)", spec.Service, img, strings.Join(spec.Images, ", ")) return res } root, err := aubUnder(spec.EmptyFiles.Under, cfg.Env) if err != nil { res.Skipped = err.Error() return res } re := regexp.MustCompile(spec.EmptyFiles.IDFromName) // The app must be up: it rebuilds its own files. deadline := time.Now().Add(aubHealthyWait) for { if m.aubRunning(name) { break } if time.Now().After(deadline) { res.Skipped = fmt.Sprintf("the app was not running and healthy within %s", aubHealthyWait) return res } m.aubSleep(5 * time.Second) } ids, err := ScanEmptyFiles(root, re) if err != nil { res.Problem = "the scan failed: " + err.Error() return res } res.Found = len(ids) if len(ids) > 0 { still := ids for attempt := 1; attempt <= 2 && len(still) > 0; attempt++ { // the repair, and at most ONE retry if perr := m.aubRepair(dir, spec, still); perr != nil { res.Problem = perr.Error() } after, serr := ScanEmptyFiles(root, re) if serr != nil { res.Problem = "the re-scan failed: " + serr.Error() break } still = intersect(still, after) } res.StillBroken = len(still) res.Repaired = res.Found - res.StillBroken } if spec.ReportLost != nil { out, lerr := m.aubExec(dir, spec.Service, spec.ReportLost.Command) switch { case lerr != nil || !strings.Contains(out, spec.ReportLost.Success): if res.Problem == "" { res.Problem = fmt.Sprintf("the lost-file check did not finish (err=%v)", lerr) } default: res.LostChecked = true lost = parseLost(out) res.Lost = len(lost) if len(lost) > aubMaxLostListed { lost = lost[:aubMaxLostListed] } } } return res } func (m *Manager) aubRepair(dir string, spec *AfterUncleanBootSpec, ids []string) error { batch := spec.Repair.Batch if batch <= 0 { batch = aubDefaultBatch } for i := 0; i < len(ids); i += batch { j := min(i+batch, len(ids)) cmd := append(append([]string{}, spec.Repair.Command...), ids[i:j]...) out, err := m.aubExec(dir, spec.Service, cmd) if err != nil || !strings.Contains(out, spec.Repair.Success) { return fmt.Errorf("the repair command did not finish for %d id(s) (err=%v)", j-i, err) } } return nil } // aubExec runs one command in the service's container. The output may carry the household's FILE NAMES (the lost-file // check prints them): it is returned to the caller only — never logged, and the error carries the exit code only. // That is why this is not afterLoadExec: compose's shared runner logs a failed command's stdout and puts its stderr in // the error, and the error text reaches the operator event (security review of 8d862e4). Tests replace afterLoadFn. // Pinned by TestAUB_AFailingCommandLeaksNoOutput. func (m *Manager) aubExec(dir, service string, cmd []string) (string, error) { args := append([]string{"exec", "-T", service}, cmd...) var out string var err error if m.afterLoadFn != nil { out, err = m.afterLoadFn(dir, args...) } else { out, err = m.quietComposeExec(dir, args...) } return out, exitOnly(err) } // quietComposeExec is composeExecCustomEnv without its logging of the command, its output and its stderr. func (m *Manager) quietComposeExec(dir string, args ...string) (string, error) { var cmd *exec.Cmd if m.composeCmd == "docker compose" { cmd = dockerexec.Command("docker", append([]string{"compose"}, args...)...) } else { cmd = dockerexec.Command("docker-compose", args...) } cmd.Dir = dir cmd.Env = m.stackEnv(dir) var stdout bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = io.Discard err := cmd.Run() return stdout.String(), err } // exitOnly keeps an error's exit code and drops any text it carries (stderr, output). func exitOnly(err error) error { if err == nil { return nil } var ee *exec.ExitError if errors.As(err, &ee) { return fmt.Errorf("exit code %d", ee.ExitCode()) } if code := regexp.MustCompile(`^exit code (-?\d+)`).FindStringSubmatch(err.Error()); code != nil { return fmt.Errorf("exit code %s", code[1]) } return errors.New("the command could not run") } // aubRunning: the app is up (its own containers running and healthy). Seam: aubRunningFn. func (m *Manager) aubRunning(name string) bool { if m.aubRunningFn != nil { return m.aubRunningFn(name) } _ = m.RefreshStatus() s, ok := m.GetStack(name) return ok && s.State == StateRunning } func (m *Manager) aubSleep(d time.Duration) { if m.aubSleepFn != nil { m.aubSleepFn(d) return } time.Sleep(d) } // DismissLostFiles hides the household's note (the record stays for the operator's counts). func (m *Manager) DismissLostFiles(name string) error { st, ok := m.GetStack(name) if !ok { return fmt.Errorf("stack %q not found", name) } dir := filepath.Dir(st.ComposePath) var had bool m.mutateAppConfig(name, dir, "after_unclean_boot_dismiss", func(cfg *AppConfig) bool { if cfg.AfterUncleanBoot == nil || len(cfg.AfterUncleanBoot.Lost) == 0 { return false } had = true cfg.AfterUncleanBoot.Dismissed = true return true }) if !had { return fmt.Errorf("no lost-file note for %s", name) } return nil } func installedRef(cfg *AppConfig, service string) string { if cfg == nil || cfg.InstalledImages == nil { return "" } return cfg.InstalledImages[service].Ref } func containsString(list []string, s string) bool { for _, x := range list { if x == s && s != "" { return true } } return false } func intersect(a, b []string) []string { in := map[string]bool{} for _, x := range b { in[x] = true } var out []string for _, x := range a { if in[x] { out = append(out, x) } } return out }