admin
3b85a0c29a
gates / gates (push) Successful in 50s
Security review of 8d862e4: compose's shared runner logs a failed command's stdout and puts its
stderr in the error, and that error became the event's `problem`. The after_unclean_boot commands now
run through a quiet runner (nothing logged, stderr discarded) and `problem` keeps the exit code only.
TestAUB_AFailingCommandLeaksNoOutput, red-proofed.
Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
485 lines
16 KiB
Go
485 lines
16 KiB
Go
package stacks
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
|
)
|
|
|
|
// ── after_unclean_boot (R-1007, `09` §3 decision 236): an app repairs the files a power cut left empty ──
|
|
//
|
|
// A power cut while photos upload leaves some of Immich's thumbnails and previews ZERO BYTES (ext4 had the name, not
|
|
// the data). Immich still lists the photos as done and never redoes them; rarely an original is gone too. The
|
|
// template declares, for its own app, where such files are, how the app is asked to rebuild THOSE (and only those),
|
|
// and how the app's own lost-file check is read:
|
|
//
|
|
// after_unclean_boot:
|
|
// service: immich-server # the commands run in this compose service
|
|
// images: ["ghcr.io/immich-app/immich-server:v3.2.4"] # the images it was PROVEN on; any other: skip
|
|
// empty_files:
|
|
// under: "${HDD_PATH}/appdata/immich/thumbs" # only ${HDD_PATH} is filled in
|
|
// id_from_name: "^([0-9a-f-]{36})_" # one capture group: the app's id for that file
|
|
// repair:
|
|
// command: ["node", "-e", "…"] # the ids are appended as arguments
|
|
// success: FELHOM_REPAIR_OK
|
|
// batch: 200
|
|
// report_lost: # optional
|
|
// command: ["node", "-e", "…"] # prints `LOST {"name":…,"taken":…}` lines
|
|
// success: FELHOM_LOST_OK
|
|
//
|
|
// The controller holds NO app knowledge: the catalog block is all of it. Ran ONLY once per controller start that
|
|
// followed an unclean host boot (internal/aftercrash decides when). Never deletes or rewrites anything itself; the
|
|
// app rebuilds its own files. A run that fails never stops the app. File names never reach a log line or an event:
|
|
// the lost list lives only in app.yaml on this box, for the household's note. Pinned by after_unclean_boot_test.go.
|
|
|
|
// AfterUncleanBootSpec is `.felhom.yml`'s `after_unclean_boot:`.
|
|
type AfterUncleanBootSpec struct {
|
|
Service string `yaml:"service" json:"service"`
|
|
Images []string `yaml:"images" json:"images"`
|
|
EmptyFiles AUBEmptyFiles `yaml:"empty_files" json:"empty_files"`
|
|
Repair AUBCommand `yaml:"repair" json:"repair"`
|
|
ReportLost *AUBCommand `yaml:"report_lost,omitempty" json:"report_lost,omitempty"`
|
|
Extra map[string]any `yaml:",inline" json:"-"` // a newer key this box does not know → skip
|
|
}
|
|
|
|
// AUBEmptyFiles says where the empty files are and how a file name maps to the app's id.
|
|
type AUBEmptyFiles struct {
|
|
Under string `yaml:"under" json:"under"`
|
|
IDFromName string `yaml:"id_from_name" json:"id_from_name"`
|
|
}
|
|
|
|
// AUBCommand is one command run in the service's container.
|
|
type AUBCommand struct {
|
|
Command []string `yaml:"command" json:"command"`
|
|
Success string `yaml:"success" json:"success"`
|
|
Batch int `yaml:"batch,omitempty" json:"batch,omitempty"`
|
|
}
|
|
|
|
// LostFile is one file the app's own check reports lost: the household's note shows it, nothing else does.
|
|
type LostFile struct {
|
|
Name string `yaml:"name" json:"name"`
|
|
Taken string `yaml:"taken,omitempty" json:"taken,omitempty"`
|
|
}
|
|
|
|
// AfterUncleanBootRecord is app.yaml's `after_unclean_boot:` — what the last run found and did.
|
|
type AfterUncleanBootRecord struct {
|
|
At string `yaml:"at" json:"at"`
|
|
BootAt string `yaml:"boot_at,omitempty" json:"boot_at,omitempty"`
|
|
Found int `yaml:"found" json:"found"`
|
|
Repaired int `yaml:"repaired" json:"repaired"`
|
|
StillBroken int `yaml:"still_broken" json:"still_broken"`
|
|
Lost []LostFile `yaml:"lost,omitempty" json:"lost,omitempty"`
|
|
LostChecked bool `yaml:"lost_checked" json:"lost_checked"`
|
|
DurationMS int64 `yaml:"duration_ms" json:"duration_ms"`
|
|
Skipped string `yaml:"skipped,omitempty" json:"skipped,omitempty"`
|
|
Problem string `yaml:"problem,omitempty" json:"problem,omitempty"`
|
|
Dismissed bool `yaml:"dismissed,omitempty" json:"dismissed,omitempty"`
|
|
}
|
|
|
|
// AfterUncleanBootResult is a run's outcome for the log and the operator event (counts only).
|
|
type AfterUncleanBootResult struct {
|
|
Found, Repaired, StillBroken, Lost int
|
|
LostChecked bool
|
|
Duration time.Duration
|
|
Skipped string // why nothing ran ("" = it ran)
|
|
Problem string // a step that failed (the counts say what was done)
|
|
}
|
|
|
|
var (
|
|
aubIDSafe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$`)
|
|
aubHealthyWait = 10 * time.Minute
|
|
aubDefaultBatch = 200
|
|
aubMaxLostListed = 200
|
|
)
|
|
|
|
// aubUnsupported names why this box cannot run the block, or "" when it can.
|
|
func aubUnsupported(s *AfterUncleanBootSpec) string {
|
|
if len(s.Extra) > 0 {
|
|
keys := make([]string, 0, len(s.Extra))
|
|
for k := range s.Extra {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
return "the template has keys this controller does not know: " + strings.Join(keys, ", ")
|
|
}
|
|
switch {
|
|
case s.Service == "" || len(s.Images) == 0:
|
|
return "the template names no service or no proven image"
|
|
case s.EmptyFiles.Under == "" || s.EmptyFiles.IDFromName == "":
|
|
return "the template names no folder or no id pattern"
|
|
case len(s.Repair.Command) == 0 || s.Repair.Success == "":
|
|
return "the template names no repair command"
|
|
case s.ReportLost != nil && (len(s.ReportLost.Command) == 0 || s.ReportLost.Success == ""):
|
|
return "the template's lost-file check is incomplete"
|
|
}
|
|
re, err := regexp.Compile(s.EmptyFiles.IDFromName)
|
|
if err != nil || re.NumSubexp() != 1 {
|
|
return "the template's id pattern is not one capture group"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// aubUnder fills ${HDD_PATH} (and nothing else) into the folder; an empty or relative result refuses.
|
|
func aubUnder(tmpl string, env map[string]string) (string, error) {
|
|
var bad []string
|
|
out := os.Expand(tmpl, func(k string) string {
|
|
if k != "HDD_PATH" || strings.TrimSpace(env["HDD_PATH"]) == "" {
|
|
bad = append(bad, k)
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(env["HDD_PATH"])
|
|
})
|
|
if len(bad) > 0 {
|
|
return "", fmt.Errorf("the folder needs %v, which this app does not have", bad)
|
|
}
|
|
out = filepath.Clean(out)
|
|
if !filepath.IsAbs(out) || out == "/" {
|
|
return "", fmt.Errorf("the folder %q is not an absolute app folder", out)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// ScanEmptyFiles walks root and returns the ids (de-duplicated, sorted) of the REGULAR, zero-byte files whose names
|
|
// match re. Symlinks are never followed nor counted; the walk stays on root's own filesystem; a name whose id is not
|
|
// plain id text is skipped. A missing root is no error (nothing to repair).
|
|
func ScanEmptyFiles(root string, re *regexp.Regexp) ([]string, error) {
|
|
rootInfo, err := os.Lstat(root)
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !rootInfo.IsDir() {
|
|
return nil, fmt.Errorf("%s is not a folder", root)
|
|
}
|
|
rootDev, haveDev := devOf(rootInfo)
|
|
seen := map[string]bool{}
|
|
err = filepath.WalkDir(root, func(p string, d fs.DirEntry, werr error) error {
|
|
if werr != nil {
|
|
if p == root {
|
|
return werr
|
|
}
|
|
return nil // an unreadable corner is skipped, not fatal
|
|
}
|
|
if d.Type()&fs.ModeSymlink != 0 {
|
|
return nil
|
|
}
|
|
info, ierr := d.Info()
|
|
if ierr != nil {
|
|
return nil
|
|
}
|
|
if d.IsDir() {
|
|
if haveDev && p != root {
|
|
if dev, ok := devOf(info); ok && dev != rootDev {
|
|
return filepath.SkipDir // another filesystem mounted inside: not the app's folder
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if !info.Mode().IsRegular() || info.Size() != 0 {
|
|
return nil
|
|
}
|
|
m := re.FindStringSubmatch(d.Name())
|
|
if len(m) != 2 || !aubIDSafe.MatchString(m[1]) {
|
|
return nil
|
|
}
|
|
seen[m[1]] = true
|
|
return nil
|
|
})
|
|
ids := make([]string, 0, len(seen))
|
|
for id := range seen {
|
|
ids = append(ids, id)
|
|
}
|
|
sort.Strings(ids)
|
|
return ids, err
|
|
}
|
|
|
|
func devOf(info fs.FileInfo) (uint64, bool) {
|
|
if st, ok := info.Sys().(*syscall.Stat_t); ok {
|
|
return uint64(st.Dev), true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// parseLost reads the lost-file check's `LOST {json}` lines.
|
|
func parseLost(out string) []LostFile {
|
|
var lost []LostFile
|
|
for _, line := range strings.Split(out, "\n") {
|
|
rest, ok := strings.CutPrefix(strings.TrimSpace(line), "LOST ")
|
|
if !ok {
|
|
continue
|
|
}
|
|
var f LostFile
|
|
if json.Unmarshal([]byte(rest), &f) == nil && strings.TrimSpace(f.Name) != "" {
|
|
f.Name = truncateStr(f.Name, 200)
|
|
f.Taken = truncateStr(f.Taken, 40)
|
|
lost = append(lost, f)
|
|
}
|
|
}
|
|
return lost
|
|
}
|
|
|
|
// AfterUncleanBootApps lists the deployed apps whose template carries the block.
|
|
func (m *Manager) AfterUncleanBootApps() []string {
|
|
var out []string
|
|
for _, st := range m.GetStacks() {
|
|
if st.Deployed && st.Meta.AfterUncleanBoot != nil {
|
|
out = append(out, st.Name)
|
|
}
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
// RunAfterUncleanBoot runs one app's block once (internal/aftercrash calls it after an unclean boot). It records
|
|
// the outcome in app.yaml and returns the counts. It never stops, restarts or changes the app.
|
|
func (m *Manager) RunAfterUncleanBoot(name string, bootAt time.Time) AfterUncleanBootResult {
|
|
t0 := time.Now()
|
|
res := AfterUncleanBootResult{}
|
|
st, ok := m.GetStack(name)
|
|
if !ok || st.Meta.AfterUncleanBoot == nil {
|
|
res.Skipped = "the app has no after_unclean_boot block"
|
|
return res
|
|
}
|
|
spec := st.Meta.AfterUncleanBoot
|
|
dir := filepath.Dir(st.ComposePath)
|
|
var lost []LostFile
|
|
defer func() {
|
|
res.Duration = time.Since(t0)
|
|
rec := &AfterUncleanBootRecord{
|
|
At: m.now().UTC().Format(time.RFC3339), Found: res.Found, Repaired: res.Repaired,
|
|
StillBroken: res.StillBroken, Lost: lost, LostChecked: res.LostChecked,
|
|
DurationMS: res.Duration.Milliseconds(), Skipped: res.Skipped, Problem: truncateStr(res.Problem, 300),
|
|
}
|
|
if !bootAt.IsZero() {
|
|
rec.BootAt = bootAt.UTC().Format(time.RFC3339)
|
|
}
|
|
m.mutateAppConfig(name, dir, "after_unclean_boot", func(cfg *AppConfig) bool { cfg.AfterUncleanBoot = rec; return true })
|
|
}()
|
|
|
|
if why := aubUnsupported(spec); why != "" {
|
|
res.Skipped = why
|
|
return res
|
|
}
|
|
cfg := LoadAppConfigDecrypted(dir, m.encKey)
|
|
if cfg == nil {
|
|
res.Skipped = "the app's settings could not be read"
|
|
return res
|
|
}
|
|
if img := installedRef(cfg, spec.Service); !containsString(spec.Images, img) {
|
|
if img == "" {
|
|
img = "unknown"
|
|
}
|
|
res.Skipped = fmt.Sprintf("%s runs %s, which the template has not proven (%s)", spec.Service, img, strings.Join(spec.Images, ", "))
|
|
return res
|
|
}
|
|
root, err := aubUnder(spec.EmptyFiles.Under, cfg.Env)
|
|
if err != nil {
|
|
res.Skipped = err.Error()
|
|
return res
|
|
}
|
|
re := regexp.MustCompile(spec.EmptyFiles.IDFromName)
|
|
|
|
// The app must be up: it rebuilds its own files.
|
|
deadline := time.Now().Add(aubHealthyWait)
|
|
for {
|
|
if m.aubRunning(name) {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
res.Skipped = fmt.Sprintf("the app was not running and healthy within %s", aubHealthyWait)
|
|
return res
|
|
}
|
|
m.aubSleep(5 * time.Second)
|
|
}
|
|
|
|
ids, err := ScanEmptyFiles(root, re)
|
|
if err != nil {
|
|
res.Problem = "the scan failed: " + err.Error()
|
|
return res
|
|
}
|
|
res.Found = len(ids)
|
|
if len(ids) > 0 {
|
|
still := ids
|
|
for attempt := 1; attempt <= 2 && len(still) > 0; attempt++ { // the repair, and at most ONE retry
|
|
if perr := m.aubRepair(dir, spec, still); perr != nil {
|
|
res.Problem = perr.Error()
|
|
}
|
|
after, serr := ScanEmptyFiles(root, re)
|
|
if serr != nil {
|
|
res.Problem = "the re-scan failed: " + serr.Error()
|
|
break
|
|
}
|
|
still = intersect(still, after)
|
|
}
|
|
res.StillBroken = len(still)
|
|
res.Repaired = res.Found - res.StillBroken
|
|
}
|
|
if spec.ReportLost != nil {
|
|
out, lerr := m.aubExec(dir, spec.Service, spec.ReportLost.Command)
|
|
switch {
|
|
case lerr != nil || !strings.Contains(out, spec.ReportLost.Success):
|
|
if res.Problem == "" {
|
|
res.Problem = fmt.Sprintf("the lost-file check did not finish (err=%v)", lerr)
|
|
}
|
|
default:
|
|
res.LostChecked = true
|
|
lost = parseLost(out)
|
|
res.Lost = len(lost)
|
|
if len(lost) > aubMaxLostListed {
|
|
lost = lost[:aubMaxLostListed]
|
|
}
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
func (m *Manager) aubRepair(dir string, spec *AfterUncleanBootSpec, ids []string) error {
|
|
batch := spec.Repair.Batch
|
|
if batch <= 0 {
|
|
batch = aubDefaultBatch
|
|
}
|
|
for i := 0; i < len(ids); i += batch {
|
|
j := min(i+batch, len(ids))
|
|
cmd := append(append([]string{}, spec.Repair.Command...), ids[i:j]...)
|
|
out, err := m.aubExec(dir, spec.Service, cmd)
|
|
if err != nil || !strings.Contains(out, spec.Repair.Success) {
|
|
return fmt.Errorf("the repair command did not finish for %d id(s) (err=%v)", j-i, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// aubExec runs one command in the service's container. The output may carry the household's FILE NAMES (the lost-file
|
|
// check prints them): it is returned to the caller only — never logged, and the error carries the exit code only.
|
|
// That is why this is not afterLoadExec: compose's shared runner logs a failed command's stdout and puts its stderr in
|
|
// the error, and the error text reaches the operator event (security review of 8d862e4). Tests replace afterLoadFn.
|
|
// Pinned by TestAUB_AFailingCommandLeaksNoOutput.
|
|
func (m *Manager) aubExec(dir, service string, cmd []string) (string, error) {
|
|
args := append([]string{"exec", "-T", service}, cmd...)
|
|
var out string
|
|
var err error
|
|
if m.afterLoadFn != nil {
|
|
out, err = m.afterLoadFn(dir, args...)
|
|
} else {
|
|
out, err = m.quietComposeExec(dir, args...)
|
|
}
|
|
return out, exitOnly(err)
|
|
}
|
|
|
|
// quietComposeExec is composeExecCustomEnv without its logging of the command, its output and its stderr.
|
|
func (m *Manager) quietComposeExec(dir string, args ...string) (string, error) {
|
|
var cmd *exec.Cmd
|
|
if m.composeCmd == "docker compose" {
|
|
cmd = dockerexec.Command("docker", append([]string{"compose"}, args...)...)
|
|
} else {
|
|
cmd = dockerexec.Command("docker-compose", args...)
|
|
}
|
|
cmd.Dir = dir
|
|
cmd.Env = m.stackEnv(dir)
|
|
var stdout bytes.Buffer
|
|
cmd.Stdout = &stdout
|
|
cmd.Stderr = io.Discard
|
|
err := cmd.Run()
|
|
return stdout.String(), err
|
|
}
|
|
|
|
// exitOnly keeps an error's exit code and drops any text it carries (stderr, output).
|
|
func exitOnly(err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
var ee *exec.ExitError
|
|
if errors.As(err, &ee) {
|
|
return fmt.Errorf("exit code %d", ee.ExitCode())
|
|
}
|
|
if code := regexp.MustCompile(`^exit code (-?\d+)`).FindStringSubmatch(err.Error()); code != nil {
|
|
return fmt.Errorf("exit code %s", code[1])
|
|
}
|
|
return errors.New("the command could not run")
|
|
}
|
|
|
|
// aubRunning: the app is up (its own containers running and healthy). Seam: aubRunningFn.
|
|
func (m *Manager) aubRunning(name string) bool {
|
|
if m.aubRunningFn != nil {
|
|
return m.aubRunningFn(name)
|
|
}
|
|
_ = m.RefreshStatus()
|
|
s, ok := m.GetStack(name)
|
|
return ok && s.State == StateRunning
|
|
}
|
|
|
|
func (m *Manager) aubSleep(d time.Duration) {
|
|
if m.aubSleepFn != nil {
|
|
m.aubSleepFn(d)
|
|
return
|
|
}
|
|
time.Sleep(d)
|
|
}
|
|
|
|
// DismissLostFiles hides the household's note (the record stays for the operator's counts).
|
|
func (m *Manager) DismissLostFiles(name string) error {
|
|
st, ok := m.GetStack(name)
|
|
if !ok {
|
|
return fmt.Errorf("stack %q not found", name)
|
|
}
|
|
dir := filepath.Dir(st.ComposePath)
|
|
var had bool
|
|
m.mutateAppConfig(name, dir, "after_unclean_boot_dismiss", func(cfg *AppConfig) bool {
|
|
if cfg.AfterUncleanBoot == nil || len(cfg.AfterUncleanBoot.Lost) == 0 {
|
|
return false
|
|
}
|
|
had = true
|
|
cfg.AfterUncleanBoot.Dismissed = true
|
|
return true
|
|
})
|
|
if !had {
|
|
return fmt.Errorf("no lost-file note for %s", name)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func installedRef(cfg *AppConfig, service string) string {
|
|
if cfg == nil || cfg.InstalledImages == nil {
|
|
return ""
|
|
}
|
|
return cfg.InstalledImages[service].Ref
|
|
}
|
|
|
|
func containsString(list []string, s string) bool {
|
|
for _, x := range list {
|
|
if x == s && s != "" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func intersect(a, b []string) []string {
|
|
in := map[string]bool{}
|
|
for _, x := range b {
|
|
in[x] = true
|
|
}
|
|
var out []string
|
|
for _, x := range a {
|
|
if in[x] {
|
|
out = append(out, x)
|
|
}
|
|
}
|
|
return out
|
|
}
|