v0.243.0: FileBrowser generated admin password (R-513); per-tier whole-guest backup truth (R-517); skip absent-storage tiers (R-518); OOM-killed worker visible (R-514)

MinAgent: 0.131.0

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/stacks/filebrowser_password.go

@@ -0,0 +1,166 @@
+package stacks
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// R-513 — FileBrowser's admin password.
+//
+// MEASURED FIRST (2026-09-15, gtstef/filebrowser:1.3.3-stable, evidence-p1fixes-2026-09-15/B1):
+//   - with no `auth.adminPassword` key and no FILEBROWSER_ADMIN_PASSWORD env — the controller's
+//     render — a new database accepts admin/admin;
+//   - the config key or the env var DOES set the password, on a fresh AND on an existing database —
+//     but it RE-APPLIES ON EVERY START: a password changed by hand afterwards is overwritten at the
+//     next restart;
+//   - the API changes it once and it sticks: `PUT /api/users?id=<id>` with
+//     `{"which":["password"],"data":{"id":<id>,"username":"admin","password":<new>}}`, the session
+//     token, and `X-Password: <current>` → 204.
+//
+// THE DECISION (one mechanism for fresh and existing boxes): the API path, never the config key. The
+// key would silently undo the password the operator set by hand on the HP and the N100 (2026-09-15)
+// and any a household sets later. Cost: on a brand-new box admin/admin works from FileBrowser's first
+// start until the next base-stack tick sets the password (seconds; before a claim there is no tunnel).
+//
+// The probe: login admin/admin → 200 ⇒ generate (password:16), PUT, verify new=200 AND admin=401, then
+// record "generated" with the encrypted value; 401 ⇒ record "operator" (somebody set it — leave it).
+// Anything else (container starting, network) ⇒ record nothing and try again next tick.
+
+const fileBrowserBaseURL = "http://filebrowser:80"
+
+// fbHTTPDo is the network seam (tests inject a fake FileBrowser).
+type fbHTTPDo func(req *http.Request) (*http.Response, error)
+
+func (m *Manager) fbDo() fbHTTPDo {
+	if m.fbHTTP != nil {
+		return m.fbHTTP
+	}
+	c := &http.Client{Timeout: 10 * time.Second}
+	return c.Do
+}
+
+// fbLogin returns (token, status, err). status 200 → token set; 401 → wrong password.
+func fbLogin(do fbHTTPDo, base, password string) (string, int, error) {
+	req, _ := http.NewRequest(http.MethodPost, base+"/api/auth/login?username=admin", nil)
+	req.Header.Set("X-Password", password)
+	resp, err := do(req)
+	if err != nil {
+		return "", 0, err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
+	if resp.StatusCode != http.StatusOK {
+		return "", resp.StatusCode, nil
+	}
+	return strings.Trim(strings.TrimSpace(string(b)), `"`), resp.StatusCode, nil
+}
+
+// EnsureFileBrowserAdminPassword makes the one-time decision. Safe to call every tick: it returns at
+// once when a decision is recorded. Returns an error only for logging.
+func (m *Manager) EnsureFileBrowserAdminPassword() error {
+	if m.settings == nil || len(m.encKey) == 0 {
+		return nil
+	}
+	if state, _, _ := m.settings.GetFileBrowserAdmin(); state != "" {
+		return nil
+	}
+	do := m.fbDo()
+	base := fileBrowserBaseURL
+	if m.fbBaseURL != "" {
+		base = m.fbBaseURL
+	}
+	now := time.Now().UTC().Format(time.RFC3339)
+
+	token, code, err := fbLogin(do, base, "admin")
+	if err != nil {
+		return fmt.Errorf("filebrowser admin probe: %w (will retry)", err)
+	}
+	switch code {
+	case http.StatusOK:
+		// default login still works — set a generated password below
+	case http.StatusUnauthorized, http.StatusForbidden:
+		m.logger.Printf("[INFO] [infra] filebrowser: admin/admin is refused (HTTP %d) — the password was set by someone; leaving it and recording \"operator\"", code)
+		return m.settings.SetFileBrowserAdmin(settings.FileBrowserAdminOperator, "", now)
+	default:
+		return fmt.Errorf("filebrowser admin probe: HTTP %d (will retry)", code)
+	}
+
+	id, err := fbSelfID(do, base, token)
+	if err != nil {
+		return fmt.Errorf("filebrowser: read admin user id: %w (will retry)", err)
+	}
+	pw, err := generateValue("password:16")
+	if err != nil {
+		return fmt.Errorf("filebrowser: generate password: %w", err)
+	}
+	body, _ := json.Marshal(map[string]any{
+		"which": []string{"password"},
+		"data":  map[string]any{"id": id, "username": "admin", "password": pw},
+	})
+	req, _ := http.NewRequest(http.MethodPut, fmt.Sprintf("%s/api/users?id=%d", base, id), bytes.NewReader(body))
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("X-Auth", token)
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("X-Password", "admin")
+	resp, err := do(req)
+	if err != nil {
+		return fmt.Errorf("filebrowser: set password: %w (will retry)", err)
+	}
+	io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
+	resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("filebrowser: set password: HTTP %d (will retry)", resp.StatusCode)
+	}
+	// Verify the consequence, both directions, before recording anything.
+	if _, c, err := fbLogin(do, base, pw); err != nil || c != http.StatusOK {
+		return fmt.Errorf("filebrowser: new password does not log in (HTTP %d, err %v) — NOT recorded, will retry", c, err)
+	}
+	if _, c, err := fbLogin(do, base, "admin"); err != nil || c == http.StatusOK {
+		return fmt.Errorf("filebrowser: admin/admin still logs in after the change (HTTP %d, err %v) — NOT recorded", c, err)
+	}
+	enc, err := crypto.Encrypt(m.encKey, pw)
+	if err != nil {
+		return fmt.Errorf("filebrowser: encrypt password: %w", err)
+	}
+	if err := m.settings.SetFileBrowserAdmin(settings.FileBrowserAdminGenerated, enc, now); err != nil {
+		// The password IS changed and we could not record it: say so loudly — the household cannot
+		// be shown a password that is not stored. Recoverable by the operator (FileBrowser CLI).
+		m.logger.Printf("[ERROR] [infra] filebrowser: password CHANGED but settings save FAILED: %v — the generated password is lost; reset it with the filebrowser CLI", err)
+		return err
+	}
+	m.logger.Printf("[INFO] [infra] filebrowser: admin/admin replaced by a generated password (value never logged); verified new=200 admin=401")
+	return nil
+}
+
+// fbSelfID reads the logged-in user's id (GET /api/users?id=self).
+func fbSelfID(do fbHTTPDo, base, token string) (int, error) {
+	req, _ := http.NewRequest(http.MethodGet, base+"/api/users?id=self", nil)
+	req.Header.Set("X-Auth", token)
+	req.Header.Set("Authorization", "Bearer "+token)
+	resp, err := do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("HTTP %d", resp.StatusCode)
+	}
+	var u struct {
+		ID int `json:"id"`
+	}
+	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&u); err != nil {
+		return 0, err
+	}
+	if u.ID <= 0 {
+		return 0, fmt.Errorf("no user id in response")
+	}
+	return u.ID, nil
+}

controller/internal/stacks/filebrowser_password_test.go

@@ -0,0 +1,147 @@
+package stacks
+
+import (
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/crypto"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// R-513 — FileBrowser accepted admin/admin on every box. The fake below behaves like the measured
+// Quantum 1.3.3 API (evidence-p1fixes-2026-09-15/B1): login by X-Password, PUT /api/users with the
+// current password in X-Password.
+
+type fakeFB struct {
+	mu       sync.Mutex
+	password string
+	puts     int
+}
+
+func (f *fakeFB) handler() http.Handler {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/api/auth/login", func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if r.Header.Get("X-Password") != f.password {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		io.WriteString(w, "tok-123")
+	})
+	mux.HandleFunc("/api/users", func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		if r.Header.Get("X-Auth") != "tok-123" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		switch r.Method {
+		case http.MethodGet:
+			io.WriteString(w, `{"id":1,"username":"admin"}`)
+		case http.MethodPut:
+			if r.Header.Get("X-Password") != f.password {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+			var body struct {
+				Which []string `json:"which"`
+				Data  struct {
+					Password string `json:"password"`
+				} `json:"data"`
+			}
+			_ = json.NewDecoder(r.Body).Decode(&body)
+			f.password = body.Data.Password
+			f.puts++
+			w.WriteHeader(http.StatusNoContent)
+		}
+	})
+	return mux
+}
+
+func fbManager(t *testing.T, base string) (*Manager, *settings.Settings, []byte) {
+	t.Helper()
+	st, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	key := make([]byte, 32)
+	for i := range key {
+		key[i] = byte(i + 1)
+	}
+	return &Manager{logger: log.New(io.Discard, "", 0), settings: st, encKey: key, fbBaseURL: base}, st, key
+}
+
+// The consequence: after one tick admin/admin no longer logs in, the stored (decrypted) password
+// does, and the state is "generated". A second tick changes nothing.
+//
+// RED-PROOF (run 2026-09-15, recorded in REPORT.md): with EnsureFileBrowserAdminPassword returning
+// nil before the PUT, admin/admin stayed valid and this failed at "admin/admin still logs in".
+func TestFileBrowserAdmin_DefaultLoginReplaced(t *testing.T) {
+	fb := &fakeFB{password: "admin"}
+	srv := httptest.NewServer(fb.handler())
+	defer srv.Close()
+	m, st, key := fbManager(t, srv.URL)
+
+	if err := m.EnsureFileBrowserAdminPassword(); err != nil {
+		t.Fatalf("ensure: %v", err)
+	}
+	if _, c, _ := fbLogin(srv.Client().Do, srv.URL, "admin"); c == http.StatusOK {
+		t.Fatalf("admin/admin still logs in — R-513 not fixed")
+	}
+	state, enc, at := st.GetFileBrowserAdmin()
+	if state != settings.FileBrowserAdminGenerated || enc == "" || at == "" {
+		t.Fatalf("decision not recorded: state=%q enc=%v at=%q", state, enc != "", at)
+	}
+	if enc == fb.password {
+		t.Fatal("the password was stored in plaintext")
+	}
+	pw, err := crypto.Decrypt(key, enc)
+	if err != nil || len(pw) != 16 {
+		t.Fatalf("stored password does not decrypt to a 16-char value (len=%d err=%v)", len(pw), err)
+	}
+	if _, c, _ := fbLogin(srv.Client().Do, srv.URL, pw); c != http.StatusOK {
+		t.Fatalf("the stored password does not log in (HTTP %d)", c)
+	}
+	_ = m.EnsureFileBrowserAdminPassword()
+	if fb.puts != 1 {
+		t.Fatalf("a recorded decision was acted on again (puts=%d)", fb.puts)
+	}
+}
+
+// A password set by hand (admin/admin refused) is NEVER overwritten.
+func TestFileBrowserAdmin_HandSetPasswordLeftAlone(t *testing.T) {
+	fb := &fakeFB{password: "operator-set-by-hand"}
+	srv := httptest.NewServer(fb.handler())
+	defer srv.Close()
+	m, st, _ := fbManager(t, srv.URL)
+	if err := m.EnsureFileBrowserAdminPassword(); err != nil {
+		t.Fatal(err)
+	}
+	if fb.puts != 0 || fb.password != "operator-set-by-hand" {
+		t.Fatalf("hand-set password was changed (puts=%d)", fb.puts)
+	}
+	if state, enc, _ := st.GetFileBrowserAdmin(); state != settings.FileBrowserAdminOperator || enc != "" {
+		t.Fatalf("want state operator with no stored value, got %q enc=%v", state, enc != "")
+	}
+}
+
+// FileBrowser not reachable → nothing recorded, so the next tick tries again.
+func TestFileBrowserAdmin_UnreachableRecordsNothing(t *testing.T) {
+	srv := httptest.NewServer(http.NotFoundHandler())
+	url := srv.URL
+	srv.Close()
+	m, st, _ := fbManager(t, url)
+	if err := m.EnsureFileBrowserAdminPassword(); err == nil {
+		t.Fatal("want an error (for the log) when FileBrowser is unreachable")
+	}
+	if state, _, _ := st.GetFileBrowserAdmin(); state != "" {
+		t.Fatalf("an unreachable FileBrowser recorded a decision: %q", state)
+	}
+}

controller/internal/stacks/infra.go

@@ -69,6 +69,9 @@ func (m *Manager) EnsureBaseStack() error {
 
 	if err := m.ensureFileBrowser(filepath.Join(base, "filebrowser")); err != nil {
 		errs = append(errs, fmt.Sprintf("filebrowser: %v", err))
+	} else if err := m.EnsureFileBrowserAdminPassword(); err != nil {
+		// R-513: one-time; retried every tick until decided. Logged, never fatal to the base stack.
+		m.logger.Printf("[WARN] [infra] %v", err)
 	}
 
 	// samba (LAN network-sharing, R-7) — conditional deploy, same shape as cloudflared: only when the

controller/internal/stacks/manager.go

@@ -211,6 +211,12 @@ type Manager struct {
 	// answering from a stale entry; pruned every refresh to the live container set. Guarded by mu
 	// (every read/write happens under refreshStatusLocked's write lock).
 	restartPolicyCache map[string]string
+	// R-514: last OOM scan (oom.go), for the dashboard.
+	oomMu    sync.Mutex
+	oomCache map[string][]string
+	// R-513: FileBrowser admin-password seams (filebrowser_password.go); nil/"" in production.
+	fbHTTP    fbHTTPDo
+	fbBaseURL string
 	// execFn replaces execCommand's process boundary in tests; nil in production.
 	execFn func(name string, args ...string) (string, error)
 

controller/internal/stacks/oom.go

@@ -0,0 +1,85 @@
+package stacks
+
+import (
+	"sort"
+	"strings"
+)
+
+// R-514 (v0.243.0) — an OOM-killed worker inside a RUNNING container is invisible to the state model.
+//
+// BIGNIGHT: Paperless's celery worker was killed by the memory cgroup inside the webserver container;
+// the container kept running (`docker inspect` → OOMKilled=true, RestartCount=0), the app read „Fut",
+// 11 documents failed and 8 waited forever. Docker's State.OOMKilled is set when ANY process in the
+// container's cgroup was OOM-killed and stays set until the container restarts — so it is the
+// observable, read for the running containers of deployed stacks.
+
+// OOMContainer is one container whose cgroup had a process OOM-killed since it started.
+type OOMContainer struct {
+	Stack     string
+	Container string
+	StartedAt string // identifies the container run — one event per run
+}
+
+// ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
+// returns those with State.OOMKilled=true. It also caches the per-stack result for the dashboard.
+func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
+	m.mu.RLock()
+	owner := map[string]string{}
+	var names []string
+	for name, st := range m.stacks {
+		if !st.Deployed {
+			continue
+		}
+		for _, c := range st.Containers {
+			if c.State == StateRunning || c.State == StateUnhealthy || c.State == StateRestarting {
+				owner[c.Name] = name
+				names = append(names, c.Name)
+			}
+		}
+	}
+	m.mu.RUnlock()
+	if len(names) == 0 {
+		m.setOOMCache(nil)
+		return nil, nil
+	}
+	sort.Strings(names)
+	args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
+	out, err := m.execCommand("docker", args...)
+	if err != nil && strings.TrimSpace(out) == "" {
+		return nil, err
+	}
+	var found []OOMContainer
+	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+		f := strings.SplitN(strings.TrimSpace(line), "|", 3)
+		if len(f) != 3 || f[1] != "true" {
+			continue
+		}
+		cname := strings.TrimPrefix(f[0], "/")
+		if st, ok := owner[cname]; ok {
+			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2]})
+		}
+	}
+	m.setOOMCache(found)
+	return found, nil
+}
+
+func (m *Manager) setOOMCache(found []OOMContainer) {
+	cache := map[string][]string{}
+	for _, o := range found {
+		cache[o.Stack] = append(cache[o.Stack], o.Container)
+	}
+	m.oomMu.Lock()
+	m.oomCache = cache
+	m.oomMu.Unlock()
+}
+
+// OOMKilledStacks returns stack name → OOM-killed container names from the last scan.
+func (m *Manager) OOMKilledStacks() map[string][]string {
+	m.oomMu.Lock()
+	defer m.oomMu.Unlock()
+	out := make(map[string][]string, len(m.oomCache))
+	for k, v := range m.oomCache {
+		out[k] = append([]string(nil), v...)
+	}
+	return out
+}

controller/internal/stacks/oom_test.go

@@ -0,0 +1,55 @@
+package stacks
+
+import (
+	"io"
+	"log"
+	"strings"
+	"testing"
+)
+
+// R-514 — a worker OOM-killed inside a RUNNING container must be seen. BIGNIGHT: paperless-webserver
+// `oomkilled=true restarts=0`, app „Fut", no event.
+//
+// RED-PROOF (run 2026-09-15, recorded in REPORT.md): with the `f[1] != "true"` filter inverted to
+// skip every "true" line, the scan returned nothing and this failed at "OOM-killed worker not seen".
+func TestScanOOMKilled_SeesKilledWorkerInRunningContainer(t *testing.T) {
+	var gotArgs []string
+	m := &Manager{
+		logger: log.New(io.Discard, "", 0),
+		stacks: map[string]*Stack{
+			"paperless-ngx": {Name: "paperless-ngx", Deployed: true, Containers: []ContainerInfo{
+				{Name: "paperless-webserver", State: StateRunning},
+				{Name: "paperless-redis", State: StateRunning},
+			}},
+			"bookstack":  {Name: "bookstack", Deployed: true, Containers: []ContainerInfo{{Name: "bookstack", State: StateRunning}}},
+			"stopped":    {Name: "stopped", Deployed: true, Containers: []ContainerInfo{{Name: "stopped-c", State: StateExited}}},
+			"undeployed": {Name: "undeployed", Deployed: false, Containers: []ContainerInfo{{Name: "u", State: StateRunning}}},
+		},
+	}
+	m.execFn = func(name string, args ...string) (string, error) {
+		gotArgs = args
+		return "/bookstack|false|2026-09-14T18:00:00Z\n/paperless-redis|false|2026-09-14T18:00:00Z\n/paperless-webserver|true|2026-09-14T18:00:01Z\n", nil
+	}
+	ooms, err := m.ScanOOMKilled()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ooms) != 1 || ooms[0].Stack != "paperless-ngx" || ooms[0].Container != "paperless-webserver" || ooms[0].StartedAt == "" {
+		t.Fatalf("OOM-killed worker not seen: %+v", ooms)
+	}
+	joined := strings.Join(gotArgs, " ")
+	if strings.Contains(joined, "stopped-c") || strings.Contains(joined, " u") {
+		t.Fatalf("inspected a stopped or undeployed container: %v", gotArgs)
+	}
+	if got := m.OOMKilledStacks(); len(got["paperless-ngx"]) != 1 || len(got) != 1 {
+		t.Fatalf("dashboard cache wrong: %v", got)
+	}
+	// Next scan clean → cache cleared (a restarted container resets OOMKilled).
+	m.execFn = func(string, ...string) (string, error) {
+		return "/bookstack|false|x\n/paperless-redis|false|x\n/paperless-webserver|false|y\n", nil
+	}
+	_, _ = m.ScanOOMKilled()
+	if got := m.OOMKilledStacks(); len(got) != 0 {
+		t.Fatalf("cache not cleared after a clean scan: %v", got)
+	}
+}