fill watch: one warning per filesystem (R-1371) — committed, not released

On a box whose guest has one data volume the two built-in fill targets, "/"
(the controller's overlay on the Docker data volume) and /mnt/sys_drive, are one
filesystem. The target list was de-duplicated by path only, so one fill would
send two warnings under two names.

fillTargets now keeps one target per filesystem; the first stays (app area,
system area, drives). The identity is system.FilesystemKey, not st_dev alone:
inside the container "/" is an overlay with its own device number, so an
overlay path takes the device of a Docker-kept file on the data-root
(/etc/hostname, /etc/hosts, /etc/resolv.conf). Measured read-only on demo-hp
9201/9202 and the N100 on 2026-11-10.

Tests: TestR1371_* (3), TestFilesystemKey_* (2). Red seen three times.
No version, no image, no floor: it waits for a release session.

Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/README.md

@@ -2220,6 +2220,10 @@ that folder is never a dead end, and an install never runs into it silently (R-6
 - **Where it lives.** `<drive>/kept/` is beside `appdata/` and `userdata/`, inside neither: no app bind, FileBrowser
   userdata source, Samba share or backup leg reads it. It is in `ProtectedHDDPaths`.
 - **The drive-full warning** ends with the kept folders on that drive and their sizes (`fillwatch.SetExtra`).
+- **One fill warning per filesystem (R-1371; committed 2026-11-10, not released).** The watch list (`fillTargets`) holds the
+  app area ("/"), the system area (`/mnt/sys_drive`) and each drive, and keeps ONE target per filesystem — the first, in that
+  order. On a box whose guest has one data volume the two areas are one filesystem, so the warning carries the app area's
+  name. The identity is `system.FilesystemKey`, not the device number: the container's "/" is an overlay.
 - **The fill warning in the household's language (v0.309.0, R-521 F7).** `disk_warning` / `disk_critical` carry the
   Hungarian sentence AND the household's own (`message_customer`), from the bundle keys `event.fill.*`; the two built-in
   areas („Alkalmazások területe" = the guest's root filesystem as the controller sees it, „Rendszer- és mentési terület")

controller/cmd/controller/main.go

@@ -2702,11 +2702,18 @@ const fillWatchInterval = 10 * time.Minute
 //     silent exhaustion R-158 measured;
 //   - the registered DRIVES hold the customer's own data.
 //
-// De-duplicated by path: on a box where a drive is not a separate mount these collapse, and warning
-// twice about one filesystem is exactly the noise the per-filesystem rule exists to prevent. A
-// DECOMMISSIONED drive is dropped — it is deliberately out of service, and its fill is not news. A
-// DISCONNECTED one is kept in the list but will read as unreadable and be skipped by §8.4, which is
-// the correct outcome: its absence already has its own alert.
+// ONE WARNING PER FILESYSTEM (R-1371). The list is de-duplicated by path AND by filesystem
+// (fillTargetsBy). By path alone it was not enough: since the guest's two data volumes became one, "/"
+// (the controller's overlay, which lies on the Docker data volume) and `/mnt/sys_drive` are ONE
+// filesystem on every box measured 2026-11-10, so one fill sent two warnings with two names. The
+// identity is system.FilesystemKey — NOT the plain device number, which an overlay hides (fskey.go).
+// The FIRST target of a filesystem stays, in the order below: the app area, the system area, then the
+// drives. A path whose filesystem cannot be identified is kept (one warning too many, never one too
+// few). Pinned by TestR1371_OneFillSendsOneWarning.
+//
+// A DECOMMISSIONED drive is dropped — it is deliberately out of service, and its fill is not news. A
+// DISCONNECTED one has no readable filesystem, so it is kept in the list, reads as unreadable and is
+// skipped by §8.4, which is the correct outcome: its absence already has its own alert.
 // fillEventPusher is the one notifier method fillEventPush needs — an interface so the R-521 F7 test
 // can record what is pushed without a hub.
 type fillEventPusher interface {
@@ -2732,13 +2739,26 @@ func fillEventPush(n fillEventPusher) func(fillwatch.Event) {
 }
 
 func fillTargets(cfg *config.Config, sett *settings.Settings) []fillwatch.Target {
+	return fillTargetsBy(cfg, sett, system.FilesystemKey)
+}
+
+// fillTargetsBy is fillTargets with the filesystem identity injected (the tests' seam). fsKey returns
+// ok=false for a path whose filesystem it cannot identify.
+func fillTargetsBy(cfg *config.Config, sett *settings.Settings, fsKey func(path string) (uint64, bool)) []fillwatch.Target {
 	seen := make(map[string]bool)
+	seenFS := make(map[uint64]bool)
 	var out []fillwatch.Target
 	add := func(path, label, labelKey string) {
 		if path == "" || seen[path] {
 			return
 		}
 		seen[path] = true
+		if key, ok := fsKey(path); ok {
+			if seenFS[key] {
+				return // this filesystem is watched already, under the target that came first
+			}
+			seenFS[key] = true
+		}
 		out = append(out, fillwatch.Target{Path: path, Label: label, LabelKey: labelKey})
 	}
 

controller/cmd/controller/r1371_one_warning_per_filesystem_test.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/system"
+)
+
+// R-1371 — one warning per filesystem.
+//
+// The box as measured 2026-11-10 (demo-hp, the N100, Tester 1): "/" and `/mnt/sys_drive` are ONE filesystem, seen
+// through two paths. The filesystem identity here is the one system.FilesystemKey gives on those boxes (its own rule
+// is pinned by TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot): equal for the two paths, another for a drive.
+//
+// RED-PROOF (observed 2026-11-10, then restored): with the filesystem check removed from fillTargetsBy this test fails
+// with "one filling filesystem sent 2 warnings".
+func boxKeys(extra map[string]uint64) func(string) (uint64, bool) {
+	keys := map[string]uint64{system.DockerVolumePath: 64519, "/mnt/sys_drive": 64519}
+	for p, k := range extra {
+		keys[p] = k
+	}
+	return func(p string) (uint64, bool) { k, ok := keys[p]; return k, ok }
+}
+
+func settingsWithDrives(t *testing.T, paths ...settings.StoragePath) *settings.Settings {
+	t.Helper()
+	sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, sp := range paths {
+		if err := sett.AddStoragePath(sp); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return sett
+}
+
+// The consequence: the one filesystem fills, and the household gets ONE warning, then ONE critical — through the real
+// watcher and the real push closure.
+func TestR1371_OneFillSendsOneWarning(t *testing.T) {
+	cfg := &config.Config{}
+	cfg.Paths.SystemDataPath = "/mnt/sys_drive"
+	sett := settingsWithDrives(t, settings.StoragePath{Path: "/mnt/felhom-drives/usb", Label: "Családi lemez", Schedulable: true})
+	key := boxKeys(map[string]uint64{"/mnt/felhom-drives/usb": 2049})
+
+	pct := 50.0
+	usage := func(p string) *fillwatch.Usage {
+		if p == "/mnt/felhom-drives/usb" {
+			return &fillwatch.Usage{UsedPercent: 20, AvailGB: 700, UsedGB: 200, TotalGB: 900}
+		}
+		// "/" and /mnt/sys_drive: the same filesystem, so the same numbers.
+		return &fillwatch.Usage{UsedPercent: pct, AvailGB: 68.7 * (100 - pct) / 100, UsedGB: 68.7 * pct / 100, TotalGB: 68.7}
+	}
+	w := fillwatch.New(filepath.Join(t.TempDir(), "fill.json"), log.New(io.Discard, "", 0),
+		func() []fillwatch.Target { return fillTargetsBy(cfg, sett, key) }, usage)
+	rec := &fakeFillPusher{}
+	w.SetNotify(fillEventPush(rec))
+
+	check := func() {
+		t.Helper()
+		for i := 0; i < 2; i++ {
+			if err := w.Check(); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+	check()
+	if len(rec.got) != 0 {
+		t.Fatalf("a half-full filesystem sent %d events", len(rec.got))
+	}
+	pct = 90
+	check()
+	if len(rec.got) != 1 {
+		t.Fatalf("one filling filesystem sent %d warnings (%+v), want exactly 1", len(rec.got), rec.got)
+	}
+	if rec.got[0].eventType != "disk_warning" {
+		t.Fatalf("event = %s, want disk_warning", rec.got[0].eventType)
+	}
+	if d, _ := rec.got[0].details.(map[string]any); d == nil || d["path"] != system.DockerVolumePath {
+		t.Fatalf("the warning is for %v, want the first target of the filesystem (%q)", rec.got[0].details, system.DockerVolumePath)
+	}
+	pct = 97
+	check()
+	if len(rec.got) != 2 || rec.got[1].eventType != "disk_critical" {
+		t.Fatalf("after 97%% the events are %+v, want one warning then one critical", rec.got)
+	}
+}
+
+// What stays and what goes, target by target.
+func TestR1371_TargetsByFilesystem(t *testing.T) {
+	cfg := &config.Config{}
+	cfg.Paths.SystemDataPath = "/mnt/sys_drive"
+	paths := func(ts []fillwatch.Target) []string {
+		var out []string
+		for _, tg := range ts {
+			out = append(out, tg.Path)
+		}
+		return out
+	}
+	eq := func(got, want []string) bool {
+		if len(got) != len(want) {
+			return false
+		}
+		for i := range got {
+			if got[i] != want[i] {
+				return false
+			}
+		}
+		return true
+	}
+
+	// One volume, no drive: one target, and it is the app area (the first one added).
+	got := fillTargetsBy(cfg, nil, boxKeys(nil))
+	if len(got) != 1 || got[0].Path != system.DockerVolumePath || got[0].LabelKey != "event.fill.label_apps" {
+		t.Fatalf("one-volume box: targets = %+v, want only the app area", got)
+	}
+
+	// The OLD layout (two volumes): both areas stay.
+	two := func(p string) (uint64, bool) {
+		return map[string]uint64{system.DockerVolumePath: 1, "/mnt/sys_drive": 2}[p], p == system.DockerVolumePath || p == "/mnt/sys_drive"
+	}
+	if got := paths(fillTargetsBy(cfg, nil, two)); !eq(got, []string{"/", "/mnt/sys_drive"}) {
+		t.Fatalf("two-volume box: targets = %v, want both areas", got)
+	}
+
+	sett := settingsWithDrives(t,
+		settings.StoragePath{Path: "/mnt/felhom-drives/a", Label: "A", Schedulable: true},
+		settings.StoragePath{Path: "/mnt/felhom-drives/b", Label: "B", Schedulable: true},         // a second folder of drive A's filesystem
+		settings.StoragePath{Path: "/mnt/felhom-drives/gone", Label: "Gone", Schedulable: true},   // unplugged: no identity
+		settings.StoragePath{Path: "/mnt/felhom-drives/gone2", Label: "Gone2", Schedulable: true}, // unplugged too
+		settings.StoragePath{Path: "/mnt/sys_drive/internal", Label: "Belső", Schedulable: true},  // a "drive" that is the system volume
+	)
+	key := boxKeys(map[string]uint64{"/mnt/felhom-drives/a": 2049, "/mnt/felhom-drives/b": 2049, "/mnt/sys_drive/internal": 64519})
+	want := []string{"/", "/mnt/felhom-drives/a", "/mnt/felhom-drives/gone", "/mnt/felhom-drives/gone2"}
+	if got := paths(fillTargetsBy(cfg, sett, key)); !eq(got, want) {
+		t.Fatalf("targets = %v, want %v — one per filesystem, and every path with no identity kept", got, want)
+	}
+
+	// No identity at all (not Linux, or nothing readable): the list is the old one, by path.
+	none := func(string) (uint64, bool) { return 0, false }
+	if got := paths(fillTargetsBy(cfg, nil, none)); !eq(got, []string{"/", "/mnt/sys_drive"}) {
+		t.Fatalf("no identity: targets = %v, want both paths (one warning too many, never one too few)", got)
+	}
+}
+
+// The wiring: the production list uses the real identity. On this machine "/mnt/sys_drive" does not exist, so it has
+// no identity and stays; a folder and its sub-folder are one filesystem and collapse.
+func TestR1371_ProductionListUsesTheRealIdentity(t *testing.T) {
+	dir := t.TempDir()
+	if _, ok := system.FilesystemKey(dir); !ok {
+		t.Skip("no filesystem identity on this machine (not Linux)")
+	}
+	cfg := &config.Config{}
+	cfg.Paths.SystemDataPath = dir
+	sett := settingsWithDrives(t, settings.StoragePath{Path: filepath.Join(dir, "sub"), Label: "Sub", Schedulable: true})
+	if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	got := fillTargets(cfg, sett)
+	for _, tg := range got {
+		if tg.Path == filepath.Join(dir, "sub") {
+			t.Fatalf("fillTargets kept %q, a sub-folder of the system area's filesystem: targets = %+v", tg.Path, got)
+		}
+	}
+	// "/" and dir are one filesystem or two; the list must hold exactly that many targets.
+	rootKey, okRoot := system.FilesystemKey(system.DockerVolumePath)
+	dirKey, _ := system.FilesystemKey(dir)
+	want := 2
+	if okRoot && rootKey == dirKey {
+		want = 1
+	}
+	if len(got) != want {
+		t.Fatalf("fillTargets = %+v, want %d target(s): one per filesystem among / and %s", got, want, dir)
+	}
+}

controller/internal/system/fskey.go

@@ -0,0 +1,59 @@
+package system
+
+// Which filesystem does a path lie on? — the identity the fill watch de-duplicates by (R-1371).
+//
+// WHY st_dev ALONE IS NOT THE ANSWER. Inside the controller's container "/" is an OVERLAY. An overlay has its own
+// device number and its own filesystem id, and both differ from the filesystem it lies on — while statfs reports that
+// filesystem's size and free space. Measured 2026-11-10 on demo-hp (guests 9201, 9202) and the N100 (felhom.eu
+// documentation/audits/space-list-2026-11-10/evidence/extra/fs-identity.txt):
+//
+//	/               type=overlayfs  dev=139    blocks=18016108 free=10674453
+//	/mnt/sys_drive  type=ext2/ext3  dev=64519  blocks=18016108 free=10674452
+//	/etc/hostname   type=ext2/ext3  dev=64519  (the same on /etc/hosts and /etc/resolv.conf)
+//
+// One filesystem, two device numbers. The size-and-free numbers cannot be the identity either: the two `free` values
+// above are from ONE read loop and already differ by one block — a filesystem that is filling is being written.
+//
+// THE RULE. A path that is not on an overlay is identified by its device number. A path on an overlay is identified by
+// the device number of a file Docker itself keeps on its data-root and bind-mounts into every container
+// (dockerDataRootAnchors): the container's overlay lies on that same data-root. If no anchor can be read, there is NO
+// key — and a caller must then treat the path as a filesystem of its own (one warning too many, never one too few).
+//
+// LIMITS, stated: this holds for a container that Docker runs with Docker's own /etc files (the product's only
+// shape). A container whose /etc/hostname is bind-mounted from another filesystem by hand would get the wrong key.
+
+// dockerDataRootAnchors are the files Docker writes under <data-root>/containers/<id>/ and bind-mounts into the
+// container. Any one of them names the data-root's real device.
+var dockerDataRootAnchors = []string{"/etc/hostname", "/etc/hosts", "/etc/resolv.conf"}
+
+// FilesystemKey returns a number that is equal for two paths exactly when they lie on the same filesystem, as far as
+// this process can tell, and ok=false when it cannot tell.
+func FilesystemKey(path string) (key uint64, ok bool) {
+	key, ok = filesystemKeyWith(path, statDev, isOverlayFS, dockerDataRootAnchors)
+	debugf("[DEBUG] [system] FilesystemKey: path=%q key=%d ok=%v", path, key, ok)
+	return key, ok
+}
+
+// filesystemKeyWith is the rule with its three inputs injected (the tests' seam): dev reads a path's device number,
+// overlay says whether a path lies on an overlay; both report ok=false for a path they cannot read.
+func filesystemKeyWith(path string, dev func(string) (uint64, bool), overlay func(string) (isOverlay, ok bool), anchors []string) (uint64, bool) {
+	if path == "" {
+		return 0, false
+	}
+	isOv, ok := overlay(path)
+	if !ok {
+		return 0, false
+	}
+	if !isOv {
+		return dev(path)
+	}
+	for _, a := range anchors {
+		if aOv, ok := overlay(a); !ok || aOv {
+			continue // unreadable, or itself on the overlay (not bind-mounted): it names nothing
+		}
+		if d, ok := dev(a); ok {
+			return d, true
+		}
+	}
+	return 0, false
+}

controller/internal/system/fskey_linux.go

@@ -0,0 +1,24 @@
+//go:build linux
+
+package system
+
+import "syscall"
+
+// overlayMagic is statfs's f_type for an overlay filesystem (OVERLAYFS_SUPER_MAGIC, linux/magic.h).
+const overlayMagic = 0x794c7630
+
+func statDev(path string) (uint64, bool) {
+	var st syscall.Stat_t
+	if err := syscall.Stat(path, &st); err != nil {
+		return 0, false
+	}
+	return uint64(st.Dev), true
+}
+
+func isOverlayFS(path string) (isOverlay, ok bool) {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(path, &st); err != nil {
+		return false, false
+	}
+	return int64(st.Type) == overlayMagic, true
+}

controller/internal/system/fskey_other.go

@@ -0,0 +1,8 @@
+//go:build !linux
+
+package system
+
+// Not Linux (dev/testing only): no filesystem identity, so nothing is ever de-duplicated.
+func statDev(_ string) (uint64, bool) { return 0, false }
+
+func isOverlayFS(_ string) (isOverlay, ok bool) { return false, false }

controller/internal/system/fskey_test.go

@@ -0,0 +1,76 @@
+package system
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+// R-1371 — the rule, with the numbers measured on demo-hp's guest 9201 (fskey.go).
+func TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot(t *testing.T) {
+	devs := map[string]uint64{"/": 139, "/mnt/sys_drive": 64519, "/etc/hostname": 64519, "/etc/hosts": 64519, "/opt/docker/stacks": 64518}
+	ovl := map[string]bool{"/": true}
+	dev := func(p string) (uint64, bool) { d, ok := devs[p]; return d, ok }
+	overlay := func(p string) (bool, bool) {
+		if _, ok := devs[p]; !ok {
+			return false, false
+		}
+		return ovl[p], true
+	}
+	key := func(p string) (uint64, bool) { return filesystemKeyWith(p, dev, overlay, dockerDataRootAnchors) }
+
+	root, ok1 := key("/")
+	sys, ok2 := key("/mnt/sys_drive")
+	if !ok1 || !ok2 || root != sys {
+		t.Fatalf("/ (%d,%v) and /mnt/sys_drive (%d,%v) are ONE filesystem on every box measured — the keys must be equal", root, ok1, sys, ok2)
+	}
+	if stacks, ok := key("/opt/docker/stacks"); !ok || stacks == root {
+		t.Fatalf("/opt/docker/stacks (%d,%v) is another filesystem — its key must differ from %d", stacks, ok, root)
+	}
+	if _, ok := key("/mnt/not-mounted"); ok {
+		t.Fatal("an unreadable path must have no key")
+	}
+	if _, ok := key(""); ok {
+		t.Fatal("an empty path must have no key")
+	}
+
+	// No anchor readable → NO key (never the overlay's own number, which would look like a filesystem of its own
+	// only by luck, and never a guess).
+	if k, ok := filesystemKeyWith("/", dev, overlay, []string{"/etc/absent"}); ok {
+		t.Fatalf("an overlay with no readable anchor got key %d — want no key", k)
+	}
+	// An anchor that is itself on the overlay (not bind-mounted) names nothing; the next one is used.
+	devs["/etc/plain"] = 139
+	ovl["/etc/plain"] = true
+	if k, ok := filesystemKeyWith("/", dev, overlay, []string{"/etc/plain", "/etc/hosts"}); !ok || k != 64519 {
+		t.Fatalf("key = %d,%v — want the first anchor that is NOT on the overlay (64519)", k, ok)
+	}
+}
+
+// The real system calls: two paths on one filesystem share a key, a path on another filesystem does not, and an
+// absent path has none. Runs on the machine's real filesystems (and, in a container, through the overlay branch).
+func TestFilesystemKey_RealSyscalls(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("the identity is Linux-only")
+	}
+	dir := t.TempDir()
+	sub := filepath.Join(dir, "sub")
+	if err := os.Mkdir(sub, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	a, okA := FilesystemKey(dir)
+	b, okB := FilesystemKey(sub)
+	if !okA || !okB || a != b {
+		t.Fatalf("a folder (%d,%v) and its sub-folder (%d,%v) must share a key", a, okA, b, okB)
+	}
+	if p, ok := FilesystemKey("/proc"); !ok || p == a {
+		t.Fatalf("/proc (%d,%v) is another filesystem than %s (%d)", p, ok, dir, a)
+	}
+	if k, ok := FilesystemKey(filepath.Join(dir, "absent")); ok {
+		t.Fatalf("an absent path got key %d", k)
+	}
+	if ov, ok := isOverlayFS("/proc"); !ok || ov {
+		t.Fatalf("/proc read as overlay=%v ok=%v", ov, ok)
+	}
+}