From 844f19f11ef88df315439a2d3710a2ce63905661 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Tue, 10 Nov 2026 14:18:36 +0100 Subject: [PATCH] =?UTF-8?q?fill=20watch:=20one=20warning=20per=20filesyste?= =?UTF-8?q?m=20(R-1371)=20=E2=80=94=20committed,=20not=20released?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a box whose guest has one data volume the two built-in fill targets, "/" (the controller's overlay on the Docker data volume) and /mnt/sys_drive, are one filesystem. The target list was de-duplicated by path only, so one fill would send two warnings under two names. fillTargets now keeps one target per filesystem; the first stays (app area, system area, drives). The identity is system.FilesystemKey, not st_dev alone: inside the container "/" is an overlay with its own device number, so an overlay path takes the device of a Docker-kept file on the data-root (/etc/hostname, /etc/hosts, /etc/resolv.conf). Measured read-only on demo-hp 9201/9202 and the N100 on 2026-11-10. Tests: TestR1371_* (3), TestFilesystemKey_* (2). Red seen three times. No version, no image, no floor: it waits for a release session. Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 33 ++++ CONTEXT.md | 5 + REPORT.md | 35 ++-- REUSE.md | 3 +- controller/README.md | 4 + controller/cmd/controller/main.go | 30 ++- .../r1371_one_warning_per_filesystem_test.go | 183 ++++++++++++++++++ controller/internal/system/fskey.go | 59 ++++++ controller/internal/system/fskey_linux.go | 24 +++ controller/internal/system/fskey_other.go | 8 + controller/internal/system/fskey_test.go | 76 ++++++++ 11 files changed, 438 insertions(+), 22 deletions(-) create mode 100644 controller/cmd/controller/r1371_one_warning_per_filesystem_test.go create mode 100644 controller/internal/system/fskey.go create mode 100644 controller/internal/system/fskey_linux.go create mode 100644 controller/internal/system/fskey_other.go create mode 100644 controller/internal/system/fskey_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d5d8f8e..3ca72f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,36 @@ +## Unreleased (2026-11-10) — R-1371: one fill warning per filesystem — COMMITTED, NOT RELEASED (no version, no image, no floor; it waits for a release session) + +### 2026-11-10 + +**MinAgent: 0.158.1** (unchanged; nothing is released by this entry). + +**Why (measured 2026-11-10, demo-hp guests 9201 and 9202 and the N100, read-only; felhom.eu +`documentation/audits/space-list-2026-11-10/evidence/extra/fs-identity.txt`).** The fill watch has two built-in targets: +"/" (the controller's overlay, which lies on the Docker data volume) and `/mnt/sys_drive`. Since the guest's two data +volumes became one they are ONE filesystem: the same size, the same free space. The target list was de-duplicated by +path only, so one fill would send two warnings with two names („Alkalmazások területe", „Rendszer- és mentési terület"). +Not seen on a box: no disk was filled. + +**The fix.** `fillTargets` now keeps one target per filesystem (`fillTargetsBy`); the first one stays, in the order app +area, system area, drives. A path whose filesystem cannot be identified is kept (one warning too many, never one too +few). + +**The identity is NOT the device number alone** (`system.FilesystemKey`, `internal/system/fskey.go`). Inside the +container "/" is an overlay: it has its own device number and its own filesystem id (139 against 64519 on demo-hp), so a +plain `st_dev` compare would have passed its tests and changed nothing on a box. Equal size-and-free numbers are no +identity either: two reads in one loop already differed by one block. An overlay path is identified by the device of a +file Docker keeps on its data-root and bind-mounts into the container (`/etc/hostname`, `/etc/hosts`, +`/etc/resolv.conf`) — measured equal to `/mnt/sys_drive`'s device on all three guests. + +**Tests.** `TestR1371_OneFillSendsOneWarning` (the real watcher and push closure: one warning, then one critical), +`TestR1371_TargetsByFilesystem`, `TestR1371_ProductionListUsesTheRealIdentity`, +`TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot` (the measured numbers), `TestFilesystemKey_RealSyscalls`. +**Red seen three times:** no filesystem check (2 warnings); the production list not wired to the real identity; an +overlay keeping its own device number. + +**Not proven.** A running controller with this code on a box (commit only). **Not fixed here:** the guest's own root +disk (`/opt/docker/stacks` lies on it) has no fill watcher — a design point in felhom.eu `16-storage-physical-identity.md`. + ## v0.351.0 (2026-11-09) — old test builds and old infrastructure images are deleted (decision 56 amended, decision 285) — RELEASED (image `sha256:1d90563c…`, from `27360d6a`, CI 2603; rc.1 on Tester 1 by hand); per-box floors 12:14 UTC, global floor 2026-11-09 12:18 UTC on the operator's yes (MinAgent 0.158.1) ### 2026-11-09 diff --git a/CONTEXT.md b/CONTEXT.md index cc8c5da9..19b0103b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,5 +1,10 @@ # CONTEXT.md — Project Memory +> **2026-11-10 (day, operator brief) — COMMITTED, NOT RELEASED: one fill warning per filesystem (felhom.eu R-1371).** +> `fillTargets` keeps one target per filesystem (`system.FilesystemKey`; an overlay "/" is identified by a Docker-kept +> file's device, not by its own). No version, no image, no floor: it rides the next controller release. Still open, by +> design question: the guest's own root disk has no fill watcher (felhom.eu `16-storage-physical-identity.md`). + > **2026-11-09 (attended) — v0.351.0 RELEASED** (global floor 12:18 UTC, MinAgent 0.158.1; rc.1 on Tester 1, then the > three test boxes through per-box floors, walk green; felhom.eu `audits/release-2026-11-09/`). Operator's word 2026-11-09 („go with A"): decision 56 amended > (an old `-rc` controller image is deleted like an old release) and decision 285 (pinned infra images keep the pin + the diff --git a/REPORT.md b/REPORT.md index fcfc3f71..64bb8b97 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,21 +1,24 @@ -# REPORT — v0.351.0 released: old test builds and old infrastructure images are deleted (2026-11-09, attended) +# REPORT — one fill warning per filesystem (R-1371) — committed, NOT released (2026-11-10, operator brief) -**Plain words first.** Each box now deletes old controller test builds and old copies of the system images (traefik, -cloudflared, FileBrowser, Samba) by itself, about 3 minutes after the controller starts. It always keeps the running -version and the one before it. Released on the operator's „go with A"; every box gets it through the floor. +**Plain words first.** The controller watches how full the disks are. On a box with one data volume it watched that +volume twice, under two names. One full disk would have sent two warnings. The controller now watches each filesystem +once. This is a commit only. No version, no image, no floor change. No box has it. It waits for the next release. + +The full report of the run is in `felhom.eu`: `REPORT-endings-kernels-fixes-2026-11-10.md`. | | | |---|---| -| Version | 0.351.0 from `27360d6a` (CI 2603), image `sha256:1d90563c443022770b2db90a7730f1ea5aea45cc5b0fa64fec9823d60d42d97f`; MinAgent 0.158.1 unchanged | -| Rulings | `09` §3 decision 56 amended, decision 285 new — operator 2026-11-09 | -| Tests | `TestControllerRetention_OldTestBuildsGo`, `_BoxOnATestBuildIsJudged`, `TestInfraRetention_*` (4); six code mutants each turned a test red | -| rc.1 live | Tester 1 (by hand): 12 controller images and 2 infrastructure images deleted, 37 → 24 images, about 1 GB freed (the test builds share most layers) | -| Release live | per-box floors 12:14 UTC; Tester 1, demo-hp, demo-felhom on 0.351.0 by 12:15; each box's pass deleted what the rule says (demo-felhom: 1 controller + 5 infrastructure images) | -| Box walk | green on all three; `https_only_check.py` rc 0 | -| Floor | global 0.351.0 / MinAgent 0.158.1 at 12:18 UTC, read back from the hub log and the configuration page; per-box floors cleared | +| Change | `fillTargets` / `fillTargetsBy` in `controller/cmd/controller/main.go`; `system.FilesystemKey` (new, `controller/internal/system/fskey*.go`) | +| Rule | One watch target per filesystem. The first stays: app area, then system area, then drives. A path that cannot be identified is kept | +| The trap | Inside the container "/" is an overlay. It has its own device number (139 against 64519 on demo-hp). A plain device compare changes nothing on a box. The key of an overlay path is the device of a file Docker keeps on its data volume (`/etc/hostname`, `/etc/hosts`, `/etc/resolv.conf`) | +| Measured | Read-only, 2026-11-10, demo-hp guests 9201 and 9202 and the N100: felhom.eu `documentation/audits/space-list-2026-11-10/evidence/extra/fs-identity.txt` | +| Tests | `TestR1371_OneFillSendsOneWarning`, `TestR1371_TargetsByFilesystem`, `TestR1371_ProductionListUsesTheRealIdentity`, `TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot`, `TestFilesystemKey_RealSyscalls` | +| Red seen | 3 times: no filesystem check (2 warnings); the production list not wired to the real identity; an overlay keeping its own device number | +| Gates | `python3 scripts/controller_gates.py --fast` | +| Not proven | On a box. No disk was filled, so two real warnings were never seen; the double count is read from the target list and the measured filesystems | +| Not fixed here | The guest's own root disk (`/opt/docker/stacks` lies on it) has no fill watcher. It is a design point in felhom.eu `16-storage-physical-identity.md` | +| Side effect | A registered drive that lies on the same filesystem as a built-in area is now watched under the area's name | +| Machines | None changed by this commit. The measurement was read-only | -Evidence: felhom.eu `documentation/audits/release-2026-11-09/`. - -**Note.** A box that updates from a test build keeps that test build as its „previous" (Tester 1 now keeps rc.1 and -deleted 0.350.0). That is the rule as written (the previous comes from the self-update's record); only test boxes run -test builds. +**Next.** A release session builds it. Then read the watch list in the controller's log on a one-volume box: it must +name the data volume once. diff --git a/REUSE.md b/REUSE.md index 7af3d489..985fb891 100644 --- a/REUSE.md +++ b/REUSE.md @@ -235,6 +235,7 @@ | `registerStoragePath` | controller/internal/web/storage_handlers.go | `(where, label, setDefault) error` | Post-enroll registration | The single funnel used by init/attach/manual-add | | `system.IsMountPoint` / `IsWritable` / `PathsOverlap` | controller/internal/system/mounts_linux.go | `(path) bool` | Mount checks | `_other.go` stubs return permissive values — Linux behavior is the real one | | `system.CheckBackupDestination` | controller/internal/system/mounts_linux.go | `(path) DestinationHealth` | Tier2/offbox target vetting | Detects same-physical-device (`SamePhysicalDevice`) | +| `system.FilesystemKey` (R-1371) | controller/internal/system/fskey.go | `(path) (key uint64, ok bool)` | "Are these two paths ONE filesystem?" — the fill watch's de-duplication | **Never compare `st_dev` (or `SamePhysicalDevice`) for a path that may be the container's "/"**: an overlay has its own device number and filesystem id. An overlay path gets the device of a Docker-kept file (`/etc/hostname` …) that lies on the data-root. `ok=false` = cannot tell → treat as a filesystem of its own. Seam: `filesystemKeyWith` | | `backup.Manager.diskRelation` | controller/internal/backup/backup.go | `(a, b) (DiskRelation, reason)` — DiskDifferent / DiskSame / DiskUnknown | EVERY Tier-2 or one-copy "same disk?" decision (v0.317.0, felhom.eu `16-storage-physical-identity.md` §4) | Asks the host agent (`SetDiskJudge`, FeatureSameDisk ≥ agent 0.157.0); an older agent or an `unmapped` path → the old `sameDevice`/st_dev rule. Never call `sameDevice` directly for a Tier-2 decision — st_dev calls two partitions of one disk "different". Pinned by diskrelation_test.go | | `web.Server.ensureInternalPlace` / `refuseInternalLifecycle` | controller/internal/web/internal_place.go | drive-gate tick hook; `(w, r, where) bool` | the internal data place (v0.318.0, `16` §5): bind on one tick, register (kind internal, default when none) on the next; refuse eject/decommission/wipe | Only an agent row of kind `internal` is enrolled without a press — a household drive never is. Pinned by internal_place_test.go + stacks d4_needs_drive_test.go (a media app lands on it) | | `web.Server.offerRelationFor` | controller/internal/web/backup_target_offer.go | `(ctx, drive) offerRelation` (sameDiskFn seam) | the whole-box backup offer's disk question (R-1084) | Older agent → today's offer; an error → no promise. Pinned by r1084_offer_identity_test.go | @@ -290,7 +291,7 @@ | `Manager.beginRunSummary` / `noteFailure` / `noteAttempted` / `emitRunSummary` / `SetRunSummaryNotify` (R-182, v0.194.0) | controller/internal/backup/runsummary.go | `(kind, runID) func()` / `(app, leg, reason)` / `(RunSummary)` | **THE per-run operator digest.** One `backup_run_failures` event at the end of a run listing every failed app, its leg and its reason — emitted ONLY when something failed | **The RECORD and the NOTIFICATION are different things and must stay so.** The per-app `recovery_unit_capture_failed` event is the record (hub routes it *record-only*, stored + logged every time); this digest is the notification. Before R-182 one event was both, and did neither: nine arrived, two were mailed, seven vanished before `LogNotification`. **Lifetime is `admissionSet`'s exactly** — absent collector means "no run in flight", never a stale answer. **A refusal is noted ONCE, inside `admitApp` where the verdict is taken**, not at the three legs that consult it: R-181's one-verdict-covers-all-three contract makes per-leg noting produce "2 of 1 apps failed". **Deliberate skips (disconnected / decommissioned) must NEVER be noted** — they have their own alert and a nightly digest about an unplugged drive is an ignored digest. **A clean run emits NOTHING**; silence is safe only because the hub's deadline check (`monitor/deadline.go:396,417`) raises a missed backup from report freshness independently — if that is ever weakened this design loses its footing. **`run_id` is unique per real run** (so the hub's 1-h cooldown cannot collapse a manual run into the nightly one) and **deliberately EMPTY on the periodic refresh sweep**, which must stay under that cooldown or a polled status page becomes a mail flood | | `Manager.admitApp` / `beginAdmissionRun` / `decideAdmission` / `estimatedWriteBytes` (R-181, v0.193.0) | controller/internal/backup/admission.go | `(stackName) bool` / `() func()` | **THE reserve gate. Call it before ANY per-app backup write** — one verdict per app per run, covering the DB dump, the volume dump and the unit capture (all three write under one per-app root) | **Decided LAZILY at the app's first write, never once at run start** — app A's dump can put app B under the reserve, so a run-start verdict reads a disk that no longer exists. **Never re-decided between an app's own legs**: that is exactly the split R-181 closed (bulk written, capture refused). **Reset per run** via the closer `beginAdmissionRun` returns. **Must sit ahead of `DumpAppVolumesSafe`**, which stops the stack as its first act — a refusal decided inside it has already bounced the app. Fires **exactly one** `unitNotify` per refused app per run. Nil admission set (periodic status refresh) → decides fresh, which is still once per app per sweep. Wiring pinned by an **AST walk** in `TestAdmission_IsWiredIntoEveryProductionWriteLeg`, not `strings.Contains` | | `Manager.floorVerdict` + `FloorUsedPercent`/`FloorFreeGiB` / `ErrCaptureFloor` / `floorReason` (R-165 B2 v0.192.0, size term R-181 v0.193.0) | controller/internal/backup/recovery_unit.go | `(*UnitSpace, estGiB float64) (*UnitSpace, floorReason)` | The pure two-question predicate behind `admitApp`: is the filesystem already below the reserve (`floorHeadroom`), and would THIS app's write take it below (`floorSize`)? | **Headroom is about the FILESYSTEM, never a per-unit cap** — a size cap is R-163 rebuilt inside one volume; the size term bounds the *delta*, not the unit. **REFUSES, never deletes:** nothing here is generational (a unit is one fixed path per app, a DB dump one fixed name), so pruning could only destroy a DIFFERENT app's only local copy — **never repurpose `pruneStalePrimaryDirs`**, which removes ORPHANED dirs from an app that moved drives and has no notion of age. Two terms (97% / 1 GiB) in `fillwatch`'s shape, deliberately BEYOND its critical band (95% / 2 GiB) so the customer is always warned first — pinned by `TestFloorSitsBelowTheCriticalWarningBand`. **`estGiB == 0` degrades to headroom-only on purpose** — refusing an app with no history makes the FIRST backup the one that can never happen. A nil reading neither refuses nor warns (§8.4). Inject `unitSpaceFn` in tests rather than manufacturing occupancy on a real disk | -| `fillwatch.Watcher` (`New`/`SetNotify`/`Check`) (R-167, v0.191.0) | controller/internal/fillwatch/fillwatch.go | `(statePath, logger, targetsFn, usageFn)` → `Check() error` | THE customer fill warning — warns BEFORE a filesystem fills, per FILESYSTEM (never per app: one full disk holding ten apps would fire ten times) | Emits the **pre-existing** `disk_warning`/`disk_critical` pair, which was allowlisted + copy'd + default-enabled with **no producer in any repo** until now — do NOT mint a new type beside it. **Two threshold terms, whichever trips first** (85% / 5 GiB; critical 95% / 2 GiB) because a percentage alone lies at both ends of this fleet's size range. **Edge-triggered on ESCALATION ONLY**, state persisted; de-escalation is silent and re-arms. Hysteresis dead zone between clear (75% / 7 GiB) and warn — pinned by `TestThresholdsKeepTheirHysteresisGap`. **A nil usage read is NEVER a warning** (§8.4). The hub has **no `customerMessages` entry** for either type on purpose — an entry would override the dynamic message and discard the drive label + free space. **Both languages (v0.309.0, R-521 F7):** the sentence is the bundle's `event.fill.*`, the household's rendering rides as `message_customer` via `Notifier.PushEventHousehold` (`SetLang`; `SetExtra` takes the language); a built-in area carries a `LabelKey`, a household's own drive label is used as written. Hungarian pinned byte-for-byte by `TestMessageHungarianWireText` (v0.309.0 dropped the formal „Kérjük," — R-516) | +| `fillwatch.Watcher` (`New`/`SetNotify`/`Check`) (R-167, v0.191.0) | controller/internal/fillwatch/fillwatch.go | `(statePath, logger, targetsFn, usageFn)` → `Check() error` | THE customer fill warning — warns BEFORE a filesystem fills, per FILESYSTEM (never per app: one full disk holding ten apps would fire ten times) | Emits the **pre-existing** `disk_warning`/`disk_critical` pair, which was allowlisted + copy'd + default-enabled with **no producer in any repo** until now — do NOT mint a new type beside it. **Two threshold terms, whichever trips first** (85% / 5 GiB; critical 95% / 2 GiB) because a percentage alone lies at both ends of this fleet's size range. **Edge-triggered on ESCALATION ONLY**, state persisted; de-escalation is silent and re-arms. Hysteresis dead zone between clear (75% / 7 GiB) and warn — pinned by `TestThresholdsKeepTheirHysteresisGap`. **A nil usage read is NEVER a warning** (§8.4). The hub has **no `customerMessages` entry** for either type on purpose — an entry would override the dynamic message and discard the drive label + free space. **Both languages (v0.309.0, R-521 F7):** the sentence is the bundle's `event.fill.*`, the household's rendering rides as `message_customer` via `Notifier.PushEventHousehold` (`SetLang`; `SetExtra` takes the language); a built-in area carries a `LabelKey`, a household's own drive label is used as written. Hungarian pinned byte-for-byte by `TestMessageHungarianWireText` (v0.309.0 dropped the formal „Kérjük," — R-516) **One target per filesystem (R-1371):** `fillTargets` (cmd/controller/main.go) de-duplicates by `system.FilesystemKey`, first target wins (app area, system area, drives); a new built-in target goes into that order on purpose. | ### Scheduler / time / UI diff --git a/controller/README.md b/controller/README.md index 0f1b615e..68b9c842 100644 --- a/controller/README.md +++ b/controller/README.md @@ -2220,6 +2220,10 @@ that folder is never a dead end, and an install never runs into it silently (R-6 - **Where it lives.** `/kept/` is beside `appdata/` and `userdata/`, inside neither: no app bind, FileBrowser userdata source, Samba share or backup leg reads it. It is in `ProtectedHDDPaths`. - **The drive-full warning** ends with the kept folders on that drive and their sizes (`fillwatch.SetExtra`). +- **One fill warning per filesystem (R-1371; committed 2026-11-10, not released).** The watch list (`fillTargets`) holds the + app area ("/"), the system area (`/mnt/sys_drive`) and each drive, and keeps ONE target per filesystem — the first, in that + order. On a box whose guest has one data volume the two areas are one filesystem, so the warning carries the app area's + name. The identity is `system.FilesystemKey`, not the device number: the container's "/" is an overlay. - **The fill warning in the household's language (v0.309.0, R-521 F7).** `disk_warning` / `disk_critical` carry the Hungarian sentence AND the household's own (`message_customer`), from the bundle keys `event.fill.*`; the two built-in areas („Alkalmazások területe" = the guest's root filesystem as the controller sees it, „Rendszer- és mentési terület") diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 6440ec53..39852852 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -2702,11 +2702,18 @@ const fillWatchInterval = 10 * time.Minute // silent exhaustion R-158 measured; // - the registered DRIVES hold the customer's own data. // -// De-duplicated by path: on a box where a drive is not a separate mount these collapse, and warning -// twice about one filesystem is exactly the noise the per-filesystem rule exists to prevent. A -// DECOMMISSIONED drive is dropped — it is deliberately out of service, and its fill is not news. A -// DISCONNECTED one is kept in the list but will read as unreadable and be skipped by §8.4, which is -// the correct outcome: its absence already has its own alert. +// ONE WARNING PER FILESYSTEM (R-1371). The list is de-duplicated by path AND by filesystem +// (fillTargetsBy). By path alone it was not enough: since the guest's two data volumes became one, "/" +// (the controller's overlay, which lies on the Docker data volume) and `/mnt/sys_drive` are ONE +// filesystem on every box measured 2026-11-10, so one fill sent two warnings with two names. The +// identity is system.FilesystemKey — NOT the plain device number, which an overlay hides (fskey.go). +// The FIRST target of a filesystem stays, in the order below: the app area, the system area, then the +// drives. A path whose filesystem cannot be identified is kept (one warning too many, never one too +// few). Pinned by TestR1371_OneFillSendsOneWarning. +// +// A DECOMMISSIONED drive is dropped — it is deliberately out of service, and its fill is not news. A +// DISCONNECTED one has no readable filesystem, so it is kept in the list, reads as unreadable and is +// skipped by §8.4, which is the correct outcome: its absence already has its own alert. // fillEventPusher is the one notifier method fillEventPush needs — an interface so the R-521 F7 test // can record what is pushed without a hub. type fillEventPusher interface { @@ -2732,13 +2739,26 @@ func fillEventPush(n fillEventPusher) func(fillwatch.Event) { } func fillTargets(cfg *config.Config, sett *settings.Settings) []fillwatch.Target { + return fillTargetsBy(cfg, sett, system.FilesystemKey) +} + +// fillTargetsBy is fillTargets with the filesystem identity injected (the tests' seam). fsKey returns +// ok=false for a path whose filesystem it cannot identify. +func fillTargetsBy(cfg *config.Config, sett *settings.Settings, fsKey func(path string) (uint64, bool)) []fillwatch.Target { seen := make(map[string]bool) + seenFS := make(map[uint64]bool) var out []fillwatch.Target add := func(path, label, labelKey string) { if path == "" || seen[path] { return } seen[path] = true + if key, ok := fsKey(path); ok { + if seenFS[key] { + return // this filesystem is watched already, under the target that came first + } + seenFS[key] = true + } out = append(out, fillwatch.Target{Path: path, Label: label, LabelKey: labelKey}) } diff --git a/controller/cmd/controller/r1371_one_warning_per_filesystem_test.go b/controller/cmd/controller/r1371_one_warning_per_filesystem_test.go new file mode 100644 index 00000000..37e01c9b --- /dev/null +++ b/controller/cmd/controller/r1371_one_warning_per_filesystem_test.go @@ -0,0 +1,183 @@ +package main + +import ( + "io" + "log" + "os" + "path/filepath" + "testing" + + "gitea.dooplex.hu/admin/felhom-controller/internal/config" + "gitea.dooplex.hu/admin/felhom-controller/internal/fillwatch" + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" + "gitea.dooplex.hu/admin/felhom-controller/internal/system" +) + +// R-1371 — one warning per filesystem. +// +// The box as measured 2026-11-10 (demo-hp, the N100, Tester 1): "/" and `/mnt/sys_drive` are ONE filesystem, seen +// through two paths. The filesystem identity here is the one system.FilesystemKey gives on those boxes (its own rule +// is pinned by TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot): equal for the two paths, another for a drive. +// +// RED-PROOF (observed 2026-11-10, then restored): with the filesystem check removed from fillTargetsBy this test fails +// with "one filling filesystem sent 2 warnings". +func boxKeys(extra map[string]uint64) func(string) (uint64, bool) { + keys := map[string]uint64{system.DockerVolumePath: 64519, "/mnt/sys_drive": 64519} + for p, k := range extra { + keys[p] = k + } + return func(p string) (uint64, bool) { k, ok := keys[p]; return k, ok } +} + +func settingsWithDrives(t *testing.T, paths ...settings.StoragePath) *settings.Settings { + t.Helper() + sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + for _, sp := range paths { + if err := sett.AddStoragePath(sp); err != nil { + t.Fatal(err) + } + } + return sett +} + +// The consequence: the one filesystem fills, and the household gets ONE warning, then ONE critical — through the real +// watcher and the real push closure. +func TestR1371_OneFillSendsOneWarning(t *testing.T) { + cfg := &config.Config{} + cfg.Paths.SystemDataPath = "/mnt/sys_drive" + sett := settingsWithDrives(t, settings.StoragePath{Path: "/mnt/felhom-drives/usb", Label: "Családi lemez", Schedulable: true}) + key := boxKeys(map[string]uint64{"/mnt/felhom-drives/usb": 2049}) + + pct := 50.0 + usage := func(p string) *fillwatch.Usage { + if p == "/mnt/felhom-drives/usb" { + return &fillwatch.Usage{UsedPercent: 20, AvailGB: 700, UsedGB: 200, TotalGB: 900} + } + // "/" and /mnt/sys_drive: the same filesystem, so the same numbers. + return &fillwatch.Usage{UsedPercent: pct, AvailGB: 68.7 * (100 - pct) / 100, UsedGB: 68.7 * pct / 100, TotalGB: 68.7} + } + w := fillwatch.New(filepath.Join(t.TempDir(), "fill.json"), log.New(io.Discard, "", 0), + func() []fillwatch.Target { return fillTargetsBy(cfg, sett, key) }, usage) + rec := &fakeFillPusher{} + w.SetNotify(fillEventPush(rec)) + + check := func() { + t.Helper() + for i := 0; i < 2; i++ { + if err := w.Check(); err != nil { + t.Fatal(err) + } + } + } + check() + if len(rec.got) != 0 { + t.Fatalf("a half-full filesystem sent %d events", len(rec.got)) + } + pct = 90 + check() + if len(rec.got) != 1 { + t.Fatalf("one filling filesystem sent %d warnings (%+v), want exactly 1", len(rec.got), rec.got) + } + if rec.got[0].eventType != "disk_warning" { + t.Fatalf("event = %s, want disk_warning", rec.got[0].eventType) + } + if d, _ := rec.got[0].details.(map[string]any); d == nil || d["path"] != system.DockerVolumePath { + t.Fatalf("the warning is for %v, want the first target of the filesystem (%q)", rec.got[0].details, system.DockerVolumePath) + } + pct = 97 + check() + if len(rec.got) != 2 || rec.got[1].eventType != "disk_critical" { + t.Fatalf("after 97%% the events are %+v, want one warning then one critical", rec.got) + } +} + +// What stays and what goes, target by target. +func TestR1371_TargetsByFilesystem(t *testing.T) { + cfg := &config.Config{} + cfg.Paths.SystemDataPath = "/mnt/sys_drive" + paths := func(ts []fillwatch.Target) []string { + var out []string + for _, tg := range ts { + out = append(out, tg.Path) + } + return out + } + eq := func(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true + } + + // One volume, no drive: one target, and it is the app area (the first one added). + got := fillTargetsBy(cfg, nil, boxKeys(nil)) + if len(got) != 1 || got[0].Path != system.DockerVolumePath || got[0].LabelKey != "event.fill.label_apps" { + t.Fatalf("one-volume box: targets = %+v, want only the app area", got) + } + + // The OLD layout (two volumes): both areas stay. + two := func(p string) (uint64, bool) { + return map[string]uint64{system.DockerVolumePath: 1, "/mnt/sys_drive": 2}[p], p == system.DockerVolumePath || p == "/mnt/sys_drive" + } + if got := paths(fillTargetsBy(cfg, nil, two)); !eq(got, []string{"/", "/mnt/sys_drive"}) { + t.Fatalf("two-volume box: targets = %v, want both areas", got) + } + + sett := settingsWithDrives(t, + settings.StoragePath{Path: "/mnt/felhom-drives/a", Label: "A", Schedulable: true}, + settings.StoragePath{Path: "/mnt/felhom-drives/b", Label: "B", Schedulable: true}, // a second folder of drive A's filesystem + settings.StoragePath{Path: "/mnt/felhom-drives/gone", Label: "Gone", Schedulable: true}, // unplugged: no identity + settings.StoragePath{Path: "/mnt/felhom-drives/gone2", Label: "Gone2", Schedulable: true}, // unplugged too + settings.StoragePath{Path: "/mnt/sys_drive/internal", Label: "Belső", Schedulable: true}, // a "drive" that is the system volume + ) + key := boxKeys(map[string]uint64{"/mnt/felhom-drives/a": 2049, "/mnt/felhom-drives/b": 2049, "/mnt/sys_drive/internal": 64519}) + want := []string{"/", "/mnt/felhom-drives/a", "/mnt/felhom-drives/gone", "/mnt/felhom-drives/gone2"} + if got := paths(fillTargetsBy(cfg, sett, key)); !eq(got, want) { + t.Fatalf("targets = %v, want %v — one per filesystem, and every path with no identity kept", got, want) + } + + // No identity at all (not Linux, or nothing readable): the list is the old one, by path. + none := func(string) (uint64, bool) { return 0, false } + if got := paths(fillTargetsBy(cfg, nil, none)); !eq(got, []string{"/", "/mnt/sys_drive"}) { + t.Fatalf("no identity: targets = %v, want both paths (one warning too many, never one too few)", got) + } +} + +// The wiring: the production list uses the real identity. On this machine "/mnt/sys_drive" does not exist, so it has +// no identity and stays; a folder and its sub-folder are one filesystem and collapse. +func TestR1371_ProductionListUsesTheRealIdentity(t *testing.T) { + dir := t.TempDir() + if _, ok := system.FilesystemKey(dir); !ok { + t.Skip("no filesystem identity on this machine (not Linux)") + } + cfg := &config.Config{} + cfg.Paths.SystemDataPath = dir + sett := settingsWithDrives(t, settings.StoragePath{Path: filepath.Join(dir, "sub"), Label: "Sub", Schedulable: true}) + if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil { + t.Fatal(err) + } + got := fillTargets(cfg, sett) + for _, tg := range got { + if tg.Path == filepath.Join(dir, "sub") { + t.Fatalf("fillTargets kept %q, a sub-folder of the system area's filesystem: targets = %+v", tg.Path, got) + } + } + // "/" and dir are one filesystem or two; the list must hold exactly that many targets. + rootKey, okRoot := system.FilesystemKey(system.DockerVolumePath) + dirKey, _ := system.FilesystemKey(dir) + want := 2 + if okRoot && rootKey == dirKey { + want = 1 + } + if len(got) != want { + t.Fatalf("fillTargets = %+v, want %d target(s): one per filesystem among / and %s", got, want, dir) + } +} diff --git a/controller/internal/system/fskey.go b/controller/internal/system/fskey.go new file mode 100644 index 00000000..e940cbaa --- /dev/null +++ b/controller/internal/system/fskey.go @@ -0,0 +1,59 @@ +package system + +// Which filesystem does a path lie on? — the identity the fill watch de-duplicates by (R-1371). +// +// WHY st_dev ALONE IS NOT THE ANSWER. Inside the controller's container "/" is an OVERLAY. An overlay has its own +// device number and its own filesystem id, and both differ from the filesystem it lies on — while statfs reports that +// filesystem's size and free space. Measured 2026-11-10 on demo-hp (guests 9201, 9202) and the N100 (felhom.eu +// documentation/audits/space-list-2026-11-10/evidence/extra/fs-identity.txt): +// +// / type=overlayfs dev=139 blocks=18016108 free=10674453 +// /mnt/sys_drive type=ext2/ext3 dev=64519 blocks=18016108 free=10674452 +// /etc/hostname type=ext2/ext3 dev=64519 (the same on /etc/hosts and /etc/resolv.conf) +// +// One filesystem, two device numbers. The size-and-free numbers cannot be the identity either: the two `free` values +// above are from ONE read loop and already differ by one block — a filesystem that is filling is being written. +// +// THE RULE. A path that is not on an overlay is identified by its device number. A path on an overlay is identified by +// the device number of a file Docker itself keeps on its data-root and bind-mounts into every container +// (dockerDataRootAnchors): the container's overlay lies on that same data-root. If no anchor can be read, there is NO +// key — and a caller must then treat the path as a filesystem of its own (one warning too many, never one too few). +// +// LIMITS, stated: this holds for a container that Docker runs with Docker's own /etc files (the product's only +// shape). A container whose /etc/hostname is bind-mounted from another filesystem by hand would get the wrong key. + +// dockerDataRootAnchors are the files Docker writes under /containers// and bind-mounts into the +// container. Any one of them names the data-root's real device. +var dockerDataRootAnchors = []string{"/etc/hostname", "/etc/hosts", "/etc/resolv.conf"} + +// FilesystemKey returns a number that is equal for two paths exactly when they lie on the same filesystem, as far as +// this process can tell, and ok=false when it cannot tell. +func FilesystemKey(path string) (key uint64, ok bool) { + key, ok = filesystemKeyWith(path, statDev, isOverlayFS, dockerDataRootAnchors) + debugf("[DEBUG] [system] FilesystemKey: path=%q key=%d ok=%v", path, key, ok) + return key, ok +} + +// filesystemKeyWith is the rule with its three inputs injected (the tests' seam): dev reads a path's device number, +// overlay says whether a path lies on an overlay; both report ok=false for a path they cannot read. +func filesystemKeyWith(path string, dev func(string) (uint64, bool), overlay func(string) (isOverlay, ok bool), anchors []string) (uint64, bool) { + if path == "" { + return 0, false + } + isOv, ok := overlay(path) + if !ok { + return 0, false + } + if !isOv { + return dev(path) + } + for _, a := range anchors { + if aOv, ok := overlay(a); !ok || aOv { + continue // unreadable, or itself on the overlay (not bind-mounted): it names nothing + } + if d, ok := dev(a); ok { + return d, true + } + } + return 0, false +} diff --git a/controller/internal/system/fskey_linux.go b/controller/internal/system/fskey_linux.go new file mode 100644 index 00000000..c7e7130c --- /dev/null +++ b/controller/internal/system/fskey_linux.go @@ -0,0 +1,24 @@ +//go:build linux + +package system + +import "syscall" + +// overlayMagic is statfs's f_type for an overlay filesystem (OVERLAYFS_SUPER_MAGIC, linux/magic.h). +const overlayMagic = 0x794c7630 + +func statDev(path string) (uint64, bool) { + var st syscall.Stat_t + if err := syscall.Stat(path, &st); err != nil { + return 0, false + } + return uint64(st.Dev), true +} + +func isOverlayFS(path string) (isOverlay, ok bool) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return false, false + } + return int64(st.Type) == overlayMagic, true +} diff --git a/controller/internal/system/fskey_other.go b/controller/internal/system/fskey_other.go new file mode 100644 index 00000000..0ab3738c --- /dev/null +++ b/controller/internal/system/fskey_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package system + +// Not Linux (dev/testing only): no filesystem identity, so nothing is ever de-duplicated. +func statDev(_ string) (uint64, bool) { return 0, false } + +func isOverlayFS(_ string) (isOverlay, ok bool) { return false, false } diff --git a/controller/internal/system/fskey_test.go b/controller/internal/system/fskey_test.go new file mode 100644 index 00000000..eef8a4fe --- /dev/null +++ b/controller/internal/system/fskey_test.go @@ -0,0 +1,76 @@ +package system + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +// R-1371 — the rule, with the numbers measured on demo-hp's guest 9201 (fskey.go). +func TestFilesystemKey_AnOverlayRootIsTheDockerDataRoot(t *testing.T) { + devs := map[string]uint64{"/": 139, "/mnt/sys_drive": 64519, "/etc/hostname": 64519, "/etc/hosts": 64519, "/opt/docker/stacks": 64518} + ovl := map[string]bool{"/": true} + dev := func(p string) (uint64, bool) { d, ok := devs[p]; return d, ok } + overlay := func(p string) (bool, bool) { + if _, ok := devs[p]; !ok { + return false, false + } + return ovl[p], true + } + key := func(p string) (uint64, bool) { return filesystemKeyWith(p, dev, overlay, dockerDataRootAnchors) } + + root, ok1 := key("/") + sys, ok2 := key("/mnt/sys_drive") + if !ok1 || !ok2 || root != sys { + t.Fatalf("/ (%d,%v) and /mnt/sys_drive (%d,%v) are ONE filesystem on every box measured — the keys must be equal", root, ok1, sys, ok2) + } + if stacks, ok := key("/opt/docker/stacks"); !ok || stacks == root { + t.Fatalf("/opt/docker/stacks (%d,%v) is another filesystem — its key must differ from %d", stacks, ok, root) + } + if _, ok := key("/mnt/not-mounted"); ok { + t.Fatal("an unreadable path must have no key") + } + if _, ok := key(""); ok { + t.Fatal("an empty path must have no key") + } + + // No anchor readable → NO key (never the overlay's own number, which would look like a filesystem of its own + // only by luck, and never a guess). + if k, ok := filesystemKeyWith("/", dev, overlay, []string{"/etc/absent"}); ok { + t.Fatalf("an overlay with no readable anchor got key %d — want no key", k) + } + // An anchor that is itself on the overlay (not bind-mounted) names nothing; the next one is used. + devs["/etc/plain"] = 139 + ovl["/etc/plain"] = true + if k, ok := filesystemKeyWith("/", dev, overlay, []string{"/etc/plain", "/etc/hosts"}); !ok || k != 64519 { + t.Fatalf("key = %d,%v — want the first anchor that is NOT on the overlay (64519)", k, ok) + } +} + +// The real system calls: two paths on one filesystem share a key, a path on another filesystem does not, and an +// absent path has none. Runs on the machine's real filesystems (and, in a container, through the overlay branch). +func TestFilesystemKey_RealSyscalls(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the identity is Linux-only") + } + dir := t.TempDir() + sub := filepath.Join(dir, "sub") + if err := os.Mkdir(sub, 0o755); err != nil { + t.Fatal(err) + } + a, okA := FilesystemKey(dir) + b, okB := FilesystemKey(sub) + if !okA || !okB || a != b { + t.Fatalf("a folder (%d,%v) and its sub-folder (%d,%v) must share a key", a, okA, b, okB) + } + if p, ok := FilesystemKey("/proc"); !ok || p == a { + t.Fatalf("/proc (%d,%v) is another filesystem than %s (%d)", p, ok, dir, a) + } + if k, ok := FilesystemKey(filepath.Join(dir, "absent")); ok { + t.Fatalf("an absent path got key %d", k) + } + if ov, ok := isOverlayFS("/proc"); !ok || ov { + t.Fatalf("/proc read as overlay=%v ok=%v", ov, ok) + } +}