From added9d226cbd1877f772a7c10e327a4868de66a Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sat, 11 Jul 2026 09:44:06 +0200 Subject: [PATCH] =?UTF-8?q?v0.81.0:=20NAS=20verify-before-commit=20?= =?UTF-8?q?=E2=80=94=20retry=3D0,=20journal=20classifier,=20detached=20ver?= =?UTF-8?q?ify=20job=20+=20auto-rollback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent half of the verify-before-commit task (SPIKE-nas-verify-2026-07-11, b57f6c1): retry=0 in the production NFS options (Q4-vi); ClassifyNetVerifyFailure on the live Q4 strings (nfs_export merges not-found/not-permitted); add = sync fast-fail (2s TCP pre-probe, nothing installed) + detached in-memory verify job judging /proc/mounts only, auto-rollback on failure; GET /netstorage/verify-status (phase none = the controller's Scenario-F rollback signal); unprivileged journalctl (systemd-journal group, NO new sudoers grants). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6 --- CHANGELOG.md | 33 +++ REUSE.md | 7 +- internal/localapi/netstorage.go | 94 +++++++-- internal/localapi/netstorage_test.go | 6 + internal/localapi/netverifyjob.go | 234 +++++++++++++++++++++ internal/localapi/netverifyjob_test.go | 272 +++++++++++++++++++++++++ internal/localapi/server.go | 20 ++ internal/storage/netmount.go | 49 ++++- internal/storage/netmount_test.go | 2 +- internal/storage/netverify.go | 61 ++++++ internal/storage/netverify_test.go | 107 ++++++++++ 11 files changed, 859 insertions(+), 26 deletions(-) create mode 100644 internal/localapi/netverifyjob.go create mode 100644 internal/localapi/netverifyjob_test.go create mode 100644 internal/storage/netverify.go create mode 100644 internal/storage/netverify_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 722674e..8261672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,36 @@ +## v0.81.0 — NAS verify-before-commit: retry=0 + detached verify job + journal classification (2026-07-11) + +Implements the agent half of the "NAS verify-before-commit" task on the SPIKE-nas-verify-2026-07-11 +evidence (felhom.eu/documentation/audits/, commit b57f6c1). `POST /netstorage/add` no longer succeeds +blind — a bogus share can no longer sit at "Készenlét" forever. + +- **`retry=0` in the production NFS option string** (`netmount.go mountOptions`; Q4-vi): a dead-NAS + on-demand access now fails clean in ~4 s (ENODEV) instead of wedging the accessing app until + systemd's 90 s start cap; verify failures classify as "No route to host" instead of a + diagnostic-free timeout. SMB string unchanged (retry is a mount.nfs option). Already-installed + units are NOT rewritten (pre-customer; re-add re-creates them). +- **`internal/storage/netverify.go`** — `ClassifyNetVerifyFailure`: pure, table-driven journal + classification on the Q4 VERBATIM substrings → `unreachable | nfs_export | smb_auth | smb_share | + timeout | mount_failed`. `nfs_export` deliberately MERGES not-found/not-permitted (NFSv4 returns + the identical string for both — Q4 ii≡iii). Everything exits rc=32, so classification is + string-based by design. +- **Detached verify job** (`localapi/netverifyjob.go`, the formatjob shape but IN-MEMORY single + slot): add = decode → role-gate → SYNC fast-fail (full spec validation + 2 s TCP pre-probe; an + unreachable server is refused with NOTHING installed) → stage SMB creds → EnsureNetworkMount → + detached verify (trigger read through the automount; mount success judged from **/proc/mounts + only** — never readability, a 0700 export EACCES is a good mount) → on failure: journal-classify + + **auto-rollback** (RemoveNetworkMount + creds file). `GET /netstorage/verify-status` reports the + slot; phase `none` after an agent restart is the controller's rollback signal (Scenario F — + deliberately not persisted). Single-flight: a second add while verifying is a 409. +- **Journal access is UNPRIVILEGED** (`journalctl -u -n 20 -o cat`, no sudo, no new sudoers + grant): requires the felhom-agent user in the `systemd-journal` group (host-install ≥ v1.12.0 + successor adds it; existing hosts: `usermod -aG systemd-journal felhom-agent`). Journal + unavailable degrades to `mount_failed` + a hint, still rolled back. +- New exported helpers: `storage.NetworkMountedAt` (autofs trigger ≠ mounted), + `storage.NetworkEndpointReachable` (the 2 s pre-probe). REUSE.md updated. +- Tests: classifier table on the live Q4 strings, §8 truth table, rollback effects, pre-probe + zero-install, single-flight + no-job shape. Red-proof outcomes recorded in REPORT.md. + ## v0.80.0 — PBS DR tier SLICE 2: the apply-bridge (2026-07-10) Consumes hub v0.44.0's `pbs_dr` desired-state descriptor (slice 1): hub tick → the box grows the diff --git a/REUSE.md b/REUSE.md index e1f3737..c7e6db1 100644 --- a/REUSE.md +++ b/REUSE.md @@ -48,7 +48,9 @@ | `GuestBinder.EnsureSharedParent` | internal/localapi/intermediary.go | `EnsureSharedParent(ctx) error` | shared-parent bind + boot unit | make-private+make-shared ONLY on first bind — re-running orphans the guest's slave; F2-a: compares script AND unit for staleness | | `StablePathForRaw` / `DriveNameFromRaw` | internal/localapi/intermediary.go | `StablePathForRaw("/mnt/") string` | raw host mount → stable guest path | Single source of truth both repos derive from | | `GuestBinder.GuestSeesMount` / `GuestBootID` | internal/localapi/intermediary.go | `GuestSeesMount(ctx, vmid, path) bool` | guest-visible (usable) signal; reboot detection | Host bind present ≠ guest sees it (non-recursive parent bind) | -| `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe) | +| `SudoHostOps.EnsureNetworkMount` / `RemoveNetworkMount` / `ListNetworkMounts` | internal/storage/netmount.go | `EnsureNetworkMount(ctx, spec) error` | NAS automount pair | rm glob confined to `mnt-felhom*` units; NAS ≠ drive (no durable-id/SMART/wipe); RemoveNetworkMount doubles as the verify-fail rollback (idempotent) | +| `NetworkMountedAt` / `NetworkEndpointReachable` | internal/storage/netmount.go | `NetworkMountedAt(where) bool` | verify mount-truth + the 2 s add pre-probe | /proc/mounts is the ONLY mount-success judge (autofs trigger ≠ mounted; readability ≠ mounted — SPIKE-nas-verify §8) | +| `ClassifyNetVerifyFailure` | internal/storage/netverify.go | `ClassifyNetVerifyFailure(journalTail, tcpReachable) (code, hint)` | NAS verify failure categories | String-based BY DESIGN (every mount failure is rc=32); substrings verbatim from SPIKE-nas-verify Q4; `nfs_export` merges not-found/not-permitted (NFSv4 identical) | ### Durable stores (atomic state) @@ -98,7 +100,8 @@ | Anti-retarget durable-id binding | internal/localapi/wipe_reresolve.go | resolve id → re-derive + exact match → re-inspect expected state → act on RE-RESOLVED device only | | Atomic single-file JSON store | internal/storage/intent.go | `Open*` loads (missing=empty, corrupt=fail-loud), mutex, tmp+rename 0600, idempotent set | | Durable append-only log + index | internal/authz/noncestore.go (`FileNonceStore`) | fsync before returning "new"; replay into index on open; expiry-only compaction | -| Injectable seam funcs on Server | internal/localapi/server.go (`reresolveWipe`, `deviceDurableID`, `boundCheck`) | prod default wired in `NewServer`; tests override — no real /dev in tests | +| Injectable seam funcs on Server | internal/localapi/server.go (`reresolveWipe`, `deviceDurableID`, `boundCheck`, net-verify: `netTrigger`/`netMounted`/`netJournal`/`netReachable`) | prod default wired in `NewServer`; tests override — no real /dev, /proc/mounts, journalctl or TCP in tests | +| Detached IN-MEMORY verify job (single slot, deliberately unpersisted) | internal/localapi/netverifyjob.go | claim slot sync (single-flight 409) → detached pipeline off baseCtx → auto-rollback on fail; restart ⇒ slot empty ⇒ the CALLER rolls back (Scenario F) — contrast formatjob (persisted+recovered) | | Optional dependency degradation | internal/localapi/server.go (`Options`) | nil dep ⇒ endpoint answers "not configured" (503), never a crash | | Root-file install via random temp | internal/localapi/intermediary.go (`installSharedParentUnit`) | `stageTemp` (os.CreateTemp) → sudoers-globbed `install -m` → pinned destination | | Detached destructive job + restart recovery | internal/localapi/formatjob.go | persist `running` → run off baseCtx → record outcome; recovery re-resolves durable id, never a path | diff --git a/internal/localapi/netstorage.go b/internal/localapi/netstorage.go index cf218a6..d717ae7 100644 --- a/internal/localapi/netstorage.go +++ b/internal/localapi/netstorage.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "time" "gitea.dooplex.hu/admin/felhom-agent/internal/storage" ) @@ -48,8 +49,12 @@ type netStorageAddRequest struct { Password string `json:"password,omitempty"` // SMB only (secret — written to creds file) } -// handleNetStorageAdd mounts a NAS share host-side. It role-gates the mount to the user-data namespace, -// writes the SMB creds file out-of-band (0600), then calls EnsureNetworkMount (automount idle-unmount). +// handleNetStorageAdd installs a NAS share host-side — verify-before-commit (SPIKE-nas-verify). +// Pipeline: decode + scope + role-gate (unchanged) → SYNC fast-fail (spec validation + a 2 s TCP +// pre-probe; an unreachable server is refused with NOTHING installed — Scenario E) → stage SMB creds +// (0600) → EnsureNetworkMount → start the DETACHED verify job → respond {verify:"started"}. The +// caller (controller orchestrator) polls GET /netstorage/verify-status; a failed verify has already +// auto-rolled-back agent-side. func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmid int) { if s.netStorage == nil { writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") @@ -83,10 +88,57 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi IdleTimeoutSec: req.IdleTimeoutSec, } - // SMB: stage the credentials out-of-band (0600) BEFORE the mount. NFS needs none (server squash). + // SYNC fast-fail 1/2 — full spec validation BEFORE anything is written or installed. For SMB the + // spec is validated against the CANONICAL creds path (a fixed POSIX constant + the + // charset-validated name), not the configured dir — the configured dir is agent-controlled, not + // user input, and EnsureNetworkMount re-validates the real spec anyway. The file itself is + // written only after every sync check has passed. + if spec.Protocol == storage.ProtocolSMB { + if req.Username == "" || req.Password == "" { + writeErr(w, http.StatusBadRequest, "smb credentials (username + password) are required") + return + } + spec.CredsRef = defaultSmbCredsDir + "/" + spec.Name + ".cred" + } + if err := storage.ValidateNetworkMountSpec(spec); err != nil { + writeErr(w, http.StatusBadRequest, err.Error()) + return + } + + // Single-flight (Scenario G): claim the one verify slot before any side effect, so two racing + // adds can never interleave creds/install/verify. + job := &netVerifyJob{ + JobID: s.nowFn().UTC().Format("20060102T150405.000Z"), + Name: spec.Name, + Where: spec.Where(), + Protocol: string(spec.Protocol), + Phase: netVerifyPhaseRunning, + StartedAt: s.nowFn().UTC().Format(time.RFC3339), + UpdatedAt: s.nowFn().UTC().Format(time.RFC3339), + } + if !s.tryStartNetVerify(job) { + writeStatus(w, http.StatusConflict, false, map[string]any{"code": "busy"}, + "a network storage add is already in progress") + return + } + + // SYNC fast-fail 2/2 (Scenario E): a dead endpoint is refused in ~2 s, BEFORE any unit or creds + // file exists. The refusal carries the category code so the UI message is exact. + if !s.netReachable(spec.Protocol, spec.Server) { + s.releaseNetVerify(job) + s.logger.Warn("local-api: network mount refused — endpoint not reachable", + "name", spec.Name, "server", spec.Server, "proto", spec.Protocol) + writeStatus(w, http.StatusBadGateway, false, + map[string]any{"code": storage.NetVerifyUnreachable}, + "NAS endpoint not reachable") + return + } + + // SMB: stage the credentials out-of-band (0600). NFS needs none (server squash). if spec.Protocol == storage.ProtocolSMB { credsPath, err := s.writeSMBCreds(spec.Name, req.Username, req.Password) if err != nil { + s.releaseNetVerify(job) s.logger.Error("local-api: writing SMB credentials failed", "name", spec.Name, "err", err) writeErr(w, http.StatusBadRequest, "could not stage SMB credentials") return @@ -100,18 +152,25 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi if spec.Protocol == storage.ProtocolSMB { s.removeSMBCreds(spec.Name) } + s.releaseNetVerify(job) writeErr(w, http.StatusBadGateway, "network mount failed: "+err.Error()) return } - s.logger.Info("local-api: network mount added", "name", spec.Name, "proto", spec.Protocol, - "server", spec.Server, "export", spec.Export, "where", spec.Where()) + + // Units installed + automount enabled — hand off to the detached verify (it rolls back on fail). + s.runNetVerify(spec, job) + s.logger.Info("local-api: network mount installed — verify started", "name", spec.Name, + "proto", spec.Protocol, "server", spec.Server, "export", spec.Export, "where", spec.Where(), + "job_id", job.JobID) writeOK(w, map[string]any{ - "name": spec.Name, - "protocol": string(spec.Protocol), - "where": spec.Where(), - "host_uid": spec.HostUID(), - "host_gid": spec.HostGID(), + "name": spec.Name, + "protocol": string(spec.Protocol), + "where": spec.Where(), + "host_uid": spec.HostUID(), + "host_gid": spec.HostGID(), "guest_path": spec.Where(), // the in-guest path the controller (A2) repoints a media app's data dir to + "verify": "started", + "job_id": job.JobID, }) } @@ -179,14 +238,10 @@ func (s *Server) writeSMBCreds(name, username, password string) (string, error) if strings.ContainsAny(username, "\n\r") || strings.ContainsAny(password, "\n\r") { return "", fmt.Errorf("smb credentials must not contain newlines") } - dir := s.smbCredsDir - if dir == "" { - dir = defaultSmbCredsDir - } - if err := os.MkdirAll(dir, 0o700); err != nil { + path := s.smbCredsPath(name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return "", fmt.Errorf("creds dir: %w", err) } - path := filepath.Join(dir, name+".cred") body := "username=" + username + "\npassword=" + password + "\n" if err := os.WriteFile(path, []byte(body), 0o600); err != nil { return "", fmt.Errorf("writing creds: %w", err) @@ -196,9 +251,14 @@ func (s *Server) writeSMBCreds(name, username, password string) (string, error) // removeSMBCreds deletes a share's creds file (best-effort; absent is fine). func (s *Server) removeSMBCreds(name string) { + _ = os.Remove(s.smbCredsPath(name)) +} + +// smbCredsPath computes a share's creds file path (pure — used by validation BEFORE the file exists). +func (s *Server) smbCredsPath(name string) string { dir := s.smbCredsDir if dir == "" { dir = defaultSmbCredsDir } - _ = os.Remove(filepath.Join(dir, name+".cred")) + return filepath.Join(dir, name+".cred") } diff --git a/internal/localapi/netstorage_test.go b/internal/localapi/netstorage_test.go index 82d4cfe..d39b11e 100644 --- a/internal/localapi/netstorage_test.go +++ b/internal/localapi/netstorage_test.go @@ -51,6 +51,12 @@ func newNetServer(t *testing.T, n NetworkStorageOps, credsDir string) *Server { if err != nil { t.Fatalf("new server: %v", err) } + // Hermetic verify seams: no TCP dial, no /proc/mounts, no journalctl. Verify-specific tests + // (netverifyjob_test.go) override the rows they exercise. + srv.netReachable = func(storage.NetworkProtocol, string) bool { return true } + srv.netTrigger = func(string) error { return nil } + srv.netMounted = func(string) bool { return true } + srv.netJournal = func(context.Context, string) (string, error) { return "", nil } return srv } diff --git a/internal/localapi/netverifyjob.go b/internal/localapi/netverifyjob.go new file mode 100644 index 0000000..3113afe --- /dev/null +++ b/internal/localapi/netverifyjob.go @@ -0,0 +1,234 @@ +package localapi + +import ( + "context" + "fmt" + "net/http" + "os" + "os/exec" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/storage" +) + +// Network-storage verify job — the agent half of verify-before-commit (SPIKE-nas-verify-2026-07-11). +// POST /netstorage/add no longer succeeds blind: after EnsureNetworkMount installs the unit pair, +// a DETACHED job triggers a real mount through the enabled automount (spike Q1: a single directory +// read wakes it), judges success from /proc/mounts (§8 truth table), classifies a failure from the +// mount unit's journal (Q4 taxonomy), and AUTO-ROLLS-BACK a failed install (RemoveNetworkMount + +// creds file) so a bogus share never lingers half-configured. +// +// The job record is an IN-MEMORY single slot, deliberately NOT persisted (unlike formatJob): if the +// agent restarts mid-verify the slot is empty and GET /netstorage/verify-status reports phase +// "none" — the CONTROLLER orchestrator treats "no job after units were installed" as verify-lost +// and rolls the add back itself (Scenario F). Persistence would buy nothing here: the trigger is +// re-runnable and the controller owns the retry; a lost verify must fail the add, never revive it. + +// netVerifyJob is the single-slot verify record served by GET /netstorage/verify-status. +type netVerifyJob struct { + JobID string `json:"job_id"` + Name string `json:"name"` + Where string `json:"where"` + Protocol string `json:"protocol"` + Phase string `json:"phase"` // running | done | failed (the empty slot reports "none") + Code string `json:"code,omitempty"` // failure category (storage.NetVerify*) + Detail string `json:"detail,omitempty"` // operator hint + raw journal fragment (English; UI maps Hungarian by Code) + StartedAt string `json:"started_at"` + UpdatedAt string `json:"updated_at"` +} + +const ( + netVerifyPhaseNone = "none" + netVerifyPhaseRunning = "running" + netVerifyPhaseDone = "done" + netVerifyPhaseFailed = "failed" +) + +// netVerifyDeadline bounds the whole verify pipeline. systemd resolves even a black-holed mount at +// its 90 s start cap (SPIKE Q4 case i), so 95 s guarantees the trigger goroutine's outcome is in by +// the time the deadline can fire — the deadline is the belt, not the expected path. +const netVerifyDeadline = 95 * time.Second + +// netRollbackTimeout bounds the auto-rollback after a failed verify (a fresh context — the verify +// deadline may already be spent when the rollback runs). +const netRollbackTimeout = 30 * time.Second + +// tryStartNetVerify claims the single verify slot for job (a copy is stored). false = another verify +// is still running (Scenario G single-flight: the caller refuses the second add). +func (s *Server) tryStartNetVerify(job *netVerifyJob) bool { + s.netVerifyMu.Lock() + defer s.netVerifyMu.Unlock() + if s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning { + return false + } + cp := *job + s.netVerifyCur = &cp + return true +} + +// releaseNetVerify frees the slot IF it still holds this job — the sync fast-fail path (pre-probe / +// Ensure failure) claims the slot first to close the race window, then releases it because no +// detached work ever started. +func (s *Server) releaseNetVerify(job *netVerifyJob) { + s.netVerifyMu.Lock() + defer s.netVerifyMu.Unlock() + if s.netVerifyCur != nil && s.netVerifyCur.JobID == job.JobID { + s.netVerifyCur = nil + } +} + +// finishNetVerify records the terminal phase (done/failed) on the slot. +func (s *Server) finishNetVerify(job *netVerifyJob, phase, code, detail string) { + s.netVerifyMu.Lock() + defer s.netVerifyMu.Unlock() + job.Phase = phase + job.Code = code + job.Detail = detail + job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) + cp := *job + s.netVerifyCur = &cp +} + +// netVerifySnapshot returns a copy of the current slot (nil = no job — the Scenario F signal). +func (s *Server) netVerifySnapshot() *netVerifyJob { + s.netVerifyMu.Lock() + defer s.netVerifyMu.Unlock() + if s.netVerifyCur == nil { + return nil + } + cp := *s.netVerifyCur + return &cp +} + +// runNetVerify runs the verify pipeline detached from the HTTP request (off baseCtx, like +// startFormatDetached). The returned channel closes when the job reaches a terminal phase (tests +// wait on it; production ignores it — the controller polls the status endpoint). +func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob) <-chan struct{} { + base := s.baseCtx + if base == nil { + base = context.Background() + } + done := make(chan struct{}) + go func() { + defer close(done) + ctx, cancel := context.WithTimeout(base, netVerifyDeadline) + defer cancel() + + // Trigger a real mount: a directory read through the enabled automount mounts the share + // (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount + // attempt (≤90 s, its start cap) — so it runs in its own goroutine, which may linger past + // our select until that cap resolves it. Bounded and accepted; never awaited twice. + trigger := make(chan error, 1) + go func() { trigger <- s.netTrigger(spec.Where()) }() + timedOut := false + select { + case <-trigger: + case <-ctx.Done(): + timedOut = true + } + + // §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted + // share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides + // writability, not the agent user's readability. + if s.netMounted(spec.Where()) { + s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where()) + s.finishNetVerify(job, netVerifyPhaseDone, "", "") + return + } + + // Mount FAILED → classify from the mount unit's journal (unprivileged read — systemd-journal + // group membership, NO sudo), then auto-rollback the install. + code, detail := s.classifyNetFailure(base, spec, timedOut) + s.rollbackNetMount(base, spec) + s.logger.Warn("netverify: verify failed — install rolled back", + "name", spec.Name, "code", code, "timed_out", timedOut) + s.finishNetVerify(job, netVerifyPhaseFailed, code, detail) + }() + return done +} + +// classifyNetFailure reads the mount unit's journal tail and classifies it per the Q4 taxonomy. +// Journal unavailable (agent user not in systemd-journal, journalctl missing) degrades to the +// generic category — with the timeout verdict preserved when the deadline fired. +func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMountSpec, timedOut bool) (code, detail string) { + fallback := storage.NetVerifyMountFailed + if timedOut { + fallback = storage.NetVerifyTimeout + } + unit, err := storage.UnitNameForMount(spec.Where()) + if err != nil { + return fallback, "internal: unit name: " + err.Error() + } + jctx, cancel := context.WithTimeout(base, 10*time.Second) + defer cancel() + tail, jerr := s.netJournal(jctx, unit) + if jerr != nil { + s.logger.Warn("netverify: journal unavailable — degraded classification (is felhom-agent in the systemd-journal group?)", + "unit", unit, "err", jerr) + return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)" + } + code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server)) + if code == storage.NetVerifyMountFailed && timedOut { + code = storage.NetVerifyTimeout + } + // Detail carries the hint + a bounded raw fragment so the UI's collapsible detail shows the truth. + frag := strings.TrimSpace(tail) + if len(frag) > 500 { + frag = frag[len(frag)-500:] + } + return code, hint + " | journal: " + frag +} + +// rollbackNetMount is the verify-fail auto-rollback: remove the unit pair + the SMB creds file. +// Idempotent + best-effort-logged — the controller may double-remove after us (Scenario F/§8), +// which is harmless. +func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMountSpec) { + ctx, cancel := context.WithTimeout(base, netRollbackTimeout) + defer cancel() + if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil { + s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)", + "name", spec.Name, "err", err) + } + if spec.Protocol == storage.ProtocolSMB { + s.removeSMBCreds(spec.Name) + } +} + +// handleNetVerifyStatus reports the single verify slot. phase "none" (no job) is a REAL signal, not +// an error: the controller reads it after an agent restart as verify-lost and rolls back (Scenario F). +func (s *Server) handleNetVerifyStatus(w http.ResponseWriter, r *http.Request, vmid int) { + if s.netStorage == nil { + writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") + return + } + job := s.netVerifySnapshot() + if job == nil { + writeOK(w, map[string]any{"vmid": vmid, "phase": netVerifyPhaseNone}) + return + } + writeOK(w, map[string]any{ + "vmid": vmid, "phase": job.Phase, "name": job.Name, "where": job.Where, + "protocol": job.Protocol, "code": job.Code, "detail": job.Detail, + "job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt, + }) +} + +// readUnitJournal is the production netJournal seam: an UNPRIVILEGED journalctl read of the mount +// unit's tail (-o cat = raw message strings for the classifier). No sudo — this works iff the +// felhom-agent user is in the systemd-journal group (host-install adds it; §9 rule 7). +func readUnitJournal(ctx context.Context, unit string) (string, error) { + out, err := exec.CommandContext(ctx, "journalctl", "-u", unit, "-n", "20", "--no-pager", "-o", "cat").CombinedOutput() + if err != nil { + return "", fmt.Errorf("journalctl -u %s: %w (%s)", unit, err, strings.TrimSpace(string(out))) + } + return string(out), nil +} + +// triggerNetMount is the production netTrigger seam: a plain directory read through the automount +// trigger path — exactly what wakes the mount (spike Q1). The error is deliberately unused by the +// caller for the verdict (§8: readability is NOT the truth source); it exists for the log line only. +func triggerNetMount(where string) error { + _, err := os.ReadDir(where) + return err +} diff --git a/internal/localapi/netverifyjob_test.go b/internal/localapi/netverifyjob_test.go new file mode 100644 index 0000000..b28b133 --- /dev/null +++ b/internal/localapi/netverifyjob_test.go @@ -0,0 +1,272 @@ +package localapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-agent/internal/storage" +) + +// Verify-job tests (SPIKE-nas-verify): the detached pipeline runs entirely through seams — no +// systemctl, no journalctl, no /proc/mounts, no network. fakeNetOps (netstorage_test.go) records +// the mount-surface effects the assertions check. + +// verifySeams configures a Server's verify pipeline for tests. +type verifySeams struct { + trigger func(where string) error + mounted func(where string) bool + journal func(ctx context.Context, unit string) (string, error) + reachable func(proto storage.NetworkProtocol, server string) bool +} + +func newVerifyServer(t *testing.T, n NetworkStorageOps, credsDir string, seams verifySeams) *Server { + t.Helper() + srv := newNetServer(t, n, credsDir) + if seams.trigger != nil { + srv.netTrigger = seams.trigger + } else { + srv.netTrigger = func(string) error { return nil } // default: instant, successful read + } + if seams.mounted != nil { + srv.netMounted = seams.mounted + } + if seams.journal != nil { + srv.netJournal = seams.journal + } else { + srv.netJournal = func(context.Context, string) (string, error) { return "", nil } + } + if seams.reachable != nil { + srv.netReachable = seams.reachable + } else { + srv.netReachable = func(storage.NetworkProtocol, string) bool { return true } + } + return srv +} + +// pollVerify polls GET /netstorage/verify-status until the job leaves `running` (or the deadline). +func pollVerify(t *testing.T, h http.Handler) map[string]any { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + w := do(t, h, "GET", "/netstorage/verify-status", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("verify-status: got %d (%s)", w.Code, w.Body.String()) + } + var resp struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode status: %v", err) + } + if p, _ := resp.Data["phase"].(string); p != netVerifyPhaseRunning { + return resp.Data + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("verify job did not reach a terminal phase in time") + return nil +} + +// --- A3: mount-FAILED ⇒ auto-rollback (units + creds) + failed{code} --------------------------------- +// Companion red-proof: drop the s.rollbackNetMount call from runNetVerify → the removed/creds +// assertions fail (the failed install would linger — today's bug shape). +func TestNetVerify_MountFailed_RollsBackAndClassifies(t *testing.T) { + credsDir := t.TempDir() + n := &fakeNetOps{} + srv := newVerifyServer(t, n, credsDir, verifySeams{ + mounted: func(string) bool { return false }, // §8: not in /proc/mounts ⇒ FAILED + journal: func(context.Context, string) (string, error) { + return "mount error(13): Permission denied", nil // Q4 iv — smb_auth + }, + }) + h := srv.Handler() + + body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}` + w := do(t, h, "POST", "/netstorage/add", "A", body) + if w.Code != http.StatusOK { + t.Fatalf("add: got %d (%s)", w.Code, w.Body.String()) + } + if len(n.ensured) != 1 { + t.Fatalf("EnsureNetworkMount calls = %d, want 1", len(n.ensured)) + } + + final := pollVerify(t, h) + if final["phase"] != netVerifyPhaseFailed { + t.Fatalf("phase = %v, want failed (%v)", final["phase"], final) + } + if final["code"] != storage.NetVerifySMBAuth { + t.Errorf("code = %v, want %q", final["code"], storage.NetVerifySMBAuth) + } + // The auto-rollback effects: the unit removal was called AND the creds file is gone. + if len(n.removed) != 1 || n.removed[0] != "vids" { + t.Errorf("RemoveNetworkMount not called for the failed install: removed=%v", n.removed) + } + if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) { + t.Errorf("creds file must be removed on verify failure (stat err = %v)", err) + } +} + +// TestNetVerify_JournalUnavailable_DegradesButStillRollsBack: journal read error ⇒ generic +// mount_failed + the systemd-journal hint, and the rollback still runs (Scenario B degradation). +func TestNetVerify_JournalUnavailable_DegradesButStillRollsBack(t *testing.T) { + n := &fakeNetOps{} + srv := newVerifyServer(t, n, t.TempDir(), verifySeams{ + mounted: func(string) bool { return false }, + journal: func(context.Context, string) (string, error) { + return "", fmt.Errorf("journalctl: permission denied") + }, + }) + h := srv.Handler() + w := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"media","protocol":"nfs","server":"10.0.0.5","export":"/srv/media","mapped_uid":1000,"mapped_gid":1000}`) + if w.Code != http.StatusOK { + t.Fatalf("add: got %d (%s)", w.Code, w.Body.String()) + } + final := pollVerify(t, h) + if final["phase"] != netVerifyPhaseFailed || final["code"] != storage.NetVerifyMountFailed { + t.Fatalf("degraded classification: phase=%v code=%v, want failed/%q", final["phase"], final["code"], storage.NetVerifyMountFailed) + } + detail, _ := final["detail"].(string) + if want := "systemd-journal"; !containsStr(detail, want) { + t.Errorf("degraded detail must carry the group hint %q: %q", want, detail) + } + if len(n.removed) != 1 { + t.Errorf("rollback must still run when the journal is unavailable: removed=%v", n.removed) + } +} + +// --- A4: the §8 truth table — /proc/mounts is the ONLY judge ----------------------------------------- +// Companion red-proof: a readability-based verdict (trigger err == nil ⇒ OK) fails BOTH rows. +func TestNetVerify_TruthTable(t *testing.T) { + t.Run("ReadDir ok but NOT mounted = FAILED (empty-dir false positive guard)", func(t *testing.T) { + n := &fakeNetOps{} + srv := newVerifyServer(t, n, t.TempDir(), verifySeams{ + trigger: func(string) error { return nil }, // the read "worked" (empty dir) + mounted: func(string) bool { return false }, // …but no nfs4/cifs in /proc/mounts + journal: func(context.Context, string) (string, error) { return "", nil }, + }) + h := srv.Handler() + w := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`) + if w.Code != http.StatusOK { + t.Fatalf("add: %d (%s)", w.Code, w.Body.String()) + } + final := pollVerify(t, h) + if final["phase"] != netVerifyPhaseFailed { + t.Fatalf("a readable-but-unmounted path must FAIL verify, got %v", final["phase"]) + } + if len(n.removed) != 1 { + t.Errorf("failed verify must roll back: removed=%v", n.removed) + } + }) + t.Run("ReadDir EACCES but mounted = OK (0700 export on a good mount)", func(t *testing.T) { + n := &fakeNetOps{} + srv := newVerifyServer(t, n, t.TempDir(), verifySeams{ + trigger: func(string) error { return os.ErrPermission }, // agent user can't read it — fine + mounted: func(string) bool { return true }, // the mount is REAL + }) + h := srv.Handler() + w := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"m2","protocol":"nfs","server":"10.0.0.5","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`) + if w.Code != http.StatusOK { + t.Fatalf("add: %d (%s)", w.Code, w.Body.String()) + } + final := pollVerify(t, h) + if final["phase"] != netVerifyPhaseDone { + t.Fatalf("an unreadable-but-mounted share must PASS agent verify (writability is the controller probe's job), got %v (%v)", final["phase"], final) + } + if len(n.removed) != 0 { + t.Errorf("a passing verify must NOT roll back: removed=%v", n.removed) + } + }) +} + +// --- A5: the sync TCP pre-probe refuses BEFORE any install (Scenario E) ------------------------------ +// Companion red-proof: move the pre-probe after EnsureNetworkMount → the zero-install assertion fails. +func TestNetVerify_UnreachablePreProbe_InstallsNothing(t *testing.T) { + credsDir := t.TempDir() + n := &fakeNetOps{} + srv := newVerifyServer(t, n, credsDir, verifySeams{ + reachable: func(storage.NetworkProtocol, string) bool { return false }, + }) + h := srv.Handler() + w := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"vids","protocol":"smb","server":"192.168.0.199","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`) + if w.Code != http.StatusBadGateway { + t.Fatalf("unreachable add: got %d want 502 (%s)", w.Code, w.Body.String()) + } + var resp struct { + OK bool `json:"ok"` + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.OK || resp.Data["code"] != storage.NetVerifyUnreachable { + t.Errorf("refusal must carry code=unreachable: %s", w.Body.String()) + } + // NOTHING installed: no Ensure call, no creds file, and the verify slot is free again. + if len(n.ensured) != 0 { + t.Errorf("EnsureNetworkMount must not run for an unreachable server: %v", n.ensured) + } + if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) { + t.Errorf("no creds file may be written for an unreachable server (stat err = %v)", err) + } + if got := srv.netVerifySnapshot(); got != nil { + t.Errorf("the verify slot must be released after a sync refusal: %+v", got) + } +} + +// --- A6: single-flight + the "no job" shape ---------------------------------------------------------- +// Companion red-proof: drop the tryStartNetVerify running-check → the 409 assertion fails. +func TestNetVerify_SingleFlight_AndNoJobShape(t *testing.T) { + // No job yet: the status endpoint must serve the Scenario-F "none" shape. + nIdle := &fakeNetOps{} + hIdle := newVerifyServer(t, nIdle, t.TempDir(), verifySeams{}).Handler() + w := do(t, hIdle, "GET", "/netstorage/verify-status", "A", "") + var idle struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &idle); err != nil { + t.Fatal(err) + } + if idle.Data["phase"] != netVerifyPhaseNone { + t.Fatalf("empty slot must report phase %q, got %v", netVerifyPhaseNone, idle.Data["phase"]) + } + + // Single-flight: hold the first verify open via a blocking trigger, then submit a second add. + release := make(chan struct{}) + n := &fakeNetOps{} + srv := newVerifyServer(t, n, t.TempDir(), verifySeams{ + trigger: func(string) error { <-release; return nil }, + mounted: func(string) bool { return true }, + }) + h := srv.Handler() + w1 := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`) + if w1.Code != http.StatusOK { + t.Fatalf("first add: %d (%s)", w1.Code, w1.Body.String()) + } + w2 := do(t, h, "POST", "/netstorage/add", "A", + `{"name":"m2","protocol":"nfs","server":"10.0.0.6","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`) + if w2.Code != http.StatusConflict { + t.Fatalf("second add while verifying: got %d want 409 (%s)", w2.Code, w2.Body.String()) + } + if len(n.ensured) != 1 { + t.Errorf("the refused second add must not reach EnsureNetworkMount: %v", n.ensured) + } + close(release) + final := pollVerify(t, h) + if final["phase"] != netVerifyPhaseDone || final["name"] != "m1" { + t.Errorf("first job must finish unaffected: %v", final) + } +} + +func containsStr(s, sub string) bool { return strings.Contains(s, sub) } diff --git a/internal/localapi/server.go b/internal/localapi/server.go index b948a1e..eee825e 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -216,6 +216,19 @@ type Server struct { swapMu sync.Mutex swapInFlight map[int]bool + // Network-storage verify job (SPIKE-nas-verify): the IN-MEMORY single slot + the seams the + // detached pipeline runs through (tests inject; production defaults set in NewServer). + netVerifyMu sync.Mutex + netVerifyCur *netVerifyJob + // netTrigger performs the mount-waking directory read through the automount path (Q1). + netTrigger func(where string) error + // netMounted judges mount success from /proc/mounts — the §8 truth source (never readability). + netMounted func(where string) bool + // netJournal reads a mount unit's journal tail UNPRIVILEGED (systemd-journal group, no sudo). + netJournal func(ctx context.Context, unit string) (string, error) + // netReachable is the 2 s TCP endpoint pre-probe (sync fast-fail + classification tiebreak). + netReachable func(proto storage.NetworkProtocol, server string) bool + baseCtx context.Context // for fire-and-forget backups; set in Run } @@ -270,6 +283,10 @@ func NewServer(o Options) (*Server, error) { s.reresolveWipe = s.reresolveDurableForWipe s.reresolveBlank = s.reresolveDurableForBlankFormat s.deviceDurableID = storage.DeviceDurableID + s.netTrigger = triggerNetMount + s.netMounted = storage.NetworkMountedAt + s.netJournal = readUnitJournal + s.netReachable = storage.NetworkEndpointReachable if o.ControllerSwap != nil { s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger) } @@ -304,8 +321,11 @@ func (s *Server) Handler() http.Handler { // Network storage (NAS) — Part A1: mount/list/remove a bulk-media NAS share host-side (automount // idle-unmount; +100000 uid recipe). A distinct class from a drive — no enroll/eject/wipe. + // Add is verify-before-commit (SPIKE-nas-verify): it starts a detached verify job the caller + // polls on /netstorage/verify-status; a failed verify auto-rolls-back agent-side. mux.HandleFunc("POST /netstorage/add", s.withGuest(s.handleNetStorageAdd)) mux.HandleFunc("GET /netstorage", s.withGuest(s.handleNetStorageList)) + mux.HandleFunc("GET /netstorage/verify-status", s.withGuest(s.handleNetVerifyStatus)) mux.HandleFunc("POST /netstorage/remove", s.withGuest(s.handleNetStorageRemove)) // agentic controller update (Phase 1): in-guest image swap + rollback, owned by the agent. diff --git a/internal/storage/netmount.go b/internal/storage/netmount.go index 3994355..3d0d93e 100644 --- a/internal/storage/netmount.go +++ b/internal/storage/netmount.go @@ -205,13 +205,20 @@ func (s NetworkMountSpec) fsType() string { return "nfs4" // vers=4.1 → nfs4 (avoids the rpcbind/lock-manager surface of v3) } -// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5): -// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev — soft is the failure-isolation knob -// (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not the -// client mount, so no uid appears here. +// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5 + +// SPIKE-nas-verify Q4-vi): +// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0 — soft is the failure-isolation +// knob (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not +// the client mount, so no uid appears here. retry=0 (SPIKE-nas-verify Q4-vi): without it a +// dead-NAS on-demand access wedges the app until systemd's 90 s start cap (measured 91 s); with +// it the access fails clean in ~3.8 s (ENODEV) AND the verify sees a classifiable +// "No route to host" instead of a diagnostic-free systemd timeout. retry only governs retrying +// a FAILED first attempt — the happy path is untouched, and each autofs re-access is a fresh +// attempt anyway. // - SMB: vers=3.0,credentials=,uid=<+100000>,gid=<+100000>,forceuid,forcegid,file_mode=0664, // dir_mode=0775,_netdev — modes are PLAIN octal (not setgid 2775); the client forces the -// guest-visible owner to the mapped id so the container reads+writes. +// guest-visible owner to the mapped id so the container reads+writes. NO retry= here — retry is +// a mount.nfs option; mount.cifs would reject it. // // Every interpolated value is pre-validated by ValidateNetworkMountSpec, so the string carries no // newline / no extra directive. NEVER a default `hard` NFS mount (it wedges) — soft is mandatory. @@ -229,7 +236,7 @@ func (s NetworkMountSpec) mountOptions() string { "_netdev", }, ",") } - return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev" + return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0" } // renderNetworkMountUnit builds the .mount unit (triggered by the .automount; deliberately NO [Install] @@ -496,6 +503,36 @@ func isNetworkMounted(fstype string) bool { } } +// NetworkEndpointReachable TCP-dials a share's NAS endpoint (NFS 2049 / SMB 445) with the short +// liveness timeout. This is the add endpoint's SYNC pre-probe (SPIKE-nas-verify Scenario E): an +// unreachable server is refused in ~2 s BEFORE any unit is installed. +func NetworkEndpointReachable(proto NetworkProtocol, server string) bool { + return endpointReachable(netEndpoint(string(proto), server)) +} + +// NetworkMountedAt reports whether a REAL network filesystem (nfs/nfs4/cifs) is currently mounted at +// where, per /proc/mounts. The autofs trigger does NOT count. This is the verify job's mount-success +// truth source (SPIKE-nas-verify §8): success is judged from /proc/mounts, NEVER from readability — +// a 0700 export owned by the squashed uid gives the agent user EACCES on a perfectly good mount. +func NetworkMountedAt(where string) bool { + data, err := os.ReadFile("/proc/mounts") + if err != nil { + return false + } + return networkMountedIn(string(data), where) +} + +// networkMountedIn is the pure core of NetworkMountedAt (unit-tested against fixture tables). +func networkMountedIn(procMounts, where string) bool { + for _, line := range strings.Split(procMounts, "\n") { + f := strings.Fields(line) + if len(f) >= 3 && f[1] == where && isNetworkMounted(f[2]) { + return true + } + } + return false +} + // endpointReachable TCP-dials a NAS endpoint with a short timeout (the liveness probe that never touches // the mount). "" endpoint → not reachable. func endpointReachable(endpoint string) bool { diff --git a/internal/storage/netmount_test.go b/internal/storage/netmount_test.go index c1552af..aeb70ac 100644 --- a/internal/storage/netmount_test.go +++ b/internal/storage/netmount_test.go @@ -21,7 +21,7 @@ func TestNetworkMount_NFSUnitRendering(t *testing.T) { } mu := renderNetworkMountUnit(spec) - wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev" + wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0" for _, want := range []string{ netUnitMarker, "What=192.168.0.180:/srv/nas-sim/media", diff --git a/internal/storage/netverify.go b/internal/storage/netverify.go new file mode 100644 index 0000000..93065b3 --- /dev/null +++ b/internal/storage/netverify.go @@ -0,0 +1,61 @@ +package storage + +import "strings" + +// Network-mount verify failure classification — the categorized refusal behind verify-before-commit. +// The categories and their journal substrings are the LIVE-MEASURED error taxonomy of +// SPIKE-nas-verify-2026-07-11 §Q4 (felhom.eu/documentation/audits/): every mount failure exits +// rc=32, so classification MUST run on the journal's message strings, never on exit codes — an +// exit-code classifier cannot even split wrong-password from wrong-share-name. + +// Verify failure category codes. The controller maps these to the customer-facing Hungarian +// messages; the codes themselves are a stable wire vocabulary — do not rename casually. +const ( + NetVerifyUnreachable = "unreachable" // no host behind the endpoint (pre-probe or No route/refused) + NetVerifyNFSExport = "nfs_export" // NFS export missing OR not permitted — MERGED (Q4 ii≡iii) + NetVerifySMBAuth = "smb_auth" // SMB wrong username/password (mount error(13)) + NetVerifySMBShare = "smb_share" // SMB share name not found (mount error(2)) + NetVerifyTimeout = "timeout" // blocked until systemd's 90 s start cap (black-holed-but-routed) + NetVerifyMountFailed = "mount_failed" // no diagnostic matched / journal unavailable +) + +// netVerifyRule is one first-match-wins row of the classification table. The substrings are +// VERBATIM from the spike's Q4 transcripts (mount.nfs4 / mount.cifs / systemd on PVE 8) — matching +// is on the stable fragment, tolerant of surrounding version drift. +type netVerifyRule struct { + substr string + code string + hint string +} + +// netVerifyRules — ordered: protocol-specific diagnostics before generic ones. Note the NFS rule +// keys on the full "reason given by server:" fragment, so SMB's "mount error(2): No such file or +// directory" can never shadow it (and vice versa). +var netVerifyRules = []netVerifyRule{ + {"No route to host", NetVerifyUnreachable, "no route to the server (retry=0 fast-fail)"}, + {"Connection refused", NetVerifyUnreachable, "the server refused the connection"}, + {"Connection timed out", NetVerifyUnreachable, "the connection timed out"}, + {"reason given by server: No such file or directory", NetVerifyNFSExport, + "NFS export not found OR not permitted for this client — NFSv4 cannot distinguish the two (SPIKE Q4 ii≡iii)"}, + {"mount error(13)", NetVerifySMBAuth, "SMB logon failure (STATUS_LOGON_FAILURE) — wrong username or password"}, + {"mount error(2)", NetVerifySMBShare, "SMB share not found (BAD_NETWORK_NAME)"}, + {"Mounting timed out. Terminating", NetVerifyTimeout, + "mount blocked until systemd's start timeout — server routed but not answering"}, +} + +// ClassifyNetVerifyFailure maps a mount unit's journal tail to a verify failure category + an +// operator-facing English hint (the Hungarian customer message is the controller's job). Pure, +// table-driven, first match wins. tcpReachable (the endpoint pre-probe result at classification +// time) only breaks the tie when NO substring matched: an empty/unmatched journal against a +// dead endpoint is still `unreachable`, not the generic `mount_failed`. +func ClassifyNetVerifyFailure(journalTail string, tcpReachable bool) (code, hint string) { + for _, r := range netVerifyRules { + if strings.Contains(journalTail, r.substr) { + return r.code, r.hint + } + } + if !tcpReachable { + return NetVerifyUnreachable, "no mount diagnostic in the journal and the endpoint is not reachable" + } + return NetVerifyMountFailed, "mount failed with no recognized diagnostic — see the raw journal detail" +} diff --git a/internal/storage/netverify_test.go b/internal/storage/netverify_test.go new file mode 100644 index 0000000..ac1edb4 --- /dev/null +++ b/internal/storage/netverify_test.go @@ -0,0 +1,107 @@ +package storage + +import ( + "strings" + "testing" +) + +// --- A1: the retry=0 knob (SPIKE-nas-verify Q4-vi) -------------------------------------------------- + +// TestMountOptions_NFSRetry0_SMBWithout: retry=0 is in the NFS option string (dead-NAS access 91 s → +// 3.8 s) and NOT in the SMB one (retry is a mount.nfs option; mount.cifs would reject the mount). +// Companion red-proof: revert the mountOptions NFS branch → the first assertion fails. +func TestMountOptions_NFSRetry0_SMBWithout(t *testing.T) { + nfs := NetworkMountSpec{Name: "m", Protocol: ProtocolNFS, Server: "s", Export: "/e", MappedUID: 1000, MappedGID: 1000} + if got := nfs.mountOptions(); !strings.Contains(got, ",retry=0") { + t.Errorf("NFS options missing retry=0 (Q4-vi): %q", got) + } + smb := NetworkMountSpec{Name: "m", Protocol: ProtocolSMB, Server: "s", Export: "e", + MappedUID: 1000, MappedGID: 1000, CredsRef: "/var/lib/felhom-agent/smb-creds/m.cred"} + if got := smb.mountOptions(); strings.Contains(got, "retry=") { + t.Errorf("SMB options must NOT carry retry= (mount.cifs rejects it): %q", got) + } +} + +// --- A2: the journal classifier (SPIKE-nas-verify Q4 — VERBATIM live strings) ----------------------- + +// TestClassifyNetVerifyFailure runs the Q4 taxonomy on the strings captured live in the spike. +// Companion red-proof: an exit-code-based classifier (every failure is rc=32) collapses smb_auth and +// smb_share into one code — modeled by replacing the body with `return NetVerifyMountFailed, ""`; +// every non-generic row fails. +func TestClassifyNetVerifyFailure(t *testing.T) { + cases := []struct { + name string + journal string + reachable bool + want string + }{ + // Q4 i′ — unreachable with retry=0 (the production unit's shape). + {"no route", "mount.nfs4: No route to host for 192.168.0.199:/srv/nope on /mnt/felhom-drives/x", false, NetVerifyUnreachable}, + {"conn refused", "mount.nfs4: Connection refused", false, NetVerifyUnreachable}, + {"conn timed out", "mount.nfs4: Connection timed out", false, NetVerifyUnreachable}, + // Q4 ii ≡ iii — the MERGED category: NFSv4 cannot distinguish no-export from not-permitted. + {"nfs export missing", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/nope failed, reason given by server: No such file or directory", true, NetVerifyNFSExport}, + {"nfs export denied (identical string)", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/q4iii failed, reason given by server: No such file or directory", true, NetVerifyNFSExport}, + // Q4 iv / v — SMB splits cleanly on the errno line. + {"smb wrong password", "mount error(13): Permission denied\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBAuth}, + {"smb wrong share", "mount error(2): No such file or directory\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBShare}, + // Q4 i (default retry) — systemd kills mount.nfs at its 90 s cap, no mount.nfs diagnostic. + {"systemd timeout", "Mounting timed out. Terminating.\nMount process exited, code=killed, status=15/TERM", true, NetVerifyTimeout}, + // Degradations: nothing matched. + {"empty journal, reachable", "", true, NetVerifyMountFailed}, + {"empty journal, unreachable", "", false, NetVerifyUnreachable}, + {"garbage, reachable", "some future mount.nfs5 wording", true, NetVerifyMountFailed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + code, hint := ClassifyNetVerifyFailure(tc.journal, tc.reachable) + if code != tc.want { + t.Errorf("ClassifyNetVerifyFailure(%q, reachable=%v) = %q, want %q", tc.journal, tc.reachable, code, tc.want) + } + if hint == "" { + t.Errorf("hint must never be empty (code %q)", code) + } + }) + } +} + +// TestClassify_SMBShareNeverShadowsNFS: the SMB error(2) line contains "No such file or directory" +// too — the NFS rule must key on the full "reason given by server:" fragment so neither shadows the +// other regardless of table order. +func TestClassify_SMBShareNeverShadowsNFS(t *testing.T) { + code, _ := ClassifyNetVerifyFailure("mount error(2): No such file or directory", true) + if code != NetVerifySMBShare { + t.Errorf("SMB error(2) classified %q, want %q", code, NetVerifySMBShare) + } + code, _ = ClassifyNetVerifyFailure("failed, reason given by server: No such file or directory", true) + if code != NetVerifyNFSExport { + t.Errorf("NFS server-reason classified %q, want %q", code, NetVerifyNFSExport) + } +} + +// --- networkMountedIn: the §8 truth source (autofs trigger ≠ mounted) -------------------------------- + +func TestNetworkMountedIn(t *testing.T) { + procMounts := `sysfs /sys sysfs rw 0 0 +systemd-1 /mnt/felhom-drives/idle autofs rw,relatime,fd=86 0 0 +systemd-1 /mnt/felhom-drives/live autofs rw,relatime,fd=86 0 0 +192.168.0.180:/srv/media /mnt/felhom-drives/live nfs4 rw,noatime,vers=4.1,soft 0 0 +//nas/share /mnt/felhom-drives/smb cifs rw,vers=3.0 0 0 +/dev/sda1 /mnt/felhom-drives/disk ext4 rw 0 0 +` + cases := []struct { + where string + want bool + }{ + {"/mnt/felhom-drives/live", true}, // real nfs4 (the autofs line for the same path must not confuse it) + {"/mnt/felhom-drives/smb", true}, // cifs + {"/mnt/felhom-drives/idle", false}, // autofs trigger ONLY — idle automount is NOT mounted + {"/mnt/felhom-drives/disk", false}, // a local fs at the path is not a network mount + {"/mnt/felhom-drives/nope", false}, // absent + } + for _, tc := range cases { + if got := networkMountedIn(procMounts, tc.where); got != tc.want { + t.Errorf("networkMountedIn(%q) = %v, want %v", tc.where, got, tc.want) + } + } +}