v0.81.0: NAS verify-before-commit — retry=0, journal classifier, detached verify job + auto-rollback
Agent half of the verify-before-commit task (SPIKE-nas-verify-2026-07-11, b57f6c1): retry=0 in the production NFS options (Q4-vi); ClassifyNetVerifyFailure on the live Q4 strings (nfs_export merges not-found/not-permitted); add = sync fast-fail (2s TCP pre-probe, nothing installed) + detached in-memory verify job judging /proc/mounts only, auto-rollback on failure; GET /netstorage/verify-status (phase none = the controller's Scenario-F rollback signal); unprivileged journalctl (systemd-journal group, NO new sudoers grants). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
@@ -48,8 +49,12 @@ type netStorageAddRequest struct {
|
||||
Password string `json:"password,omitempty"` // SMB only (secret — written to creds file)
|
||||
}
|
||||
|
||||
// handleNetStorageAdd mounts a NAS share host-side. It role-gates the mount to the user-data namespace,
|
||||
// writes the SMB creds file out-of-band (0600), then calls EnsureNetworkMount (automount idle-unmount).
|
||||
// handleNetStorageAdd installs a NAS share host-side — verify-before-commit (SPIKE-nas-verify).
|
||||
// Pipeline: decode + scope + role-gate (unchanged) → SYNC fast-fail (spec validation + a 2 s TCP
|
||||
// pre-probe; an unreachable server is refused with NOTHING installed — Scenario E) → stage SMB creds
|
||||
// (0600) → EnsureNetworkMount → start the DETACHED verify job → respond {verify:"started"}. The
|
||||
// caller (controller orchestrator) polls GET /netstorage/verify-status; a failed verify has already
|
||||
// auto-rolled-back agent-side.
|
||||
func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.netStorage == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host")
|
||||
@@ -83,10 +88,57 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi
|
||||
IdleTimeoutSec: req.IdleTimeoutSec,
|
||||
}
|
||||
|
||||
// SMB: stage the credentials out-of-band (0600) BEFORE the mount. NFS needs none (server squash).
|
||||
// SYNC fast-fail 1/2 — full spec validation BEFORE anything is written or installed. For SMB the
|
||||
// spec is validated against the CANONICAL creds path (a fixed POSIX constant + the
|
||||
// charset-validated name), not the configured dir — the configured dir is agent-controlled, not
|
||||
// user input, and EnsureNetworkMount re-validates the real spec anyway. The file itself is
|
||||
// written only after every sync check has passed.
|
||||
if spec.Protocol == storage.ProtocolSMB {
|
||||
if req.Username == "" || req.Password == "" {
|
||||
writeErr(w, http.StatusBadRequest, "smb credentials (username + password) are required")
|
||||
return
|
||||
}
|
||||
spec.CredsRef = defaultSmbCredsDir + "/" + spec.Name + ".cred"
|
||||
}
|
||||
if err := storage.ValidateNetworkMountSpec(spec); err != nil {
|
||||
writeErr(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Single-flight (Scenario G): claim the one verify slot before any side effect, so two racing
|
||||
// adds can never interleave creds/install/verify.
|
||||
job := &netVerifyJob{
|
||||
JobID: s.nowFn().UTC().Format("20060102T150405.000Z"),
|
||||
Name: spec.Name,
|
||||
Where: spec.Where(),
|
||||
Protocol: string(spec.Protocol),
|
||||
Phase: netVerifyPhaseRunning,
|
||||
StartedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if !s.tryStartNetVerify(job) {
|
||||
writeStatus(w, http.StatusConflict, false, map[string]any{"code": "busy"},
|
||||
"a network storage add is already in progress")
|
||||
return
|
||||
}
|
||||
|
||||
// SYNC fast-fail 2/2 (Scenario E): a dead endpoint is refused in ~2 s, BEFORE any unit or creds
|
||||
// file exists. The refusal carries the category code so the UI message is exact.
|
||||
if !s.netReachable(spec.Protocol, spec.Server) {
|
||||
s.releaseNetVerify(job)
|
||||
s.logger.Warn("local-api: network mount refused — endpoint not reachable",
|
||||
"name", spec.Name, "server", spec.Server, "proto", spec.Protocol)
|
||||
writeStatus(w, http.StatusBadGateway, false,
|
||||
map[string]any{"code": storage.NetVerifyUnreachable},
|
||||
"NAS endpoint not reachable")
|
||||
return
|
||||
}
|
||||
|
||||
// SMB: stage the credentials out-of-band (0600). NFS needs none (server squash).
|
||||
if spec.Protocol == storage.ProtocolSMB {
|
||||
credsPath, err := s.writeSMBCreds(spec.Name, req.Username, req.Password)
|
||||
if err != nil {
|
||||
s.releaseNetVerify(job)
|
||||
s.logger.Error("local-api: writing SMB credentials failed", "name", spec.Name, "err", err)
|
||||
writeErr(w, http.StatusBadRequest, "could not stage SMB credentials")
|
||||
return
|
||||
@@ -100,18 +152,25 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request, vmi
|
||||
if spec.Protocol == storage.ProtocolSMB {
|
||||
s.removeSMBCreds(spec.Name)
|
||||
}
|
||||
s.releaseNetVerify(job)
|
||||
writeErr(w, http.StatusBadGateway, "network mount failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: network mount added", "name", spec.Name, "proto", spec.Protocol,
|
||||
"server", spec.Server, "export", spec.Export, "where", spec.Where())
|
||||
|
||||
// Units installed + automount enabled — hand off to the detached verify (it rolls back on fail).
|
||||
s.runNetVerify(spec, job)
|
||||
s.logger.Info("local-api: network mount installed — verify started", "name", spec.Name,
|
||||
"proto", spec.Protocol, "server", spec.Server, "export", spec.Export, "where", spec.Where(),
|
||||
"job_id", job.JobID)
|
||||
writeOK(w, map[string]any{
|
||||
"name": spec.Name,
|
||||
"protocol": string(spec.Protocol),
|
||||
"where": spec.Where(),
|
||||
"host_uid": spec.HostUID(),
|
||||
"host_gid": spec.HostGID(),
|
||||
"name": spec.Name,
|
||||
"protocol": string(spec.Protocol),
|
||||
"where": spec.Where(),
|
||||
"host_uid": spec.HostUID(),
|
||||
"host_gid": spec.HostGID(),
|
||||
"guest_path": spec.Where(), // the in-guest path the controller (A2) repoints a media app's data dir to
|
||||
"verify": "started",
|
||||
"job_id": job.JobID,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -179,14 +238,10 @@ func (s *Server) writeSMBCreds(name, username, password string) (string, error)
|
||||
if strings.ContainsAny(username, "\n\r") || strings.ContainsAny(password, "\n\r") {
|
||||
return "", fmt.Errorf("smb credentials must not contain newlines")
|
||||
}
|
||||
dir := s.smbCredsDir
|
||||
if dir == "" {
|
||||
dir = defaultSmbCredsDir
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
path := s.smbCredsPath(name)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return "", fmt.Errorf("creds dir: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, name+".cred")
|
||||
body := "username=" + username + "\npassword=" + password + "\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
return "", fmt.Errorf("writing creds: %w", err)
|
||||
@@ -196,9 +251,14 @@ func (s *Server) writeSMBCreds(name, username, password string) (string, error)
|
||||
|
||||
// removeSMBCreds deletes a share's creds file (best-effort; absent is fine).
|
||||
func (s *Server) removeSMBCreds(name string) {
|
||||
_ = os.Remove(s.smbCredsPath(name))
|
||||
}
|
||||
|
||||
// smbCredsPath computes a share's creds file path (pure — used by validation BEFORE the file exists).
|
||||
func (s *Server) smbCredsPath(name string) string {
|
||||
dir := s.smbCredsDir
|
||||
if dir == "" {
|
||||
dir = defaultSmbCredsDir
|
||||
}
|
||||
_ = os.Remove(filepath.Join(dir, name+".cred"))
|
||||
return filepath.Join(dir, name+".cred")
|
||||
}
|
||||
|
||||
@@ -51,6 +51,12 @@ func newNetServer(t *testing.T, n NetworkStorageOps, credsDir string) *Server {
|
||||
if err != nil {
|
||||
t.Fatalf("new server: %v", err)
|
||||
}
|
||||
// Hermetic verify seams: no TCP dial, no /proc/mounts, no journalctl. Verify-specific tests
|
||||
// (netverifyjob_test.go) override the rows they exercise.
|
||||
srv.netReachable = func(storage.NetworkProtocol, string) bool { return true }
|
||||
srv.netTrigger = func(string) error { return nil }
|
||||
srv.netMounted = func(string) bool { return true }
|
||||
srv.netJournal = func(context.Context, string) (string, error) { return "", nil }
|
||||
return srv
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// Network-storage verify job — the agent half of verify-before-commit (SPIKE-nas-verify-2026-07-11).
|
||||
// POST /netstorage/add no longer succeeds blind: after EnsureNetworkMount installs the unit pair,
|
||||
// a DETACHED job triggers a real mount through the enabled automount (spike Q1: a single directory
|
||||
// read wakes it), judges success from /proc/mounts (§8 truth table), classifies a failure from the
|
||||
// mount unit's journal (Q4 taxonomy), and AUTO-ROLLS-BACK a failed install (RemoveNetworkMount +
|
||||
// creds file) so a bogus share never lingers half-configured.
|
||||
//
|
||||
// The job record is an IN-MEMORY single slot, deliberately NOT persisted (unlike formatJob): if the
|
||||
// agent restarts mid-verify the slot is empty and GET /netstorage/verify-status reports phase
|
||||
// "none" — the CONTROLLER orchestrator treats "no job after units were installed" as verify-lost
|
||||
// and rolls the add back itself (Scenario F). Persistence would buy nothing here: the trigger is
|
||||
// re-runnable and the controller owns the retry; a lost verify must fail the add, never revive it.
|
||||
|
||||
// netVerifyJob is the single-slot verify record served by GET /netstorage/verify-status.
|
||||
type netVerifyJob struct {
|
||||
JobID string `json:"job_id"`
|
||||
Name string `json:"name"`
|
||||
Where string `json:"where"`
|
||||
Protocol string `json:"protocol"`
|
||||
Phase string `json:"phase"` // running | done | failed (the empty slot reports "none")
|
||||
Code string `json:"code,omitempty"` // failure category (storage.NetVerify*)
|
||||
Detail string `json:"detail,omitempty"` // operator hint + raw journal fragment (English; UI maps Hungarian by Code)
|
||||
StartedAt string `json:"started_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
netVerifyPhaseNone = "none"
|
||||
netVerifyPhaseRunning = "running"
|
||||
netVerifyPhaseDone = "done"
|
||||
netVerifyPhaseFailed = "failed"
|
||||
)
|
||||
|
||||
// netVerifyDeadline bounds the whole verify pipeline. systemd resolves even a black-holed mount at
|
||||
// its 90 s start cap (SPIKE Q4 case i), so 95 s guarantees the trigger goroutine's outcome is in by
|
||||
// the time the deadline can fire — the deadline is the belt, not the expected path.
|
||||
const netVerifyDeadline = 95 * time.Second
|
||||
|
||||
// netRollbackTimeout bounds the auto-rollback after a failed verify (a fresh context — the verify
|
||||
// deadline may already be spent when the rollback runs).
|
||||
const netRollbackTimeout = 30 * time.Second
|
||||
|
||||
// tryStartNetVerify claims the single verify slot for job (a copy is stored). false = another verify
|
||||
// is still running (Scenario G single-flight: the caller refuses the second add).
|
||||
func (s *Server) tryStartNetVerify(job *netVerifyJob) bool {
|
||||
s.netVerifyMu.Lock()
|
||||
defer s.netVerifyMu.Unlock()
|
||||
if s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning {
|
||||
return false
|
||||
}
|
||||
cp := *job
|
||||
s.netVerifyCur = &cp
|
||||
return true
|
||||
}
|
||||
|
||||
// releaseNetVerify frees the slot IF it still holds this job — the sync fast-fail path (pre-probe /
|
||||
// Ensure failure) claims the slot first to close the race window, then releases it because no
|
||||
// detached work ever started.
|
||||
func (s *Server) releaseNetVerify(job *netVerifyJob) {
|
||||
s.netVerifyMu.Lock()
|
||||
defer s.netVerifyMu.Unlock()
|
||||
if s.netVerifyCur != nil && s.netVerifyCur.JobID == job.JobID {
|
||||
s.netVerifyCur = nil
|
||||
}
|
||||
}
|
||||
|
||||
// finishNetVerify records the terminal phase (done/failed) on the slot.
|
||||
func (s *Server) finishNetVerify(job *netVerifyJob, phase, code, detail string) {
|
||||
s.netVerifyMu.Lock()
|
||||
defer s.netVerifyMu.Unlock()
|
||||
job.Phase = phase
|
||||
job.Code = code
|
||||
job.Detail = detail
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
cp := *job
|
||||
s.netVerifyCur = &cp
|
||||
}
|
||||
|
||||
// netVerifySnapshot returns a copy of the current slot (nil = no job — the Scenario F signal).
|
||||
func (s *Server) netVerifySnapshot() *netVerifyJob {
|
||||
s.netVerifyMu.Lock()
|
||||
defer s.netVerifyMu.Unlock()
|
||||
if s.netVerifyCur == nil {
|
||||
return nil
|
||||
}
|
||||
cp := *s.netVerifyCur
|
||||
return &cp
|
||||
}
|
||||
|
||||
// runNetVerify runs the verify pipeline detached from the HTTP request (off baseCtx, like
|
||||
// startFormatDetached). The returned channel closes when the job reaches a terminal phase (tests
|
||||
// wait on it; production ignores it — the controller polls the status endpoint).
|
||||
func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob) <-chan struct{} {
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
ctx, cancel := context.WithTimeout(base, netVerifyDeadline)
|
||||
defer cancel()
|
||||
|
||||
// Trigger a real mount: a directory read through the enabled automount mounts the share
|
||||
// (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount
|
||||
// attempt (≤90 s, its start cap) — so it runs in its own goroutine, which may linger past
|
||||
// our select until that cap resolves it. Bounded and accepted; never awaited twice.
|
||||
trigger := make(chan error, 1)
|
||||
go func() { trigger <- s.netTrigger(spec.Where()) }()
|
||||
timedOut := false
|
||||
select {
|
||||
case <-trigger:
|
||||
case <-ctx.Done():
|
||||
timedOut = true
|
||||
}
|
||||
|
||||
// §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted
|
||||
// share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides
|
||||
// writability, not the agent user's readability.
|
||||
if s.netMounted(spec.Where()) {
|
||||
s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where())
|
||||
s.finishNetVerify(job, netVerifyPhaseDone, "", "")
|
||||
return
|
||||
}
|
||||
|
||||
// Mount FAILED → classify from the mount unit's journal (unprivileged read — systemd-journal
|
||||
// group membership, NO sudo), then auto-rollback the install.
|
||||
code, detail := s.classifyNetFailure(base, spec, timedOut)
|
||||
s.rollbackNetMount(base, spec)
|
||||
s.logger.Warn("netverify: verify failed — install rolled back",
|
||||
"name", spec.Name, "code", code, "timed_out", timedOut)
|
||||
s.finishNetVerify(job, netVerifyPhaseFailed, code, detail)
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
// classifyNetFailure reads the mount unit's journal tail and classifies it per the Q4 taxonomy.
|
||||
// Journal unavailable (agent user not in systemd-journal, journalctl missing) degrades to the
|
||||
// generic category — with the timeout verdict preserved when the deadline fired.
|
||||
func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMountSpec, timedOut bool) (code, detail string) {
|
||||
fallback := storage.NetVerifyMountFailed
|
||||
if timedOut {
|
||||
fallback = storage.NetVerifyTimeout
|
||||
}
|
||||
unit, err := storage.UnitNameForMount(spec.Where())
|
||||
if err != nil {
|
||||
return fallback, "internal: unit name: " + err.Error()
|
||||
}
|
||||
jctx, cancel := context.WithTimeout(base, 10*time.Second)
|
||||
defer cancel()
|
||||
tail, jerr := s.netJournal(jctx, unit)
|
||||
if jerr != nil {
|
||||
s.logger.Warn("netverify: journal unavailable — degraded classification (is felhom-agent in the systemd-journal group?)",
|
||||
"unit", unit, "err", jerr)
|
||||
return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)"
|
||||
}
|
||||
code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server))
|
||||
if code == storage.NetVerifyMountFailed && timedOut {
|
||||
code = storage.NetVerifyTimeout
|
||||
}
|
||||
// Detail carries the hint + a bounded raw fragment so the UI's collapsible detail shows the truth.
|
||||
frag := strings.TrimSpace(tail)
|
||||
if len(frag) > 500 {
|
||||
frag = frag[len(frag)-500:]
|
||||
}
|
||||
return code, hint + " | journal: " + frag
|
||||
}
|
||||
|
||||
// rollbackNetMount is the verify-fail auto-rollback: remove the unit pair + the SMB creds file.
|
||||
// Idempotent + best-effort-logged — the controller may double-remove after us (Scenario F/§8),
|
||||
// which is harmless.
|
||||
func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMountSpec) {
|
||||
ctx, cancel := context.WithTimeout(base, netRollbackTimeout)
|
||||
defer cancel()
|
||||
if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil {
|
||||
s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)",
|
||||
"name", spec.Name, "err", err)
|
||||
}
|
||||
if spec.Protocol == storage.ProtocolSMB {
|
||||
s.removeSMBCreds(spec.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// handleNetVerifyStatus reports the single verify slot. phase "none" (no job) is a REAL signal, not
|
||||
// an error: the controller reads it after an agent restart as verify-lost and rolls back (Scenario F).
|
||||
func (s *Server) handleNetVerifyStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.netStorage == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host")
|
||||
return
|
||||
}
|
||||
job := s.netVerifySnapshot()
|
||||
if job == nil {
|
||||
writeOK(w, map[string]any{"vmid": vmid, "phase": netVerifyPhaseNone})
|
||||
return
|
||||
}
|
||||
writeOK(w, map[string]any{
|
||||
"vmid": vmid, "phase": job.Phase, "name": job.Name, "where": job.Where,
|
||||
"protocol": job.Protocol, "code": job.Code, "detail": job.Detail,
|
||||
"job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// readUnitJournal is the production netJournal seam: an UNPRIVILEGED journalctl read of the mount
|
||||
// unit's tail (-o cat = raw message strings for the classifier). No sudo — this works iff the
|
||||
// felhom-agent user is in the systemd-journal group (host-install adds it; §9 rule 7).
|
||||
func readUnitJournal(ctx context.Context, unit string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, "journalctl", "-u", unit, "-n", "20", "--no-pager", "-o", "cat").CombinedOutput()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("journalctl -u %s: %w (%s)", unit, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// triggerNetMount is the production netTrigger seam: a plain directory read through the automount
|
||||
// trigger path — exactly what wakes the mount (spike Q1). The error is deliberately unused by the
|
||||
// caller for the verdict (§8: readability is NOT the truth source); it exists for the log line only.
|
||||
func triggerNetMount(where string) error {
|
||||
_, err := os.ReadDir(where)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// Verify-job tests (SPIKE-nas-verify): the detached pipeline runs entirely through seams — no
|
||||
// systemctl, no journalctl, no /proc/mounts, no network. fakeNetOps (netstorage_test.go) records
|
||||
// the mount-surface effects the assertions check.
|
||||
|
||||
// verifySeams configures a Server's verify pipeline for tests.
|
||||
type verifySeams struct {
|
||||
trigger func(where string) error
|
||||
mounted func(where string) bool
|
||||
journal func(ctx context.Context, unit string) (string, error)
|
||||
reachable func(proto storage.NetworkProtocol, server string) bool
|
||||
}
|
||||
|
||||
func newVerifyServer(t *testing.T, n NetworkStorageOps, credsDir string, seams verifySeams) *Server {
|
||||
t.Helper()
|
||||
srv := newNetServer(t, n, credsDir)
|
||||
if seams.trigger != nil {
|
||||
srv.netTrigger = seams.trigger
|
||||
} else {
|
||||
srv.netTrigger = func(string) error { return nil } // default: instant, successful read
|
||||
}
|
||||
if seams.mounted != nil {
|
||||
srv.netMounted = seams.mounted
|
||||
}
|
||||
if seams.journal != nil {
|
||||
srv.netJournal = seams.journal
|
||||
} else {
|
||||
srv.netJournal = func(context.Context, string) (string, error) { return "", nil }
|
||||
}
|
||||
if seams.reachable != nil {
|
||||
srv.netReachable = seams.reachable
|
||||
} else {
|
||||
srv.netReachable = func(storage.NetworkProtocol, string) bool { return true }
|
||||
}
|
||||
return srv
|
||||
}
|
||||
|
||||
// pollVerify polls GET /netstorage/verify-status until the job leaves `running` (or the deadline).
|
||||
func pollVerify(t *testing.T, h http.Handler) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
w := do(t, h, "GET", "/netstorage/verify-status", "A", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("verify-status: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode status: %v", err)
|
||||
}
|
||||
if p, _ := resp.Data["phase"].(string); p != netVerifyPhaseRunning {
|
||||
return resp.Data
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("verify job did not reach a terminal phase in time")
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- A3: mount-FAILED ⇒ auto-rollback (units + creds) + failed{code} ---------------------------------
|
||||
// Companion red-proof: drop the s.rollbackNetMount call from runNetVerify → the removed/creds
|
||||
// assertions fail (the failed install would linger — today's bug shape).
|
||||
func TestNetVerify_MountFailed_RollsBackAndClassifies(t *testing.T) {
|
||||
credsDir := t.TempDir()
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, credsDir, verifySeams{
|
||||
mounted: func(string) bool { return false }, // §8: not in /proc/mounts ⇒ FAILED
|
||||
journal: func(context.Context, string) (string, error) {
|
||||
return "mount error(13): Permission denied", nil // Q4 iv — smb_auth
|
||||
},
|
||||
})
|
||||
h := srv.Handler()
|
||||
|
||||
body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`
|
||||
w := do(t, h, "POST", "/netstorage/add", "A", body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if len(n.ensured) != 1 {
|
||||
t.Fatalf("EnsureNetworkMount calls = %d, want 1", len(n.ensured))
|
||||
}
|
||||
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed {
|
||||
t.Fatalf("phase = %v, want failed (%v)", final["phase"], final)
|
||||
}
|
||||
if final["code"] != storage.NetVerifySMBAuth {
|
||||
t.Errorf("code = %v, want %q", final["code"], storage.NetVerifySMBAuth)
|
||||
}
|
||||
// The auto-rollback effects: the unit removal was called AND the creds file is gone.
|
||||
if len(n.removed) != 1 || n.removed[0] != "vids" {
|
||||
t.Errorf("RemoveNetworkMount not called for the failed install: removed=%v", n.removed)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) {
|
||||
t.Errorf("creds file must be removed on verify failure (stat err = %v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetVerify_JournalUnavailable_DegradesButStillRollsBack: journal read error ⇒ generic
|
||||
// mount_failed + the systemd-journal hint, and the rollback still runs (Scenario B degradation).
|
||||
func TestNetVerify_JournalUnavailable_DegradesButStillRollsBack(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
mounted: func(string) bool { return false },
|
||||
journal: func(context.Context, string) (string, error) {
|
||||
return "", fmt.Errorf("journalctl: permission denied")
|
||||
},
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"media","protocol":"nfs","server":"10.0.0.5","export":"/srv/media","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed || final["code"] != storage.NetVerifyMountFailed {
|
||||
t.Fatalf("degraded classification: phase=%v code=%v, want failed/%q", final["phase"], final["code"], storage.NetVerifyMountFailed)
|
||||
}
|
||||
detail, _ := final["detail"].(string)
|
||||
if want := "systemd-journal"; !containsStr(detail, want) {
|
||||
t.Errorf("degraded detail must carry the group hint %q: %q", want, detail)
|
||||
}
|
||||
if len(n.removed) != 1 {
|
||||
t.Errorf("rollback must still run when the journal is unavailable: removed=%v", n.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A4: the §8 truth table — /proc/mounts is the ONLY judge -----------------------------------------
|
||||
// Companion red-proof: a readability-based verdict (trigger err == nil ⇒ OK) fails BOTH rows.
|
||||
func TestNetVerify_TruthTable(t *testing.T) {
|
||||
t.Run("ReadDir ok but NOT mounted = FAILED (empty-dir false positive guard)", func(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { return nil }, // the read "worked" (empty dir)
|
||||
mounted: func(string) bool { return false }, // …but no nfs4/cifs in /proc/mounts
|
||||
journal: func(context.Context, string) (string, error) { return "", nil },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed {
|
||||
t.Fatalf("a readable-but-unmounted path must FAIL verify, got %v", final["phase"])
|
||||
}
|
||||
if len(n.removed) != 1 {
|
||||
t.Errorf("failed verify must roll back: removed=%v", n.removed)
|
||||
}
|
||||
})
|
||||
t.Run("ReadDir EACCES but mounted = OK (0700 export on a good mount)", func(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { return os.ErrPermission }, // agent user can't read it — fine
|
||||
mounted: func(string) bool { return true }, // the mount is REAL
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m2","protocol":"nfs","server":"10.0.0.5","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseDone {
|
||||
t.Fatalf("an unreadable-but-mounted share must PASS agent verify (writability is the controller probe's job), got %v (%v)", final["phase"], final)
|
||||
}
|
||||
if len(n.removed) != 0 {
|
||||
t.Errorf("a passing verify must NOT roll back: removed=%v", n.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --- A5: the sync TCP pre-probe refuses BEFORE any install (Scenario E) ------------------------------
|
||||
// Companion red-proof: move the pre-probe after EnsureNetworkMount → the zero-install assertion fails.
|
||||
func TestNetVerify_UnreachablePreProbe_InstallsNothing(t *testing.T) {
|
||||
credsDir := t.TempDir()
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, credsDir, verifySeams{
|
||||
reachable: func(storage.NetworkProtocol, string) bool { return false },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"vids","protocol":"smb","server":"192.168.0.199","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`)
|
||||
if w.Code != http.StatusBadGateway {
|
||||
t.Fatalf("unreachable add: got %d want 502 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
OK bool `json:"ok"`
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if resp.OK || resp.Data["code"] != storage.NetVerifyUnreachable {
|
||||
t.Errorf("refusal must carry code=unreachable: %s", w.Body.String())
|
||||
}
|
||||
// NOTHING installed: no Ensure call, no creds file, and the verify slot is free again.
|
||||
if len(n.ensured) != 0 {
|
||||
t.Errorf("EnsureNetworkMount must not run for an unreachable server: %v", n.ensured)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) {
|
||||
t.Errorf("no creds file may be written for an unreachable server (stat err = %v)", err)
|
||||
}
|
||||
if got := srv.netVerifySnapshot(); got != nil {
|
||||
t.Errorf("the verify slot must be released after a sync refusal: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A6: single-flight + the "no job" shape ----------------------------------------------------------
|
||||
// Companion red-proof: drop the tryStartNetVerify running-check → the 409 assertion fails.
|
||||
func TestNetVerify_SingleFlight_AndNoJobShape(t *testing.T) {
|
||||
// No job yet: the status endpoint must serve the Scenario-F "none" shape.
|
||||
nIdle := &fakeNetOps{}
|
||||
hIdle := newVerifyServer(t, nIdle, t.TempDir(), verifySeams{}).Handler()
|
||||
w := do(t, hIdle, "GET", "/netstorage/verify-status", "A", "")
|
||||
var idle struct {
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &idle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if idle.Data["phase"] != netVerifyPhaseNone {
|
||||
t.Fatalf("empty slot must report phase %q, got %v", netVerifyPhaseNone, idle.Data["phase"])
|
||||
}
|
||||
|
||||
// Single-flight: hold the first verify open via a blocking trigger, then submit a second add.
|
||||
release := make(chan struct{})
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { <-release; return nil },
|
||||
mounted: func(string) bool { return true },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w1 := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w1.Code != http.StatusOK {
|
||||
t.Fatalf("first add: %d (%s)", w1.Code, w1.Body.String())
|
||||
}
|
||||
w2 := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m2","protocol":"nfs","server":"10.0.0.6","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w2.Code != http.StatusConflict {
|
||||
t.Fatalf("second add while verifying: got %d want 409 (%s)", w2.Code, w2.Body.String())
|
||||
}
|
||||
if len(n.ensured) != 1 {
|
||||
t.Errorf("the refused second add must not reach EnsureNetworkMount: %v", n.ensured)
|
||||
}
|
||||
close(release)
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseDone || final["name"] != "m1" {
|
||||
t.Errorf("first job must finish unaffected: %v", final)
|
||||
}
|
||||
}
|
||||
|
||||
func containsStr(s, sub string) bool { return strings.Contains(s, sub) }
|
||||
@@ -216,6 +216,19 @@ type Server struct {
|
||||
swapMu sync.Mutex
|
||||
swapInFlight map[int]bool
|
||||
|
||||
// Network-storage verify job (SPIKE-nas-verify): the IN-MEMORY single slot + the seams the
|
||||
// detached pipeline runs through (tests inject; production defaults set in NewServer).
|
||||
netVerifyMu sync.Mutex
|
||||
netVerifyCur *netVerifyJob
|
||||
// netTrigger performs the mount-waking directory read through the automount path (Q1).
|
||||
netTrigger func(where string) error
|
||||
// netMounted judges mount success from /proc/mounts — the §8 truth source (never readability).
|
||||
netMounted func(where string) bool
|
||||
// netJournal reads a mount unit's journal tail UNPRIVILEGED (systemd-journal group, no sudo).
|
||||
netJournal func(ctx context.Context, unit string) (string, error)
|
||||
// netReachable is the 2 s TCP endpoint pre-probe (sync fast-fail + classification tiebreak).
|
||||
netReachable func(proto storage.NetworkProtocol, server string) bool
|
||||
|
||||
baseCtx context.Context // for fire-and-forget backups; set in Run
|
||||
}
|
||||
|
||||
@@ -270,6 +283,10 @@ func NewServer(o Options) (*Server, error) {
|
||||
s.reresolveWipe = s.reresolveDurableForWipe
|
||||
s.reresolveBlank = s.reresolveDurableForBlankFormat
|
||||
s.deviceDurableID = storage.DeviceDurableID
|
||||
s.netTrigger = triggerNetMount
|
||||
s.netMounted = storage.NetworkMountedAt
|
||||
s.netJournal = readUnitJournal
|
||||
s.netReachable = storage.NetworkEndpointReachable
|
||||
if o.ControllerSwap != nil {
|
||||
s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger)
|
||||
}
|
||||
@@ -304,8 +321,11 @@ func (s *Server) Handler() http.Handler {
|
||||
|
||||
// Network storage (NAS) — Part A1: mount/list/remove a bulk-media NAS share host-side (automount
|
||||
// idle-unmount; +100000 uid recipe). A distinct class from a drive — no enroll/eject/wipe.
|
||||
// Add is verify-before-commit (SPIKE-nas-verify): it starts a detached verify job the caller
|
||||
// polls on /netstorage/verify-status; a failed verify auto-rolls-back agent-side.
|
||||
mux.HandleFunc("POST /netstorage/add", s.withGuest(s.handleNetStorageAdd))
|
||||
mux.HandleFunc("GET /netstorage", s.withGuest(s.handleNetStorageList))
|
||||
mux.HandleFunc("GET /netstorage/verify-status", s.withGuest(s.handleNetVerifyStatus))
|
||||
mux.HandleFunc("POST /netstorage/remove", s.withGuest(s.handleNetStorageRemove))
|
||||
|
||||
// agentic controller update (Phase 1): in-guest image swap + rollback, owned by the agent.
|
||||
|
||||
@@ -205,13 +205,20 @@ func (s NetworkMountSpec) fsType() string {
|
||||
return "nfs4" // vers=4.1 → nfs4 (avoids the rpcbind/lock-manager surface of v3)
|
||||
}
|
||||
|
||||
// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5):
|
||||
// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev — soft is the failure-isolation knob
|
||||
// (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not the
|
||||
// client mount, so no uid appears here.
|
||||
// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5 +
|
||||
// SPIKE-nas-verify Q4-vi):
|
||||
// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0 — soft is the failure-isolation
|
||||
// knob (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not
|
||||
// the client mount, so no uid appears here. retry=0 (SPIKE-nas-verify Q4-vi): without it a
|
||||
// dead-NAS on-demand access wedges the app until systemd's 90 s start cap (measured 91 s); with
|
||||
// it the access fails clean in ~3.8 s (ENODEV) AND the verify sees a classifiable
|
||||
// "No route to host" instead of a diagnostic-free systemd timeout. retry only governs retrying
|
||||
// a FAILED first attempt — the happy path is untouched, and each autofs re-access is a fresh
|
||||
// attempt anyway.
|
||||
// - SMB: vers=3.0,credentials=<file>,uid=<+100000>,gid=<+100000>,forceuid,forcegid,file_mode=0664,
|
||||
// dir_mode=0775,_netdev — modes are PLAIN octal (not setgid 2775); the client forces the
|
||||
// guest-visible owner to the mapped id so the container reads+writes.
|
||||
// guest-visible owner to the mapped id so the container reads+writes. NO retry= here — retry is
|
||||
// a mount.nfs option; mount.cifs would reject it.
|
||||
//
|
||||
// Every interpolated value is pre-validated by ValidateNetworkMountSpec, so the string carries no
|
||||
// newline / no extra directive. NEVER a default `hard` NFS mount (it wedges) — soft is mandatory.
|
||||
@@ -229,7 +236,7 @@ func (s NetworkMountSpec) mountOptions() string {
|
||||
"_netdev",
|
||||
}, ",")
|
||||
}
|
||||
return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev"
|
||||
return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0"
|
||||
}
|
||||
|
||||
// renderNetworkMountUnit builds the .mount unit (triggered by the .automount; deliberately NO [Install]
|
||||
@@ -496,6 +503,36 @@ func isNetworkMounted(fstype string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkEndpointReachable TCP-dials a share's NAS endpoint (NFS 2049 / SMB 445) with the short
|
||||
// liveness timeout. This is the add endpoint's SYNC pre-probe (SPIKE-nas-verify Scenario E): an
|
||||
// unreachable server is refused in ~2 s BEFORE any unit is installed.
|
||||
func NetworkEndpointReachable(proto NetworkProtocol, server string) bool {
|
||||
return endpointReachable(netEndpoint(string(proto), server))
|
||||
}
|
||||
|
||||
// NetworkMountedAt reports whether a REAL network filesystem (nfs/nfs4/cifs) is currently mounted at
|
||||
// where, per /proc/mounts. The autofs trigger does NOT count. This is the verify job's mount-success
|
||||
// truth source (SPIKE-nas-verify §8): success is judged from /proc/mounts, NEVER from readability —
|
||||
// a 0700 export owned by the squashed uid gives the agent user EACCES on a perfectly good mount.
|
||||
func NetworkMountedAt(where string) bool {
|
||||
data, err := os.ReadFile("/proc/mounts")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return networkMountedIn(string(data), where)
|
||||
}
|
||||
|
||||
// networkMountedIn is the pure core of NetworkMountedAt (unit-tested against fixture tables).
|
||||
func networkMountedIn(procMounts, where string) bool {
|
||||
for _, line := range strings.Split(procMounts, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 3 && f[1] == where && isNetworkMounted(f[2]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// endpointReachable TCP-dials a NAS endpoint with a short timeout (the liveness probe that never touches
|
||||
// the mount). "" endpoint → not reachable.
|
||||
func endpointReachable(endpoint string) bool {
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestNetworkMount_NFSUnitRendering(t *testing.T) {
|
||||
}
|
||||
mu := renderNetworkMountUnit(spec)
|
||||
|
||||
wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev"
|
||||
wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0"
|
||||
for _, want := range []string{
|
||||
netUnitMarker,
|
||||
"What=192.168.0.180:/srv/nas-sim/media",
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package storage
|
||||
|
||||
import "strings"
|
||||
|
||||
// Network-mount verify failure classification — the categorized refusal behind verify-before-commit.
|
||||
// The categories and their journal substrings are the LIVE-MEASURED error taxonomy of
|
||||
// SPIKE-nas-verify-2026-07-11 §Q4 (felhom.eu/documentation/audits/): every mount failure exits
|
||||
// rc=32, so classification MUST run on the journal's message strings, never on exit codes — an
|
||||
// exit-code classifier cannot even split wrong-password from wrong-share-name.
|
||||
|
||||
// Verify failure category codes. The controller maps these to the customer-facing Hungarian
|
||||
// messages; the codes themselves are a stable wire vocabulary — do not rename casually.
|
||||
const (
|
||||
NetVerifyUnreachable = "unreachable" // no host behind the endpoint (pre-probe or No route/refused)
|
||||
NetVerifyNFSExport = "nfs_export" // NFS export missing OR not permitted — MERGED (Q4 ii≡iii)
|
||||
NetVerifySMBAuth = "smb_auth" // SMB wrong username/password (mount error(13))
|
||||
NetVerifySMBShare = "smb_share" // SMB share name not found (mount error(2))
|
||||
NetVerifyTimeout = "timeout" // blocked until systemd's 90 s start cap (black-holed-but-routed)
|
||||
NetVerifyMountFailed = "mount_failed" // no diagnostic matched / journal unavailable
|
||||
)
|
||||
|
||||
// netVerifyRule is one first-match-wins row of the classification table. The substrings are
|
||||
// VERBATIM from the spike's Q4 transcripts (mount.nfs4 / mount.cifs / systemd on PVE 8) — matching
|
||||
// is on the stable fragment, tolerant of surrounding version drift.
|
||||
type netVerifyRule struct {
|
||||
substr string
|
||||
code string
|
||||
hint string
|
||||
}
|
||||
|
||||
// netVerifyRules — ordered: protocol-specific diagnostics before generic ones. Note the NFS rule
|
||||
// keys on the full "reason given by server:" fragment, so SMB's "mount error(2): No such file or
|
||||
// directory" can never shadow it (and vice versa).
|
||||
var netVerifyRules = []netVerifyRule{
|
||||
{"No route to host", NetVerifyUnreachable, "no route to the server (retry=0 fast-fail)"},
|
||||
{"Connection refused", NetVerifyUnreachable, "the server refused the connection"},
|
||||
{"Connection timed out", NetVerifyUnreachable, "the connection timed out"},
|
||||
{"reason given by server: No such file or directory", NetVerifyNFSExport,
|
||||
"NFS export not found OR not permitted for this client — NFSv4 cannot distinguish the two (SPIKE Q4 ii≡iii)"},
|
||||
{"mount error(13)", NetVerifySMBAuth, "SMB logon failure (STATUS_LOGON_FAILURE) — wrong username or password"},
|
||||
{"mount error(2)", NetVerifySMBShare, "SMB share not found (BAD_NETWORK_NAME)"},
|
||||
{"Mounting timed out. Terminating", NetVerifyTimeout,
|
||||
"mount blocked until systemd's start timeout — server routed but not answering"},
|
||||
}
|
||||
|
||||
// ClassifyNetVerifyFailure maps a mount unit's journal tail to a verify failure category + an
|
||||
// operator-facing English hint (the Hungarian customer message is the controller's job). Pure,
|
||||
// table-driven, first match wins. tcpReachable (the endpoint pre-probe result at classification
|
||||
// time) only breaks the tie when NO substring matched: an empty/unmatched journal against a
|
||||
// dead endpoint is still `unreachable`, not the generic `mount_failed`.
|
||||
func ClassifyNetVerifyFailure(journalTail string, tcpReachable bool) (code, hint string) {
|
||||
for _, r := range netVerifyRules {
|
||||
if strings.Contains(journalTail, r.substr) {
|
||||
return r.code, r.hint
|
||||
}
|
||||
}
|
||||
if !tcpReachable {
|
||||
return NetVerifyUnreachable, "no mount diagnostic in the journal and the endpoint is not reachable"
|
||||
}
|
||||
return NetVerifyMountFailed, "mount failed with no recognized diagnostic — see the raw journal detail"
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- A1: the retry=0 knob (SPIKE-nas-verify Q4-vi) --------------------------------------------------
|
||||
|
||||
// TestMountOptions_NFSRetry0_SMBWithout: retry=0 is in the NFS option string (dead-NAS access 91 s →
|
||||
// 3.8 s) and NOT in the SMB one (retry is a mount.nfs option; mount.cifs would reject the mount).
|
||||
// Companion red-proof: revert the mountOptions NFS branch → the first assertion fails.
|
||||
func TestMountOptions_NFSRetry0_SMBWithout(t *testing.T) {
|
||||
nfs := NetworkMountSpec{Name: "m", Protocol: ProtocolNFS, Server: "s", Export: "/e", MappedUID: 1000, MappedGID: 1000}
|
||||
if got := nfs.mountOptions(); !strings.Contains(got, ",retry=0") {
|
||||
t.Errorf("NFS options missing retry=0 (Q4-vi): %q", got)
|
||||
}
|
||||
smb := NetworkMountSpec{Name: "m", Protocol: ProtocolSMB, Server: "s", Export: "e",
|
||||
MappedUID: 1000, MappedGID: 1000, CredsRef: "/var/lib/felhom-agent/smb-creds/m.cred"}
|
||||
if got := smb.mountOptions(); strings.Contains(got, "retry=") {
|
||||
t.Errorf("SMB options must NOT carry retry= (mount.cifs rejects it): %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A2: the journal classifier (SPIKE-nas-verify Q4 — VERBATIM live strings) -----------------------
|
||||
|
||||
// TestClassifyNetVerifyFailure runs the Q4 taxonomy on the strings captured live in the spike.
|
||||
// Companion red-proof: an exit-code-based classifier (every failure is rc=32) collapses smb_auth and
|
||||
// smb_share into one code — modeled by replacing the body with `return NetVerifyMountFailed, ""`;
|
||||
// every non-generic row fails.
|
||||
func TestClassifyNetVerifyFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
journal string
|
||||
reachable bool
|
||||
want string
|
||||
}{
|
||||
// Q4 i′ — unreachable with retry=0 (the production unit's shape).
|
||||
{"no route", "mount.nfs4: No route to host for 192.168.0.199:/srv/nope on /mnt/felhom-drives/x", false, NetVerifyUnreachable},
|
||||
{"conn refused", "mount.nfs4: Connection refused", false, NetVerifyUnreachable},
|
||||
{"conn timed out", "mount.nfs4: Connection timed out", false, NetVerifyUnreachable},
|
||||
// Q4 ii ≡ iii — the MERGED category: NFSv4 cannot distinguish no-export from not-permitted.
|
||||
{"nfs export missing", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/nope failed, reason given by server: No such file or directory", true, NetVerifyNFSExport},
|
||||
{"nfs export denied (identical string)", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/q4iii failed, reason given by server: No such file or directory", true, NetVerifyNFSExport},
|
||||
// Q4 iv / v — SMB splits cleanly on the errno line.
|
||||
{"smb wrong password", "mount error(13): Permission denied\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBAuth},
|
||||
{"smb wrong share", "mount error(2): No such file or directory\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBShare},
|
||||
// Q4 i (default retry) — systemd kills mount.nfs at its 90 s cap, no mount.nfs diagnostic.
|
||||
{"systemd timeout", "Mounting timed out. Terminating.\nMount process exited, code=killed, status=15/TERM", true, NetVerifyTimeout},
|
||||
// Degradations: nothing matched.
|
||||
{"empty journal, reachable", "", true, NetVerifyMountFailed},
|
||||
{"empty journal, unreachable", "", false, NetVerifyUnreachable},
|
||||
{"garbage, reachable", "some future mount.nfs5 wording", true, NetVerifyMountFailed},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, hint := ClassifyNetVerifyFailure(tc.journal, tc.reachable)
|
||||
if code != tc.want {
|
||||
t.Errorf("ClassifyNetVerifyFailure(%q, reachable=%v) = %q, want %q", tc.journal, tc.reachable, code, tc.want)
|
||||
}
|
||||
if hint == "" {
|
||||
t.Errorf("hint must never be empty (code %q)", code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassify_SMBShareNeverShadowsNFS: the SMB error(2) line contains "No such file or directory"
|
||||
// too — the NFS rule must key on the full "reason given by server:" fragment so neither shadows the
|
||||
// other regardless of table order.
|
||||
func TestClassify_SMBShareNeverShadowsNFS(t *testing.T) {
|
||||
code, _ := ClassifyNetVerifyFailure("mount error(2): No such file or directory", true)
|
||||
if code != NetVerifySMBShare {
|
||||
t.Errorf("SMB error(2) classified %q, want %q", code, NetVerifySMBShare)
|
||||
}
|
||||
code, _ = ClassifyNetVerifyFailure("failed, reason given by server: No such file or directory", true)
|
||||
if code != NetVerifyNFSExport {
|
||||
t.Errorf("NFS server-reason classified %q, want %q", code, NetVerifyNFSExport)
|
||||
}
|
||||
}
|
||||
|
||||
// --- networkMountedIn: the §8 truth source (autofs trigger ≠ mounted) --------------------------------
|
||||
|
||||
func TestNetworkMountedIn(t *testing.T) {
|
||||
procMounts := `sysfs /sys sysfs rw 0 0
|
||||
systemd-1 /mnt/felhom-drives/idle autofs rw,relatime,fd=86 0 0
|
||||
systemd-1 /mnt/felhom-drives/live autofs rw,relatime,fd=86 0 0
|
||||
192.168.0.180:/srv/media /mnt/felhom-drives/live nfs4 rw,noatime,vers=4.1,soft 0 0
|
||||
//nas/share /mnt/felhom-drives/smb cifs rw,vers=3.0 0 0
|
||||
/dev/sda1 /mnt/felhom-drives/disk ext4 rw 0 0
|
||||
`
|
||||
cases := []struct {
|
||||
where string
|
||||
want bool
|
||||
}{
|
||||
{"/mnt/felhom-drives/live", true}, // real nfs4 (the autofs line for the same path must not confuse it)
|
||||
{"/mnt/felhom-drives/smb", true}, // cifs
|
||||
{"/mnt/felhom-drives/idle", false}, // autofs trigger ONLY — idle automount is NOT mounted
|
||||
{"/mnt/felhom-drives/disk", false}, // a local fs at the path is not a network mount
|
||||
{"/mnt/felhom-drives/nope", false}, // absent
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := networkMountedIn(procMounts, tc.where); got != tc.want {
|
||||
t.Errorf("networkMountedIn(%q) = %v, want %v", tc.where, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user