v0.81.0: NAS verify-before-commit — retry=0, journal classifier, detached verify job + auto-rollback
Agent half of the verify-before-commit task (SPIKE-nas-verify-2026-07-11, b57f6c1): retry=0 in the production NFS options (Q4-vi); ClassifyNetVerifyFailure on the live Q4 strings (nfs_export merges not-found/not-permitted); add = sync fast-fail (2s TCP pre-probe, nothing installed) + detached in-memory verify job judging /proc/mounts only, auto-rollback on failure; GET /netstorage/verify-status (phase none = the controller's Scenario-F rollback signal); unprivileged journalctl (systemd-journal group, NO new sudoers grants). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -205,13 +205,20 @@ func (s NetworkMountSpec) fsType() string {
|
||||
return "nfs4" // vers=4.1 → nfs4 (avoids the rpcbind/lock-manager surface of v3)
|
||||
}
|
||||
|
||||
// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5):
|
||||
// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev — soft is the failure-isolation knob
|
||||
// (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not the
|
||||
// client mount, so no uid appears here.
|
||||
// mountOptions returns the exact, validated option set for the protocol (SPIKE Q2/Q5 +
|
||||
// SPIKE-nas-verify Q4-vi):
|
||||
// - NFS: vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0 — soft is the failure-isolation
|
||||
// knob (clean EIO, never a wedge); the +100000 squash is the EXPORT's job (anonuid=101000), not
|
||||
// the client mount, so no uid appears here. retry=0 (SPIKE-nas-verify Q4-vi): without it a
|
||||
// dead-NAS on-demand access wedges the app until systemd's 90 s start cap (measured 91 s); with
|
||||
// it the access fails clean in ~3.8 s (ENODEV) AND the verify sees a classifiable
|
||||
// "No route to host" instead of a diagnostic-free systemd timeout. retry only governs retrying
|
||||
// a FAILED first attempt — the happy path is untouched, and each autofs re-access is a fresh
|
||||
// attempt anyway.
|
||||
// - SMB: vers=3.0,credentials=<file>,uid=<+100000>,gid=<+100000>,forceuid,forcegid,file_mode=0664,
|
||||
// dir_mode=0775,_netdev — modes are PLAIN octal (not setgid 2775); the client forces the
|
||||
// guest-visible owner to the mapped id so the container reads+writes.
|
||||
// guest-visible owner to the mapped id so the container reads+writes. NO retry= here — retry is
|
||||
// a mount.nfs option; mount.cifs would reject it.
|
||||
//
|
||||
// Every interpolated value is pre-validated by ValidateNetworkMountSpec, so the string carries no
|
||||
// newline / no extra directive. NEVER a default `hard` NFS mount (it wedges) — soft is mandatory.
|
||||
@@ -229,7 +236,7 @@ func (s NetworkMountSpec) mountOptions() string {
|
||||
"_netdev",
|
||||
}, ",")
|
||||
}
|
||||
return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev"
|
||||
return "vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0"
|
||||
}
|
||||
|
||||
// renderNetworkMountUnit builds the .mount unit (triggered by the .automount; deliberately NO [Install]
|
||||
@@ -496,6 +503,36 @@ func isNetworkMounted(fstype string) bool {
|
||||
}
|
||||
}
|
||||
|
||||
// NetworkEndpointReachable TCP-dials a share's NAS endpoint (NFS 2049 / SMB 445) with the short
|
||||
// liveness timeout. This is the add endpoint's SYNC pre-probe (SPIKE-nas-verify Scenario E): an
|
||||
// unreachable server is refused in ~2 s BEFORE any unit is installed.
|
||||
func NetworkEndpointReachable(proto NetworkProtocol, server string) bool {
|
||||
return endpointReachable(netEndpoint(string(proto), server))
|
||||
}
|
||||
|
||||
// NetworkMountedAt reports whether a REAL network filesystem (nfs/nfs4/cifs) is currently mounted at
|
||||
// where, per /proc/mounts. The autofs trigger does NOT count. This is the verify job's mount-success
|
||||
// truth source (SPIKE-nas-verify §8): success is judged from /proc/mounts, NEVER from readability —
|
||||
// a 0700 export owned by the squashed uid gives the agent user EACCES on a perfectly good mount.
|
||||
func NetworkMountedAt(where string) bool {
|
||||
data, err := os.ReadFile("/proc/mounts")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return networkMountedIn(string(data), where)
|
||||
}
|
||||
|
||||
// networkMountedIn is the pure core of NetworkMountedAt (unit-tested against fixture tables).
|
||||
func networkMountedIn(procMounts, where string) bool {
|
||||
for _, line := range strings.Split(procMounts, "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) >= 3 && f[1] == where && isNetworkMounted(f[2]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// endpointReachable TCP-dials a NAS endpoint with a short timeout (the liveness probe that never touches
|
||||
// the mount). "" endpoint → not reachable.
|
||||
func endpointReachable(endpoint string) bool {
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestNetworkMount_NFSUnitRendering(t *testing.T) {
|
||||
}
|
||||
mu := renderNetworkMountUnit(spec)
|
||||
|
||||
wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev"
|
||||
wantOpts := "Options=vers=4.1,soft,timeo=50,retrans=2,noatime,_netdev,retry=0"
|
||||
for _, want := range []string{
|
||||
netUnitMarker,
|
||||
"What=192.168.0.180:/srv/nas-sim/media",
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package storage
|
||||
|
||||
import "strings"
|
||||
|
||||
// Network-mount verify failure classification — the categorized refusal behind verify-before-commit.
|
||||
// The categories and their journal substrings are the LIVE-MEASURED error taxonomy of
|
||||
// SPIKE-nas-verify-2026-07-11 §Q4 (felhom.eu/documentation/audits/): every mount failure exits
|
||||
// rc=32, so classification MUST run on the journal's message strings, never on exit codes — an
|
||||
// exit-code classifier cannot even split wrong-password from wrong-share-name.
|
||||
|
||||
// Verify failure category codes. The controller maps these to the customer-facing Hungarian
|
||||
// messages; the codes themselves are a stable wire vocabulary — do not rename casually.
|
||||
const (
|
||||
NetVerifyUnreachable = "unreachable" // no host behind the endpoint (pre-probe or No route/refused)
|
||||
NetVerifyNFSExport = "nfs_export" // NFS export missing OR not permitted — MERGED (Q4 ii≡iii)
|
||||
NetVerifySMBAuth = "smb_auth" // SMB wrong username/password (mount error(13))
|
||||
NetVerifySMBShare = "smb_share" // SMB share name not found (mount error(2))
|
||||
NetVerifyTimeout = "timeout" // blocked until systemd's 90 s start cap (black-holed-but-routed)
|
||||
NetVerifyMountFailed = "mount_failed" // no diagnostic matched / journal unavailable
|
||||
)
|
||||
|
||||
// netVerifyRule is one first-match-wins row of the classification table. The substrings are
|
||||
// VERBATIM from the spike's Q4 transcripts (mount.nfs4 / mount.cifs / systemd on PVE 8) — matching
|
||||
// is on the stable fragment, tolerant of surrounding version drift.
|
||||
type netVerifyRule struct {
|
||||
substr string
|
||||
code string
|
||||
hint string
|
||||
}
|
||||
|
||||
// netVerifyRules — ordered: protocol-specific diagnostics before generic ones. Note the NFS rule
|
||||
// keys on the full "reason given by server:" fragment, so SMB's "mount error(2): No such file or
|
||||
// directory" can never shadow it (and vice versa).
|
||||
var netVerifyRules = []netVerifyRule{
|
||||
{"No route to host", NetVerifyUnreachable, "no route to the server (retry=0 fast-fail)"},
|
||||
{"Connection refused", NetVerifyUnreachable, "the server refused the connection"},
|
||||
{"Connection timed out", NetVerifyUnreachable, "the connection timed out"},
|
||||
{"reason given by server: No such file or directory", NetVerifyNFSExport,
|
||||
"NFS export not found OR not permitted for this client — NFSv4 cannot distinguish the two (SPIKE Q4 ii≡iii)"},
|
||||
{"mount error(13)", NetVerifySMBAuth, "SMB logon failure (STATUS_LOGON_FAILURE) — wrong username or password"},
|
||||
{"mount error(2)", NetVerifySMBShare, "SMB share not found (BAD_NETWORK_NAME)"},
|
||||
{"Mounting timed out. Terminating", NetVerifyTimeout,
|
||||
"mount blocked until systemd's start timeout — server routed but not answering"},
|
||||
}
|
||||
|
||||
// ClassifyNetVerifyFailure maps a mount unit's journal tail to a verify failure category + an
|
||||
// operator-facing English hint (the Hungarian customer message is the controller's job). Pure,
|
||||
// table-driven, first match wins. tcpReachable (the endpoint pre-probe result at classification
|
||||
// time) only breaks the tie when NO substring matched: an empty/unmatched journal against a
|
||||
// dead endpoint is still `unreachable`, not the generic `mount_failed`.
|
||||
func ClassifyNetVerifyFailure(journalTail string, tcpReachable bool) (code, hint string) {
|
||||
for _, r := range netVerifyRules {
|
||||
if strings.Contains(journalTail, r.substr) {
|
||||
return r.code, r.hint
|
||||
}
|
||||
}
|
||||
if !tcpReachable {
|
||||
return NetVerifyUnreachable, "no mount diagnostic in the journal and the endpoint is not reachable"
|
||||
}
|
||||
return NetVerifyMountFailed, "mount failed with no recognized diagnostic — see the raw journal detail"
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- A1: the retry=0 knob (SPIKE-nas-verify Q4-vi) --------------------------------------------------
|
||||
|
||||
// TestMountOptions_NFSRetry0_SMBWithout: retry=0 is in the NFS option string (dead-NAS access 91 s →
|
||||
// 3.8 s) and NOT in the SMB one (retry is a mount.nfs option; mount.cifs would reject the mount).
|
||||
// Companion red-proof: revert the mountOptions NFS branch → the first assertion fails.
|
||||
func TestMountOptions_NFSRetry0_SMBWithout(t *testing.T) {
|
||||
nfs := NetworkMountSpec{Name: "m", Protocol: ProtocolNFS, Server: "s", Export: "/e", MappedUID: 1000, MappedGID: 1000}
|
||||
if got := nfs.mountOptions(); !strings.Contains(got, ",retry=0") {
|
||||
t.Errorf("NFS options missing retry=0 (Q4-vi): %q", got)
|
||||
}
|
||||
smb := NetworkMountSpec{Name: "m", Protocol: ProtocolSMB, Server: "s", Export: "e",
|
||||
MappedUID: 1000, MappedGID: 1000, CredsRef: "/var/lib/felhom-agent/smb-creds/m.cred"}
|
||||
if got := smb.mountOptions(); strings.Contains(got, "retry=") {
|
||||
t.Errorf("SMB options must NOT carry retry= (mount.cifs rejects it): %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A2: the journal classifier (SPIKE-nas-verify Q4 — VERBATIM live strings) -----------------------
|
||||
|
||||
// TestClassifyNetVerifyFailure runs the Q4 taxonomy on the strings captured live in the spike.
|
||||
// Companion red-proof: an exit-code-based classifier (every failure is rc=32) collapses smb_auth and
|
||||
// smb_share into one code — modeled by replacing the body with `return NetVerifyMountFailed, ""`;
|
||||
// every non-generic row fails.
|
||||
func TestClassifyNetVerifyFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
journal string
|
||||
reachable bool
|
||||
want string
|
||||
}{
|
||||
// Q4 i′ — unreachable with retry=0 (the production unit's shape).
|
||||
{"no route", "mount.nfs4: No route to host for 192.168.0.199:/srv/nope on /mnt/felhom-drives/x", false, NetVerifyUnreachable},
|
||||
{"conn refused", "mount.nfs4: Connection refused", false, NetVerifyUnreachable},
|
||||
{"conn timed out", "mount.nfs4: Connection timed out", false, NetVerifyUnreachable},
|
||||
// Q4 ii ≡ iii — the MERGED category: NFSv4 cannot distinguish no-export from not-permitted.
|
||||
{"nfs export missing", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/nope failed, reason given by server: No such file or directory", true, NetVerifyNFSExport},
|
||||
{"nfs export denied (identical string)", "mount.nfs4: mounting 192.168.0.180:/srv/nas-spike2/q4iii failed, reason given by server: No such file or directory", true, NetVerifyNFSExport},
|
||||
// Q4 iv / v — SMB splits cleanly on the errno line.
|
||||
{"smb wrong password", "mount error(13): Permission denied\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBAuth},
|
||||
{"smb wrong share", "mount error(2): No such file or directory\nRefer to the mount.cifs(8) manual page", true, NetVerifySMBShare},
|
||||
// Q4 i (default retry) — systemd kills mount.nfs at its 90 s cap, no mount.nfs diagnostic.
|
||||
{"systemd timeout", "Mounting timed out. Terminating.\nMount process exited, code=killed, status=15/TERM", true, NetVerifyTimeout},
|
||||
// Degradations: nothing matched.
|
||||
{"empty journal, reachable", "", true, NetVerifyMountFailed},
|
||||
{"empty journal, unreachable", "", false, NetVerifyUnreachable},
|
||||
{"garbage, reachable", "some future mount.nfs5 wording", true, NetVerifyMountFailed},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
code, hint := ClassifyNetVerifyFailure(tc.journal, tc.reachable)
|
||||
if code != tc.want {
|
||||
t.Errorf("ClassifyNetVerifyFailure(%q, reachable=%v) = %q, want %q", tc.journal, tc.reachable, code, tc.want)
|
||||
}
|
||||
if hint == "" {
|
||||
t.Errorf("hint must never be empty (code %q)", code)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestClassify_SMBShareNeverShadowsNFS: the SMB error(2) line contains "No such file or directory"
|
||||
// too — the NFS rule must key on the full "reason given by server:" fragment so neither shadows the
|
||||
// other regardless of table order.
|
||||
func TestClassify_SMBShareNeverShadowsNFS(t *testing.T) {
|
||||
code, _ := ClassifyNetVerifyFailure("mount error(2): No such file or directory", true)
|
||||
if code != NetVerifySMBShare {
|
||||
t.Errorf("SMB error(2) classified %q, want %q", code, NetVerifySMBShare)
|
||||
}
|
||||
code, _ = ClassifyNetVerifyFailure("failed, reason given by server: No such file or directory", true)
|
||||
if code != NetVerifyNFSExport {
|
||||
t.Errorf("NFS server-reason classified %q, want %q", code, NetVerifyNFSExport)
|
||||
}
|
||||
}
|
||||
|
||||
// --- networkMountedIn: the §8 truth source (autofs trigger ≠ mounted) --------------------------------
|
||||
|
||||
func TestNetworkMountedIn(t *testing.T) {
|
||||
procMounts := `sysfs /sys sysfs rw 0 0
|
||||
systemd-1 /mnt/felhom-drives/idle autofs rw,relatime,fd=86 0 0
|
||||
systemd-1 /mnt/felhom-drives/live autofs rw,relatime,fd=86 0 0
|
||||
192.168.0.180:/srv/media /mnt/felhom-drives/live nfs4 rw,noatime,vers=4.1,soft 0 0
|
||||
//nas/share /mnt/felhom-drives/smb cifs rw,vers=3.0 0 0
|
||||
/dev/sda1 /mnt/felhom-drives/disk ext4 rw 0 0
|
||||
`
|
||||
cases := []struct {
|
||||
where string
|
||||
want bool
|
||||
}{
|
||||
{"/mnt/felhom-drives/live", true}, // real nfs4 (the autofs line for the same path must not confuse it)
|
||||
{"/mnt/felhom-drives/smb", true}, // cifs
|
||||
{"/mnt/felhom-drives/idle", false}, // autofs trigger ONLY — idle automount is NOT mounted
|
||||
{"/mnt/felhom-drives/disk", false}, // a local fs at the path is not a network mount
|
||||
{"/mnt/felhom-drives/nope", false}, // absent
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := networkMountedIn(procMounts, tc.where); got != tc.want {
|
||||
t.Errorf("networkMountedIn(%q) = %v, want %v", tc.where, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user