v0.81.0: NAS verify-before-commit — retry=0, journal classifier, detached verify job + auto-rollback
Agent half of the verify-before-commit task (SPIKE-nas-verify-2026-07-11, b57f6c1): retry=0 in the production NFS options (Q4-vi); ClassifyNetVerifyFailure on the live Q4 strings (nfs_export merges not-found/not-permitted); add = sync fast-fail (2s TCP pre-probe, nothing installed) + detached in-memory verify job judging /proc/mounts only, auto-rollback on failure; GET /netstorage/verify-status (phase none = the controller's Scenario-F rollback signal); unprivileged journalctl (systemd-journal group, NO new sudoers grants). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// Verify-job tests (SPIKE-nas-verify): the detached pipeline runs entirely through seams — no
|
||||
// systemctl, no journalctl, no /proc/mounts, no network. fakeNetOps (netstorage_test.go) records
|
||||
// the mount-surface effects the assertions check.
|
||||
|
||||
// verifySeams configures a Server's verify pipeline for tests.
|
||||
type verifySeams struct {
|
||||
trigger func(where string) error
|
||||
mounted func(where string) bool
|
||||
journal func(ctx context.Context, unit string) (string, error)
|
||||
reachable func(proto storage.NetworkProtocol, server string) bool
|
||||
}
|
||||
|
||||
func newVerifyServer(t *testing.T, n NetworkStorageOps, credsDir string, seams verifySeams) *Server {
|
||||
t.Helper()
|
||||
srv := newNetServer(t, n, credsDir)
|
||||
if seams.trigger != nil {
|
||||
srv.netTrigger = seams.trigger
|
||||
} else {
|
||||
srv.netTrigger = func(string) error { return nil } // default: instant, successful read
|
||||
}
|
||||
if seams.mounted != nil {
|
||||
srv.netMounted = seams.mounted
|
||||
}
|
||||
if seams.journal != nil {
|
||||
srv.netJournal = seams.journal
|
||||
} else {
|
||||
srv.netJournal = func(context.Context, string) (string, error) { return "", nil }
|
||||
}
|
||||
if seams.reachable != nil {
|
||||
srv.netReachable = seams.reachable
|
||||
} else {
|
||||
srv.netReachable = func(storage.NetworkProtocol, string) bool { return true }
|
||||
}
|
||||
return srv
|
||||
}
|
||||
|
||||
// pollVerify polls GET /netstorage/verify-status until the job leaves `running` (or the deadline).
|
||||
func pollVerify(t *testing.T, h http.Handler) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
w := do(t, h, "GET", "/netstorage/verify-status", "A", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("verify-status: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode status: %v", err)
|
||||
}
|
||||
if p, _ := resp.Data["phase"].(string); p != netVerifyPhaseRunning {
|
||||
return resp.Data
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("verify job did not reach a terminal phase in time")
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- A3: mount-FAILED ⇒ auto-rollback (units + creds) + failed{code} ---------------------------------
|
||||
// Companion red-proof: drop the s.rollbackNetMount call from runNetVerify → the removed/creds
|
||||
// assertions fail (the failed install would linger — today's bug shape).
|
||||
func TestNetVerify_MountFailed_RollsBackAndClassifies(t *testing.T) {
|
||||
credsDir := t.TempDir()
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, credsDir, verifySeams{
|
||||
mounted: func(string) bool { return false }, // §8: not in /proc/mounts ⇒ FAILED
|
||||
journal: func(context.Context, string) (string, error) {
|
||||
return "mount error(13): Permission denied", nil // Q4 iv — smb_auth
|
||||
},
|
||||
})
|
||||
h := srv.Handler()
|
||||
|
||||
body := `{"name":"vids","protocol":"smb","server":"nas","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`
|
||||
w := do(t, h, "POST", "/netstorage/add", "A", body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if len(n.ensured) != 1 {
|
||||
t.Fatalf("EnsureNetworkMount calls = %d, want 1", len(n.ensured))
|
||||
}
|
||||
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed {
|
||||
t.Fatalf("phase = %v, want failed (%v)", final["phase"], final)
|
||||
}
|
||||
if final["code"] != storage.NetVerifySMBAuth {
|
||||
t.Errorf("code = %v, want %q", final["code"], storage.NetVerifySMBAuth)
|
||||
}
|
||||
// The auto-rollback effects: the unit removal was called AND the creds file is gone.
|
||||
if len(n.removed) != 1 || n.removed[0] != "vids" {
|
||||
t.Errorf("RemoveNetworkMount not called for the failed install: removed=%v", n.removed)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) {
|
||||
t.Errorf("creds file must be removed on verify failure (stat err = %v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetVerify_JournalUnavailable_DegradesButStillRollsBack: journal read error ⇒ generic
|
||||
// mount_failed + the systemd-journal hint, and the rollback still runs (Scenario B degradation).
|
||||
func TestNetVerify_JournalUnavailable_DegradesButStillRollsBack(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
mounted: func(string) bool { return false },
|
||||
journal: func(context.Context, string) (string, error) {
|
||||
return "", fmt.Errorf("journalctl: permission denied")
|
||||
},
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"media","protocol":"nfs","server":"10.0.0.5","export":"/srv/media","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: got %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed || final["code"] != storage.NetVerifyMountFailed {
|
||||
t.Fatalf("degraded classification: phase=%v code=%v, want failed/%q", final["phase"], final["code"], storage.NetVerifyMountFailed)
|
||||
}
|
||||
detail, _ := final["detail"].(string)
|
||||
if want := "systemd-journal"; !containsStr(detail, want) {
|
||||
t.Errorf("degraded detail must carry the group hint %q: %q", want, detail)
|
||||
}
|
||||
if len(n.removed) != 1 {
|
||||
t.Errorf("rollback must still run when the journal is unavailable: removed=%v", n.removed)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A4: the §8 truth table — /proc/mounts is the ONLY judge -----------------------------------------
|
||||
// Companion red-proof: a readability-based verdict (trigger err == nil ⇒ OK) fails BOTH rows.
|
||||
func TestNetVerify_TruthTable(t *testing.T) {
|
||||
t.Run("ReadDir ok but NOT mounted = FAILED (empty-dir false positive guard)", func(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { return nil }, // the read "worked" (empty dir)
|
||||
mounted: func(string) bool { return false }, // …but no nfs4/cifs in /proc/mounts
|
||||
journal: func(context.Context, string) (string, error) { return "", nil },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseFailed {
|
||||
t.Fatalf("a readable-but-unmounted path must FAIL verify, got %v", final["phase"])
|
||||
}
|
||||
if len(n.removed) != 1 {
|
||||
t.Errorf("failed verify must roll back: removed=%v", n.removed)
|
||||
}
|
||||
})
|
||||
t.Run("ReadDir EACCES but mounted = OK (0700 export on a good mount)", func(t *testing.T) {
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { return os.ErrPermission }, // agent user can't read it — fine
|
||||
mounted: func(string) bool { return true }, // the mount is REAL
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m2","protocol":"nfs","server":"10.0.0.5","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("add: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseDone {
|
||||
t.Fatalf("an unreadable-but-mounted share must PASS agent verify (writability is the controller probe's job), got %v (%v)", final["phase"], final)
|
||||
}
|
||||
if len(n.removed) != 0 {
|
||||
t.Errorf("a passing verify must NOT roll back: removed=%v", n.removed)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --- A5: the sync TCP pre-probe refuses BEFORE any install (Scenario E) ------------------------------
|
||||
// Companion red-proof: move the pre-probe after EnsureNetworkMount → the zero-install assertion fails.
|
||||
func TestNetVerify_UnreachablePreProbe_InstallsNothing(t *testing.T) {
|
||||
credsDir := t.TempDir()
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, credsDir, verifySeams{
|
||||
reachable: func(storage.NetworkProtocol, string) bool { return false },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"vids","protocol":"smb","server":"192.168.0.199","export":"vids","mapped_uid":1000,"mapped_gid":1000,"username":"u","password":"p"}`)
|
||||
if w.Code != http.StatusBadGateway {
|
||||
t.Fatalf("unreachable add: got %d want 502 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
var resp struct {
|
||||
OK bool `json:"ok"`
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if resp.OK || resp.Data["code"] != storage.NetVerifyUnreachable {
|
||||
t.Errorf("refusal must carry code=unreachable: %s", w.Body.String())
|
||||
}
|
||||
// NOTHING installed: no Ensure call, no creds file, and the verify slot is free again.
|
||||
if len(n.ensured) != 0 {
|
||||
t.Errorf("EnsureNetworkMount must not run for an unreachable server: %v", n.ensured)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(credsDir, "vids.cred")); !os.IsNotExist(err) {
|
||||
t.Errorf("no creds file may be written for an unreachable server (stat err = %v)", err)
|
||||
}
|
||||
if got := srv.netVerifySnapshot(); got != nil {
|
||||
t.Errorf("the verify slot must be released after a sync refusal: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- A6: single-flight + the "no job" shape ----------------------------------------------------------
|
||||
// Companion red-proof: drop the tryStartNetVerify running-check → the 409 assertion fails.
|
||||
func TestNetVerify_SingleFlight_AndNoJobShape(t *testing.T) {
|
||||
// No job yet: the status endpoint must serve the Scenario-F "none" shape.
|
||||
nIdle := &fakeNetOps{}
|
||||
hIdle := newVerifyServer(t, nIdle, t.TempDir(), verifySeams{}).Handler()
|
||||
w := do(t, hIdle, "GET", "/netstorage/verify-status", "A", "")
|
||||
var idle struct {
|
||||
Data map[string]any `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &idle); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if idle.Data["phase"] != netVerifyPhaseNone {
|
||||
t.Fatalf("empty slot must report phase %q, got %v", netVerifyPhaseNone, idle.Data["phase"])
|
||||
}
|
||||
|
||||
// Single-flight: hold the first verify open via a blocking trigger, then submit a second add.
|
||||
release := make(chan struct{})
|
||||
n := &fakeNetOps{}
|
||||
srv := newVerifyServer(t, n, t.TempDir(), verifySeams{
|
||||
trigger: func(string) error { <-release; return nil },
|
||||
mounted: func(string) bool { return true },
|
||||
})
|
||||
h := srv.Handler()
|
||||
w1 := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m1","protocol":"nfs","server":"10.0.0.5","export":"/srv/m1","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w1.Code != http.StatusOK {
|
||||
t.Fatalf("first add: %d (%s)", w1.Code, w1.Body.String())
|
||||
}
|
||||
w2 := do(t, h, "POST", "/netstorage/add", "A",
|
||||
`{"name":"m2","protocol":"nfs","server":"10.0.0.6","export":"/srv/m2","mapped_uid":1000,"mapped_gid":1000}`)
|
||||
if w2.Code != http.StatusConflict {
|
||||
t.Fatalf("second add while verifying: got %d want 409 (%s)", w2.Code, w2.Body.String())
|
||||
}
|
||||
if len(n.ensured) != 1 {
|
||||
t.Errorf("the refused second add must not reach EnsureNetworkMount: %v", n.ensured)
|
||||
}
|
||||
close(release)
|
||||
final := pollVerify(t, h)
|
||||
if final["phase"] != netVerifyPhaseDone || final["name"] != "m1" {
|
||||
t.Errorf("first job must finish unaffected: %v", final)
|
||||
}
|
||||
}
|
||||
|
||||
func containsStr(s, sub string) bool { return strings.Contains(s, sub) }
|
||||
Reference in New Issue
Block a user