netstorage: verify-before-commit orchestration — agentapi verify fields, uid-1000 re-exec probe, detached add job, orphan rows
Controller half of the verify pipeline (SPIKE-nas-verify b57f6c1): AddNetStorage gains verify/job fields + typed NetAddRefusedError; NetVerifyStatus polls the agent slot; --netprobe hidden re-exec mode (SysProcAttr.Credential uid/gid 1000, no shell) proves in-guest writability; the add handler starts a detached single-flight job (agent_add → verifying → probing → registering LAST) with full rollback on any failure incl. verify-lost-after-restart (Scenario F); §3.2 Hungarian error map server-side; live-but-unregistered shares surface as remove- only 'Árva megosztás' rows. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6
This commit is contained in:
@@ -0,0 +1,356 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// Orchestration tests (verify-before-commit): everything runs through the netAgent / netProbeFn /
|
||||
// netListFn seams — no docker, no TLS, no re-exec.
|
||||
|
||||
// fakeNetAgent is the netAgent seam fake: scripted add/verify results + call recording.
|
||||
type fakeNetAgent struct {
|
||||
mu sync.Mutex
|
||||
addRes agentapi.NetStorageAddResult
|
||||
addErr error
|
||||
verify agentapi.NetVerifyStatus
|
||||
verifyErr error
|
||||
addCalls int
|
||||
removes []string
|
||||
verifyPoll int
|
||||
}
|
||||
|
||||
func (f *fakeNetAgent) AddNetStorage(_ context.Context, req agentapi.AddNetStorageRequest) (agentapi.NetStorageAddResult, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.addCalls++
|
||||
return f.addRes, f.addErr
|
||||
}
|
||||
func (f *fakeNetAgent) NetVerifyStatus(_ context.Context) (agentapi.NetVerifyStatus, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.verifyPoll++
|
||||
return f.verify, f.verifyErr
|
||||
}
|
||||
func (f *fakeNetAgent) RemoveNetStorage(_ context.Context, name string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.removes = append(f.removes, name)
|
||||
return nil
|
||||
}
|
||||
func (f *fakeNetAgent) removed() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string(nil), f.removes...)
|
||||
}
|
||||
|
||||
// okAddRes is the standard successful agent add result (units installed, verify started).
|
||||
func okAddRes(name string) agentapi.NetStorageAddResult {
|
||||
return agentapi.NetStorageAddResult{
|
||||
Name: name, Protocol: "nfs", Where: settings.NetworkMountRoot + "/" + name,
|
||||
GuestPath: settings.NetworkMountRoot + "/" + name, HostUID: 101000, HostGID: 101000,
|
||||
Verify: "started", JobID: "job-1",
|
||||
}
|
||||
}
|
||||
|
||||
// netAddReq builds the standard orchestrator input.
|
||||
func netAddReq(name string) agentapi.AddNetStorageRequest {
|
||||
return agentapi.AddNetStorageRequest{
|
||||
Name: name, Protocol: "nfs", Server: "10.0.0.5", Export: "/srv/" + name,
|
||||
MappedUID: 1000, MappedGID: 1000,
|
||||
}
|
||||
}
|
||||
|
||||
// waitNetAdd polls the job slot until it reaches a terminal phase.
|
||||
func waitNetAdd(t *testing.T, s *Server) *netAddJob {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if j := s.netAdd.snapshot(); j != nil && (j.Phase == netAddPhaseDone || j.Phase == netAddPhaseFailed) {
|
||||
return j
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("net add job did not finish (last: %+v)", s.netAdd.snapshot())
|
||||
return nil
|
||||
}
|
||||
|
||||
// networkPathCount counts registered Kind=network paths.
|
||||
func networkPathCount(s *Server) int {
|
||||
n := 0
|
||||
for _, sp := range s.settings.GetStoragePaths() {
|
||||
if sp.IsNetwork() {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// --- C1: happy path — register ONLY after probe-ok, exactly once ------------------------------------
|
||||
// Companion red-proof: move AddStoragePath before the probe → registeredAtProbe becomes 1 → FAIL.
|
||||
func TestNetAdd_HappyPath_RegisterOnlyAfterProbe(t *testing.T) {
|
||||
s := testServer(t)
|
||||
agent := &fakeNetAgent{addRes: okAddRes("media"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}}
|
||||
registeredAtProbe := -1
|
||||
s.netProbeFn = func(_ context.Context, dir string) probeOutcome {
|
||||
registeredAtProbe = networkPathCount(s) // MUST be 0 — registration is the LAST step
|
||||
if dir != settings.NetworkMountRoot+"/media" {
|
||||
t.Errorf("probe dir = %q, want the guest path", dir)
|
||||
}
|
||||
return probeOutcome{OK: true}
|
||||
}
|
||||
|
||||
if !s.startNetAdd(agent, netAddReq("media"), "NAS media") {
|
||||
t.Fatal("startNetAdd refused with a free slot")
|
||||
}
|
||||
job := waitNetAdd(t, s)
|
||||
if job.Phase != netAddPhaseDone {
|
||||
t.Fatalf("phase = %s (category=%s detail=%s), want done", job.Phase, job.Category, job.Detail)
|
||||
}
|
||||
if registeredAtProbe != 0 {
|
||||
t.Errorf("AddStoragePath ran BEFORE the probe (count at probe = %d, want 0)", registeredAtProbe)
|
||||
}
|
||||
if got := networkPathCount(s); got != 1 {
|
||||
t.Errorf("registered network paths = %d, want exactly 1", got)
|
||||
}
|
||||
if len(agent.removed()) != 0 {
|
||||
t.Errorf("happy path must not roll back: removes=%v", agent.removed())
|
||||
}
|
||||
}
|
||||
|
||||
// --- C2: probe-fail ⇒ rollback + NOT registered + failed{not_writable} -------------------------------
|
||||
// Companion red-proof: drop the rollback call in the probe-fail branch → the removes assertion fails.
|
||||
func TestNetAdd_ProbeFail_RollsBackNotRegistered(t *testing.T) {
|
||||
s := testServer(t)
|
||||
agent := &fakeNetAgent{addRes: okAddRes("media"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}}
|
||||
s.netProbeFn = func(context.Context, string) probeOutcome {
|
||||
return probeOutcome{OK: false, Category: "not_writable", Detail: "uid-1000 write probe: create/write refused"}
|
||||
}
|
||||
|
||||
if !s.startNetAdd(agent, netAddReq("media"), "NAS media") {
|
||||
t.Fatal("startNetAdd refused")
|
||||
}
|
||||
job := waitNetAdd(t, s)
|
||||
if job.Phase != netAddPhaseFailed || job.Category != "not_writable" {
|
||||
t.Fatalf("phase/category = %s/%s, want failed/not_writable", job.Phase, job.Category)
|
||||
}
|
||||
// The §3.2 Route-A message, with the computed host id (1000 + 100000).
|
||||
if !strings.Contains(job.Message, "minden felhasználó leképezése") || !strings.Contains(job.Message, "101000") {
|
||||
t.Errorf("not_writable message must carry the map-all-users guidance + host id 101000: %q", job.Message)
|
||||
}
|
||||
if got := agent.removed(); len(got) != 1 || got[0] != "media" {
|
||||
t.Errorf("probe-fail must roll the agent install back: removes=%v", got)
|
||||
}
|
||||
if got := networkPathCount(s); got != 0 {
|
||||
t.Errorf("a probe-failed share must NOT be registered (got %d paths)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- C3: agent verify failed{code} ⇒ mapped Hungarian message, NOT registered ------------------------
|
||||
func TestNetAdd_AgentVerifyFailed_MappedMessage(t *testing.T) {
|
||||
s := testServer(t)
|
||||
agent := &fakeNetAgent{
|
||||
addRes: okAddRes("media"),
|
||||
verify: agentapi.NetVerifyStatus{Phase: "failed", Code: "nfs_export", Detail: "reason given by server: No such file or directory", JobID: "job-1"},
|
||||
}
|
||||
probeRan := false
|
||||
s.netProbeFn = func(context.Context, string) probeOutcome { probeRan = true; return probeOutcome{OK: true} }
|
||||
|
||||
if !s.startNetAdd(agent, netAddReq("media"), "NAS media") {
|
||||
t.Fatal("startNetAdd refused")
|
||||
}
|
||||
job := waitNetAdd(t, s)
|
||||
if job.Phase != netAddPhaseFailed || job.Category != "nfs_export" {
|
||||
t.Fatalf("phase/category = %s/%s, want failed/nfs_export", job.Phase, job.Category)
|
||||
}
|
||||
// The EXACT §3.2 merged message (NFSv4 cannot distinguish not-found from not-permitted).
|
||||
want := "A megosztás nem található, vagy a NAS nem engedélyezi ennek a gépnek a hozzáférését. Ellenőrizze az export útvonalát, és hogy a NAS engedélyezi-e a Felhom gép IP-címét."
|
||||
if job.Message != want {
|
||||
t.Errorf("nfs_export message:\n got %q\nwant %q", job.Message, want)
|
||||
}
|
||||
if probeRan {
|
||||
t.Error("the probe must not run after a failed agent verify")
|
||||
}
|
||||
if got := networkPathCount(s); got != 0 {
|
||||
t.Errorf("a verify-failed share must NOT be registered (got %d)", got)
|
||||
}
|
||||
// The AGENT already auto-rolled-back — the controller must not double-remove on this branch.
|
||||
if got := agent.removed(); len(got) != 0 {
|
||||
t.Errorf("verify-failed is agent-rolled-back; controller removes=%v want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- C4: agent "no job" after install ⇒ rollback + failed (Scenario F) --------------------------------
|
||||
// Companion red-proof: treat "none" as success (or skip the rollback) → assertions fail.
|
||||
func TestNetAdd_VerifyLost_RollsBack(t *testing.T) {
|
||||
s := testServer(t)
|
||||
agent := &fakeNetAgent{
|
||||
addRes: okAddRes("media"),
|
||||
verify: agentapi.NetVerifyStatus{Phase: "none"}, // the agent restarted mid-verify
|
||||
}
|
||||
s.netProbeFn = func(context.Context, string) probeOutcome {
|
||||
t.Error("probe must not run when the verify was lost")
|
||||
return probeOutcome{OK: true}
|
||||
}
|
||||
|
||||
if !s.startNetAdd(agent, netAddReq("media"), "NAS media") {
|
||||
t.Fatal("startNetAdd refused")
|
||||
}
|
||||
job := waitNetAdd(t, s)
|
||||
if job.Phase != netAddPhaseFailed {
|
||||
t.Fatalf("phase = %s, want failed", job.Phase)
|
||||
}
|
||||
if got := agent.removed(); len(got) != 1 || got[0] != "media" {
|
||||
t.Errorf("verify-lost must roll back the install: removes=%v", got)
|
||||
}
|
||||
if got := networkPathCount(s); got != 0 {
|
||||
t.Errorf("a verify-lost share must NOT be registered (got %d)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// --- C5: the --netprobe child body (pure file logic, t.TempDir) ---------------------------------------
|
||||
func TestNetProbeChild(t *testing.T) {
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if got := NetProbeChild(dir); got != netProbeExitOK {
|
||||
t.Fatalf("exit = %d, want 0", got)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 0 {
|
||||
t.Errorf("probe must clean up its file, left: %v", entries)
|
||||
}
|
||||
})
|
||||
t.Run("unwritable dir → exit 2 → not_writable", func(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "nope") // nonexistent → create fails everywhere
|
||||
if runtime.GOOS != "windows" {
|
||||
dir = t.TempDir()
|
||||
if err := os.Chmod(dir, 0o555); err != nil { // read-only — the squash-trap shape
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = os.Chmod(dir, 0o755) })
|
||||
}
|
||||
got := NetProbeChild(dir)
|
||||
if got != netProbeExitNoWrite {
|
||||
t.Fatalf("exit = %d, want %d", got, netProbeExitNoWrite)
|
||||
}
|
||||
if v := netProbeVerdict(got, ""); v.OK || v.Category != "not_writable" {
|
||||
t.Errorf("verdict = %+v, want not_writable", v)
|
||||
}
|
||||
})
|
||||
t.Run("nonce tamper → exit 3 → probe_io", func(t *testing.T) {
|
||||
orig := netProbeReadBack
|
||||
netProbeReadBack = func(path string) ([]byte, error) { return []byte("tampered"), nil }
|
||||
t.Cleanup(func() { netProbeReadBack = orig })
|
||||
got := NetProbeChild(t.TempDir())
|
||||
if got != netProbeExitMismatch {
|
||||
t.Fatalf("exit = %d, want %d", got, netProbeExitMismatch)
|
||||
}
|
||||
if v := netProbeVerdict(got, ""); v.OK || v.Category != "probe_io" {
|
||||
t.Errorf("verdict = %+v, want probe_io", v)
|
||||
}
|
||||
})
|
||||
t.Run("cleanup fail → exit 4 → OK with warn", func(t *testing.T) {
|
||||
orig := netProbeReadBack
|
||||
// Read back correctly but DELETE the file first — the child's own Remove then fails.
|
||||
netProbeReadBack = func(path string) ([]byte, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
return data, nil
|
||||
}
|
||||
t.Cleanup(func() { netProbeReadBack = orig })
|
||||
got := NetProbeChild(t.TempDir())
|
||||
if got != netProbeExitCleanup {
|
||||
t.Fatalf("exit = %d, want %d", got, netProbeExitCleanup)
|
||||
}
|
||||
v := netProbeVerdict(got, "")
|
||||
if !v.OK || v.Warn == "" {
|
||||
t.Errorf("cleanup-fail must be OK-with-warn (§8), got %+v", v)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// --- C6: single-flight (Scenario G) -------------------------------------------------------------------
|
||||
// Companion red-proof: drop the acquire() running-check → the second add is accepted → FAIL.
|
||||
func TestNetAdd_SingleFlight(t *testing.T) {
|
||||
s := testServer(t)
|
||||
release := make(chan struct{})
|
||||
agent := &fakeNetAgent{addRes: okAddRes("m1"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}}
|
||||
s.netAgentFn = func() (netAgent, error) { return agent, nil }
|
||||
s.netProbeFn = func(context.Context, string) probeOutcome { <-release; return probeOutcome{OK: true} }
|
||||
|
||||
if !s.startNetAdd(agent, netAddReq("m1"), "NAS m1") {
|
||||
t.Fatal("first add refused")
|
||||
}
|
||||
// Second add through the HTTP handler while the first is blocked in probing.
|
||||
body := `{"name":"m2","protocol":"nfs","server":"10.0.0.6","export":"/srv/m2"}`
|
||||
r := httptest.NewRequest(http.MethodPost, "/api/storage/netstorage/add", strings.NewReader(body))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleNetStorageAdd(w, r)
|
||||
if w.Code != http.StatusConflict {
|
||||
t.Fatalf("second add: got %d want 409 (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "már folyamatban van egy csatlakoztatás") {
|
||||
t.Errorf("409 must carry the Hungarian busy message: %s", w.Body.String())
|
||||
}
|
||||
close(release)
|
||||
job := waitNetAdd(t, s)
|
||||
if job.Phase != netAddPhaseDone || job.Name != "m1" {
|
||||
t.Errorf("first job must finish unaffected: %+v", job)
|
||||
}
|
||||
}
|
||||
|
||||
// --- C7: orphan surfacing — live-but-unregistered share renders as a remove-only row ------------------
|
||||
// Companion red-proof: a registry-only filter (drop the live-map sweep) loses the ghost row → FAIL.
|
||||
func TestNetStorage_OrphanRow(t *testing.T) {
|
||||
s := testServer(t)
|
||||
addNetworkPath(t, s, "media") // registered + live
|
||||
s.netListFn = func(context.Context) ([]agentapi.NetworkMountStatus, error) {
|
||||
return []agentapi.NetworkMountStatus{
|
||||
{Name: "media", Protocol: "nfs", Server: "10.0.0.5", Where: settings.NetworkMountRoot + "/media", Configured: true, Mounted: true, Reachable: true, Health: "ok"},
|
||||
{Name: "ghost", Protocol: "nfs", Server: "10.0.0.5", Where: settings.NetworkMountRoot + "/ghost", Configured: true, Mounted: false, Reachable: true, Health: "idle"},
|
||||
}, nil
|
||||
}
|
||||
items := s.networkStorageItems(context.Background())
|
||||
if len(items) != 2 {
|
||||
t.Fatalf("items = %d, want 2 (registered + orphan): %+v", len(items), items)
|
||||
}
|
||||
var ghost *networkStorageItem
|
||||
for i := range items {
|
||||
if items[i].Name == "ghost" {
|
||||
ghost = &items[i]
|
||||
} else if items[i].Orphan {
|
||||
t.Errorf("registered share %q must not be an orphan", items[i].Name)
|
||||
}
|
||||
}
|
||||
if ghost == nil {
|
||||
t.Fatalf("live-but-unregistered share missing from the list: %+v", items)
|
||||
}
|
||||
if !ghost.Orphan || ghost.Label != "Árva megosztás: ghost" {
|
||||
t.Errorf("ghost row must be Orphan with the árva label, got %+v", *ghost)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetAdd_StatusEndpoint_NoJob: the poll endpoint's empty shape.
|
||||
func TestNetAdd_StatusEndpoint_NoJob(t *testing.T) {
|
||||
s := testServer(t)
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/storage/netstorage/add/status", nil)
|
||||
w := httptest.NewRecorder()
|
||||
s.handleNetStorageAddStatus(w, r)
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"phase":"none"`) {
|
||||
t.Fatalf("empty status: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user