diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 3fac70a..2cc47f4 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -57,6 +57,14 @@ var ( ) func main() { + // Hidden re-exec mode (NAS verify-before-commit): the web server re-execs THIS binary as + // `felhom-controller --netprobe ` with uid/gid-1000 credentials to prove a media app can + // write through a freshly-verified network share (SPIKE-nas-verify Q2). Handled before flag + // parsing; the exit code is the probe verdict (see web/netprobe.go). + if len(os.Args) == 3 && os.Args[1] == "--netprobe" { + os.Exit(web.NetProbeChild(os.Args[2])) + } + configPath := flag.String("config", "/opt/docker/felhom-controller/controller.yaml", "Path to configuration file") showVersion := flag.Bool("version", false, "Show version and exit") flag.Parse() diff --git a/controller/internal/agentapi/client.go b/controller/internal/agentapi/client.go index 67f3711..99843cf 100644 --- a/controller/internal/agentapi/client.go +++ b/controller/internal/agentapi/client.go @@ -652,7 +652,9 @@ type AddNetStorageRequest struct { Password string `json:"password,omitempty"` // SMB secret — pass-through, never persisted } -// NetStorageAddResult mirrors the agent's add response (the in-guest path the media app's data dir points at). +// NetStorageAddResult mirrors the agent's add response (the in-guest path the media app's data dir +// points at). Since agent v0.81.0 (verify-before-commit) a successful add means "units installed, +// verify STARTED" — Verify/JobID carry the detached verify job the caller polls via NetVerifyStatus. type NetStorageAddResult struct { Name string `json:"name"` Protocol string `json:"protocol"` @@ -660,18 +662,67 @@ type NetStorageAddResult struct { GuestPath string `json:"guest_path"` HostUID int `json:"host_uid"` HostGID int `json:"host_gid"` + Verify string `json:"verify"` // "started" on a verify-before-commit agent (v0.81.0+) + JobID string `json:"job_id"` + Code string `json:"code"` // set on a categorized SYNC refusal (e.g. "unreachable", "busy") } -// AddNetStorage mounts a NAS share host-side (the agent automounts it; it propagates into this guest via -// the shared bind). Returns the in-guest path the media app's data dir is pointed at. +// NetAddRefusedError is the agent's CATEGORIZED sync refusal of a netstorage add (the 2 s TCP +// pre-probe "unreachable", or "busy" single-flight). Code is the verify-category vocabulary the UI +// maps to Hungarian; nothing was installed agent-side. +type NetAddRefusedError struct { + Code string + Msg string +} + +func (e *NetAddRefusedError) Error() string { + return "agentapi: netstorage add refused (" + e.Code + "): " + e.Msg +} + +// NetVerifyStatus mirrors the agent's GET /netstorage/verify-status: the single verify slot. +// Phase "none" is a REAL signal — after an agent restart the in-memory job is gone; the caller +// treats none-after-install as verify-lost and rolls the add back (Scenario F). +type NetVerifyStatus struct { + Phase string `json:"phase"` // none | running | done | failed + Name string `json:"name"` + Where string `json:"where"` + Protocol string `json:"protocol"` + Code string `json:"code"` // failure category (the agent's classifier vocabulary) + Detail string `json:"detail"` // operator hint + raw journal fragment + JobID string `json:"job_id"` +} + +// AddNetStorage installs a NAS share host-side and starts the agent's detached verify job (agent +// v0.81.0 verify-before-commit). A categorized sync refusal returns *NetAddRefusedError carrying +// the category code (the result body also carries it); other failures are plain errors. func (c *Client) AddNetStorage(ctx context.Context, req AddNetStorageRequest) (NetStorageAddResult, error) { var out NetStorageAddResult - body, err := c.post(ctx, "/netstorage/add", req) + env, status, err := c.postWithStatus(ctx, "/netstorage/add", req) + if err != nil { + return out, err + } + if len(env.Data) > 0 { + _ = json.Unmarshal(env.Data, &out) // best-effort; the refusal body carries {code} + } + if rerr := refusalError("/netstorage/add", status, env); rerr != nil { + if out.Code != "" { + return out, &NetAddRefusedError{Code: out.Code, Msg: truncateErr(env.Error, 300)} + } + return out, rerr + } + return out, nil +} + +// NetVerifyStatus polls the agent's verify slot (short GET — fits the client's global 15 s timeout; +// the LONG wait lives in the caller's poll loop, never in one HTTP call). +func (c *Client) NetVerifyStatus(ctx context.Context) (NetVerifyStatus, error) { + var out NetVerifyStatus + body, err := c.get(ctx, "/netstorage/verify-status") if err != nil { return out, err } if err := json.Unmarshal(body, &out); err != nil { - return out, fmt.Errorf("agentapi: decode /netstorage/add: %w", err) + return out, fmt.Errorf("agentapi: decode /netstorage/verify-status: %w", err) } return out, nil } diff --git a/controller/internal/web/netprobe.go b/controller/internal/web/netprobe.go new file mode 100644 index 0000000..f229e57 --- /dev/null +++ b/controller/internal/web/netprobe.go @@ -0,0 +1,81 @@ +package web + +import ( + "crypto/rand" + "encoding/hex" + "os" + "path/filepath" +) + +// In-guest uid-1000 write probe (NAS verify-before-commit, SPIKE-nas-verify Q2/Q3). The agent's +// verify proves the MOUNT is real; this probe proves a uid/gid-1000 app can actually WRITE through +// it — the squash trap (Q3 row c: an export that mounts fine but denies every write). The probe is +// a RE-EXEC of this binary (`felhom-controller --netprobe `) dropped to uid/gid 1000 via +// SysProcAttr.Credential (spike Q2: SETUID/SETGID caps present in the container) — no shell, no +// in-process setuid (which would poison the whole Go runtime's credentials). + +// Probe child exit codes (the parent maps them to verify categories). +const ( + netProbeExitOK = 0 // write + readback + delete all fine + netProbeExitNoWrite = 2 // create/write failed → not_writable (the squash trap) + netProbeExitMismatch = 3 // readback failed or differed → probe_io + netProbeExitCleanup = 4 // wrote fine but delete failed → OK + warn (never a failure) +) + +// netProbeReadBack is the child's readback seam (package var — the child is a re-exec'd process in +// production, so a struct seam can't reach it; tests override in-process). +var netProbeReadBack = os.ReadFile + +// NetProbeChild is the --netprobe body, run AS uid/gid 1000 by the re-exec parent: create a +// dot-file with a random name + nonce in dir, read it back, compare, remove. Pure file logic — +// unit-tested directly in t.TempDir(). Exposed for cmd/controller's hidden mode. +func NetProbeChild(dir string) int { + name := filepath.Join(dir, ".felhom-proba-"+randHexToken(8)) + nonce := randHexToken(32) + if err := os.WriteFile(name, []byte(nonce), 0o644); err != nil { + return netProbeExitNoWrite + } + back, err := netProbeReadBack(name) + if err != nil || string(back) != nonce { + _ = os.Remove(name) // best-effort — the verdict is already mismatch + return netProbeExitMismatch + } + if err := os.Remove(name); err != nil { + return netProbeExitCleanup + } + return netProbeExitOK +} + +// probeOutcome is the parent-side verdict of one probe run. +type probeOutcome struct { + OK bool + Category string // failure category (not_writable | probe_io) when !OK + Detail string + Warn string // set on OK when cleanup failed (§8: a failed delete is a WARN, not a failure) +} + +// netProbeVerdict maps the child's exit code to the outcome (pure — unit-tested). +func netProbeVerdict(exitCode int, output string) probeOutcome { + switch exitCode { + case netProbeExitOK: + return probeOutcome{OK: true} + case netProbeExitCleanup: + return probeOutcome{OK: true, Warn: "a próbafájl törlése nem sikerült a megosztáson"} + case netProbeExitNoWrite: + return probeOutcome{OK: false, Category: "not_writable", Detail: "uid-1000 write probe: create/write refused | " + output} + case netProbeExitMismatch: + return probeOutcome{OK: false, Category: "probe_io", Detail: "uid-1000 write probe: readback failed or differed | " + output} + default: + return probeOutcome{OK: false, Category: "probe_io", Detail: "uid-1000 write probe: unexpected exit | " + output} + } +} + +// randHexToken returns n random bytes hex-encoded (2n chars); crypto/rand, panics never — a rand +// failure degrades to a constant (the probe still functions, names just stop being random). +func randHexToken(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "felhom-static" + } + return hex.EncodeToString(b) +} diff --git a/controller/internal/web/netprobe_linux.go b/controller/internal/web/netprobe_linux.go new file mode 100644 index 0000000..91a0d04 --- /dev/null +++ b/controller/internal/web/netprobe_linux.go @@ -0,0 +1,43 @@ +//go:build linux + +package web + +import ( + "context" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +// netProbeTimeout bounds one probe run (LAN write+readback is sub-second; a wedged share must not +// hold the orchestrator — the NFS soft/retry=0 options error out well inside this). +const netProbeTimeout = 30 * time.Second + +// runNetProbe re-execs this binary as `felhom-controller --netprobe ` with uid/gid 1000 +// credentials (supplementary groups CLEARED — the probe must see exactly what a media app sees) +// and maps the exit code to a verdict. This is the production netProbeFn seam value. +func runNetProbe(ctx context.Context, dir string) probeOutcome { + exe, err := os.Executable() + if err != nil { + return probeOutcome{OK: false, Category: "probe_io", Detail: "probe re-exec: executable path: " + err.Error()} + } + pctx, cancel := context.WithTimeout(ctx, netProbeTimeout) + defer cancel() + cmd := exec.CommandContext(pctx, exe, "--netprobe", dir) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 1000, Gid: 1000, Groups: []uint32{}}, + } + out, err := cmd.CombinedOutput() + output := strings.TrimSpace(string(out)) + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return netProbeVerdict(ee.ExitCode(), output) + } + // Spawn-level failure (EPERM on setuid = capability dropped somewhere — spike Q2 proved the + // default container HAS the caps, so this is a real config regression worth the raw detail). + return probeOutcome{OK: false, Category: "probe_io", Detail: "probe spawn failed: " + err.Error() + " | " + output} + } + return netProbeVerdict(0, output) +} diff --git a/controller/internal/web/netprobe_other.go b/controller/internal/web/netprobe_other.go new file mode 100644 index 0000000..12a814e --- /dev/null +++ b/controller/internal/web/netprobe_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package web + +import "context" + +// runNetProbe is Linux-only (SysProcAttr.Credential). Off-linux (dev/test hosts) the seam must be +// injected; reaching this stub is a wiring error, reported as a failed probe — never a false PASS. +func runNetProbe(_ context.Context, _ string) probeOutcome { + return probeOutcome{OK: false, Category: "probe_io", Detail: "uid-1000 probe unavailable on this platform"} +} diff --git a/controller/internal/web/netstorage_handlers.go b/controller/internal/web/netstorage_handlers.go index 2863156..2a8eecb 100644 --- a/controller/internal/web/netstorage_handlers.go +++ b/controller/internal/web/netstorage_handlers.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "net/http" + "sort" "strings" "time" @@ -22,6 +23,8 @@ import ( const defaultMediaUID = 1000 // networkStorageItem is the UI row: the registered descriptor + live per-share health from the agent. +// Orphan marks an agent-configured share with NO registry entry (a crash-window leftover — Scenario +// F's visible closure): the row renders with ONLY the remove action. type networkStorageItem struct { Name string `json:"name"` Label string `json:"label"` @@ -33,10 +36,12 @@ type networkStorageItem struct { Reachable bool `json:"reachable"` Mounted bool `json:"mounted"` Configured bool `json:"configured"` + Orphan bool `json:"orphan"` // agent-configured but NOT registered — remove is the only action } -// handleNetStorageAdd proxies POST /api/storage/netstorage/add → agent /netstorage/add, then registers a -// Kind=network StoragePath (no password persisted). +// handleNetStorageAdd starts the verify-before-commit orchestration: sync input validation (a bad +// request changes nothing), then the detached job (agent add → verify poll → uid-1000 probe → +// register LAST). The response is {started:true}; the UI polls /api/storage/netstorage/add/status. func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request) { var req struct { Name string `json:"name"` @@ -81,46 +86,42 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request) { gid = defaultMediaUID } - agent, err := s.agentClient() + agent, err := s.netAgentForAdd() if err != nil { writeDiskJSON(w, http.StatusServiceUnavailable, false, err.Error(), nil) return } - res, err := agent.AddNetStorage(r.Context(), agentapi.AddNetStorageRequest{ - Name: name, Protocol: proto, Server: server, Export: export, - MappedUID: uid, MappedGID: gid, IdleTimeoutSec: req.IdleTimeoutSec, - Username: req.Username, Password: req.Password, - }) - if err != nil { - s.logger.Printf("[ERROR] [web] netstorage add %q via agent failed: %v", name, err) - writeDiskJSON(w, http.StatusBadGateway, false, err.Error(), nil) - return - } - label := strings.TrimSpace(req.Label) if label == "" { label = "Hálózati tárhely: " + name } - // Register the Kind=network path. NO password is persisted — only the non-secret descriptors. - sp := settings.StoragePath{ - Path: res.GuestPath, - Label: label, - Schedulable: true, - AddedAt: time.Now().UTC().Format(time.RFC3339), - Kind: settings.StorageKindNetwork, - Protocol: proto, - Server: server, - Export: export, - MappedUID: uid, - MappedGID: gid, - } - if err := s.settings.AddStoragePath(sp); err != nil { - s.logger.Printf("[ERROR] [web] netstorage register %q failed: %v", name, err) - writeDiskJSON(w, http.StatusInternalServerError, false, "regisztráció sikertelen", nil) + // The password rides INSIDE the request straight to the agent (0600 creds file) — it is never + // persisted controller-side and never appears in the job status. + started := s.startNetAdd(agent, agentapi.AddNetStorageRequest{ + Name: name, Protocol: proto, Server: server, Export: export, + MappedUID: uid, MappedGID: gid, IdleTimeoutSec: req.IdleTimeoutSec, + Username: req.Username, Password: req.Password, + }, label) + if !started { + writeDiskJSON(w, http.StatusConflict, false, "már folyamatban van egy csatlakoztatás", nil) return } - s.logger.Printf("[INFO] [web] network storage added: %s (%s %s:%s) → %s", name, proto, server, export, res.GuestPath) - writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"registered": true, "name": name, "path": res.GuestPath}) + s.logger.Printf("[INFO] [web] network storage add started: %s (%s %s:%s)", name, proto, server, export) + writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"started": true, "name": name}) +} + +// handleNetStorageAddStatus reports the last / in-flight add job (deep copy; "none" when never ran). +func (s *Server) handleNetStorageAddStatus(w http.ResponseWriter, r *http.Request) { + job := s.netAdd.snapshot() + if job == nil { + writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"phase": "none"}) + return + } + writeDiskJSON(w, http.StatusOK, true, "", map[string]any{ + "phase": job.Phase, "name": job.Name, "category": job.Category, + "message": job.Message, "detail": trimNetDetail(job.Detail), "warn": job.Warn, + "path": job.Path, "started_at": job.StartedAt, "updated_at": job.UpdatedAt, + }) } // networkStorageItems returns the registered network shares merged with the agent's live per-share @@ -128,23 +129,23 @@ func (s *Server) handleNetStorageAdd(w http.ResponseWriter, r *http.Request) { // Shared by the JSON list handler and the settings page render. func (s *Server) networkStorageItems(ctx context.Context) []networkStorageItem { live := map[string]agentapi.NetworkMountStatus{} - if agent, err := s.agentClient(); err == nil { - lctx, cancel := context.WithTimeout(ctx, 5*time.Second) // never let a slow agent hang the page - defer cancel() - if mounts, lerr := agent.ListNetStorage(lctx); lerr == nil { - for _, m := range mounts { - live[m.Name] = m - } - } else { - s.logger.Printf("[WARN] [web] netstorage live health unavailable: %v", lerr) + lctx, cancel := context.WithTimeout(ctx, 5*time.Second) // never let a slow agent hang the page + defer cancel() + if mounts, lerr := s.listNetStorage(lctx); lerr == nil { + for _, m := range mounts { + live[m.Name] = m } + } else { + s.logger.Printf("[WARN] [web] netstorage live health unavailable: %v", lerr) } items := make([]networkStorageItem, 0) + registered := map[string]bool{} for _, sp := range s.settings.GetStoragePaths() { if !sp.IsNetwork() { continue } name := pathBase(sp.Path) + registered[name] = true it := networkStorageItem{ Name: name, Label: sp.Label, Protocol: sp.Protocol, Server: sp.Server, Export: sp.Export, Path: sp.Path, Health: "unknown", @@ -157,6 +158,26 @@ func (s *Server) networkStorageItems(ctx context.Context) []networkStorageItem { } items = append(items, it) } + // Orphans (Scenario F's visible closure): an agent-configured share with NO registry entry is a + // crash-window leftover (e.g. controller died between agent-add and register). Surface it with + // ONLY the remove action — re-adding the same name is the repair path (EnsureNetworkMount is + // idempotent, verify runs again). + var orphans []string + for name := range live { + if !registered[name] { + orphans = append(orphans, name) + } + } + sort.Strings(orphans) + for _, name := range orphans { + m := live[name] + items = append(items, networkStorageItem{ + Name: name, Label: "Árva megosztás: " + name, Protocol: m.Protocol, + Server: m.Server, Export: m.Export, Path: m.Where, + Health: m.Health, Reachable: m.Reachable, Mounted: m.Mounted, + Configured: m.Configured, Orphan: true, + }) + } return items } @@ -198,6 +219,18 @@ func (s *Server) handleNetStorageRemove(w http.ResponseWriter, r *http.Request) writeDiskJSON(w, http.StatusOK, true, "", map[string]any{"removed": true, "name": name}) } +// listNetStorage resolves the agent's live share list (test seam first, then the shared client). +func (s *Server) listNetStorage(ctx context.Context) ([]agentapi.NetworkMountStatus, error) { + if s.netListFn != nil { + return s.netListFn(ctx) + } + agent, err := s.agentClient() + if err != nil { + return nil, err + } + return agent.ListNetStorage(ctx) +} + // pathBase returns the last path segment (the share name) of a /mnt/felhom-drives/ path. func pathBase(p string) string { p = strings.TrimRight(p, "/") diff --git a/controller/internal/web/netstorage_job.go b/controller/internal/web/netstorage_job.go new file mode 100644 index 0000000..216de35 --- /dev/null +++ b/controller/internal/web/netstorage_job.go @@ -0,0 +1,323 @@ +package web + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/agentapi" + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// NAS add orchestration (verify-before-commit). POST /api/storage/netstorage/add no longer +// registers blind: it validates sync, then a DETACHED job (migrate.go shape — sync-validate-then- +// goroutine, single-flight, deep-copied status) drives: +// +// agent_add → the agent installs units + starts ITS detached verify (auto-rollback on fail) +// verifying → poll the agent's verify slot every 2 s; "none" after install = agent restarted +// mid-verify ⇒ WE roll back (Scenario F) +// probing → in-guest uid-1000 write probe (the squash trap — a mountable-but-unwritable +// export must never register) +// registering→ AddStoragePath — the LAST step, so the worst crash outcome is an agent-side +// orphan (surfaced by the list), never a registered-but-broken path +// done | failed +// +// The job runs on context.Background() with an overall deadline — a closed browser tab can never +// abort a half-done add mid-rollback. Rollback (agent RemoveNetStorage) is idempotent; a +// double-remove after the agent's own auto-rollback is harmless. + +// netAgent is the agent surface the orchestrator needs (seam — tests inject a fake; production is +// the shared *agentapi.Client). +type netAgent interface { + AddNetStorage(ctx context.Context, req agentapi.AddNetStorageRequest) (agentapi.NetStorageAddResult, error) + NetVerifyStatus(ctx context.Context) (agentapi.NetVerifyStatus, error) + RemoveNetStorage(ctx context.Context, name string) error +} + +// netAddJob is the poll-visible orchestration state (GET /api/storage/netstorage/add/status). +type netAddJob struct { + Name string `json:"name"` + Phase string `json:"phase"` // agent_add | verifying | probing | registering | done | failed + Category string `json:"category,omitempty"` // failure category (agent classifier vocabulary + not_writable/probe_io) + Message string `json:"message,omitempty"` // the category's Hungarian customer message (§3.2) + Detail string `json:"detail,omitempty"` // raw English/journal detail (the collapsible) + Warn string `json:"warn,omitempty"` // non-fatal note on a successful add (e.g. probe cleanup) + Path string `json:"path,omitempty"` // the registered in-guest path (done only) + StartedAt time.Time `json:"started_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +const ( + netAddPhaseAgentAdd = "agent_add" + netAddPhaseVerifying = "verifying" + netAddPhaseProbing = "probing" + netAddPhaseRegistering = "registering" + netAddPhaseDone = "done" + netAddPhaseFailed = "failed" +) + +const ( + // netAddDeadline bounds the WHOLE orchestration (§8: detached ctx + ~150 s). The dominant term + // is the agent verify's own 95 s worst case (black-holed-but-routed server). + netAddDeadline = 150 * time.Second + // netVerifyPollEvery is the agent verify-slot poll cadence. + netVerifyPollEvery = 2 * time.Second + // netRollbackBudget bounds a rollback call on a possibly-expired job context. + netRollbackBudget = 30 * time.Second +) + +// netAddState is the single-flight slot (migrate.go's acquire/release + deep-copy status shape). +type netAddState struct { + mu sync.Mutex + running bool + cur *netAddJob +} + +func (s *netAddState) acquire(job *netAddJob) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.running { + return false + } + s.running = true + cp := *job + s.cur = &cp + return true +} + +func (s *netAddState) release() { + s.mu.Lock() + s.running = false + s.mu.Unlock() +} + +func (s *netAddState) set(job *netAddJob) { + s.mu.Lock() + cp := *job + s.cur = &cp + s.mu.Unlock() +} + +// snapshot returns a copy of the last / in-flight job (nil = never ran). +func (s *netAddState) snapshot() *netAddJob { + s.mu.Lock() + defer s.mu.Unlock() + if s.cur == nil { + return nil + } + cp := *s.cur + return &cp +} + +// netAgentForAdd resolves the orchestrator's agent surface (test seam first, then the shared client). +func (s *Server) netAgentForAdd() (netAgent, error) { + if s.netAgentFn != nil { + return s.netAgentFn() + } + c, err := s.agentClient() + if err != nil { + return nil, err + } + return c, nil +} + +// netProbe runs the in-guest uid-1000 write probe (test seam first, then the re-exec runner). +func (s *Server) netProbe(ctx context.Context, dir string) probeOutcome { + if s.netProbeFn != nil { + return s.netProbeFn(ctx, dir) + } + return runNetProbe(ctx, dir) +} + +// startNetAdd claims the single-flight slot and launches the detached orchestration. false = an add +// is already in flight (Scenario G). +func (s *Server) startNetAdd(agent netAgent, req agentapi.AddNetStorageRequest, label string) bool { + job := &netAddJob{ + Name: req.Name, + Phase: netAddPhaseAgentAdd, + StartedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + if !s.netAdd.acquire(job) { + return false + } + go s.runNetAdd(agent, req, label, job) + return true +} + +// runNetAdd drives the phase machine on a DETACHED context (a closed tab can't abort a rollback). +func (s *Server) runNetAdd(agent netAgent, req agentapi.AddNetStorageRequest, label string, job *netAddJob) { + defer s.netAdd.release() + ctx, cancel := context.WithTimeout(context.Background(), netAddDeadline) + defer cancel() + + setPhase := func(p string) { + job.Phase = p + job.UpdatedAt = time.Now().UTC() + s.netAdd.set(job) + } + fail := func(category, detail string) { + job.Phase = netAddPhaseFailed + job.Category = category + job.Message = netAddMessage(category, req.Server, req.MappedUID) + job.Detail = detail + job.UpdatedAt = time.Now().UTC() + s.netAdd.set(job) + s.logger.Printf("[WARN] [web] netstorage add %q failed: category=%s detail=%s", req.Name, category, detail) + } + rollback := func(why string) { + rctx, rcancel := context.WithTimeout(context.Background(), netRollbackBudget) + defer rcancel() + if err := agent.RemoveNetStorage(rctx, req.Name); err != nil { + // Best-effort by design: the agent may have auto-rolled-back already (double-remove is + // harmless) — but log it, a REAL leftover shows up as an orphan row in the list. + s.logger.Printf("[WARN] [web] netstorage add %q rollback (%s): remove: %v", req.Name, why, err) + } + } + + // Phase 1 — agent add (install units + start the agent-side verify). + res, err := agent.AddNetStorage(ctx, req) + if err != nil { + var refused *agentapi.NetAddRefusedError + if errors.As(err, &refused) { + fail(refused.Code, refused.Msg) // categorized sync refusal — NOTHING was installed + return + } + fail("agent_error", err.Error()) + return + } + + // Phase 2 — poll the agent's verify slot. On a pre-verify agent (no job started) skip straight + // to the probe: the mount-trigger check then happens implicitly through the probe's write. + if res.Verify == "started" { + setPhase(netAddPhaseVerifying) + verdict, verr := s.pollAgentVerify(ctx, agent, res.JobID) + switch { + case verr != nil: + rollback("verify poll failed") + fail("agent_error", "verify status unavailable: "+verr.Error()) + return + case verdict.Phase == "failed": + // The agent has ALREADY auto-rolled-back (units + creds) — no controller rollback needed. + fail(verdict.Code, verdict.Detail) + return + case verdict.Phase == "done": + // verified — proceed + default: + // "none" (agent restarted mid-verify — Scenario F) or an alien job id: the verify is + // LOST, the units may linger — roll back and fail loud. + rollback("verify lost (agent restart?)") + fail("mount_failed", fmt.Sprintf("agent verify job lost after install (phase=%q) — rolled back", verdict.Phase)) + return + } + } + + // Phase 3 — the in-guest uid-1000 write probe (the squash trap). + setPhase(netAddPhaseProbing) + outcome := s.netProbe(ctx, res.GuestPath) + if !outcome.OK { + rollback("probe failed") + fail(outcome.Category, outcome.Detail) + return + } + + // Phase 4 — register. THE LAST STEP (worst crash outcome = an agent-side orphan, never a + // registered-but-broken path). + setPhase(netAddPhaseRegistering) + sp := settings.StoragePath{ + Path: res.GuestPath, + Label: label, + Schedulable: true, + AddedAt: time.Now().UTC().Format(time.RFC3339), + Kind: settings.StorageKindNetwork, + Protocol: req.Protocol, + Server: req.Server, + Export: req.Export, + MappedUID: req.MappedUID, + MappedGID: req.MappedGID, + } + if err := s.settings.AddStoragePath(sp); err != nil { + rollback("register failed") + fail("register_failed", err.Error()) + return + } + + job.Phase = netAddPhaseDone + job.Path = res.GuestPath + job.Warn = outcome.Warn + job.UpdatedAt = time.Now().UTC() + s.netAdd.set(job) + s.logger.Printf("[INFO] [web] network storage added + verified: %s (%s %s:%s) → %s (warn=%q)", + req.Name, req.Protocol, req.Server, req.Export, res.GuestPath, outcome.Warn) +} + +// pollAgentVerify polls the agent's verify slot until it leaves `running` (or ctx expires). +// Transient poll errors are tolerated up to a small budget — the agent may be busy mounting. +func (s *Server) pollAgentVerify(ctx context.Context, agent netAgent, jobID string) (agentapi.NetVerifyStatus, error) { + var last agentapi.NetVerifyStatus + consecutiveErrs := 0 + t := time.NewTicker(netVerifyPollEvery) + defer t.Stop() + for { + st, err := agent.NetVerifyStatus(ctx) + if err != nil { + consecutiveErrs++ + if consecutiveErrs >= 5 { + return last, err + } + } else { + consecutiveErrs = 0 + last = st + if st.Phase != "running" { + // A different job id in the slot means OUR job is gone (restart + a newer add) — + // report it as the lost-verify shape, not a false done/failed. + if st.Phase != "none" && jobID != "" && st.JobID != jobID { + last.Phase = "none" + } + return last, nil + } + } + select { + case <-ctx.Done(): + return last, ctx.Err() + case <-t.C: + } + } +} + +// netAddMessage maps a failure category to the EXACT Hungarian customer message (§3.2). server and +// mappedUID parameterize the unreachable/not_writable texts (host id = mapped uid + 100000). +func netAddMessage(category, server string, mappedUID int) string { + switch category { + case "unreachable": + return "A szerver nem érhető el (" + server + "). Ellenőrizze az IP-címet, és hogy a NAS be van-e kapcsolva." + case "nfs_export": + return "A megosztás nem található, vagy a NAS nem engedélyezi ennek a gépnek a hozzáférését. Ellenőrizze az export útvonalát, és hogy a NAS engedélyezi-e a Felhom gép IP-címét." + case "smb_auth": + return "Hibás SMB felhasználónév vagy jelszó." + case "smb_share": + return "A megadott SMB-megosztás nem található a szerveren. Ellenőrizze a megosztás nevét." + case "timeout": + return "Időtúllépés a csatolás közben — a szerver elérhető a hálózaton, de a megosztás nem csatolható. Ellenőrizze a NAS NFS/SMB szolgáltatását." + case "not_writable": + return "A megosztás csatolható, de az alkalmazások nem tudnak rá írni. NFS esetén kapcsolja be a NAS-on a „minden felhasználó leképezése” (map all users / all squash) beállítást a megosztáson — vagy állítsa a fájlok tulajdonosát a(z) " + fmt.Sprint(mappedUID+100000) + " azonosítóra. SMB esetén ellenőrizze, hogy a felhasználónak írási joga van a megosztáson." + case "probe_io": + return "Írási hiba a megosztáson (az adat nem olvasható vissza hibátlanul). Ellenőrizze a megosztást és a hálózatot." + case "busy": + return "Már folyamatban van egy csatlakoztatás. Várja meg, amíg befejeződik." + default: // mount_failed + agent_error + register_failed + any future agent code + return "A csatolás sikertelen. Részletek alább." + } +} + +// trimNetDetail bounds the raw detail shown in the UI collapsible. +func trimNetDetail(d string) string { + d = strings.TrimSpace(d) + if len(d) > 800 { + return d[:800] + "…" + } + return d +} diff --git a/controller/internal/web/netstorage_job_test.go b/controller/internal/web/netstorage_job_test.go new file mode 100644 index 0000000..3bf6da3 --- /dev/null +++ b/controller/internal/web/netstorage_job_test.go @@ -0,0 +1,356 @@ +package web + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/agentapi" + "gitea.dooplex.hu/admin/felhom-controller/internal/settings" +) + +// Orchestration tests (verify-before-commit): everything runs through the netAgent / netProbeFn / +// netListFn seams — no docker, no TLS, no re-exec. + +// fakeNetAgent is the netAgent seam fake: scripted add/verify results + call recording. +type fakeNetAgent struct { + mu sync.Mutex + addRes agentapi.NetStorageAddResult + addErr error + verify agentapi.NetVerifyStatus + verifyErr error + addCalls int + removes []string + verifyPoll int +} + +func (f *fakeNetAgent) AddNetStorage(_ context.Context, req agentapi.AddNetStorageRequest) (agentapi.NetStorageAddResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.addCalls++ + return f.addRes, f.addErr +} +func (f *fakeNetAgent) NetVerifyStatus(_ context.Context) (agentapi.NetVerifyStatus, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.verifyPoll++ + return f.verify, f.verifyErr +} +func (f *fakeNetAgent) RemoveNetStorage(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removes = append(f.removes, name) + return nil +} +func (f *fakeNetAgent) removed() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.removes...) +} + +// okAddRes is the standard successful agent add result (units installed, verify started). +func okAddRes(name string) agentapi.NetStorageAddResult { + return agentapi.NetStorageAddResult{ + Name: name, Protocol: "nfs", Where: settings.NetworkMountRoot + "/" + name, + GuestPath: settings.NetworkMountRoot + "/" + name, HostUID: 101000, HostGID: 101000, + Verify: "started", JobID: "job-1", + } +} + +// netAddReq builds the standard orchestrator input. +func netAddReq(name string) agentapi.AddNetStorageRequest { + return agentapi.AddNetStorageRequest{ + Name: name, Protocol: "nfs", Server: "10.0.0.5", Export: "/srv/" + name, + MappedUID: 1000, MappedGID: 1000, + } +} + +// waitNetAdd polls the job slot until it reaches a terminal phase. +func waitNetAdd(t *testing.T, s *Server) *netAddJob { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if j := s.netAdd.snapshot(); j != nil && (j.Phase == netAddPhaseDone || j.Phase == netAddPhaseFailed) { + return j + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("net add job did not finish (last: %+v)", s.netAdd.snapshot()) + return nil +} + +// networkPathCount counts registered Kind=network paths. +func networkPathCount(s *Server) int { + n := 0 + for _, sp := range s.settings.GetStoragePaths() { + if sp.IsNetwork() { + n++ + } + } + return n +} + +// --- C1: happy path — register ONLY after probe-ok, exactly once ------------------------------------ +// Companion red-proof: move AddStoragePath before the probe → registeredAtProbe becomes 1 → FAIL. +func TestNetAdd_HappyPath_RegisterOnlyAfterProbe(t *testing.T) { + s := testServer(t) + agent := &fakeNetAgent{addRes: okAddRes("media"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}} + registeredAtProbe := -1 + s.netProbeFn = func(_ context.Context, dir string) probeOutcome { + registeredAtProbe = networkPathCount(s) // MUST be 0 — registration is the LAST step + if dir != settings.NetworkMountRoot+"/media" { + t.Errorf("probe dir = %q, want the guest path", dir) + } + return probeOutcome{OK: true} + } + + if !s.startNetAdd(agent, netAddReq("media"), "NAS media") { + t.Fatal("startNetAdd refused with a free slot") + } + job := waitNetAdd(t, s) + if job.Phase != netAddPhaseDone { + t.Fatalf("phase = %s (category=%s detail=%s), want done", job.Phase, job.Category, job.Detail) + } + if registeredAtProbe != 0 { + t.Errorf("AddStoragePath ran BEFORE the probe (count at probe = %d, want 0)", registeredAtProbe) + } + if got := networkPathCount(s); got != 1 { + t.Errorf("registered network paths = %d, want exactly 1", got) + } + if len(agent.removed()) != 0 { + t.Errorf("happy path must not roll back: removes=%v", agent.removed()) + } +} + +// --- C2: probe-fail ⇒ rollback + NOT registered + failed{not_writable} ------------------------------- +// Companion red-proof: drop the rollback call in the probe-fail branch → the removes assertion fails. +func TestNetAdd_ProbeFail_RollsBackNotRegistered(t *testing.T) { + s := testServer(t) + agent := &fakeNetAgent{addRes: okAddRes("media"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}} + s.netProbeFn = func(context.Context, string) probeOutcome { + return probeOutcome{OK: false, Category: "not_writable", Detail: "uid-1000 write probe: create/write refused"} + } + + if !s.startNetAdd(agent, netAddReq("media"), "NAS media") { + t.Fatal("startNetAdd refused") + } + job := waitNetAdd(t, s) + if job.Phase != netAddPhaseFailed || job.Category != "not_writable" { + t.Fatalf("phase/category = %s/%s, want failed/not_writable", job.Phase, job.Category) + } + // The §3.2 Route-A message, with the computed host id (1000 + 100000). + if !strings.Contains(job.Message, "minden felhasználó leképezése") || !strings.Contains(job.Message, "101000") { + t.Errorf("not_writable message must carry the map-all-users guidance + host id 101000: %q", job.Message) + } + if got := agent.removed(); len(got) != 1 || got[0] != "media" { + t.Errorf("probe-fail must roll the agent install back: removes=%v", got) + } + if got := networkPathCount(s); got != 0 { + t.Errorf("a probe-failed share must NOT be registered (got %d paths)", got) + } +} + +// --- C3: agent verify failed{code} ⇒ mapped Hungarian message, NOT registered ------------------------ +func TestNetAdd_AgentVerifyFailed_MappedMessage(t *testing.T) { + s := testServer(t) + agent := &fakeNetAgent{ + addRes: okAddRes("media"), + verify: agentapi.NetVerifyStatus{Phase: "failed", Code: "nfs_export", Detail: "reason given by server: No such file or directory", JobID: "job-1"}, + } + probeRan := false + s.netProbeFn = func(context.Context, string) probeOutcome { probeRan = true; return probeOutcome{OK: true} } + + if !s.startNetAdd(agent, netAddReq("media"), "NAS media") { + t.Fatal("startNetAdd refused") + } + job := waitNetAdd(t, s) + if job.Phase != netAddPhaseFailed || job.Category != "nfs_export" { + t.Fatalf("phase/category = %s/%s, want failed/nfs_export", job.Phase, job.Category) + } + // The EXACT §3.2 merged message (NFSv4 cannot distinguish not-found from not-permitted). + want := "A megosztás nem található, vagy a NAS nem engedélyezi ennek a gépnek a hozzáférését. Ellenőrizze az export útvonalát, és hogy a NAS engedélyezi-e a Felhom gép IP-címét." + if job.Message != want { + t.Errorf("nfs_export message:\n got %q\nwant %q", job.Message, want) + } + if probeRan { + t.Error("the probe must not run after a failed agent verify") + } + if got := networkPathCount(s); got != 0 { + t.Errorf("a verify-failed share must NOT be registered (got %d)", got) + } + // The AGENT already auto-rolled-back — the controller must not double-remove on this branch. + if got := agent.removed(); len(got) != 0 { + t.Errorf("verify-failed is agent-rolled-back; controller removes=%v want none", got) + } +} + +// --- C4: agent "no job" after install ⇒ rollback + failed (Scenario F) -------------------------------- +// Companion red-proof: treat "none" as success (or skip the rollback) → assertions fail. +func TestNetAdd_VerifyLost_RollsBack(t *testing.T) { + s := testServer(t) + agent := &fakeNetAgent{ + addRes: okAddRes("media"), + verify: agentapi.NetVerifyStatus{Phase: "none"}, // the agent restarted mid-verify + } + s.netProbeFn = func(context.Context, string) probeOutcome { + t.Error("probe must not run when the verify was lost") + return probeOutcome{OK: true} + } + + if !s.startNetAdd(agent, netAddReq("media"), "NAS media") { + t.Fatal("startNetAdd refused") + } + job := waitNetAdd(t, s) + if job.Phase != netAddPhaseFailed { + t.Fatalf("phase = %s, want failed", job.Phase) + } + if got := agent.removed(); len(got) != 1 || got[0] != "media" { + t.Errorf("verify-lost must roll back the install: removes=%v", got) + } + if got := networkPathCount(s); got != 0 { + t.Errorf("a verify-lost share must NOT be registered (got %d)", got) + } +} + +// --- C5: the --netprobe child body (pure file logic, t.TempDir) --------------------------------------- +func TestNetProbeChild(t *testing.T) { + t.Run("ok", func(t *testing.T) { + dir := t.TempDir() + if got := NetProbeChild(dir); got != netProbeExitOK { + t.Fatalf("exit = %d, want 0", got) + } + entries, _ := os.ReadDir(dir) + if len(entries) != 0 { + t.Errorf("probe must clean up its file, left: %v", entries) + } + }) + t.Run("unwritable dir → exit 2 → not_writable", func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "nope") // nonexistent → create fails everywhere + if runtime.GOOS != "windows" { + dir = t.TempDir() + if err := os.Chmod(dir, 0o555); err != nil { // read-only — the squash-trap shape + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + } + got := NetProbeChild(dir) + if got != netProbeExitNoWrite { + t.Fatalf("exit = %d, want %d", got, netProbeExitNoWrite) + } + if v := netProbeVerdict(got, ""); v.OK || v.Category != "not_writable" { + t.Errorf("verdict = %+v, want not_writable", v) + } + }) + t.Run("nonce tamper → exit 3 → probe_io", func(t *testing.T) { + orig := netProbeReadBack + netProbeReadBack = func(path string) ([]byte, error) { return []byte("tampered"), nil } + t.Cleanup(func() { netProbeReadBack = orig }) + got := NetProbeChild(t.TempDir()) + if got != netProbeExitMismatch { + t.Fatalf("exit = %d, want %d", got, netProbeExitMismatch) + } + if v := netProbeVerdict(got, ""); v.OK || v.Category != "probe_io" { + t.Errorf("verdict = %+v, want probe_io", v) + } + }) + t.Run("cleanup fail → exit 4 → OK with warn", func(t *testing.T) { + orig := netProbeReadBack + // Read back correctly but DELETE the file first — the child's own Remove then fails. + netProbeReadBack = func(path string) ([]byte, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + _ = os.Remove(path) + return data, nil + } + t.Cleanup(func() { netProbeReadBack = orig }) + got := NetProbeChild(t.TempDir()) + if got != netProbeExitCleanup { + t.Fatalf("exit = %d, want %d", got, netProbeExitCleanup) + } + v := netProbeVerdict(got, "") + if !v.OK || v.Warn == "" { + t.Errorf("cleanup-fail must be OK-with-warn (§8), got %+v", v) + } + }) +} + +// --- C6: single-flight (Scenario G) ------------------------------------------------------------------- +// Companion red-proof: drop the acquire() running-check → the second add is accepted → FAIL. +func TestNetAdd_SingleFlight(t *testing.T) { + s := testServer(t) + release := make(chan struct{}) + agent := &fakeNetAgent{addRes: okAddRes("m1"), verify: agentapi.NetVerifyStatus{Phase: "done", JobID: "job-1"}} + s.netAgentFn = func() (netAgent, error) { return agent, nil } + s.netProbeFn = func(context.Context, string) probeOutcome { <-release; return probeOutcome{OK: true} } + + if !s.startNetAdd(agent, netAddReq("m1"), "NAS m1") { + t.Fatal("first add refused") + } + // Second add through the HTTP handler while the first is blocked in probing. + body := `{"name":"m2","protocol":"nfs","server":"10.0.0.6","export":"/srv/m2"}` + r := httptest.NewRequest(http.MethodPost, "/api/storage/netstorage/add", strings.NewReader(body)) + w := httptest.NewRecorder() + s.handleNetStorageAdd(w, r) + if w.Code != http.StatusConflict { + t.Fatalf("second add: got %d want 409 (%s)", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "már folyamatban van egy csatlakoztatás") { + t.Errorf("409 must carry the Hungarian busy message: %s", w.Body.String()) + } + close(release) + job := waitNetAdd(t, s) + if job.Phase != netAddPhaseDone || job.Name != "m1" { + t.Errorf("first job must finish unaffected: %+v", job) + } +} + +// --- C7: orphan surfacing — live-but-unregistered share renders as a remove-only row ------------------ +// Companion red-proof: a registry-only filter (drop the live-map sweep) loses the ghost row → FAIL. +func TestNetStorage_OrphanRow(t *testing.T) { + s := testServer(t) + addNetworkPath(t, s, "media") // registered + live + s.netListFn = func(context.Context) ([]agentapi.NetworkMountStatus, error) { + return []agentapi.NetworkMountStatus{ + {Name: "media", Protocol: "nfs", Server: "10.0.0.5", Where: settings.NetworkMountRoot + "/media", Configured: true, Mounted: true, Reachable: true, Health: "ok"}, + {Name: "ghost", Protocol: "nfs", Server: "10.0.0.5", Where: settings.NetworkMountRoot + "/ghost", Configured: true, Mounted: false, Reachable: true, Health: "idle"}, + }, nil + } + items := s.networkStorageItems(context.Background()) + if len(items) != 2 { + t.Fatalf("items = %d, want 2 (registered + orphan): %+v", len(items), items) + } + var ghost *networkStorageItem + for i := range items { + if items[i].Name == "ghost" { + ghost = &items[i] + } else if items[i].Orphan { + t.Errorf("registered share %q must not be an orphan", items[i].Name) + } + } + if ghost == nil { + t.Fatalf("live-but-unregistered share missing from the list: %+v", items) + } + if !ghost.Orphan || ghost.Label != "Árva megosztás: ghost" { + t.Errorf("ghost row must be Orphan with the árva label, got %+v", *ghost) + } +} + +// TestNetAdd_StatusEndpoint_NoJob: the poll endpoint's empty shape. +func TestNetAdd_StatusEndpoint_NoJob(t *testing.T) { + s := testServer(t) + r := httptest.NewRequest(http.MethodGet, "/api/storage/netstorage/add/status", nil) + w := httptest.NewRecorder() + s.handleNetStorageAddStatus(w, r) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"phase":"none"`) { + t.Fatalf("empty status: %d %s", w.Code, w.Body.String()) + } +} diff --git a/controller/internal/web/server.go b/controller/internal/web/server.go index a26caa8..618ad63 100644 --- a/controller/internal/web/server.go +++ b/controller/internal/web/server.go @@ -68,6 +68,13 @@ type Server struct { // escrowed (DELETE /escrow/stage-secret). nil → the default agentClient()-backed impl; tests inject. wipeStagedEscrowFn func(ctx context.Context) error + // NAS add orchestration (verify-before-commit): the single-flight job slot + the two seams. + // netAgentFn nil → the shared agentClient(); netProbeFn nil → runNetProbe (the uid-1000 re-exec). + netAdd netAddState + netAgentFn func() (netAgent, error) + netProbeFn func(ctx context.Context, dir string) probeOutcome + netListFn func(ctx context.Context) ([]agentapi.NetworkMountStatus, error) + // Asset syncer for Hub-managed assets (optional) assetsSyncer *assets.Syncer diff --git a/controller/internal/web/storage_handlers.go b/controller/internal/web/storage_handlers.go index 8713564..cc0e009 100644 --- a/controller/internal/web/storage_handlers.go +++ b/controller/internal/web/storage_handlers.go @@ -288,8 +288,11 @@ func (s *Server) ServeStorageAPI(w http.ResponseWriter, r *http.Request) { case r.URL.Path == "/api/storage/decommission" && r.Method == http.MethodPost: s.handleStorageDecommission(w, r) // NAS network storage (Part A2) — distinct from the drive lifecycle above (proxy to agent /netstorage/*). + // Add is verify-before-commit: it starts a detached orchestration job the UI polls on add/status. case r.URL.Path == "/api/storage/netstorage/add" && r.Method == http.MethodPost: s.handleNetStorageAdd(w, r) + case r.URL.Path == "/api/storage/netstorage/add/status" && r.Method == http.MethodGet: + s.handleNetStorageAddStatus(w, r) case r.URL.Path == "/api/storage/netstorage" && r.Method == http.MethodGet: s.handleNetStorageList(w, r) case r.URL.Path == "/api/storage/netstorage/remove" && r.Method == http.MethodPost: