package localapi import ( "context" "fmt" "net/http" "os" "os/exec" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/storage" ) // Network-storage verify job — the agent half of verify-before-commit (SPIKE-nas-verify-2026-07-11). // POST /netstorage/add no longer succeeds blind: after EnsureNetworkMount installs the unit pair, // a DETACHED job triggers a real mount through the enabled automount (spike Q1: a single directory // read wakes it), judges success from /proc/mounts (§8 truth table), classifies a failure from the // mount unit's journal (Q4 taxonomy), and AUTO-ROLLS-BACK a failed install (RemoveNetworkMount + // creds file) so a bogus share never lingers half-configured. // // The job record is an IN-MEMORY single slot, deliberately NOT persisted (unlike formatJob): if the // agent restarts mid-verify the slot is empty and GET /netstorage/verify-status reports phase // "none" — the CONTROLLER orchestrator treats "no job after units were installed" as verify-lost // and rolls the add back itself (Scenario F). Persistence would buy nothing here: the trigger is // re-runnable and the controller owns the retry; a lost verify must fail the add, never revive it. // netVerifyJob is the single-slot verify record served by GET /netstorage/verify-status. type netVerifyJob struct { JobID string `json:"job_id"` Name string `json:"name"` Where string `json:"where"` Protocol string `json:"protocol"` Phase string `json:"phase"` // running | done | failed (the empty slot reports "none") Code string `json:"code,omitempty"` // failure category (storage.NetVerify*) Detail string `json:"detail,omitempty"` // operator hint + raw journal fragment (English; UI maps Hungarian by Code) StartedAt string `json:"started_at"` UpdatedAt string `json:"updated_at"` } const ( netVerifyPhaseNone = "none" netVerifyPhaseRunning = "running" netVerifyPhaseDone = "done" netVerifyPhaseFailed = "failed" ) // netVerifyDeadline bounds the whole verify pipeline. systemd resolves even a black-holed mount at // its 90 s start cap (SPIKE Q4 case i), so 95 s guarantees the trigger goroutine's outcome is in by // the time the deadline can fire — the deadline is the belt, not the expected path. const netVerifyDeadline = 95 * time.Second // netRollbackTimeout bounds the auto-rollback after a failed verify (a fresh context — the verify // deadline may already be spent when the rollback runs). const netRollbackTimeout = 30 * time.Second // tryStartNetVerify claims the single verify slot for job (a copy is stored). false = another verify // is still running (Scenario G single-flight: the caller refuses the second add). func (s *Server) tryStartNetVerify(job *netVerifyJob) bool { s.netVerifyMu.Lock() defer s.netVerifyMu.Unlock() if s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning { return false } cp := *job s.netVerifyCur = &cp return true } // releaseNetVerify frees the slot IF it still holds this job — the sync fast-fail path (pre-probe / // Ensure failure) claims the slot first to close the race window, then releases it because no // detached work ever started. func (s *Server) releaseNetVerify(job *netVerifyJob) { s.netVerifyMu.Lock() defer s.netVerifyMu.Unlock() if s.netVerifyCur != nil && s.netVerifyCur.JobID == job.JobID { s.netVerifyCur = nil } } // finishNetVerify records the terminal phase (done/failed) on the slot. func (s *Server) finishNetVerify(job *netVerifyJob, phase, code, detail string) { s.netVerifyMu.Lock() defer s.netVerifyMu.Unlock() job.Phase = phase job.Code = code job.Detail = detail job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339) cp := *job s.netVerifyCur = &cp } // netVerifySnapshot returns a copy of the current slot (nil = no job — the Scenario F signal). func (s *Server) netVerifySnapshot() *netVerifyJob { s.netVerifyMu.Lock() defer s.netVerifyMu.Unlock() if s.netVerifyCur == nil { return nil } cp := *s.netVerifyCur return &cp } // runNetVerify runs the verify pipeline detached from the HTTP request (off baseCtx, like // startFormatDetached). The returned channel closes when the job reaches a terminal phase (tests // wait on it; production ignores it — the controller polls the status endpoint). func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob) <-chan struct{} { base := s.baseCtx if base == nil { base = context.Background() } done := make(chan struct{}) go func() { defer close(done) start := time.Now() ctx, cancel := context.WithTimeout(base, netVerifyDeadline) defer cancel() s.logger.Debug("netverify: job started", "job_id", job.JobID, "name", spec.Name, "where", spec.Where(), "proto", spec.Protocol, "deadline_s", int(netVerifyDeadline.Seconds())) // Trigger a real mount: a directory read through the enabled automount mounts the share // (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount // attempt (≤90 s, its start cap) — so it runs in its own goroutine, which may linger past // our select until that cap resolves it. Bounded and accepted; never awaited twice. trigger := make(chan error, 1) go func() { trigger <- s.netTrigger(spec.Where()) }() timedOut := false select { case terr := <-trigger: // The read error is NOT the verdict (§8) — logged for the flow trace only. s.logger.Debug("netverify: automount trigger returned", "name", spec.Name, "read_err", fmt.Sprint(terr)) case <-ctx.Done(): timedOut = true s.logger.Debug("netverify: deadline fired before the trigger resolved", "name", spec.Name) } // §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted // share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides // writability, not the agent user's readability. mounted := s.netMounted(spec.Where()) s.logger.Debug("netverify: /proc/mounts verdict", "name", spec.Name, "where", spec.Where(), "mounted", mounted, "timed_out", timedOut) if mounted { s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where(), "duration_ms", time.Since(start).Milliseconds()) s.finishNetVerify(job, netVerifyPhaseDone, "", "") return } // Mount FAILED → classify from the mount unit's journal (unprivileged read — systemd-journal // group membership, NO sudo), then auto-rollback the install. code, detail := s.classifyNetFailure(base, spec, timedOut) s.rollbackNetMount(base, spec) s.logger.Warn("netverify: verify failed — install rolled back", "name", spec.Name, "code", code, "timed_out", timedOut, "duration_ms", time.Since(start).Milliseconds()) s.finishNetVerify(job, netVerifyPhaseFailed, code, detail) }() return done } // classifyNetFailure reads the mount unit's journal tail and classifies it per the Q4 taxonomy. // Journal unavailable (agent user not in systemd-journal, journalctl missing) degrades to the // generic category — with the timeout verdict preserved when the deadline fired. func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMountSpec, timedOut bool) (code, detail string) { fallback := storage.NetVerifyMountFailed if timedOut { fallback = storage.NetVerifyTimeout } unit, err := storage.UnitNameForMount(spec.Where()) if err != nil { return fallback, "internal: unit name: " + err.Error() } jctx, cancel := context.WithTimeout(base, 10*time.Second) defer cancel() tail, jerr := s.netJournal(jctx, unit) if jerr != nil { s.logger.Warn("netverify: journal unavailable — degraded classification (is felhom-agent in the systemd-journal group?)", "unit", unit, "err", jerr) return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)" } s.logger.Debug("netverify: journal tail read for classification", "unit", unit, "bytes", len(tail)) code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server)) s.logger.Debug("netverify: failure classified", "name", spec.Name, "code", code, "timed_out", timedOut) if code == storage.NetVerifyMountFailed && timedOut { code = storage.NetVerifyTimeout } // Detail carries the hint + a bounded raw fragment so the UI's collapsible detail shows the truth. frag := strings.TrimSpace(tail) if len(frag) > 500 { frag = frag[len(frag)-500:] } return code, hint + " | journal: " + frag } // rollbackNetMount is the verify-fail auto-rollback: remove the unit pair + the SMB creds file. // Idempotent + best-effort-logged — the controller may double-remove after us (Scenario F/§8), // which is harmless. func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMountSpec) { ctx, cancel := context.WithTimeout(base, netRollbackTimeout) defer cancel() if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil { s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)", "name", spec.Name, "err", err) } else { s.logger.Info("netverify: failed install rolled back (unit pair removed)", "name", spec.Name) } if spec.Protocol == storage.ProtocolSMB { s.removeSMBCreds(spec.Name) } } // handleNetVerifyStatus reports the single verify slot. phase "none" (no job) is a REAL signal, not // an error: the controller reads it after an agent restart as verify-lost and rolls back (Scenario F). func (s *Server) handleNetVerifyStatus(w http.ResponseWriter, r *http.Request, vmid int) { if s.netStorage == nil { writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host") return } job := s.netVerifySnapshot() if job == nil { writeOK(w, map[string]any{"vmid": vmid, "phase": netVerifyPhaseNone}) return } writeOK(w, map[string]any{ "vmid": vmid, "phase": job.Phase, "name": job.Name, "where": job.Where, "protocol": job.Protocol, "code": job.Code, "detail": job.Detail, "job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt, }) } // readUnitJournal is the production netJournal seam: an UNPRIVILEGED journalctl read of the mount // unit's tail (-o cat = raw message strings for the classifier). No sudo — this works iff the // felhom-agent user is in the systemd-journal group (host-install adds it; §9 rule 7). func readUnitJournal(ctx context.Context, unit string) (string, error) { out, err := exec.CommandContext(ctx, "journalctl", "-u", unit, "-n", "20", "--no-pager", "-o", "cat").CombinedOutput() if err != nil { return "", fmt.Errorf("journalctl -u %s: %w (%s)", unit, err, strings.TrimSpace(string(out))) } return string(out), nil } // triggerNetMount is the production netTrigger seam: a plain directory read through the automount // trigger path — exactly what wakes the mount (spike Q1). The error is deliberately unused by the // caller for the verdict (§8: readability is NOT the truth source); it exists for the log line only. func triggerNetMount(where string) error { _, err := os.ReadDir(where) return err }