felhom-agent/internal/localapi/netverifyjob.go

package localapi

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)

// Network-storage verify job — the agent half of verify-before-commit (SPIKE-nas-verify-2026-07-11).
// POST /netstorage/add no longer succeeds blind: after EnsureNetworkMount installs the unit pair,
// a DETACHED job triggers a real mount through the enabled automount (spike Q1: a single directory
// read wakes it), judges success from /proc/mounts (§8 truth table), classifies a failure from the
// mount unit's journal (Q4 taxonomy), and AUTO-ROLLS-BACK a failed install (RemoveNetworkMount +
// creds file) so a bogus share never lingers half-configured.
//
// The job record is an IN-MEMORY single slot, deliberately NOT persisted (unlike formatJob): if the
// agent restarts mid-verify the slot is empty and GET /netstorage/verify-status reports phase
// "none" — the CONTROLLER orchestrator treats "no job after units were installed" as verify-lost
// and rolls the add back itself (Scenario F). Persistence would buy nothing here: the trigger is
// re-runnable and the controller owns the retry; a lost verify must fail the add, never revive it.

// netVerifyJob is the single-slot verify record served by GET /netstorage/verify-status.
type netVerifyJob struct {
	JobID     string `json:"job_id"`
	Name      string `json:"name"`
	Where     string `json:"where"`
	Protocol  string `json:"protocol"`
	Phase     string `json:"phase"`            // running | done | failed  (the empty slot reports "none")
	Code      string `json:"code,omitempty"`   // failure category (storage.NetVerify*)
	Detail    string `json:"detail,omitempty"` // operator hint + raw journal fragment (English; UI maps Hungarian by Code)
	StartedAt string `json:"started_at"`
	UpdatedAt string `json:"updated_at"`
}

const (
	netVerifyPhaseNone    = "none"
	netVerifyPhaseRunning = "running"
	netVerifyPhaseDone    = "done"
	netVerifyPhaseFailed  = "failed"
)

// netVerifyDeadline bounds the whole verify pipeline. systemd resolves even a black-holed mount at
// its 90 s start cap (SPIKE Q4 case i), so 95 s guarantees the trigger goroutine's outcome is in by
// the time the deadline can fire — the deadline is the belt, not the expected path.
const netVerifyDeadline = 95 * time.Second

// netRollbackTimeout bounds the auto-rollback after a failed verify (a fresh context — the verify
// deadline may already be spent when the rollback runs).
const netRollbackTimeout = 30 * time.Second

// tryStartNetVerify claims the single verify slot for job (a copy is stored). false = another verify
// is still running (Scenario G single-flight: the caller refuses the second add).
func (s *Server) tryStartNetVerify(job *netVerifyJob) bool {
	s.netVerifyMu.Lock()
	defer s.netVerifyMu.Unlock()
	if s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning {
		return false
	}
	cp := *job
	s.netVerifyCur = &cp
	return true
}

// releaseNetVerify frees the slot IF it still holds this job — the sync fast-fail path (pre-probe /
// Ensure failure) claims the slot first to close the race window, then releases it because no
// detached work ever started.
func (s *Server) releaseNetVerify(job *netVerifyJob) {
	s.netVerifyMu.Lock()
	defer s.netVerifyMu.Unlock()
	if s.netVerifyCur != nil && s.netVerifyCur.JobID == job.JobID {
		s.netVerifyCur = nil
	}
}

// finishNetVerify records the terminal phase (done/failed) on the slot.
func (s *Server) finishNetVerify(job *netVerifyJob, phase, code, detail string) {
	s.netVerifyMu.Lock()
	defer s.netVerifyMu.Unlock()
	job.Phase = phase
	job.Code = code
	job.Detail = detail
	job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
	cp := *job
	s.netVerifyCur = &cp
}

// netVerifySnapshot returns a copy of the current slot (nil = no job — the Scenario F signal).
func (s *Server) netVerifySnapshot() *netVerifyJob {
	s.netVerifyMu.Lock()
	defer s.netVerifyMu.Unlock()
	if s.netVerifyCur == nil {
		return nil
	}
	cp := *s.netVerifyCur
	return &cp
}

// runNetVerify runs the verify pipeline detached from the HTTP request (off baseCtx, like
// startFormatDetached). The returned channel closes when the job reaches a terminal phase (tests
// wait on it; production ignores it — the controller polls the status endpoint).
func (s *Server) runNetVerify(spec storage.NetworkMountSpec, job *netVerifyJob) <-chan struct{} {
	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	done := make(chan struct{})
	go func() {
		defer close(done)
		start := time.Now()
		ctx, cancel := context.WithTimeout(base, netVerifyDeadline)
		defer cancel()
		s.logger.Debug("netverify: job started",
			"job_id", job.JobID, "name", spec.Name, "where", spec.Where(),
			"proto", spec.Protocol, "deadline_s", int(netVerifyDeadline.Seconds()))

		// Trigger a real mount: a directory read through the enabled automount mounts the share
		// (spike Q1, ~1 s on a healthy LAN). The read may block until systemd resolves the mount
		// attempt (≤90 s, its start cap) — so it runs in its own goroutine, which may linger past
		// our select until that cap resolves it. Bounded and accepted; never awaited twice.
		trigger := make(chan error, 1)
		go func() { trigger <- s.netTrigger(spec.Where()) }()
		timedOut := false
		select {
		case terr := <-trigger:
			// The read error is NOT the verdict (§8) — logged for the flow trace only.
			s.logger.Debug("netverify: automount trigger returned",
				"name", spec.Name, "read_err", fmt.Sprint(terr))
		case <-ctx.Done():
			timedOut = true
			s.logger.Debug("netverify: deadline fired before the trigger resolved", "name", spec.Name)
		}

		// §8 truth table: /proc/mounts is the ONLY success judge. A trigger read error on a mounted
		// share (EACCES on a 0700 export) is a GOOD mount — the controller's uid-1000 probe decides
		// writability, not the agent user's readability.
		mounted := s.netMounted(spec.Where())
		s.logger.Debug("netverify: /proc/mounts verdict",
			"name", spec.Name, "where", spec.Where(), "mounted", mounted, "timed_out", timedOut)
		if mounted {
			s.logger.Info("netverify: mount verified", "name", spec.Name, "where", spec.Where(),
				"duration_ms", time.Since(start).Milliseconds())
			s.finishNetVerify(job, netVerifyPhaseDone, "", "")
			return
		}

		// Mount FAILED → classify from the mount unit's journal (unprivileged read — systemd-journal
		// group membership, NO sudo), then auto-rollback the install.
		code, detail := s.classifyNetFailure(base, spec, timedOut)
		s.rollbackNetMount(base, spec)
		s.logger.Warn("netverify: verify failed — install rolled back",
			"name", spec.Name, "code", code, "timed_out", timedOut,
			"duration_ms", time.Since(start).Milliseconds())
		s.finishNetVerify(job, netVerifyPhaseFailed, code, detail)
	}()
	return done
}

// classifyNetFailure reads the mount unit's journal tail and classifies it per the Q4 taxonomy.
// Journal unavailable (agent user not in systemd-journal, journalctl missing) degrades to the
// generic category — with the timeout verdict preserved when the deadline fired.
func (s *Server) classifyNetFailure(base context.Context, spec storage.NetworkMountSpec, timedOut bool) (code, detail string) {
	fallback := storage.NetVerifyMountFailed
	if timedOut {
		fallback = storage.NetVerifyTimeout
	}
	unit, err := storage.UnitNameForMount(spec.Where())
	if err != nil {
		return fallback, "internal: unit name: " + err.Error()
	}
	jctx, cancel := context.WithTimeout(base, 10*time.Second)
	defer cancel()
	tail, jerr := s.netJournal(jctx, unit)
	if jerr != nil {
		s.logger.Warn("netverify: journal unavailable — degraded classification (is felhom-agent in the systemd-journal group?)",
			"unit", unit, "err", jerr)
		return fallback, "journal unavailable — add the felhom-agent user to the systemd-journal group (usermod -aG systemd-journal felhom-agent)"
	}
	s.logger.Debug("netverify: journal tail read for classification", "unit", unit, "bytes", len(tail))
	code, hint := storage.ClassifyNetVerifyFailure(tail, s.netReachable(spec.Protocol, spec.Server))
	s.logger.Debug("netverify: failure classified", "name", spec.Name, "code", code, "timed_out", timedOut)
	if code == storage.NetVerifyMountFailed && timedOut {
		code = storage.NetVerifyTimeout
	}
	// Detail carries the hint + a bounded raw fragment so the UI's collapsible detail shows the truth.
	frag := strings.TrimSpace(tail)
	if len(frag) > 500 {
		frag = frag[len(frag)-500:]
	}
	return code, hint + " | journal: " + frag
}

// rollbackNetMount is the verify-fail auto-rollback: remove the unit pair + the SMB creds file.
// Idempotent + best-effort-logged — the controller may double-remove after us (Scenario F/§8),
// which is harmless.
func (s *Server) rollbackNetMount(base context.Context, spec storage.NetworkMountSpec) {
	ctx, cancel := context.WithTimeout(base, netRollbackTimeout)
	defer cancel()
	if err := s.netStorage.RemoveNetworkMount(ctx, spec.Name); err != nil {
		s.logger.Error("netverify: rollback RemoveNetworkMount failed (manual cleanup may be needed)",
			"name", spec.Name, "err", err)
	} else {
		s.logger.Info("netverify: failed install rolled back (unit pair removed)", "name", spec.Name)
	}
	if spec.Protocol == storage.ProtocolSMB {
		s.removeSMBCreds(spec.Name)
	}
}

// handleNetVerifyStatus reports the single verify slot. phase "none" (no job) is a REAL signal, not
// an error: the controller reads it after an agent restart as verify-lost and rolls back (Scenario F).
func (s *Server) handleNetVerifyStatus(w http.ResponseWriter, r *http.Request, vmid int) {
	if s.netStorage == nil {
		writeErr(w, http.StatusServiceUnavailable, "network storage not configured on this host")
		return
	}
	job := s.netVerifySnapshot()
	if job == nil {
		writeOK(w, map[string]any{"vmid": vmid, "phase": netVerifyPhaseNone})
		return
	}
	writeOK(w, map[string]any{
		"vmid": vmid, "phase": job.Phase, "name": job.Name, "where": job.Where,
		"protocol": job.Protocol, "code": job.Code, "detail": job.Detail,
		"job_id": job.JobID, "started_at": job.StartedAt, "updated_at": job.UpdatedAt,
	})
}

// readUnitJournal is the production netJournal seam: an UNPRIVILEGED journalctl read of the mount
// unit's tail (-o cat = raw message strings for the classifier). No sudo — this works iff the
// felhom-agent user is in the systemd-journal group (host-install adds it; §9 rule 7).
func readUnitJournal(ctx context.Context, unit string) (string, error) {
	out, err := exec.CommandContext(ctx, "journalctl", "-u", unit, "-n", "20", "--no-pager", "-o", "cat").CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("journalctl -u %s: %w (%s)", unit, err, strings.TrimSpace(string(out)))
	}
	return string(out), nil
}

// triggerNetMount is the production netTrigger seam: a plain directory read through the automount
// trigger path — exactly what wakes the mount (spike Q1). The error is deliberately unused by the
// caller for the verdict (§8: readability is NOT the truth source); it exists for the log line only.
func triggerNetMount(where string) error {
	_, err := os.ReadDir(where)
	return err
}