felhom-agent/internal/localapi/server.go

package localapi

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
	"gitea.dooplex.hu/admin/felhom-agent/internal/escrow"
	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)

// GuestAPI is the narrow Proxmox surface the local API needs. Satisfied by *proxmox.Client.
// Every method here is invoked ONLY with the VMID resolved from the caller's token — never a
// caller-supplied id — so the proxmox op is structurally self-scoped.
type GuestAPI interface {
	GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
	Snapshot(ctx context.Context, vmid int, snapname, description string) (string, error)
	Rollback(ctx context.Context, vmid int, snapname string) (string, error)
	WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
}

// PrivilegedRunner runs a fenced root wrapper. The seam exists so the backup-target move is testable
// without sudo: the wrapper IS the security boundary, so tests substitute it, never bypass it.
type PrivilegedRunner interface {
	Run(ctx context.Context, name string, args ...string) (stdout, stderr []byte, err error)
}

// BackupService enqueues a vzdump/PBS backup of a guest. Satisfied by *backup.BackupRunner.
// BackupWithSnapshotHook (8B.2) invokes onSnapshot once mid-backup when the storage snapshot is
// taken (snapshot mode only) so the controller can resume its app early; in stop mode it is never
// called.
type BackupService interface {
	BackupWithSnapshotHook(ctx context.Context, vmid int, onSnapshot func()) (hub.Backup, error)
}

// BackupArchiveLister is an OPTIONAL extension to BackupService: "when did a backup last LAND on
// this tier's storage?", answered by the storage rather than by memory. *backup.BackupRunner
// satisfies it.
//
// R-84: the agent's backup Store is in-memory, so after every restart the due-check saw nothing and
// the controller took a redundant backup — a wasted multi-hour WAN upload on the offsite tier after
// every agent deploy. Consulting the storage makes the cold path truthful without persisting
// anything, and it self-corrects: a pruned archive correctly stops counting.
type BackupArchiveLister interface {
	NewestArchiveTime(ctx context.Context, vmid int) (time.Time, bool, error)
}

// BackupTier (R-82) binds ONE backup tier's runner to its own policy. The agent builds one per
// resolved config tier; the local API serves each independently so "local daily + PBS weekly" is
// expressible over the wire, not just in config.
//
// COMPATIBILITY CONTRACT (load-bearing — the agent and controller deploy independently):
// the tier whose Primary is true is what EVERY untargeted endpoint acts on. An old controller
// never sends `?target=`, so it sees exactly the pre-R-82 behaviour and response bytes.
type BackupTier struct {
	TargetID string
	Cadence  time.Duration
	// WaitTimeout bounds the fire-and-forget backup context. It MUST be >= the runner's own wait
	// bound, or the outer context cancels first and the tier reports a false failure while the
	// vzdump keeps running (observed live 2026-07-26 with a fixed 2h outer bound).
	WaitTimeout time.Duration
	Primary     bool
	Service     BackupService
}

// BackupStore records + reads the latest backup/restore-test state. Satisfied by *backup.Store.
type BackupStore interface {
	RecordBackup(hub.Backup)
	Backups(ctx context.Context) []hub.Backup
	RestoreTests(ctx context.Context) []hub.RestoreTest
}

// StorageView yields the host's observed storage targets (for mapping a mount's storage id →
// fast/slow class). Satisfied by *storage.Observer.
type StorageView interface {
	Observe(ctx context.Context) ([]hub.StorageTarget, error)
}

// SmartReader (v0.95.0, Fix B) reads per-disk SMART for the /disks union path so registry/USB drives
// that ride the union (not Observe's enrich) still get a health verdict. A zero-value summary
// (Health "") means "could not read". Satisfied by *storage.SmartReader.
type SmartReader interface {
	SMARTForBacking(ctx context.Context, backingDevice string) hub.SmartSummary
}

// TokenAuthority resolves a presented bearer token to its guest VMID. Satisfied by *TokenStore.
type TokenAuthority interface {
	Lookup(token string) (int, bool)
}

// HostMetricsProvider does a FRESH host-metrics collect (cpu%/mem/load/uptime/cpu-temp) for
// GET /host/metrics (slice 9). Satisfied by *hub.Collector (which reuses the slice-4 collector —
// no duplicate collection). Optional: when nil, /host/metrics reports "not configured".
type HostMetricsProvider interface {
	HostMetricsNow(ctx context.Context) (hub.HostMetrics, error)
}

// Options configures a Server.
type Options struct {
	ListenAddr string // bridge IP:port
	Cert       tls.Certificate
	Guests     GuestAPI
	Backups    BackupService
	Store      BackupStore
	Storage    StorageView
	// DriveTargets (Impl-2a, optional) yields registry+units-sourced drives for the /disks view, so a
	// drive with NO PVE storage still appears. Unioned with Storage.Observe (deduped by mount path).
	DriveTargets storage.KnownTargets
	// Smart (v0.95.0, Fix B) reads per-disk SMART for the /disks UNION path — registry/USB drives ride
	// the union (not Observe's enrich), so without this they carry no health verdict. OPTIONAL; nil →
	// union rows have no SMART (pre-v0.95.0 behavior). Satisfied by *storage.SmartReader.
	Smart  SmartReader
	Tokens TokenAuthority
	// BackupCadence is the per-guest backup interval driving GET /backup/due (slice 8B). A guest
	// is "due" when no successful backup is recorded OR the newest one is older than this. 0 → a
	// safe default (24h). The hub-served policy is slice 10; this is the agent-local cadence.
	// When BackupTiers is supplied this is IGNORED (the primary tier carries its own cadence).
	BackupCadence time.Duration
	// BackupTiers (R-82) is the resolved multi-tier policy, primary first. OPTIONAL: nil → one tier
	// synthesized from Backups + BackupCadence, i.e. exactly the pre-R-82 behaviour.
	BackupTiers []BackupTier
	// InFlight (R-85) is the host-wide one-heavy-operation gate shared with the restore-test
	// scheduler. OPTIONAL: nil → no cross-gating (pre-R-85 behaviour). See backup.InFlight.
	InFlight *backup.InFlight
	// Disk management (slice 8C) — OPTIONAL. When Disks + DiskGate are set, the /disks endpoints
	// are served; otherwise they report "not configured". DiskGate authorizes the destructive
	// (data-bearing) format path; Guests lists guests for the eject dependent-warning.
	Disks    DiskOps
	DiskGate StorageGate
	Guests2  GuestLister
	// GuestAttach binds an enrolled user-data drive's felhom-data namespace into the guest (slice 10
	// P2, Model A). OPTIONAL — when nil, POST /disks/guest-attach reports "not configured".
	GuestAttach GuestAttacher
	// Memory is the guest-RAM-resize Proxmox surface (v0.90.0, R-24). OPTIONAL — when nil, the
	// /guest/memory endpoints report "not configured". Satisfied by *proxmox.Client.
	Memory MemoryOps
	// NetStorage is the privileged network-mount (NAS) surface (Part A1). OPTIONAL — when nil, the
	// /netstorage endpoints report "not configured". Satisfied by *storage.SudoHostOps.
	NetStorage NetworkStorageOps
	// Privileged runs the fenced root wrappers (E-2a: felhom-backup-target-apply). OPTIONAL — when
	// nil, POST /backup/target reports "not configured". Satisfied by *proxmox.ExecRunner.
	Privileged PrivilegedRunner
	// ConfigPath is agent.json, so the backup-target move can repoint the primary tier. "" (env-only
	// config) → the move reports it cannot persist rather than pretending it did.
	ConfigPath string
	// StateDir is where a pre-write recovery copy of agent.json is parked. "" → no copy is parked.
	StateDir string
	// SmbCredsDir is where the agent writes the 0600 SMB credentials files (out-of-band). "" →
	// /var/lib/felhom-agent/smb-creds.
	SmbCredsDir string
	// EscrowStagePath is the 0600 file where POST /escrow/stage-secret transiently stages the
	// controller-pushed restic repo password (fork-4). "" → escrow.StagedResticPasswordPath() (the
	// canonical path the escrow-create ceremony reads). Injectable so the stage handler is testable.
	EscrowStagePath string
	// EscrowCeremony wires the controller-driven ceremony endpoints (v0.88.0): POST /escrow/ceremony
	// (+/status, /claim one-shot R) and GET /escrow/preflight. OPTIONAL — when nil, those endpoints
	// report "not configured".
	EscrowCeremony *EscrowCeremonyConfig
	// ControllerSwap runs guest commands (pct exec) for the agentic controller-update swap (Phase 1).
	// OPTIONAL — when nil, POST /controller/swap reports "not configured". Satisfied by *GuestBinder.
	ControllerSwap GuestExecutor
	// StaleLock recovers a guest left with a stale vzdump lock by a reboot-during-backup (F2-b), run at
	// startup by RecoverStaleLockedGuests. OPTIONAL — when nil, the recovery is a no-op.
	StaleLock StaleLockController
	// ControllerSwapStateDir holds the per-guest swap state file (crash-safety). "" → /var/lib/felhom-agent.
	ControllerSwapStateDir string
	// Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL —
	// when nil, no intent is recorded (self-heal runs ungated).
	Intent IntentRecorder
	// GuestBinds persists which user-data drives (by durable-id) are enrolled into each guest, so the
	// startup re-assert (ReassertGuestBinds) can restore a bind that a re-provision dropped (F9).
	// OPTIONAL — when nil, guest binds are not recorded and the startup re-assert is a no-op.
	GuestBinds *GuestBindStore
	// FormatJobs persists the in-flight/last disk-format job so mkfs runs detached from the request
	// (F20-BUG3: a request deadline can't kill it) and survives an agent restart (RecoverFormatJob).
	// OPTIONAL — when nil, formats still run detached but are not persisted/recovered.
	FormatJobs *FormatJobStore
	// HostReader is the root-free host topology reader used to classify a device/mount's protection
	// ROLE (it backs SystemDisks for the eject role-gate + the /disks role hints). OPTIONAL — when nil
	// it defaults to the production *storage.ProcHostReader. Injectable so the role-gate is testable.
	HostReader storage.HostReader
	// HostMetrics serves GET /host/metrics (slice 9) — host-wide health (cpu%/mem/load/uptime/
	// cpu-temp) + per-storage capacity, host-wide and token-authed (one-customer-per-host). When
	// nil the endpoint reports "not configured" (host still reports/reconciles).
	HostMetrics HostMetricsProvider
	// HostID is this agent's host id — surfaced in a data-bearing-format pending-op so the operator
	// signs an op bound to THIS host (slice 10B anti-retarget). Optional (only used for the hint).
	HostID string
	// AgentVersion is this agent's build version (main.version). When set, EVERY local-API response
	// carries it in the X-Felhom-Agent-Version header — the controller's capability channel: its
	// Supports() compares this against a per-feature MinAgent table instead of route-probing
	// (v0.82.0; the probe stays as the fallback for header-less agents). Optional.
	AgentVersion string
	// LogRing is the agent's always-DEBUG capture ring (v0.83.0 observability), served by
	// GET /debug/logs. OPTIONAL — when nil the endpoint reports "not configured".
	LogRing *applog.Ring
	Logger  *slog.Logger
}

// defaultBackupCadence is the fallback /backup/due window when none is configured.
const defaultBackupCadence = 24 * time.Hour

// Backup phase vocabulary reported by GET /backup/status (slice 8B). The 8B.2 fast-follow adds a
// `snapshotted` phase (vzdump --mode snapshot) so the controller can unquiesce at snapshot-taken.
const (
	PhaseIdle        = "idle"
	PhaseRunning     = "running"
	PhaseSnapshotted = "snapshotted" // 8B.2: storage snapshot taken — app may resume; backup continues
	PhaseDone        = "done"
	PhaseFailed      = "failed"
)

// backupJob is the in-flight/last backup job for one guest (drives /backup/status phases).
type backupJob struct {
	JobID      string
	Phase      string
	StartedAt  time.Time
	FinishedAt time.Time
	Archive    string
	Error      string
}

// backupJobKey identifies one guest's job on ONE tier (R-82).
type backupJobKey struct {
	vmid   int
	target string
}

// Server is the per-guest local API (doc 03 §6). It serves the agent's pinned self-signed leaf
// and authorizes every request against the token's guest only.
type Server struct {
	addr         string
	cert         tls.Certificate
	guests       GuestAPI
	backups      BackupService
	store        BackupStore
	storage      StorageView
	driveTargets storage.KnownTargets // Impl-2a: registry+units drives for /disks (optional)
	smart        SmartReader          // v0.95.0 Fix B: SMART for the /disks union path (optional)
	tokens       TokenAuthority
	cadence      time.Duration
	// tiers (R-82) is the resolved backup-tier list, PRIMARY FIRST. Always non-empty: when the
	// caller supplies no tiers it holds exactly one, synthesized from Backups+BackupCadence, which
	// is the pre-R-82 shape.
	tiers []BackupTier
	// inFlight (R-85) is shared with the restore-test scheduler so the two never run together.
	inFlight *backup.InFlight
	logger   *slog.Logger
	now      func() time.Time

	disks           DiskOps             // slice 8C (optional)
	diskGate        StorageGate         // slice 8C (optional)
	guestList       GuestLister         // slice 8C (optional)
	guestAttach     GuestAttacher       // slice 10 P2 (optional)
	mem             MemoryOps           // v0.90.0 R-24 guest RAM resize (optional)
	memMu           sync.Mutex          // single-flight around a resize apply (one customer per host)
	netStorage      NetworkStorageOps   // Part A1: NAS network mounts (optional)
	netMountRoot    string              // the user-data namespace root for the network-mount role gate
	smbCredsDir     string              // where SMB creds files are written (out-of-band, 0600)
	escrowStagePath string              // fork-4: 0600 staging file for the pushed restic repo password
	intent          IntentRecorder      // slice 10 P3 (optional)
	guestBinds      *GuestBindStore     // F9 startup bind re-assert record (optional)
	formatJobs      *FormatJobStore     // F20-BUG3 detached-format job record (optional)
	staleLock       StaleLockController // F2-b startup stale-lock recovery (optional)
	// guestPower (F-REBOOT) is per-guest start-attempt state for the guest-power watchdog.
	// Guarded by guestPowerMu in guestpower.go; in-memory on purpose (see guestPowerState).
	guestPower map[int]guestPowerState

	// guestPowerSweeps counts completed guest-power sweeps, for the liveness observable. Touched only
	// from GuestPowerTick, which the ticker calls serially.
	guestPowerSweeps int
	host             storage.HostReader // role classification source (optional; defaults to ProcHostReader)

	hostMetrics  HostMetricsProvider // slice 9 (optional)
	hostID       string              // slice 10B: for the data-bearing-format pending-op hint
	agentVersion string              // v0.82.0: the X-Felhom-Agent-Version response header value
	logRing      *applog.Ring        // v0.83.0: GET /debug/logs source (optional)

	// reresolveWipe performs the [AGENT-001] anti-retarget re-resolution before an
	// inline customer-confirmed wipe (durable id → current device, re-derive+match,
	// re-inspect). Defaults to s.reresolveDurableForWipe (real storage funcs); tests
	// override it to avoid touching real /dev.
	// resolveStorageDevice maps a durable id (uuid:<fs-uuid>) to its /dev node for the /disks union
	// path. Defaults to storage.ResolveStorageDevice (hits /dev/disk/by-*); tests override it.
	resolveStorageDevice func(durableID string) (string, error)

	reresolveWipe func(ctx context.Context, durableID string) (string, error)

	// reresolveBlank is the BLANK-format sibling (audit D3): same anti-retarget
	// sequence, but requires the re-inspected device to STILL be blank. Defaults
	// to s.reresolveDurableForBlankFormat; tests override it.
	reresolveBlank func(ctx context.Context, durableID string) (string, error)

	// deviceDurableID derives the WIPE-binding durable id of a block device (the byid:/byuuid: scheme
	// the wipe gate resolves against). F20-BUG2: BOTH the /disks list (DiskInfo.WipeDurableID) and the
	// format gate use this single seam, so the id the customer copies from the list is exactly the id
	// the gate accepts (no more uuid: vs byid: binding_mismatch). Defaults to storage.DeviceDurableID;
	// tests override it. (DiskInfo.DurableID stays the uuid: storage id — that one feeds /disks/assign.)
	deviceDurableID func(device string) (string, error)

	// boundCheck reports whether a stable guest path has felhom-data bound under it (intermediary model).
	// Optional — nil defaults to the real host mount-table read (isHostMountpoint); tests inject a fake.
	boundCheck func(string) bool

	// deviceCheck reports whether a drive's RAW host mount is still mounted — the agent's device-presence
	// signal (R-113). Deliberately separate from boundCheck: the raw mount is device-bound (a systemd
	// mount unit that dies with its device) while the agent's own bind under the shared parent is NOT,
	// so only the raw mount distinguishes "device present" from "the bind outlived the device".
	// Optional — nil defaults to isHostMountpoint; tests inject a fake.
	deviceCheck func(string) bool

	// livenessCheck answers whether the bind at a stable guest path is USABLE, not merely present — the
	// third term of the BoundUnderParent conjunction (R-117). Deliberately separate from boundCheck and
	// deviceCheck because it is the only one of the three that compares them: boundCheck asks "does the
	// guest see a mount by that name", deviceCheck asks "is the raw mount still there", and BOTH are
	// satisfied by a bind that names the drive that went away while the raw mount healed onto the
	// returning one. Optional — nil defaults to bindLiveness. Prefer redirecting procSelfMountinfo at a
	// captured fixture over injecting here: that exercises the real parser and predicate.
	livenessCheck func(stable, raw string) BindLiveness

	jobsMu sync.Mutex
	// jobs is per-guest-PER-TARGET backup job state (slice 8B; keyed by target too since R-82).
	// Keying by vmid alone would let a PBS backup started inside the same quiesce window collide
	// with the local one's single-flight and hand the caller the WRONG job id.
	jobs map[backupJobKey]*backupJob

	// agentic controller update (Phase 1): the swapper + per-guest single-flight gate.
	swap         *ControllerSwapper
	swapMu       sync.Mutex
	swapInFlight map[int]bool

	// Network-storage verify job (SPIKE-nas-verify): the IN-MEMORY single slot + the seams the
	// detached pipeline runs through (tests inject; production defaults set in NewServer).
	netVerifyMu  sync.Mutex
	netVerifyCur *netVerifyJob
	// netTrigger performs the mount-waking directory read through the automount path (Q1).
	netTrigger func(where string) error
	// netMounted judges mount success from /proc/mounts — the §8 truth source (never readability).
	netMounted func(where string) bool
	// netJournal reads a mount unit's journal tail UNPRIVILEGED (systemd-journal group, no sudo).
	netJournal func(ctx context.Context, unit string) (string, error)
	// netReachable is the 2 s TCP endpoint pre-probe (sync fast-fail + classification tiebreak).
	netReachable func(proto storage.NetworkProtocol, server string) bool

	// Controller-driven escrow ceremony (v0.88.0): the single job slot + the ONE-SHOT in-memory R
	// holder. R lives ONLY in escrowR (never in the job struct — snapshots must be structurally
	// incapable of carrying it) and is zeroed on claim, supersede, or TTL expiry. See
	// escrow_ceremony.go for the custody rules.
	escrowCeremony *EscrowCeremonyConfig
	escrowMu       sync.Mutex
	escrowJob      *escrowCeremonyJob
	escrowR        []byte
	escrowRClaimed bool
	escrowRExpiry  time.Time
	escrowDone     <-chan struct{} // closes when the detached job finishes (tests wait on it)
	// ceremonyRun executes the fixed-argv sudo self-invocation (tests inject canned JSON).
	ceremonyRun ceremonyRunner
	privileged  PrivilegedRunner
	configPath  string
	stateDir    string
	// escrowSudoCheck is the preflight's list-mode grant probe (`sudo -n -l -- <argv>`).
	escrowSudoCheck func(ctx context.Context) error
	// escrowLookPath resolves a binary on PATH for preflight (tests inject).
	escrowLookPath func(file string) (string, error)
	// statFile reports whether a path exists (preflight's staged-secret item; tests inject).
	statFile func(path string) bool

	baseCtx context.Context // for fire-and-forget backups; set in Run
}

// NewServer builds a Server. It does not bind a socket until Run.
func NewServer(o Options) (*Server, error) {
	if o.ListenAddr == "" {
		return nil, fmt.Errorf("localapi: listen addr required")
	}
	if o.Guests == nil || o.Backups == nil || o.Store == nil || o.Storage == nil || o.Tokens == nil {
		return nil, fmt.Errorf("localapi: all dependencies (guests, backups, store, storage, tokens) are required")
	}
	if o.Logger == nil {
		o.Logger = slog.Default()
	}
	cadence := o.BackupCadence
	if cadence <= 0 {
		cadence = defaultBackupCadence
	}
	s := &Server{
		addr:            o.ListenAddr,
		cert:            o.Cert,
		guests:          o.Guests,
		backups:         o.Backups,
		store:           o.Store,
		storage:         o.Storage,
		driveTargets:    o.DriveTargets,
		smart:           o.Smart,
		tokens:          o.Tokens,
		cadence:         cadence,
		logger:          o.Logger,
		now:             func() time.Time { return time.Now().UTC() },
		disks:           o.Disks,
		diskGate:        o.DiskGate,
		guestList:       o.Guests2,
		guestAttach:     o.GuestAttach,
		mem:             o.Memory,
		netStorage:      o.NetStorage,
		privileged:      o.Privileged,
		configPath:      o.ConfigPath,
		stateDir:        o.StateDir,
		netMountRoot:    storage.NetworkMountRoot,
		smbCredsDir:     o.SmbCredsDir,
		escrowStagePath: o.EscrowStagePath,
		intent:          o.Intent,
		guestBinds:      o.GuestBinds,
		formatJobs:      o.FormatJobs,
		staleLock:       o.StaleLock,
		host:            o.HostReader,
		hostMetrics:     o.HostMetrics,
		hostID:          o.HostID,
		agentVersion:    o.AgentVersion,
		logRing:         o.LogRing,
		jobs:            map[backupJobKey]*backupJob{},
		swapInFlight:    map[int]bool{},
	}
	// R-82 tier resolution. Options.BackupTiers is authoritative when supplied; otherwise ONE tier
	// is synthesized from Backups + BackupCadence — the pre-R-82 shape, so every existing caller
	// (and every existing test) keeps working untouched. Exactly one tier is marked primary, and
	// the primary is always first, because that is what the untargeted endpoints act on.
	s.tiers = normalizeBackupTiers(o.BackupTiers, o.Backups, cadence)
	s.inFlight = o.InFlight
	if s.backups == nil && len(s.tiers) > 0 {
		s.backups = s.tiers[0].Service
	}
	if s.escrowStagePath == "" {
		s.escrowStagePath = escrow.StagedResticPasswordPath()
	}
	s.reresolveWipe = s.reresolveDurableForWipe
	s.reresolveBlank = s.reresolveDurableForBlankFormat
	s.deviceDurableID = storage.DeviceDurableID
	s.resolveStorageDevice = storage.ResolveStorageDevice
	s.netTrigger = triggerNetMount
	s.netMounted = storage.NetworkMountedAt
	s.netJournal = readUnitJournal
	s.netReachable = storage.NetworkEndpointReachable
	// Controller-driven escrow ceremony (v0.88.0): production seams; tests inject fakes.
	s.escrowCeremony = o.EscrowCeremony
	if s.escrowCeremony != nil {
		s.ceremonyRun = runCeremonySubprocess(s.escrowCeremony.SudoPath)
		s.escrowSudoCheck = checkCeremonySudoGrant(s.escrowCeremony.SudoPath)
	}
	s.escrowLookPath = exec.LookPath
	s.statFile = func(path string) bool { _, err := os.Stat(path); return err == nil }
	if o.ControllerSwap != nil {
		s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger)
	}
	return s, nil
}

// Handler builds the routed mux (exposed for tests via httptest).
func (s *Server) Handler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /storage", s.withGuest(s.handleStorage))
	mux.HandleFunc("POST /snapshot", s.withGuest(s.handleSnapshot))
	mux.HandleFunc("POST /rollback", s.withGuest(s.handleRollback))
	mux.HandleFunc("POST /backup", s.withGuest(s.handleBackup))
	mux.HandleFunc("GET /backup/due", s.withGuest(s.handleBackupDue))
	mux.HandleFunc("GET /backup/tiers", s.withGuest(s.handleBackupTiers))
	mux.HandleFunc("POST /backup/target", s.withGuest(s.handleSetBackupTarget))
	mux.HandleFunc("GET /backup/status", s.withGuest(s.handleBackupStatus))
	mux.HandleFunc("GET /restore-test/status", s.withGuest(s.handleRestoreTestStatus))
	// Host metrics (slice 9): host-wide health + per-storage capacity for the customer's monitoring
	// view. Host-wide, token-authed, fresh (a live collect, not the 15-min hub snapshot).
	mux.HandleFunc("GET /host/metrics", s.withGuest(s.handleHostMetrics))
	// Disk management (slice 8C) — self-scoped; format routes through the data-bearing classifier+gate.
	mux.HandleFunc("GET /disks", s.withGuest(s.handleDisks))
	mux.HandleFunc("GET /disks/candidates", s.withGuest(s.handleDiskCandidates))
	mux.HandleFunc("POST /disks/assign", s.withGuest(s.handleDiskAssign))
	mux.HandleFunc("POST /disks/eject", s.withGuest(s.handleDiskEject))
	mux.HandleFunc("POST /disks/decommission", s.withGuest(s.handleDiskDecommission))
	mux.HandleFunc("POST /disks/format", s.withGuest(s.handleDiskFormat))
	mux.HandleFunc("GET /disks/format/status", s.withGuest(s.handleDiskFormatStatus))
	// Guest data-drive passthrough (slice 10 P2): bind an enrolled drive's felhom-data namespace in.
	mux.HandleFunc("POST /disks/guest-attach", s.withGuest(s.handleDiskGuestAttach))
	// Guest reboot (slice 10 P2 activation): user-triggered restart to activate pending drive binds.
	mux.HandleFunc("POST /guest/reboot", s.withGuest(s.handleGuestReboot))
	// Guest RAM resize (v0.90.0, R-24): read current allocation + bounds, and apply a bounded resize
	// (live cgroup apply, no reboot). Self-scoped; the agent enforces every bound fresh per request.
	mux.HandleFunc("GET /guest/memory", s.withGuest(s.handleGuestMemory))
	mux.HandleFunc("POST /guest/memory", s.withGuest(s.handleGuestMemoryResize))

	// Network storage (NAS) — Part A1: mount/list/remove a bulk-media NAS share host-side (automount
	// idle-unmount; +100000 uid recipe). A distinct class from a drive — no enroll/eject/wipe.
	// Add is verify-before-commit (SPIKE-nas-verify): it starts a detached verify job the caller
	// polls on /netstorage/verify-status; a failed verify auto-rolls-back agent-side.
	mux.HandleFunc("POST /netstorage/add", s.withGuest(s.handleNetStorageAdd))
	mux.HandleFunc("GET /netstorage", s.withGuest(s.handleNetStorageList))
	mux.HandleFunc("GET /netstorage/verify-status", s.withGuest(s.handleNetVerifyStatus))
	mux.HandleFunc("POST /netstorage/remove", s.withGuest(s.handleNetStorageRemove))

	// agentic controller update (Phase 1): in-guest image swap + rollback, owned by the agent.
	mux.HandleFunc("POST /controller/swap", s.withGuest(s.handleControllerSwap))
	mux.HandleFunc("GET /controller/swap/status", s.withGuest(s.handleControllerSwapStatus))

	// fork-4: stage the controller-pushed offsite restic repo password for the escrow-create ceremony.
	mux.HandleFunc("POST /escrow/stage-secret", s.withGuest(s.handleStageEscrowSecret))
	// fork-4 hygiene: wipe the staged secret once escrowed (controller calls this on confirm). Idempotent.
	mux.HandleFunc("DELETE /escrow/stage-secret", s.withGuest(s.handleWipeStagedEscrowSecret))

	// Controller-driven escrow ceremony (v0.88.0): preflight checklist, the detached root ceremony
	// job (fixed-argv sudo self-invocation), its status, and the ONE-SHOT in-memory R claim.
	mux.HandleFunc("GET /escrow/preflight", s.withGuest(s.handleEscrowPreflight))
	mux.HandleFunc("POST /escrow/ceremony", s.withGuest(s.handleEscrowCeremonyStart))
	mux.HandleFunc("GET /escrow/ceremony/status", s.withGuest(s.handleEscrowCeremonyStatus))
	mux.HandleFunc("POST /escrow/ceremony/claim", s.withGuest(s.handleEscrowCeremonyClaim))

	// v0.83.0 observability: the agent's always-DEBUG capture ring, for the controller's
	// Debug page agent tab (same auth/self-scoping wrap as every sibling route).
	mux.HandleFunc("GET /debug/logs", s.withGuest(s.handleDebugLogs))

	// v0.82.0 version channel: EVERY response (any route, any status — including auth failures)
	// carries X-Felhom-Agent-Version, so the controller learns the agent version passively from its
	// ordinary traffic and can capability-gate by comparison instead of route-probing. Header-less
	// (pre-0.82) agents keep working — the controller falls back to the probe.
	// v0.83.0: wrapped in the request-level DEBUG log middleware (method/path/status/duration).
	return s.logRequests(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if s.agentVersion != "" {
			w.Header().Set("X-Felhom-Agent-Version", s.agentVersion)
		}
		mux.ServeHTTP(w, r)
	}))
}

// Run binds the bridge socket, serves TLS, and shuts down gracefully on ctx cancellation. It
// returns nil on a clean shutdown (mirrors the other daemon loops' ctx-cancel contract).
func (s *Server) Run(ctx context.Context) error {
	s.baseCtx = ctx
	srv := &http.Server{
		Addr:    s.addr,
		Handler: s.Handler(),
		TLSConfig: &tls.Config{
			Certificates: []tls.Certificate{s.cert},
			MinVersion:   tls.VersionTLS12,
		},
		ReadHeaderTimeout: 10 * time.Second,
	}
	ln, err := net.Listen("tcp", s.addr)
	if err != nil {
		return fmt.Errorf("localapi: bind %s: %w", s.addr, err)
	}
	s.logger.Info("local-api server listening", "addr", s.addr)

	errc := make(chan error, 1)
	go func() { errc <- srv.ServeTLS(ln, "", "") }()

	select {
	case <-ctx.Done():
		shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		_ = srv.Shutdown(shutCtx)
		return nil
	case err := <-errc:
		if errors.Is(err, http.ErrServerClosed) {
			return nil
		}
		return err
	}
}

// ---- auth + self-scoping ----------------------------------------------------------------

// withGuest wraps a handler with bearer auth and self-scoping: it resolves the token → VMID
// (401 on absent/unknown), and refuses any explicit `vmid` that disagrees with the token's
// guest (403, cross-guest). The wrapped handler is invoked ONLY with the token's VMID, so a
// proxmox op is never issued for another guest. Self-scoping is by the token→guest map; a
// caller-supplied id is only ever a consistency check, never the authority.
func (s *Server) withGuest(fn func(w http.ResponseWriter, r *http.Request, vmid int)) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		tok := bearer(r)
		if tok == "" {
			writeErr(w, http.StatusUnauthorized, "missing bearer token")
			return
		}
		vmid, ok := s.tokens.Lookup(tok)
		if !ok {
			writeErr(w, http.StatusUnauthorized, "unknown token")
			return
		}
		// Cross-guest probe via an explicit query vmid → 403 (the op is not run).
		if q := strings.TrimSpace(r.URL.Query().Get("vmid")); q != "" {
			if want, err := strconv.Atoi(q); err != nil || want != vmid {
				s.logger.Warn("local-api: cross-guest request refused",
					"token_guest", vmid, "requested", q, "path", r.URL.Path)
				writeErr(w, http.StatusForbidden, "token is not scoped to that guest")
				return
			}
		}
		fn(w, r, vmid)
	}
}

// bearer extracts the Authorization: Bearer <token> value.
func bearer(r *http.Request) string {
	h := r.Header.Get("Authorization")
	const p = "Bearer "
	if len(h) > len(p) && strings.EqualFold(h[:len(p)], p) {
		return strings.TrimSpace(h[len(p):])
	}
	return ""
}

// scopedFromBody enforces self-scoping for a POST body that may carry an explicit vmid: a
// non-zero body vmid that disagrees with the token's guest → 403 (false return; caller stops).
func (s *Server) scopedFromBody(w http.ResponseWriter, bodyVMID, tokenVMID int, path string) bool {
	if bodyVMID != 0 && bodyVMID != tokenVMID {
		s.logger.Warn("local-api: cross-guest request refused (body)",
			"token_guest", tokenVMID, "requested", bodyVMID, "path", path)
		writeErr(w, http.StatusForbidden, "token is not scoped to that guest")
		return false
	}
	return true
}

// ---- handlers ---------------------------------------------------------------------------

// MountInfo is one of the guest's attached mountpoints with its placement class.
type MountInfo struct {
	Key        string `json:"key"`         // mp0, mp1, …
	Storage    string `json:"storage"`     // PVE storage id
	MountPoint string `json:"mount_point"` // in-guest path
	Class      string `json:"class"`       // fast | slow | "" (unknown) — from the host storage view
	Backup     bool   `json:"backup"`      // included in vzdump (backup=1)
}

// StorageResponse is GET /storage — this guest's mounts + class so the controller can place
// hot vs bulk volumes per .felhom.yml.
type StorageResponse struct {
	VMID   int         `json:"vmid"`
	Mounts []MountInfo `json:"mounts"`
}

func (s *Server) handleStorage(w http.ResponseWriter, r *http.Request, vmid int) {
	cfg, err := s.guests.GuestConfig(r.Context(), vmid)
	if err != nil {
		s.logger.Error("local-api: /storage guest config", "vmid", vmid, "err", err)
		writeErr(w, http.StatusBadGateway, "could not read guest config")
		return
	}
	classByStore := s.classByStorage(r.Context())
	mounts := make([]MountInfo, 0)
	for key, val := range cfg.MountPoints() {
		store, mp, backup := parseMount(val)
		mounts = append(mounts, MountInfo{
			Key: key, Storage: store, MountPoint: mp,
			Class: classByStore[store], Backup: backup,
		})
	}
	writeOK(w, StorageResponse{VMID: vmid, Mounts: mounts})
}

type snapshotRequest struct {
	VMID        int    `json:"vmid"` // optional; must match the token's guest if set
	Snapname    string `json:"snapname"`
	Description string `json:"description"`
}

func (s *Server) handleSnapshot(w http.ResponseWriter, r *http.Request, vmid int) {
	var req snapshotRequest
	if !decodeBody(w, r, &req) {
		return
	}
	if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
		return
	}
	snap := strings.TrimSpace(req.Snapname)
	if snap == "" {
		snap = "felhom-" + strconv.FormatInt(time.Now().UTC().Unix(), 10)
	}
	if !validSnapname(snap) {
		writeErr(w, http.StatusBadRequest, "invalid snapname (allowed: letters, digits, '_', '-')")
		return
	}
	upid, err := s.guests.Snapshot(r.Context(), vmid, snap, req.Description)
	if err != nil {
		s.logger.Error("local-api: snapshot", "vmid", vmid, "err", err)
		writeErr(w, http.StatusBadGateway, "snapshot failed")
		return
	}
	if _, err := s.guests.WaitTask(r.Context(), upid, proxmox.WaitOptions{}); err != nil {
		s.logger.Error("local-api: snapshot task", "vmid", vmid, "err", err)
		writeErr(w, http.StatusBadGateway, "snapshot task failed")
		return
	}
	writeOK(w, map[string]any{"vmid": vmid, "snapname": snap})
}

type rollbackRequest struct {
	VMID     int    `json:"vmid"`
	Snapname string `json:"snapname"`
}

func (s *Server) handleRollback(w http.ResponseWriter, r *http.Request, vmid int) {
	var req rollbackRequest
	if !decodeBody(w, r, &req) {
		return
	}
	if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
		return
	}
	snap := strings.TrimSpace(req.Snapname)
	if snap == "" || !validSnapname(snap) {
		writeErr(w, http.StatusBadRequest, "snapname is required (letters, digits, '_', '-')")
		return
	}
	upid, err := s.guests.Rollback(r.Context(), vmid, snap)
	if err != nil {
		s.logger.Error("local-api: rollback", "vmid", vmid, "snap", snap, "err", err)
		writeErr(w, http.StatusBadGateway, "rollback failed")
		return
	}
	if _, err := s.guests.WaitTask(r.Context(), upid, proxmox.WaitOptions{}); err != nil {
		s.logger.Error("local-api: rollback task", "vmid", vmid, "err", err)
		writeErr(w, http.StatusBadGateway, "rollback task failed")
		return
	}
	writeOK(w, map[string]any{"vmid": vmid, "rolled_back_to": snap})
}

type backupRequest struct {
	VMID int `json:"vmid"`
}

// BackupResponse is POST /backup. The controller polls GET /backup/status on job_id to completion.
type BackupResponse struct {
	VMID  int    `json:"vmid"`
	JobID string `json:"job_id"`
	Phase string `json:"phase"`
	// Target (R-82) echoes the tier; empty + omitted for an untargeted request (pre-R-82 bytes).
	Target string `json:"target,omitempty"`
}

func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) {
	// Body is optional; if present its vmid must match the token's guest.
	if r.ContentLength != 0 {
		var req backupRequest
		if !decodeBody(w, r, &req) {
			return
		}
		if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
			return
		}
	}
	tier, echo, ok := s.tierFromRequest(w, r)
	if !ok {
		return
	}
	key := backupJobKey{vmid: vmid, target: tier.TargetID}

	// ONE BACKUP AT A TIME PER GUEST, ACROSS ALL TIERS (operator ruling 2026-07-26: "other backup
	// shouldn't start until finished"). vzdump takes a guest lock, so a concurrent second backup
	// could not succeed anyway — but without this guard it would be ATTEMPTED, fail on the lock, and
	// record a spurious failure that leaves the tier permanently due.
	//
	// Two distinct cases, deliberately answered differently:
	//   - SAME tier already in flight  → return THAT job (202). Idempotent: the caller re-polls it.
	//   - DIFFERENT tier in flight     → 409. Not a new job, and NOT the other tier's job either —
	//     handing back a foreign job id is how a caller comes to believe its own backup ran.
	//
	// "In flight" includes `snapshotted`, not just `running`: after the storage snapshot the vzdump
	// is still uploading and still holding the lock. Checking only `running` (the pre-R-82 code)
	// left a window where a second POST would start a real second vzdump.
	s.jobsMu.Lock()
	if cur := s.jobs[key]; cur != nil && backupInFlight(cur.Phase) {
		job := *cur
		s.jobsMu.Unlock()
		writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: job.JobID, Phase: job.Phase, Target: echo}, "")
		return
	}
	if busyTarget, busyJob, busy := s.otherTierInFlight(vmid, tier.TargetID); busy {
		s.jobsMu.Unlock()
		s.logger.Info("local-api: backup refused — another tier is still in flight",
			"vmid", vmid, "requested_target", tier.TargetID, "busy_target", busyTarget, "busy_job", busyJob)
		writeStatus(w, http.StatusConflict, false, nil,
			"a backup is already in flight on target "+busyTarget+" (job "+busyJob+") — only one backup runs at a time per guest")
		return
	}
	// Job ids must be unique PER TIER, and by construction rather than by clock luck: two tiers
	// started inside the same nanosecond (the weekly both-due night, or any injected clock) would
	// otherwise collide and hand the second caller the first tier's id. The PRIMARY keeps the
	// pre-R-82 format byte-for-byte — an old controller stores this string and polls with it — so
	// only the additive tiers carry the target segment.
	// R-85 Scenario F: a backup and a restore-test must never run together — both move multi-GB over
	// the same tunnel. Acquired here (still holding jobsMu is fine: TryAcquire never blocks) and
	// released when the fire-and-forget goroutine finishes.
	release, busy, free := s.inFlight.TryAcquire("backup:" + tier.TargetID)
	if !free {
		s.jobsMu.Unlock()
		s.logger.Info("local-api: backup refused — a heavy operation is already in flight",
			"vmid", vmid, "requested_target", tier.TargetID, "busy", busy)
		writeStatus(w, http.StatusConflict, false, nil,
			"a heavy operation is already in flight ("+busy+") — only one runs at a time on this host")
		return
	}

	jobID := "backup-" + strconv.Itoa(vmid) + "-" + strconv.FormatInt(s.now().UnixNano(), 10)
	if !tier.Primary && tier.TargetID != "" {
		jobID = "backup-" + strconv.Itoa(vmid) + "-" + tier.TargetID + "-" + strconv.FormatInt(s.now().UnixNano(), 10)
	}
	s.jobs[key] = &backupJob{JobID: jobID, Phase: PhaseRunning, StartedAt: s.now()}
	s.jobsMu.Unlock()

	// Enqueue: a vzdump runs for minutes, so we do not block the request. The backup runs on the
	// server's base context (cancelled on daemon shutdown), updates the job phase, and records
	// into the store; the controller polls GET /backup/status. This is the host-side half of the
	// 8B app-consistent path — the controller quiesces (stops its stacks) BEFORE calling this, so
	// the vzdump captures a clean-shutdown-consistent state.
	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	go func() {
		defer release() // R-85: free the host-wide gate when this backup finishes, however it ends
		// Outer bound = the tier's own wait bound + headroom for the pre/post work around WaitTask.
		// A fixed 2h here would silently cap a 6h offsite tier.
		outer := tier.WaitTimeout
		if outer <= 0 {
			outer = 2 * time.Hour
		}
		bctx, cancel := context.WithTimeout(base, outer+15*time.Minute)
		defer cancel()
		// 8B.2: flip the job to `snapshotted` when the storage snapshot is taken, so the
		// controller resumes its app early (snapshot mode only; in stop mode this never fires).
		b, err := tier.Service.BackupWithSnapshotHook(bctx, vmid, func() { s.markSnapshotted(key, jobID) })
		if err != nil {
			b.VMID = vmid
			b.Success = false
			if b.Error == "" {
				b.Error = err.Error()
			}
			// TargetID is what the hub attributes the record to; a failed run must still say which
			// tier failed, and the runner may not have set it on the error path.
			if b.TargetID == "" {
				b.TargetID = tier.TargetID
			}
			s.logger.Error("local-api: backup job failed", "vmid", vmid, "target", tier.TargetID, "job", jobID, "err", err)
		} else {
			s.logger.Info("local-api: backup job complete", "vmid", vmid, "target", tier.TargetID, "job", jobID, "archive", b.Archive)
		}
		s.store.RecordBackup(b)
		s.finishJob(key, jobID, b)
	}()
	writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: jobID, Phase: PhaseRunning}, "")
}

// backupInFlight reports whether a phase means "this backup still holds the guest".
// `snapshotted` counts: the storage snapshot is taken but the vzdump is still uploading.
func backupInFlight(phase string) bool {
	return phase == PhaseRunning || phase == PhaseSnapshotted
}

// otherTierInFlight reports whether a DIFFERENT tier has an in-flight backup for this guest.
// Caller must hold s.jobsMu.
func (s *Server) otherTierInFlight(vmid int, target string) (busyTarget, busyJob string, busy bool) {
	for k, j := range s.jobs {
		if k.vmid != vmid || k.target == target || j == nil {
			continue
		}
		if backupInFlight(j.Phase) {
			return k.target, j.JobID, true
		}
	}
	return "", "", false
}

// markSnapshotted flips the guest's running job to the `snapshotted` phase (8B.2) — only if it is
// still the current job and still running (don't regress done/failed, and don't touch a newer job).
func (s *Server) markSnapshotted(key backupJobKey, jobID string) {
	s.jobsMu.Lock()
	defer s.jobsMu.Unlock()
	cur := s.jobs[key]
	if cur == nil || cur.JobID != jobID || cur.Phase != PhaseRunning {
		return
	}
	cur.Phase = PhaseSnapshotted
	s.logger.Info("local-api: backup reached snapshotted (app may resume)", "vmid", key.vmid, "target", key.target, "job", jobID)
}

// finishJob transitions the guest's job to done/failed (only if it is still the current job — a
// later job started after a single-flight gap must not be overwritten by an older one's result).
func (s *Server) finishJob(key backupJobKey, jobID string, b hub.Backup) {
	s.jobsMu.Lock()
	defer s.jobsMu.Unlock()
	cur := s.jobs[key]
	if cur == nil || cur.JobID != jobID {
		return
	}
	cur.FinishedAt = s.now()
	if b.Success {
		cur.Phase = PhaseDone
		cur.Archive = b.Archive
	} else {
		cur.Phase = PhaseFailed
		cur.Error = b.Error
	}
}

// jobSnapshot returns a copy of the guest's current job (ok=false if none).
func (s *Server) jobSnapshot(key backupJobKey) (backupJob, bool) {
	s.jobsMu.Lock()
	defer s.jobsMu.Unlock()
	if j := s.jobs[key]; j != nil {
		return *j, true
	}
	return backupJob{}, false
}

// BackupDueResponse is GET /backup/due (slice 8B). A guest is due when no successful backup is
// recorded OR the newest successful one is older than the agent-local cadence. A successful
// POST /backup flips this to false for the window, so the controller won't re-quiesce in a loop.
// The hub-served policy is slice 10.
// BackupAgeState (R-88 Part 2) says WHY AgeSecs is what it is — the distinction the type system
// could not previously express.
//
// Before this, a storage read ERROR and a genuine never-backed-up both produced a nil AgeSecs with
// the same `Reason`, byte-identical on the wire. The controller therefore fired its window-gate
// safety valve ("no backup yet — never withhold the first one") on an unreadable storage, quiescing
// customer app stacks OUTSIDE the backup window. Absence of a signal, read as a specific value —
// the fourth instance of that class in this codebase.
//
// A STRING enum, not a bool: the zero value must mean "legacy agent, no information", and "" says
// that unambiguously where `false` would silently masquerade as a real answer.
type BackupAgeState string

const (
	// AgeStateKnown — AgeSecs is set and meaningful.
	AgeStateKnown BackupAgeState = "known"
	// AgeStateAbsent — a POSITIVE determination that no backup has ever landed for this tier. This is
	// the only state that may fire the controller's safety valve.
	AgeStateAbsent BackupAgeState = "absent"
	// AgeStateUnknown — the agent could not determine the age (storage unreadable, timestamp
	// unparseable). Still DUE (an unreadable storage must never suppress a backup), but the window
	// gate must NOT be bypassed on it.
	AgeStateUnknown BackupAgeState = "unknown"
)

type BackupDueResponse struct {
	VMID    int    `json:"vmid"`
	Due     bool   `json:"due"`
	Reason  string `json:"reason"`
	AgeSecs *int64 `json:"age_seconds,omitempty"` // age of the newest successful backup; null if none
	// Target (R-82) echoes the tier this verdict is about. EMPTY (and omitted) for an untargeted
	// request, which is what keeps the pre-R-82 response bytes identical for old controllers.
	Target string `json:"target,omitempty"`
	// AgeState (R-88 Part 2) disambiguates a nil AgeSecs. Additive: an OLD controller ignores it and
	// behaves exactly as before. An EMPTY value on the wire means the agent is pre-v0.105.0 — the
	// controller must treat that as "legacy, no information", never as AgeStateUnknown.
	AgeState BackupAgeState `json:"age_state,omitempty"`
}

// archiveLookup is the three-state result of asking a tier's storage when a backup last landed.
// It exists because the old (time.Time, bool) signature could not distinguish "nothing there" from
// "I could not look" — the doc comment on newestArchiveOn promised that distinction for months while
// the type made it impossible.
type archiveLookup int

const (
	archiveFound   archiveLookup = iota // a backup exists; the time is valid
	archiveAbsent                       // read succeeded, no backup for this guest on this tier
	archiveUnknown                      // could not read (error, or the service has no lister)
)

func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid int) {
	tier, echo, ok := s.tierFromRequest(w, r)
	if !ok {
		return
	}
	// A tier whose TARGET STORAGE does not exist yet is DEFERRED, not due (R-82 Slice D).
	//
	// A fresh box carries the offsite tier in its installer defaults, but `felhom-pbs` only appears
	// when the hub provisions the DR tier (`felhom-pbs-apply`). Reporting "due" in that window would
	// have the controller quiesce the apps and fire a vzdump at a storage that does not exist —
	// every cadence, until provisioning happens. Deferring keeps the tier silent until it is real,
	// and it goes live with NO restart the moment the storage appears.
	//
	// Fail-safe: a storage-view ERROR does not defer. Unknown must never suppress a backup.
	if tier.TargetID != "" && !s.targetStoragePresent(r.Context(), tier.TargetID) {
		writeOK(w, BackupDueResponse{VMID: vmid, Due: false,
			Reason: "target storage not present yet — tier deferred until it is provisioned", Target: echo})
		return
	}
	// Newest backup for THIS tier: the in-memory record if this process took one, otherwise the
	// storage itself (R-84 — see BackupArchiveLister). Whichever is newer wins.
	var newest time.Time
	var haveNewest bool
	var unparseable bool
	if latest := s.latestSuccessfulBackupForTarget(r.Context(), vmid, tier.TargetID); latest != nil {
		if t, ok2 := backupAge2(latest.StartedAt); ok2 {
			newest, haveNewest = t, true
		} else {
			unparseable = true
		}
	}
	t, lookup := s.newestArchiveOn(r.Context(), tier, vmid)
	if lookup == archiveFound && (!haveNewest || t.After(newest)) {
		newest, haveNewest = t, true
		unparseable = false // ground truth supersedes an unreadable in-memory timestamp
	}
	if !haveNewest {
		// R-88 Part 2: THREE distinct reasons for a nil age, each with its own state. Only ABSENT is a
		// positive claim of "never backed up"; only that one may license the controller to bypass its
		// backup window. All three stay DUE — an agent that cannot tell must never suppress a backup.
		switch {
		case unparseable:
			writeOK(w, BackupDueResponse{VMID: vmid, Due: true, AgeState: AgeStateUnknown,
				Reason: "last backup time unparseable — treating as due", Target: echo})
		case lookup == archiveUnknown:
			// The storage could not be read AND this process holds no record. Previously this emitted
			// "no successful backup recorded yet" — a positive claim built out of two absences, which
			// is what fired the window-gate valve during the 2026-07-27 PBS outage.
			writeOK(w, BackupDueResponse{VMID: vmid, Due: true, AgeState: AgeStateUnknown,
				Reason: "backup storage unreadable and no in-memory record — age UNKNOWN, treating as due", Target: echo})
		default:
			writeOK(w, BackupDueResponse{VMID: vmid, Due: true, AgeState: AgeStateAbsent,
				Reason: "no successful backup recorded yet", Target: echo})
		}
		return
	}
	age := s.now().Sub(newest)
	ageSecs := int64(age.Seconds())
	if age >= tier.Cadence {
		writeOK(w, BackupDueResponse{VMID: vmid, Due: true, AgeSecs: &ageSecs, AgeState: AgeStateKnown,
			Reason: "older than cadence", Target: echo})
		return
	}
	writeOK(w, BackupDueResponse{VMID: vmid, Due: false, AgeSecs: &ageSecs, AgeState: AgeStateKnown,
		Reason: "within cadence window", Target: echo})
}

// BackupTiersResponse is GET /backup/tiers (R-82): the tiers this agent serves, primary first.
// A controller that gets 404 here is talking to a PRE-R-82 agent and must fall back to the single
// untargeted tier — that 404 is the designed capability probe.
type BackupTiersResponse struct {
	VMID  int              `json:"vmid"`
	Tiers []BackupTierInfo `json:"tiers"`
}

// BackupTierInfo is one tier as advertised to the controller.
type BackupTierInfo struct {
	Target         string `json:"target"`
	CadenceSeconds int64  `json:"cadence_seconds"`
	Primary        bool   `json:"primary"`
}

func (s *Server) handleBackupTiers(w http.ResponseWriter, r *http.Request, vmid int) {
	resp := BackupTiersResponse{VMID: vmid, Tiers: make([]BackupTierInfo, 0, len(s.tiers))}
	for _, t := range s.tiers {
		resp.Tiers = append(resp.Tiers, BackupTierInfo{
			Target:         t.TargetID,
			CadenceSeconds: int64(t.Cadence.Seconds()),
			Primary:        t.Primary,
		})
	}
	writeOK(w, resp)
}

// tierFromRequest resolves the `?target=` query parameter to a tier.
//
// THE COMPATIBILITY RULE (§4): NO target parameter → the PRIMARY tier, and the echoed target is
// EMPTY so the response marshals byte-identically to pre-R-82 (BackupDueResponse.Target is
// omitempty). An old controller cannot tell this agent from the old one.
//
// An UNKNOWN target is a 400, never a silent fallback to the primary: a controller asking about a
// tier this agent does not serve must find out, not be told about a different tier's freshness.
func (s *Server) tierFromRequest(w http.ResponseWriter, r *http.Request) (BackupTier, string, bool) {
	want := strings.TrimSpace(r.URL.Query().Get("target"))
	if want == "" {
		return s.primaryTier(), "", true
	}
	for _, t := range s.tiers {
		if t.TargetID == want {
			return t, t.TargetID, true
		}
	}
	writeStatus(w, http.StatusBadRequest, false, nil, "unknown backup target: "+want)
	return BackupTier{}, "", false
}

// primaryTier returns the tier every untargeted endpoint acts on. tiers is never empty (New
// synthesizes one), but this stays defensive: a zero tier would silently disable backups.
func (s *Server) primaryTier() BackupTier {
	for _, t := range s.tiers {
		if t.Primary {
			return t
		}
	}
	if len(s.tiers) > 0 {
		return s.tiers[0]
	}
	return BackupTier{TargetID: "", Cadence: defaultBackupCadence, Service: s.backups}
}

// BackupStatusResponse is GET /backup/status (slice 8B): the current/last job phase + the latest
// recorded backup. Phase is idle when no job has run this process lifetime.
type BackupStatusResponse struct {
	VMID   int         `json:"vmid"`
	Phase  string      `json:"phase"` // idle | running | done | failed
	JobID  string      `json:"job_id,omitempty"`
	Error  string      `json:"error,omitempty"`
	Backup *hub.Backup `json:"backup,omitempty"` // latest recorded backup for this guest
	// Target (R-82) echoes the tier; empty + omitted when untargeted (pre-R-82 bytes).
	Target string `json:"target,omitempty"`
}

func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid int) {
	tier, echo, ok0 := s.tierFromRequest(w, r)
	if !ok0 {
		return
	}
	// Untargeted keeps the pre-R-82 meaning EXACTLY: the primary tier's job, and the newest backup
	// across ANY target (echo == "" → pickLatestBackup's match-any path).
	resp := BackupStatusResponse{VMID: vmid, Phase: PhaseIdle, Target: echo,
		Backup: s.pickLatestBackup(r.Context(), vmid, false, echo)}
	if job, ok := s.jobSnapshot(backupJobKey{vmid: vmid, target: tier.TargetID}); ok {
		resp.Phase = job.Phase
		resp.JobID = job.JobID
		resp.Error = job.Error
	}
	writeOK(w, resp)
}

func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request, vmid int) {
	// The self-restore-test is host-level (it tests the newest backup in a throwaway scratch),
	// not per-guest — the controller surfaces it in its UI. Return the latest (0 or 1).
	tests := s.store.RestoreTests(r.Context())
	var latest *hub.RestoreTest
	if len(tests) > 0 {
		latest = &tests[0]
	}
	writeOK(w, map[string]any{"restore_test": latest})
}

// ---- helpers ----------------------------------------------------------------------------

// latestBackupFor returns this guest's most recent backup from the store (nil if none).
func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup {
	return s.pickLatestBackup(ctx, vmid, false, "")
}

// latestSuccessfulBackupFor returns this guest's most recent SUCCESSFUL backup (nil if none) —
// the basis for /backup/due (a failed backup must not satisfy the cadence).
func (s *Server) latestSuccessfulBackupFor(ctx context.Context, vmid int) *hub.Backup {
	return s.pickLatestBackup(ctx, vmid, true, "")
}

// latestSuccessfulBackupForTarget is the R-82 per-tier twin: a tier's due-ness must be judged
// against ITS OWN newest successful backup. The store is already keyed by target, so this is a
// filter, not a data-model change — but WITHOUT it a fresh local backup would satisfy the PBS
// tier's cadence and the DR tier would never run.
func (s *Server) latestSuccessfulBackupForTarget(ctx context.Context, vmid int, target string) *hub.Backup {
	return s.pickLatestBackup(ctx, vmid, true, target)
}

// pickLatestBackup returns the newest matching record. target "" matches ANY target (the pre-R-82
// behaviour, kept for the untargeted status endpoint).
func (s *Server) pickLatestBackup(ctx context.Context, vmid int, successOnly bool, target string) *hub.Backup {
	var latest *hub.Backup
	for _, b := range s.store.Backups(ctx) {
		if b.VMID != vmid || (successOnly && !b.Success) {
			continue
		}
		if target != "" && b.TargetID != target {
			continue
		}
		bb := b
		if latest == nil || bb.StartedAt > latest.StartedAt { // RFC3339 sorts lexically
			latest = &bb
		}
	}
	return latest
}

// newestArchiveOn asks THIS TIER's storage when a backup last landed (R-84). Errors and
// unsupported services degrade to "unknown", never to "no backup" — an unreadable storage must not
// make the tier look freshly backed up, and it must not suppress a backup either: the caller falls
// back to the in-memory record, whose absence means DUE.
func (s *Server) newestArchiveOn(ctx context.Context, tier BackupTier, vmid int) (time.Time, archiveLookup) {
	lister, ok := tier.Service.(BackupArchiveLister)
	if !ok {
		// NO LISTER = the pre-R-84 world, and it must stay ABSENT — not unknown.
		//
		// "Unknown" is the tempting answer (we cannot consult storage, so we do not know) and it is
		// WRONG here, because it would regress Scenario D: the controller fires its first-backup
		// safety valve only on ABSENT, so a genuinely new box on a no-lister build would never take
		// its first backup outside the window, and nobody would notice for weeks. A loud bug traded
		// for a silent one.
		//
		// The honest reading: on this path the in-memory record is the ONLY registry that exists, so
		// its absence means "no backup recorded" in the only terms available — exactly the claim this
		// path has always made. UNKNOWN is reserved for a lister that was asked and could not answer.
		return time.Time{}, archiveAbsent
	}
	t, found, err := lister.NewestArchiveTime(ctx, vmid)
	if err != nil {
		s.logger.Warn("local-api: could not read the backup storage for the due-check — falling back to the in-memory record",
			"vmid", vmid, "target", tier.TargetID, "err", err)
		return time.Time{}, archiveUnknown
	}
	if !found {
		return time.Time{}, archiveAbsent
	}
	return t, archiveFound
}

// backupAge2 parses an RFC3339 backup start time, returning it as a time.
func backupAge2(startedAt string) (time.Time, bool) {
	t, err := time.Parse(time.RFC3339, startedAt)
	if err != nil {
		return time.Time{}, false
	}
	return t.UTC(), true
}

// backupAge parses an RFC3339 backup start time and returns its age relative to now.
func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
	t, err := time.Parse(time.RFC3339, startedAt)
	if err != nil {
		return 0, false
	}
	return now.Sub(t), true
}

// targetStoragePresent reports whether a backup target exists on this host RIGHT NOW.
//
// Returns TRUE on a storage-view error: "I could not check" must never be read as "not there", or a
// transient probe failure would silently suppress backups — the absence-is-not-failure rule this
// project keeps relearning (R-80, R-81).
func (s *Server) targetStoragePresent(ctx context.Context, target string) bool {
	if s.storage == nil {
		return true
	}
	targets, err := s.storage.Observe(ctx)
	if err != nil {
		s.logger.Warn("local-api: storage view unavailable for the backup-target presence check — assuming present", "target", target, "err", err)
		return true
	}
	for _, t := range targets {
		if t.Name == target {
			return true
		}
	}
	return false
}

// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
func (s *Server) classByStorage(ctx context.Context) map[string]string {
	out := map[string]string{}
	targets, err := s.storage.Observe(ctx)
	if err != nil {
		s.logger.Warn("local-api: storage view unavailable for class hints", "err", err)
		return out
	}
	for _, t := range targets {
		out[t.Name] = t.ClassHint
	}
	return out
}

// parseMount splits a PVE mpN value like "local-lvm:1,mp=/mnt/bulk,backup=0" into its storage
// id, in-guest mount path, and backup flag.
func parseMount(val string) (storage, mountPoint string, backup bool) {
	parts := strings.Split(val, ",")
	if len(parts) > 0 {
		if i := strings.IndexByte(parts[0], ':'); i >= 0 {
			storage = parts[0][:i]
		} else {
			storage = parts[0]
		}
	}
	for _, p := range parts[1:] {
		switch {
		case strings.HasPrefix(p, "mp="):
			mountPoint = strings.TrimPrefix(p, "mp=")
		case strings.HasPrefix(p, "backup="):
			backup = strings.TrimPrefix(p, "backup=") == "1"
		}
	}
	return storage, mountPoint, backup
}

// validSnapname allows PVE-safe snapshot names (letters, digits, '_' and '-').
func validSnapname(s string) bool {
	if s == "" || len(s) > 64 {
		return false
	}
	for _, c := range s {
		if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' {
			continue
		}
		return false
	}
	return true
}

func decodeBody(w http.ResponseWriter, r *http.Request, v any) bool {
	if r.Body == nil || r.ContentLength == 0 {
		return true // empty body is allowed; fields stay zero
	}
	dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16))
	dec.DisallowUnknownFields()
	if err := dec.Decode(v); err != nil {
		writeErr(w, http.StatusBadRequest, "invalid JSON body")
		return false
	}
	return true
}

// ---- response envelope (matches the controller's {ok,data,error} style) -----------------

type apiResponse struct {
	OK    bool   `json:"ok"`
	Data  any    `json:"data,omitempty"`
	Error string `json:"error,omitempty"`
}

func writeOK(w http.ResponseWriter, data any) {
	writeStatus(w, http.StatusOK, true, data, "")
}

func writeErr(w http.ResponseWriter, code int, msg string) {
	writeStatus(w, code, false, nil, msg)
}

func writeStatus(w http.ResponseWriter, code int, ok bool, data any, errMsg string) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	_ = json.NewEncoder(w).Encode(apiResponse{OK: ok, Data: data, Error: errMsg})
}