v0.131.0: controller supervisor (R-523); per-tier backup status + tier storage presence (R-517/R-518)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

internal/localapi/server.go

@@ -185,6 +185,9 @@ type Options struct {
 	// StaleLock recovers a guest left with a stale vzdump lock by a reboot-during-backup (F2-b), run at
 	// startup by RecoverStaleLockedGuests. OPTIONAL — when nil, the recovery is a no-op.
 	StaleLock StaleLockController
+	// GuestsStateDir (R-523) is the agent's per-guest state dir holding <vmid>/bootstrap and the
+	// controller-parked marker. "" → /var/lib/felhom-agent/guests.
+	GuestsStateDir string
 	// ControllerSwapStateDir holds the per-guest swap state file (crash-safety). "" → /var/lib/felhom-agent.
 	ControllerSwapStateDir string
 	// Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL —
@@ -362,6 +365,12 @@ type Server struct {
 	swapMu       sync.Mutex
 	swapInFlight map[int]bool
 
+	// R-523: the in-guest controller supervisor (controllersupervisor.go). guestExec is the same
+	// GuestExecutor the swap uses; guestsDir is the agent's per-guest state dir ("" → default).
+	guestExec GuestExecutor
+	guestsDir string
+	ctrlSup   controllerSupervisor
+
 	// Network-storage verify job (SPIKE-nas-verify): the IN-MEMORY single slot + the seams the
 	// detached pipeline runs through (tests inject; production defaults set in NewServer).
 	netVerifyMu  sync.Mutex
@@ -484,7 +493,9 @@ func NewServer(o Options) (*Server, error) {
 	s.statFile = func(path string) bool { _, err := os.Stat(path); return err == nil }
 	if o.ControllerSwap != nil {
 		s.swap = NewControllerSwapper(o.ControllerSwap, o.ControllerSwapStateDir, o.Logger)
+		s.guestExec = o.ControllerSwap
 	}
+	s.guestsDir = o.GuestsStateDir
 	return s, nil
 }
 
@@ -1084,6 +1095,10 @@ type BackupTierInfo struct {
 	Target         string `json:"target"`
 	CadenceSeconds int64  `json:"cadence_seconds"`
 	Primary        bool   `json:"primary"`
+	// Storage (R-517/R-518, v0.131.0) says whether the tier's Proxmox storage exists on this host
+	// RIGHT NOW: "present" | "absent" | "unknown" (storage view unreadable). Additive — an older
+	// controller ignores it. "unknown" is never "absent": a probe failure must not skip a backup.
+	Storage string `json:"storage,omitempty"`
 }
 
 func (s *Server) handleBackupTiers(w http.ResponseWriter, r *http.Request, vmid int) {
@@ -1093,11 +1108,83 @@ func (s *Server) handleBackupTiers(w http.ResponseWriter, r *http.Request, vmid
 			Target:         t.TargetID,
 			CadenceSeconds: int64(t.Cadence.Seconds()),
 			Primary:        t.Primary,
+			Storage:        s.storagePresence(r.Context(), t.TargetID),
 		})
 	}
 	writeOK(w, resp)
 }
 
+// storagePresence is the tri-state twin of targetStoragePresent (which must stay fail-OPEN for the
+// backup path): "present", "absent", or "unknown" when the storage view cannot be read. Only a
+// successful read that does not list the storage is "absent".
+func (s *Server) storagePresence(ctx context.Context, target string) string {
+	if s.storage == nil || target == "" {
+		return StoragePresenceUnknown
+	}
+	targets, err := s.storage.Observe(ctx)
+	if err != nil {
+		s.logger.Warn("local-api: storage view unavailable for the tier presence report", "target", target, "err", err)
+		return StoragePresenceUnknown
+	}
+	for _, t := range targets {
+		if t.Name == target {
+			return StoragePresencePresent
+		}
+	}
+	return StoragePresenceAbsent
+}
+
+const (
+	StoragePresencePresent = "present"
+	StoragePresenceAbsent  = "absent"
+	StoragePresenceUnknown = "unknown"
+)
+
+// TierBackupState (R-517, v0.131.0) is one tier's truth for the customer's backup page: the newest
+// SUCCESSFUL backup and the last ATTEMPT, kept apart — so a failed attempt can never stand in for a
+// result ("presence is not success").
+type TierBackupState struct {
+	Target  string `json:"target"`
+	Primary bool   `json:"primary"`
+	Storage string `json:"storage"` // present | absent | unknown
+	// LastSuccess is the newest successful backup on this tier. From the in-memory record when there
+	// is one; otherwise from the tier's storage (after an agent restart the record is empty — the
+	// BIGNIGHT F2 page showed no backup at all), in which case only started_at is known and
+	// LastSuccessSource is "storage".
+	LastSuccess       *hub.Backup  `json:"last_success,omitempty"`
+	LastSuccessSource string       `json:"last_success_source,omitempty"` // record | storage
+	LastAttempt       *TierAttempt `json:"last_attempt,omitempty"`
+}
+
+// TierAttempt is the newest recorded attempt on a tier, successful or not.
+type TierAttempt struct {
+	StartedAt string `json:"started_at"`
+	Success   bool   `json:"success"`
+	Error     string `json:"error,omitempty"`
+}
+
+// tierBackupStates builds the per-tier view for one guest.
+func (s *Server) tierBackupStates(ctx context.Context, vmid int) []TierBackupState {
+	out := make([]TierBackupState, 0, len(s.tiers))
+	for _, t := range s.tiers {
+		st := TierBackupState{Target: t.TargetID, Primary: t.Primary, Storage: s.storagePresence(ctx, t.TargetID)}
+		if b := s.pickLatestBackup(ctx, vmid, true, t.TargetID); b != nil {
+			st.LastSuccess, st.LastSuccessSource = b, "record"
+		} else if st.Storage != StoragePresenceAbsent {
+			if when, look := s.newestArchiveOn(ctx, t, vmid); look == archiveFound {
+				st.LastSuccess = &hub.Backup{TargetID: t.TargetID, VMID: vmid, Success: true,
+					StartedAt: when.UTC().Format(time.RFC3339)}
+				st.LastSuccessSource = "storage"
+			}
+		}
+		if a := s.pickLatestBackup(ctx, vmid, false, t.TargetID); a != nil {
+			st.LastAttempt = &TierAttempt{StartedAt: a.StartedAt, Success: a.Success, Error: a.Error}
+		}
+		out = append(out, st)
+	}
+	return out
+}
+
 // tierFromRequest resolves the `?target=` query parameter to a tier.
 //
 // THE COMPATIBILITY RULE (§4): NO target parameter → the PRIMARY tier, and the echoed target is
@@ -1144,6 +1231,10 @@ type BackupStatusResponse struct {
 	Backup *hub.Backup `json:"backup,omitempty"` // latest recorded backup for this guest
 	// Target (R-82) echoes the tier; empty + omitted when untargeted (pre-R-82 bytes).
 	Target string `json:"target,omitempty"`
+	// Tiers (R-517, v0.131.0) is the per-tier truth — newest success, last attempt, storage
+	// presence. Served on the UNTARGETED request only; additive, so an older controller reads the
+	// response exactly as before.
+	Tiers []TierBackupState `json:"tiers,omitempty"`
 }
 
 func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid int) {
@@ -1155,6 +1246,9 @@ func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid
 	// across ANY target (echo == "" → pickLatestBackup's match-any path).
 	resp := BackupStatusResponse{VMID: vmid, Phase: PhaseIdle, Target: echo,
 		Backup: s.pickLatestBackup(r.Context(), vmid, false, echo)}
+	if echo == "" {
+		resp.Tiers = s.tierBackupStates(r.Context(), vmid)
+	}
 	if job, ok := s.jobSnapshot(backupJobKey{vmid: vmid, target: tier.TargetID}); ok {
 		resp.Phase = job.Phase
 		resp.JobID = job.JobID