F20-BUG3: run mkfs detached (survives request deadline + agent restart); v0.31.0

The format ran mkfs under the HTTP request context, so the controller's 15s client
timeout cancelled it → SIGKILL mid-write → corrupt disk. Now mkfs runs DETACHED off
s.baseCtx (a dropped request can't kill it) via a persisted formatJob record; the handler
still waits to return the synchronous result (backward-compatible with the v0.62.0
controller) but abandoning the wait on client-disconnect leaves the mkfs running to
completion. New GET /disks/format/status surfaces the job (additive). RecoverFormatJob
runs on agent startup: a record left 'running' (agent died mid-format) is re-resolved by
durable-id (anti-retarget — absent/swapped disk NOT re-formatted) and the mkfs re-run; a
blank/path-bound interrupted format is marked failed (retry), never auto-re-run.

Tests: detached run persists running→done + binds durable-id; status endpoint; recovery
re-runs an interrupted durable-id-bound format; skips blank; skips unresolvable durable-id.
Version 0.30.0 → 0.31.0.

internal/localapi/disks.go

@@ -2,6 +2,7 @@ package localapi
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -425,6 +426,44 @@ type PendingOp struct {
 	FSType    string `json:"fstype"`     // the filesystem to mkfs after the wipe
 }
 
+// errFormatClientGone signals the request context was cancelled (client/controller deadline) while the
+// detached mkfs keeps running — the handler returns without writing; the job record records the outcome.
+var errFormatClientGone = fmt.Errorf("format client gone (mkfs continues detached)")
+
+// awaitFormat waits for the detached mkfs result, OR returns errFormatClientGone if the request context
+// is cancelled first. Crucially the mkfs itself runs off s.baseCtx, so a cancelled request never kills it
+// (F20-BUG3) — abandoning the wait here only abandons the HTTP response, not the format.
+func (s *Server) awaitFormat(reqCtx context.Context, done <-chan error, vmid int, device string) error {
+	select {
+	case err := <-done:
+		return err
+	case <-reqCtx.Done():
+		s.logger.Warn("local-api: format client disconnected — mkfs continues detached (poll GET /disks/format/status)",
+			"vmid", vmid, "device", device)
+		return errFormatClientGone
+	}
+}
+
+// handleDiskFormatStatus reports the most-recent/in-flight format job (F20-BUG3), so a controller whose
+// request timed out (or that reconnects after an agent restart) can learn the real outcome instead of
+// assuming failure. Self-scoped (benign read).
+func (s *Server) handleDiskFormatStatus(w http.ResponseWriter, r *http.Request, vmid int) {
+	if s.formatJobs == nil {
+		writeOK(w, map[string]any{"vmid": vmid, "phase": "idle"})
+		return
+	}
+	job := s.formatJobs.get()
+	if job == nil {
+		writeOK(w, map[string]any{"vmid": vmid, "phase": "idle"})
+		return
+	}
+	writeOK(w, map[string]any{
+		"vmid": vmid, "phase": job.Phase, "device": job.Device, "fstype": job.FSType,
+		"durable_id": job.DurableID, "error": job.Error, "started_at": job.StartedAt, "updated_at": job.UpdatedAt,
+		"job_id": job.JobID,
+	})
+}
+
 // handleDiskFormat is the security centerpiece. The agent INSPECTS the device; if it is
 // data-bearing it is classified destructive and the gate refuses it `pending_signature` — the
 // caller's claim is never trusted. Only a device the agent itself reads as blank is formatted.
@@ -456,8 +495,14 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
 		// inspect error → fail-safe data-bearing (probe.DataBearing() is true on !Probed)
 	}
 	if !probe.DataBearing() {
-		// Blank device → benign → mkfs (role is irrelevant; there is nothing to destroy).
-		if err := s.disks.Format(r.Context(), req.Device, req.FSType); err != nil {
+		// Blank device → benign → mkfs (role is irrelevant; there is nothing to destroy). F20-BUG3: run
+		// it DETACHED off s.baseCtx so a request/client deadline can't SIGKILL mkfs mid-write; we still
+		// wait here to return the synchronous result (backward-compatible with the controller's client).
+		done := s.startFormatDetached(req.Device, "", req.FSType)
+		if err := s.awaitFormat(r.Context(), done, vmid, req.Device); err != nil {
+			if err == errFormatClientGone {
+				return // client gone; mkfs continues detached + the job record records the outcome
+			}
 			s.logger.Error("local-api: format", "vmid", vmid, "device", req.Device, "err", err)
 			writeErr(w, http.StatusBadGateway, "format failed: "+err.Error())
 			return
@@ -496,7 +541,14 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
 				"wipe refused (device may have changed since confirmation): "+rerr.Error())
 			return
 		}
-		if err := s.disks.Format(r.Context(), device, req.FSType); err != nil {
+		// F20-BUG3: run the destructive mkfs DETACHED off s.baseCtx (bound durable id recorded for
+		// restart-recovery), so a request/client deadline can never SIGKILL it mid-write and corrupt the
+		// disk. We still wait to return the synchronous result (backward-compatible with the controller).
+		done := s.startFormatDetached(device, deviceDurable, req.FSType)
+		if err := s.awaitFormat(r.Context(), done, vmid, device); err != nil {
+			if err == errFormatClientGone {
+				return // client gone; the wipe continues detached + survives a restart via the job record
+			}
 			s.logger.Error("local-api: customer-confirmed format", "vmid", vmid, "device", device, "err", err)
 			writeErr(w, http.StatusBadGateway, "format failed: "+err.Error())
 			return