F20-BUG3: run mkfs detached (survives request deadline + agent restart); v0.31.0

The format ran mkfs under the HTTP request context, so the controller's 15s client timeout cancelled it → SIGKILL mid-write → corrupt disk. Now mkfs runs DETACHED off s.baseCtx (a dropped request can't kill it) via a persisted formatJob record; the handler still waits to return the synchronous result (backward-compatible with the v0.62.0 controller) but abandoning the wait on client-disconnect leaves the mkfs running to completion. New GET /disks/format/status surfaces the job (additive). RecoverFormatJob runs on agent startup: a record left 'running' (agent died mid-format) is re-resolved by durable-id (anti-retarget — absent/swapped disk NOT re-formatted) and the mkfs re-run; a blank/path-bound interrupted format is marked failed (retry), never auto-re-run. Tests: detached run persists running→done + binds durable-id; status endpoint; recovery re-runs an interrupted durable-id-bound format; skips blank; skips unresolvable durable-id. Version 0.30.0 → 0.31.0.
2026-06-14 15:16:01 +02:00
parent 4cd1d024e9
commit 4777f8a221
5 changed files with 417 additions and 6 deletions
@@ -0,0 +1,168 @@
+package localapi
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
+)
+
+func tempFormatStore(t *testing.T) *FormatJobStore {
+	t.Helper()
+	fj, err := OpenFormatJobStore(filepath.Join(t.TempDir(), "format-job.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return fj
+}
+
+// formatServer builds a *Server with the destructive-format path wired (data-bearing probe, a gate, the
+// format-job store), plus the test stubs (reresolveWipe/deviceDurableID) so no real /dev is touched.
+func formatServer(t *testing.T, d *fakeDiskOps, g StorageGate, fj *FormatJobStore) *Server {
+	t.Helper()
+	srv, err := NewServer(Options{
+		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
+		Storage: fakeStorage{}, Tokens: staticTokens{"A": 8200},
+		Disks: d, DiskGate: g, FormatJobs: fj, HostReader: sysOnSDA(),
+		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv.baseCtx = context.Background()
+	srv.reresolveWipe = func(_ context.Context, _ string) (string, error) { return "/dev/sdb", nil }
+	srv.deviceDurableID = func(device string) (string, error) { return "byid:wwn-" + device, nil }
+	return srv
+}
+
+func waitFormatPhase(t *testing.T, fj *FormatJobStore, want string) *formatJob {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if j := fj.get(); j != nil && j.Phase == want {
+			return j
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	got := fj.get()
+	t.Fatalf("format job did not reach phase %q in time (got %+v)", want, got)
+	return nil
+}
+
+// TestFormat_DetachedPersistsJobRecord asserts F20-BUG3: a customer-confirmed wipe runs through the
+// detached runner and a persisted job record reaches `done` with the device + durable-id bound (so it
+// can be polled / recovered). Also confirms backward-compat: the handler still returns 200 synchronously.
+func TestFormat_DetachedPersistsJobRecord(t *testing.T) {
+	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
+	g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
+	fj := tempFormatStore(t)
+	srv := formatServer(t, d, g, fj)
+	h := srv.Handler()
+
+	w := do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
+	if w.Code != http.StatusOK {
+		t.Fatalf("confirmed format: %d (%s)", w.Code, w.Body.String())
+	}
+	job := waitFormatPhase(t, fj, formatPhaseDone)
+	if job.Device != "/dev/sdb" { // the anti-retarget re-resolved device (stub returns /dev/sdb)
+		t.Fatalf("job device = %q, want /dev/sdb (re-resolved)", job.Device)
+	}
+	if job.DurableID == "" {
+		t.Fatalf("job durable-id empty — recovery could not re-bind")
+	}
+	if len(d.formatted()) != 1 {
+		t.Fatalf("mkfs called %d times, want 1", len(d.formatted()))
+	}
+}
+
+// TestFormatStatus reports the persisted job via GET /disks/format/status.
+func TestFormatStatus(t *testing.T) {
+	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
+	g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
+	fj := tempFormatStore(t)
+	srv := formatServer(t, d, g, fj)
+	h := srv.Handler()
+
+	do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
+	waitFormatPhase(t, fj, formatPhaseDone)
+
+	w := do(t, h, "GET", "/disks/format/status", "A", "")
+	if w.Code != http.StatusOK {
+		t.Fatalf("status: %d", w.Code)
+	}
+	var resp struct {
+		Data struct {
+			Phase  string `json:"phase"`
+			Device string `json:"device"`
+		} `json:"data"`
+	}
+	_ = json.Unmarshal(w.Body.Bytes(), &resp)
+	if resp.Data.Phase != formatPhaseDone {
+		t.Fatalf("status phase = %q, want done", resp.Data.Phase)
+	}
+}
+
+// TestRecoverFormatJob_ReRunsInterrupted asserts the operator's "survive an agent restart" decision: a
+// record left in `running` (agent died mid-format) is re-resolved by durable-id and the mkfs re-run.
+func TestRecoverFormatJob_ReRunsInterrupted(t *testing.T) {
+	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
+	fj := tempFormatStore(t)
+	// Pre-seed an interrupted, durable-id-bound running job (as if the agent died mid-mkfs).
+	if err := fj.save(&formatJob{JobID: "j1", Device: "/dev/sdb", DurableID: "byid:wwn-x", FSType: "ext4", Phase: formatPhaseRunning}); err != nil {
+		t.Fatal(err)
+	}
+	srv := formatServer(t, d, &fakeGate{}, fj)
+
+	srv.RecoverFormatJob(context.Background())
+	waitFormatPhase(t, fj, formatPhaseDone)
+	if got := d.formatted(); len(got) != 1 || got[0] != "/dev/sdb" {
+		t.Fatalf("recovery should have re-run mkfs on /dev/sdb (re-resolved), got %v", got)
+	}
+}
+
+// TestRecoverFormatJob_SkipsBlank: an interrupted BLANK (no durable-id) format is NOT auto-re-run (never
+// format a mutable /dev path on recovery) — it is marked failed for the caller to retry.
+func TestRecoverFormatJob_SkipsBlank(t *testing.T) {
+	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
+	fj := tempFormatStore(t)
+	_ = fj.save(&formatJob{JobID: "j2", Device: "/dev/sdb", DurableID: "", FSType: "ext4", Phase: formatPhaseRunning})
+	srv := formatServer(t, d, &fakeGate{}, fj)
+
+	srv.RecoverFormatJob(context.Background())
+	if got := d.formatted(); len(got) != 0 {
+		t.Fatalf("recovery must NOT re-run a path-bound (blank) format, but mkfs ran: %v", got)
+	}
+	if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
+		t.Fatalf("blank interrupted job should be marked failed, got %+v", j)
+	}
+}
+
+// TestRecoverFormatJob_SkipsUnresolvable: if the durable-id no longer resolves (drive removed/replaced),
+// recovery must NOT format anything (anti-retarget) and marks the job failed.
+func TestRecoverFormatJob_SkipsUnresolvable(t *testing.T) {
+	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
+	fj := tempFormatStore(t)
+	_ = fj.save(&formatJob{JobID: "j3", Device: "/dev/sdb", DurableID: "byid:wwn-gone", FSType: "ext4", Phase: formatPhaseRunning})
+	srv := formatServer(t, d, &fakeGate{}, fj)
+	srv.reresolveWipe = func(_ context.Context, _ string) (string, error) {
+		return "", context.DeadlineExceeded // simulate "durable-id no longer resolves"
+	}
+
+	srv.RecoverFormatJob(context.Background())
+	if got := d.formatted(); len(got) != 0 {
+		t.Fatalf("recovery must NOT format when the durable-id is unresolvable: %v", got)
+	}
+	if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
+		t.Fatalf("unresolvable interrupted job should be marked failed, got %+v", j)
+	}
+}
+
+func deviceProbeDataBearing() storage.DeviceProbe {
+	return storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}
+}