felhom-agent/internal/localapi/formatjob_test.go

package localapi

import (
	"context"
	"encoding/json"
	"io"
	"log/slog"
	"net/http"
	"path/filepath"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
)

func tempFormatStore(t *testing.T) *FormatJobStore {
	t.Helper()
	fj, err := OpenFormatJobStore(filepath.Join(t.TempDir(), "format-job.json"))
	if err != nil {
		t.Fatal(err)
	}
	return fj
}

// formatServer builds a *Server with the destructive-format path wired (data-bearing probe, a gate, the
// format-job store), plus the test stubs (reresolveWipe/deviceDurableID) so no real /dev is touched.
func formatServer(t *testing.T, d *fakeDiskOps, g StorageGate, fj *FormatJobStore) *Server {
	t.Helper()
	srv, err := NewServer(Options{
		ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
		Storage: fakeStorage{}, Tokens: staticTokens{"A": 8200},
		Disks: d, DiskGate: g, FormatJobs: fj, HostReader: sysOnSDA(),
		Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
	})
	if err != nil {
		t.Fatal(err)
	}
	srv.baseCtx = context.Background()
	srv.reresolveWipe = func(_ context.Context, _ string) (string, error) { return "/dev/sdb", nil }
	srv.deviceDurableID = func(device string) (string, error) { return "byid:wwn-" + device, nil }
	return srv
}

func waitFormatPhase(t *testing.T, fj *FormatJobStore, want string) *formatJob {
	t.Helper()
	deadline := time.Now().Add(2 * time.Second)
	for time.Now().Before(deadline) {
		if j := fj.get(); j != nil && j.Phase == want {
			return j
		}
		time.Sleep(10 * time.Millisecond)
	}
	got := fj.get()
	t.Fatalf("format job did not reach phase %q in time (got %+v)", want, got)
	return nil
}

// TestFormat_DetachedPersistsJobRecord asserts F20-BUG3: a customer-confirmed wipe runs through the
// detached runner and a persisted job record reaches `done` with the device + durable-id bound (so it
// can be polled / recovered). Also confirms backward-compat: the handler still returns 200 synchronously.
func TestFormat_DetachedPersistsJobRecord(t *testing.T) {
	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
	g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
	fj := tempFormatStore(t)
	srv := formatServer(t, d, g, fj)
	h := srv.Handler()

	w := do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
	if w.Code != http.StatusOK {
		t.Fatalf("confirmed format: %d (%s)", w.Code, w.Body.String())
	}
	job := waitFormatPhase(t, fj, formatPhaseDone)
	if job.Device != "/dev/sdb" { // the anti-retarget re-resolved device (stub returns /dev/sdb)
		t.Fatalf("job device = %q, want /dev/sdb (re-resolved)", job.Device)
	}
	if job.DurableID == "" {
		t.Fatalf("job durable-id empty — recovery could not re-bind")
	}
	if len(d.formatted()) != 1 {
		t.Fatalf("mkfs called %d times, want 1", len(d.formatted()))
	}
}

// TestFormatStatus reports the persisted job via GET /disks/format/status.
func TestFormatStatus(t *testing.T) {
	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
	g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
	fj := tempFormatStore(t)
	srv := formatServer(t, d, g, fj)
	h := srv.Handler()

	do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
	waitFormatPhase(t, fj, formatPhaseDone)

	w := do(t, h, "GET", "/disks/format/status", "A", "")
	if w.Code != http.StatusOK {
		t.Fatalf("status: %d", w.Code)
	}
	var resp struct {
		Data struct {
			Phase  string `json:"phase"`
			Device string `json:"device"`
		} `json:"data"`
	}
	_ = json.Unmarshal(w.Body.Bytes(), &resp)
	if resp.Data.Phase != formatPhaseDone {
		t.Fatalf("status phase = %q, want done", resp.Data.Phase)
	}
}

// TestRecoverFormatJob_ReRunsInterrupted asserts the operator's "survive an agent restart" decision: a
// record left in `running` (agent died mid-format) is re-resolved by durable-id and the mkfs re-run.
func TestRecoverFormatJob_ReRunsInterrupted(t *testing.T) {
	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
	fj := tempFormatStore(t)
	// Pre-seed an interrupted, durable-id-bound running job (as if the agent died mid-mkfs).
	if err := fj.save(&formatJob{JobID: "j1", Device: "/dev/sdb", DurableID: "byid:wwn-x", FSType: "ext4", Phase: formatPhaseRunning}); err != nil {
		t.Fatal(err)
	}
	srv := formatServer(t, d, &fakeGate{}, fj)

	srv.RecoverFormatJob(context.Background())
	waitFormatPhase(t, fj, formatPhaseDone)
	if got := d.formatted(); len(got) != 1 || got[0] != "/dev/sdb" {
		t.Fatalf("recovery should have re-run mkfs on /dev/sdb (re-resolved), got %v", got)
	}
}

// TestRecoverFormatJob_SkipsBlank: an interrupted BLANK (no durable-id) format is NOT auto-re-run (never
// format a mutable /dev path on recovery) — it is marked failed for the caller to retry.
func TestRecoverFormatJob_SkipsBlank(t *testing.T) {
	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
	fj := tempFormatStore(t)
	_ = fj.save(&formatJob{JobID: "j2", Device: "/dev/sdb", DurableID: "", FSType: "ext4", Phase: formatPhaseRunning})
	srv := formatServer(t, d, &fakeGate{}, fj)

	srv.RecoverFormatJob(context.Background())
	if got := d.formatted(); len(got) != 0 {
		t.Fatalf("recovery must NOT re-run a path-bound (blank) format, but mkfs ran: %v", got)
	}
	if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
		t.Fatalf("blank interrupted job should be marked failed, got %+v", j)
	}
}

// TestRecoverFormatJob_SkipsUnresolvable: if the durable-id no longer resolves (drive removed/replaced),
// recovery must NOT format anything (anti-retarget) and marks the job failed.
func TestRecoverFormatJob_SkipsUnresolvable(t *testing.T) {
	d := &fakeDiskOps{probe: deviceProbeDataBearing()}
	fj := tempFormatStore(t)
	_ = fj.save(&formatJob{JobID: "j3", Device: "/dev/sdb", DurableID: "byid:wwn-gone", FSType: "ext4", Phase: formatPhaseRunning})
	srv := formatServer(t, d, &fakeGate{}, fj)
	srv.reresolveWipe = func(_ context.Context, _ string) (string, error) {
		return "", context.DeadlineExceeded // simulate "durable-id no longer resolves"
	}

	srv.RecoverFormatJob(context.Background())
	if got := d.formatted(); len(got) != 0 {
		t.Fatalf("recovery must NOT format when the durable-id is unresolvable: %v", got)
	}
	if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
		t.Fatalf("unresolvable interrupted job should be marked failed, got %+v", j)
	}
}

func deviceProbeDataBearing() storage.DeviceProbe {
	return storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}
}