felhom-agent/internal/localapi/formatjob.go

package localapi

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"sync"
	"time"
)

// formatJob is the persisted record of the most recent / in-flight disk format (F20-BUG3). It exists so
// that (a) an mkfs runs DETACHED from the HTTP request — a client/request deadline can no longer SIGKILL
// an in-progress mkfs and corrupt the disk — and (b) an in-progress format SURVIVES an agent restart:
// on boot, a record still in `running` is re-resolved by durable-id and re-run (mkfs is idempotent).
type formatJob struct {
	JobID     string `json:"job_id"`
	Device    string `json:"device"`
	DurableID string `json:"durable_id"` // durable-id binding; "" only in legacy records (never auto-recovered)
	FSType    string `json:"fstype"`
	Blank     bool   `json:"blank,omitempty"` // audit D3: blank (benign) format — recovery re-checks STILL-blank, not data-bearing
	Phase     string `json:"phase"`           // running | done | failed
	Error     string `json:"error,omitempty"`
	// FSUUID is the filesystem UUID of the NEW filesystem, read by the agent right after mkfs on the
	// device the bound durable id still resolves to (R-25). "" = not verified (the controller must not
	// read that as a UUID). Set only on phase done.
	FSUUID    string `json:"fs_uuid,omitempty"`
	StartedAt string `json:"started_at"`
	UpdatedAt string `json:"updated_at"`
}

const (
	formatPhaseRunning = "running"
	formatPhaseDone    = "done"
	formatPhaseFailed  = "failed"
)

// FormatJobStore persists the single most-recent format job (atomic tmp+rename, 0600). One slot: a host
// formats one device at a time. Mirrors storage.IntentStore.
type FormatJobStore struct {
	path string
	mu   sync.Mutex
	cur  *formatJob
}

// OpenFormatJobStore loads (or initializes) the store. Missing file = no job; corrupt file = error.
func OpenFormatJobStore(path string) (*FormatJobStore, error) {
	s := &FormatJobStore{path: path}
	data, err := os.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return s, nil
		}
		return nil, fmt.Errorf("format-job store: read %s: %w", path, err)
	}
	if len(data) > 0 {
		var j formatJob
		if err := json.Unmarshal(data, &j); err != nil {
			return nil, fmt.Errorf("format-job store: parse %s: %w", path, err)
		}
		s.cur = &j
	}
	return s, nil
}

func (s *FormatJobStore) get() *formatJob {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.cur == nil {
		return nil
	}
	cp := *s.cur
	return &cp
}

func (s *FormatJobStore) save(j *formatJob) error {
	s.mu.Lock()
	defer s.mu.Unlock()
	cp := *j
	s.cur = &cp
	data, err := json.MarshalIndent(s.cur, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
		return err
	}
	tmp := s.path + ".tmp"
	if err := os.WriteFile(tmp, data, 0o600); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, s.path)
}

// startFormatDetached persists a `running` record and runs mkfs in a goroutine off s.baseCtx (NOT the
// request context), with a long bound. It returns a channel that yields the format error (nil on
// success). The caller may stop waiting (client disconnect) without killing the mkfs — the goroutine
// runs to completion and records the outcome. device is the ALREADY anti-retarget-resolved device; the
// record carries durableID so a restart can re-resolve + re-run. blank marks a benign (blank-device)
// format, so restart recovery re-checks STILL-blank rather than data-bearing (audit D3).
//
// The returned job may be read (FSUUID) only AFTER a value arrives on done — the goroutine writes it
// before the send, which is the happens-before edge.
func (s *Server) startFormatDetached(device, durableID, fstype string, blank bool) (*formatJob, <-chan error) {
	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	job := &formatJob{
		JobID: s.nowFn().UTC().Format("20060102T150405Z"), Device: device, DurableID: durableID,
		FSType: fstype, Blank: blank, Phase: formatPhaseRunning,
		StartedAt: s.nowFn().UTC().Format(time.RFC3339), UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
	}
	if s.formatJobs != nil {
		if err := s.formatJobs.save(job); err != nil {
			s.logger.Warn("format-job: could not persist running record (format will still run)", "device", device, "err", err)
		}
	}
	done := make(chan error, 1)
	go func() {
		ctx, cancel := context.WithTimeout(base, 60*time.Minute)
		defer cancel()
		err := s.disks.Format(ctx, device, fstype)
		if err == nil {
			job.FSUUID = s.formattedFSUUID(ctx, device, durableID, fstype)
		}
		s.finishFormatJob(job, err)
		done <- err
	}()
	return job, done
}

// formattedFSUUID reads the UUID of the filesystem the agent has JUST made (R-25). The caller used to
// re-resolve the UUID from the mutable /dev path afterwards, over separate requests — a re-enumeration
// in that window could hand it ANOTHER disk's filesystem to mount. Here the bound durable id must still
// resolve to the very device that was formatted (and re-derive to the same id), and the superblock must
// carry the fstype that was asked for; anything else returns "" (not verified), never a guess.
func (s *Server) formattedFSUUID(ctx context.Context, device, durableID, fstype string) string {
	if durableID == "" || s.reresolveWipe == nil {
		return ""
	}
	// The device now holds a filesystem, so the data-bearing anti-retarget re-resolve is the right one.
	now, err := s.reresolveWipe(ctx, durableID)
	if err != nil || now != device {
		s.logger.Warn("format: new filesystem UUID NOT reported — bound durable id no longer resolves to the formatted device",
			"device", device, "durable_id", durableID, "resolves_to", now, "err", err)
		return ""
	}
	probe, err := s.disks.InspectDevice(ctx, device)
	if err != nil || !probe.Probed || probe.FSType != fstype || probe.FSUUID == "" {
		s.logger.Warn("format: new filesystem UUID NOT reported — superblock did not read back as the requested filesystem",
			"device", device, "want_fstype", fstype, "got_fstype", probe.FSType, "has_uuid", probe.FSUUID != "", "err", err)
		return ""
	}
	s.logger.Info("format: new filesystem bound to its durable id", "device", device, "durable_id", durableID, "fs_uuid", probe.FSUUID)
	return probe.FSUUID
}

// finishFormatJob updates the persisted record to done/failed.
func (s *Server) finishFormatJob(job *formatJob, err error) {
	if s.formatJobs == nil {
		return
	}
	job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
	if err != nil {
		job.Phase = formatPhaseFailed
		job.Error = err.Error()
	} else {
		job.Phase = formatPhaseDone
	}
	if serr := s.formatJobs.save(job); serr != nil {
		s.logger.Warn("format-job: could not persist final record", "device", job.Device, "phase", job.Phase, "err", serr)
	}
}

// RecoverFormatJob, on agent startup, completes a format that an agent restart interrupted (F20-BUG3 +
// the operator's "survive a restart" decision). A record still in `running` is re-resolved by its
// durable-id (anti-retarget — a swapped/absent disk is NOT re-formatted) and the mkfs is re-run detached
// (mkfs is idempotent). A blank (no-durable-id) format is NOT auto-re-run — it is marked failed for the
// caller to retry, so recovery never formats a mutable /dev path.
func (s *Server) RecoverFormatJob(ctx context.Context) {
	if s.formatJobs == nil {
		return
	}
	job := s.formatJobs.get()
	if job == nil || job.Phase != formatPhaseRunning {
		return
	}
	if job.DurableID == "" {
		s.logger.Warn("format-job recover: interrupted format has no durable id (legacy record) — marking failed (retry needed; not auto-re-running a path-bound format)", "device", job.Device)
		s.finishFormatJob(job, fmt.Errorf("interrupted by agent restart; retry the format"))
		return
	}
	// Audit D3: a blank format authorized "nothing to destroy" — its recovery re-check must assert
	// STILL-blank (an interrupted mkfs may leave partial signatures; if the re-resolved device probes
	// data-bearing the blank re-check refuses fail-safe and the caller retries). The customer-confirmed
	// wipe path keeps the data-bearing re-check as before.
	reresolve := s.reresolveWipe
	if job.Blank {
		reresolve = s.reresolveBlank
	}
	device, err := reresolve(ctx, job.DurableID)
	if err != nil {
		s.logger.Warn("format-job recover: durable-id did not re-resolve cleanly — NOT re-formatting (anti-retarget)", "durable_id", job.DurableID, "blank", job.Blank, "err", err)
		s.finishFormatJob(job, fmt.Errorf("durable-id %s did not re-resolve after restart: %w", job.DurableID, err))
		return
	}
	s.logger.Warn("format-job recover: re-running interrupted format detached", "durable_id", job.DurableID, "device", device, "fstype", job.FSType, "blank", job.Blank)
	_, _ = s.startFormatDetached(device, job.DurableID, job.FSType, job.Blank) // detached; updates the record on completion
}

// nowFn returns the server clock (testable), defaulting to time.Now.
func (s *Server) nowFn() time.Time {
	if s.now != nil {
		return s.now()
	}
	return time.Now()
}