Compare commits
3 Commits
b566fbe57f
...
4777f8a221
| Author | SHA1 | Date | |
|---|---|---|---|
| 4777f8a221 | |||
| 4cd1d024e9 | |||
| a2a76e7624 |
@@ -43,7 +43,7 @@ import (
|
||||
|
||||
// version is the agent version. Overridable at build time with
|
||||
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
||||
var version = "0.30.0"
|
||||
var version = "0.31.0"
|
||||
|
||||
func main() {
|
||||
var (
|
||||
@@ -340,6 +340,20 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
} else {
|
||||
intentReader = intentStore
|
||||
}
|
||||
// F9: per-guest enrolled-bind record, replayed by ReassertGuestBinds at startup to restore a
|
||||
// guest data-drive bind that a re-provision dropped (durable-id-keyed; absent/swapped drives skipped).
|
||||
guestBindStore, gbErr := localapi.OpenGuestBindStore(filepath.Join(intentStateDir, "guest-binds.json"))
|
||||
if gbErr != nil {
|
||||
logger.Warn("storage: guest-bind store unavailable — startup bind re-assert disabled (F9)", "err", gbErr)
|
||||
guestBindStore = nil
|
||||
}
|
||||
// F20-BUG3: persisted disk-format job — lets mkfs run detached from the request and survive an agent
|
||||
// restart (RecoverFormatJob re-runs an interrupted durable-id-bound format).
|
||||
formatJobStore, fjErr := localapi.OpenFormatJobStore(filepath.Join(intentStateDir, "format-job.json"))
|
||||
if fjErr != nil {
|
||||
logger.Warn("storage: format-job store unavailable — format restart-recovery disabled (F20-BUG3)", "err", fjErr)
|
||||
formatJobStore = nil
|
||||
}
|
||||
watchdog := storage.NewWatchdog(storage.WatchdogOptions{
|
||||
Targets: storage.NewCachingKnownTargets(observer, cfg.Storage.KnownRefresh()),
|
||||
Liveness: storage.NewHostLiveness(hostReader, 0),
|
||||
@@ -424,7 +438,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec}, cfg.Hub.HostID, logger)
|
||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||
|
||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, collector, intentRec, logger, &localTokens)
|
||||
localSrv := buildLocalAPIServer(cfg, px, backupStore, observer, hostOps, gate, collector, intentRec, guestBindStore, formatJobStore, logger, &localTokens)
|
||||
if localTokens != nil {
|
||||
defer localTokens.Close()
|
||||
}
|
||||
@@ -462,6 +476,12 @@ func runDaemon(cfg config.Config, logger *slog.Logger) int {
|
||||
go func() { errc <- pbsLoop.Run(ctx) }()
|
||||
if localSrv != nil {
|
||||
localServers = 1
|
||||
// F9: on startup (the host's bring-up/reconcile trigger), re-assert any enrolled guest data-drive
|
||||
// bind that a re-provision dropped — before serving, so the drive is back in the guest config
|
||||
// (activates on the guest's next reboot). On-durable-id-match; absent/swapped drives are skipped.
|
||||
localSrv.ReassertGuestBinds(ctx)
|
||||
// F20-BUG3: complete a disk format that an agent restart interrupted (durable-id-bound, re-resolved).
|
||||
localSrv.RecoverFormatJob(ctx)
|
||||
go func() { errc <- localSrv.Run(ctx) }()
|
||||
}
|
||||
if lanLoop != nil {
|
||||
@@ -576,7 +596,7 @@ func buildRestoreTestScheduler(cfg config.Config, px *proxmox.Client, engine *re
|
||||
// leaf (stable fingerprint). Any failure DISABLES the server (returns nil) WITHOUT crashing the
|
||||
// daemon — the host still reports/reconciles; only the controller channel is unavailable until
|
||||
// fixed. The opened token store is returned via outTokens so the caller can Close it.
|
||||
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps storage.HostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
|
||||
func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.Store, observer *storage.Observer, hostOps storage.HostOps, gate *reconcile.Gate, collector *hub.Collector, intent localapi.IntentRecorder, guestBinds *localapi.GuestBindStore, formatJobs *localapi.FormatJobStore, logger *slog.Logger, outTokens **localapi.TokenStore) *localapi.Server {
|
||||
if !cfg.LocalAPI.Enabled() {
|
||||
return nil
|
||||
}
|
||||
@@ -620,6 +640,8 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
||||
Guests2: px,
|
||||
GuestAttach: guestBinder, // slice 10 P2: bind enrolled data drives into the guest
|
||||
Intent: intent, // slice 10 P3: record enroll/eject intent for self-heal
|
||||
GuestBinds: guestBinds, // F9: per-guest bind record for the startup re-assert
|
||||
FormatJobs: formatJobs, // F20-BUG3: detached-format job record + restart recovery
|
||||
|
||||
// Host metrics (slice 9): the shared collector serves GET /host/metrics — a fresh host +
|
||||
// per-storage view to the customer's monitoring page (reuses the slice-4 collector).
|
||||
|
||||
+187
-10
@@ -2,6 +2,7 @@ package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -103,6 +104,17 @@ type DiskInfo struct {
|
||||
// controller strips the "uuid:" prefix to get the fs UUID it passes to POST /disks/assign —
|
||||
// the only way the (de-privileged) controller can learn the mount key it cannot read itself.
|
||||
DurableID string `json:"durable_id,omitempty"`
|
||||
// WipeDurableID is the device's WIPE-binding durable id in the SAME scheme the format gate resolves
|
||||
// against (byid:<wwn>/byuuid:<uuid>). F20-BUG2: the customer must confirm a data-bearing wipe with
|
||||
// THIS id, not DurableID (uuid:) — passing the uuid: id was rejected as a binding_mismatch. "" when
|
||||
// the device has no durable identity (a wipe of it can't be bound anyway). Distinct from DurableID
|
||||
// (uuid:, used for /disks/assign) on purpose; both are derived from the agent's own device read.
|
||||
WipeDurableID string `json:"wipe_durable_id,omitempty"`
|
||||
// GuestAttached reports whether THIS guest (the token's vmid) actually has the drive's felhom-data
|
||||
// namespace bound into its config — i.e. the drive is usable IN the guest, not merely present on the
|
||||
// host. F9: host presence (State=attached) != guest-usable; this is the missing signal that made the
|
||||
// HDD look available when it wasn't bound. Only meaningful for user-data drives.
|
||||
GuestAttached bool `json:"guest_attached"`
|
||||
}
|
||||
|
||||
// handleDisks lists the host's drives + data-bearing flags (read-only/benign).
|
||||
@@ -119,16 +131,20 @@ func (s *Server) handleDisks(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
// Resolve the OS/system disks ONCE for this request — role classification is agent-authoritative
|
||||
// (the agent's own mount/topology read, never the caller's claim).
|
||||
sysDisks, sysKnown := storage.SystemDisks(s.hostReader())
|
||||
// F9: which host mount paths are actually BOUND into THIS guest's config (guest-usable, not just
|
||||
// host-present). A bind's mp= equals the guest path, which is the drive's host mount path (`where`).
|
||||
boundPaths := s.guestBoundPaths(r.Context(), vmid)
|
||||
out := make([]DiskInfo, 0, len(targets))
|
||||
for _, t := range targets {
|
||||
di := DiskInfo{
|
||||
Name: t.Name, Type: t.Type, State: t.State,
|
||||
BackingDevice: t.BackingDevice, MountPath: t.MountPath, Class: t.ClassHint,
|
||||
DurableID: t.DurableID,
|
||||
Role: string(storage.RoleForStorage(t.Type, t.BackingDevice, sysDisks, sysKnown)),
|
||||
TotalBytes: t.TotalBytes,
|
||||
UsedBytes: t.UsedBytes,
|
||||
UsedFraction: t.UsedFraction,
|
||||
DurableID: t.DurableID,
|
||||
Role: string(storage.RoleForStorage(t.Type, t.BackingDevice, sysDisks, sysKnown)),
|
||||
TotalBytes: t.TotalBytes,
|
||||
UsedBytes: t.UsedBytes,
|
||||
UsedFraction: t.UsedFraction,
|
||||
GuestAttached: t.MountPath != "" && boundPaths[t.MountPath],
|
||||
}
|
||||
// Inspect the backing device for the UI's data-bearing hint (the authoritative check
|
||||
// is re-run at format time on the actual device).
|
||||
@@ -140,6 +156,11 @@ func (s *Server) handleDisks(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
di.DataBearing = true // fail-safe
|
||||
di.DataReason = "could not inspect device"
|
||||
}
|
||||
// F20-BUG2: surface the gate-scheme wipe id (byid:/byuuid:) so a customer-confirmed wipe
|
||||
// binds with the id the gate accepts. Same seam the gate uses → guaranteed to match.
|
||||
if wid, werr := s.deviceDurableID(t.BackingDevice); werr == nil {
|
||||
di.WipeDurableID = wid
|
||||
}
|
||||
}
|
||||
out = append(out, di)
|
||||
}
|
||||
@@ -266,6 +287,7 @@ func (s *Server) handleDiskGuestAttach(w http.ResponseWriter, r *http.Request, v
|
||||
for key, spec := range mounts {
|
||||
if _, mp, _ := parseMount(spec); mp == where {
|
||||
s.recordIntent(r.Context(), where, "enrolled")
|
||||
s.recordGuestBind(r.Context(), vmid, where)
|
||||
s.logger.Info("local-api: guest-attach idempotent (already bound)", "vmid", vmid, "where", where, "slot", key)
|
||||
writeOK(w, map[string]any{"vmid": vmid, "attached": where, "slot": key, "already": true})
|
||||
return
|
||||
@@ -281,8 +303,10 @@ func (s *Server) handleDiskGuestAttach(w http.ResponseWriter, r *http.Request, v
|
||||
writeErr(w, http.StatusBadGateway, "guest-attach failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
// Record the drive as ENROLLED so the self-heal watchdog will reconcile it (P3).
|
||||
// Record the drive as ENROLLED so the self-heal watchdog will reconcile it (P3), and persist the
|
||||
// per-guest bind so the startup re-assert can restore it after a re-provision (F9).
|
||||
s.recordIntent(r.Context(), where, "enrolled")
|
||||
s.recordGuestBind(r.Context(), vmid, where)
|
||||
writeOK(w, map[string]any{"vmid": vmid, "attached": where, "slot": slot})
|
||||
}
|
||||
|
||||
@@ -402,6 +426,44 @@ type PendingOp struct {
|
||||
FSType string `json:"fstype"` // the filesystem to mkfs after the wipe
|
||||
}
|
||||
|
||||
// errFormatClientGone signals the request context was cancelled (client/controller deadline) while the
|
||||
// detached mkfs keeps running — the handler returns without writing; the job record records the outcome.
|
||||
var errFormatClientGone = fmt.Errorf("format client gone (mkfs continues detached)")
|
||||
|
||||
// awaitFormat waits for the detached mkfs result, OR returns errFormatClientGone if the request context
|
||||
// is cancelled first. Crucially the mkfs itself runs off s.baseCtx, so a cancelled request never kills it
|
||||
// (F20-BUG3) — abandoning the wait here only abandons the HTTP response, not the format.
|
||||
func (s *Server) awaitFormat(reqCtx context.Context, done <-chan error, vmid int, device string) error {
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-reqCtx.Done():
|
||||
s.logger.Warn("local-api: format client disconnected — mkfs continues detached (poll GET /disks/format/status)",
|
||||
"vmid", vmid, "device", device)
|
||||
return errFormatClientGone
|
||||
}
|
||||
}
|
||||
|
||||
// handleDiskFormatStatus reports the most-recent/in-flight format job (F20-BUG3), so a controller whose
|
||||
// request timed out (or that reconnects after an agent restart) can learn the real outcome instead of
|
||||
// assuming failure. Self-scoped (benign read).
|
||||
func (s *Server) handleDiskFormatStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.formatJobs == nil {
|
||||
writeOK(w, map[string]any{"vmid": vmid, "phase": "idle"})
|
||||
return
|
||||
}
|
||||
job := s.formatJobs.get()
|
||||
if job == nil {
|
||||
writeOK(w, map[string]any{"vmid": vmid, "phase": "idle"})
|
||||
return
|
||||
}
|
||||
writeOK(w, map[string]any{
|
||||
"vmid": vmid, "phase": job.Phase, "device": job.Device, "fstype": job.FSType,
|
||||
"durable_id": job.DurableID, "error": job.Error, "started_at": job.StartedAt, "updated_at": job.UpdatedAt,
|
||||
"job_id": job.JobID,
|
||||
})
|
||||
}
|
||||
|
||||
// handleDiskFormat is the security centerpiece. The agent INSPECTS the device; if it is
|
||||
// data-bearing it is classified destructive and the gate refuses it `pending_signature` — the
|
||||
// caller's claim is never trusted. Only a device the agent itself reads as blank is formatted.
|
||||
@@ -433,8 +495,14 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
|
||||
// inspect error → fail-safe data-bearing (probe.DataBearing() is true on !Probed)
|
||||
}
|
||||
if !probe.DataBearing() {
|
||||
// Blank device → benign → mkfs (role is irrelevant; there is nothing to destroy).
|
||||
if err := s.disks.Format(r.Context(), req.Device, req.FSType); err != nil {
|
||||
// Blank device → benign → mkfs (role is irrelevant; there is nothing to destroy). F20-BUG3: run
|
||||
// it DETACHED off s.baseCtx so a request/client deadline can't SIGKILL mkfs mid-write; we still
|
||||
// wait here to return the synchronous result (backward-compatible with the controller's client).
|
||||
done := s.startFormatDetached(req.Device, "", req.FSType)
|
||||
if err := s.awaitFormat(r.Context(), done, vmid, req.Device); err != nil {
|
||||
if err == errFormatClientGone {
|
||||
return // client gone; mkfs continues detached + the job record records the outcome
|
||||
}
|
||||
s.logger.Error("local-api: format", "vmid", vmid, "device", req.Device, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "format failed: "+err.Error())
|
||||
return
|
||||
@@ -447,7 +515,7 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
|
||||
// never the caller's claim). The agent also re-resolves the device's durable id; the customer's
|
||||
// confirmation must bind to it.
|
||||
role := s.deviceRole(r.Context(), req.Device)
|
||||
deviceDurable, derr := storage.DeviceDurableID(req.Device)
|
||||
deviceDurable, derr := s.deviceDurableID(req.Device)
|
||||
if derr != nil {
|
||||
deviceDurable = "" // refusal still stands; binding/pending-op just lack the id
|
||||
}
|
||||
@@ -473,7 +541,14 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
|
||||
"wipe refused (device may have changed since confirmation): "+rerr.Error())
|
||||
return
|
||||
}
|
||||
if err := s.disks.Format(r.Context(), device, req.FSType); err != nil {
|
||||
// F20-BUG3: run the destructive mkfs DETACHED off s.baseCtx (bound durable id recorded for
|
||||
// restart-recovery), so a request/client deadline can never SIGKILL it mid-write and corrupt the
|
||||
// disk. We still wait to return the synchronous result (backward-compatible with the controller).
|
||||
done := s.startFormatDetached(device, deviceDurable, req.FSType)
|
||||
if err := s.awaitFormat(r.Context(), done, vmid, device); err != nil {
|
||||
if err == errFormatClientGone {
|
||||
return // client gone; the wipe continues detached + survives a restart via the job record
|
||||
}
|
||||
s.logger.Error("local-api: customer-confirmed format", "vmid", vmid, "device", device, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "format failed: "+err.Error())
|
||||
return
|
||||
@@ -517,6 +592,108 @@ func (s *Server) handleDiskFormat(w http.ResponseWriter, r *http.Request, vmid i
|
||||
"device is system/backup-protected — format requires an operator signature ("+dec.Reason+")")
|
||||
}
|
||||
|
||||
// guestBoundPaths returns the set of guest mountpoint paths (the `mp=` of each entry in the guest's
|
||||
// config) — i.e. the host drives actually BOUND into the guest. F9: this is the guest-attached signal
|
||||
// (`GuestAttached`) that distinguishes a guest-usable drive from one merely present on the host. A bind
|
||||
// created by AttachBind has `mp=<where>` where `where` is the drive's host mount path, so a storage
|
||||
// target is guest-attached iff its MountPath is in this set. Best-effort: a config-read error yields an
|
||||
// empty set (reported as not-attached — the safe direction).
|
||||
func (s *Server) guestBoundPaths(ctx context.Context, vmid int) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if s.guests == nil {
|
||||
return out
|
||||
}
|
||||
cfg, err := s.guests.GuestConfig(ctx, vmid)
|
||||
if err != nil {
|
||||
s.logger.Warn("local-api: guest-attached check — could not read guest config", "vmid", vmid, "err", err)
|
||||
return out
|
||||
}
|
||||
for _, spec := range cfg.MountPoints() {
|
||||
if _, mp, _ := parseMount(spec); mp != "" {
|
||||
out[mp] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// recordGuestBind persists that the drive at `where` (by its durable-id) is enrolled into `vmid`, so the
|
||||
// startup re-assert (ReassertGuestBinds) can restore the bind after a re-provision (F9). Best-effort.
|
||||
func (s *Server) recordGuestBind(ctx context.Context, vmid int, where string) {
|
||||
if s.guestBinds == nil {
|
||||
return
|
||||
}
|
||||
id := s.durableIDForMount(ctx, where)
|
||||
if id == "" {
|
||||
s.logger.Warn("local-api: guest-bind not recorded — durable-id unresolved", "vmid", vmid, "where", where)
|
||||
return
|
||||
}
|
||||
if err := s.guestBinds.Record(vmid, id); err != nil {
|
||||
s.logger.Warn("local-api: guest-bind record failed", "vmid", vmid, "where", where, "durable_id", id, "err", err)
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: guest-bind recorded for startup re-assert", "vmid", vmid, "where", where, "durable_id", id)
|
||||
}
|
||||
|
||||
// ReassertGuestBinds re-adds, on agent startup (the host's bring-up/reconcile trigger), any enrolled
|
||||
// user-data drive bind a guest is MISSING from its config (F9 — a re-provision drops the mp, and nothing
|
||||
// previously restored it). For each recorded (vmid, durable-id): only when the durable-id STILL resolves
|
||||
// to a present, mounted drive AND the guest lacks the bind, it re-runs AttachBind. "On durable-id match"
|
||||
// — a swapped or absent drive is never auto-bound. The re-added bind is config state; it activates on the
|
||||
// guest's next reboot (logged), exactly like the enroll flow. Safe to call repeatedly (idempotent).
|
||||
func (s *Server) ReassertGuestBinds(ctx context.Context) {
|
||||
if s.guestBinds == nil || s.guestAttach == nil || s.guests == nil {
|
||||
return
|
||||
}
|
||||
// durable-id -> current host mount path (present drives only), from the agent's own storage view.
|
||||
mountByDurable := map[string]string{}
|
||||
if targets, err := s.storage.Observe(ctx); err == nil {
|
||||
for _, t := range targets {
|
||||
if t.DurableID != "" && t.MountPath != "" {
|
||||
mountByDurable[t.DurableID] = t.MountPath
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.logger.Warn("F9 re-assert: storage view unavailable — skipping", "err", err)
|
||||
return
|
||||
}
|
||||
for vmid, ids := range s.guestBinds.Guests() {
|
||||
cfg, err := s.guests.GuestConfig(ctx, vmid)
|
||||
if err != nil {
|
||||
s.logger.Warn("F9 re-assert: skip guest (config read failed)", "vmid", vmid, "err", err)
|
||||
continue
|
||||
}
|
||||
mounts := cfg.MountPoints()
|
||||
boundPaths := map[string]bool{}
|
||||
for _, spec := range mounts {
|
||||
if _, mp, _ := parseMount(spec); mp != "" {
|
||||
boundPaths[mp] = true
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
where, present := mountByDurable[id]
|
||||
if !present {
|
||||
s.logger.Warn("F9 re-assert: enrolled drive not present (durable-id absent) — skipping", "vmid", vmid, "durable_id", id)
|
||||
continue
|
||||
}
|
||||
if boundPaths[where] {
|
||||
continue // already bound — nothing to re-assert
|
||||
}
|
||||
slot, ok := freeMountSlot(mounts)
|
||||
if !ok {
|
||||
s.logger.Warn("F9 re-assert: no free mountpoint slot on guest", "vmid", vmid, "where", where)
|
||||
continue
|
||||
}
|
||||
if err := s.guestAttach.AttachBind(ctx, vmid, slot, where); err != nil {
|
||||
s.logger.Error("F9 re-assert: AttachBind failed", "vmid", vmid, "where", where, "slot", slot, "err", err)
|
||||
continue
|
||||
}
|
||||
mounts[slot] = where // reserve the slot so a second enrolled drive takes the next one
|
||||
s.logger.Warn("F9 re-assert: re-attached enrolled drive into guest config (reboot to activate)",
|
||||
"vmid", vmid, "where", where, "slot", slot, "durable_id", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// durableIDForMount resolves the durable-id of the storage mounted at `where` (from the agent's own
|
||||
// storage view) — the key the intent store records enroll/eject against. "" if not resolvable.
|
||||
func (s *Server) durableIDForMount(ctx context.Context, where string) string {
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
// decodeDisks pulls the disks array out of a GET /disks response.
|
||||
func decodeDisks(t *testing.T, body []byte) []DiskInfo {
|
||||
t.Helper()
|
||||
var resp struct {
|
||||
Data struct {
|
||||
Disks []DiskInfo `json:"disks"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
t.Fatalf("decode /disks: %v (body=%s)", err, string(body))
|
||||
}
|
||||
return resp.Data.Disks
|
||||
}
|
||||
|
||||
// TestDisks_WipeDurableID_GateScheme asserts F20-BUG2: /disks surfaces a wipe_durable_id in the gate's
|
||||
// scheme (byid:/byuuid:, via the SAME s.deviceDurableID seam the format gate uses), DISTINCT from the
|
||||
// uuid: durable_id (which feeds /disks/assign). Pre-fix the only id was uuid:, which the gate rejected
|
||||
// as a binding_mismatch.
|
||||
func TestDisks_WipeDurableID_GateScheme(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}}
|
||||
sv := fakeStorage{targets: []hub.StorageTarget{
|
||||
{Name: "bulk", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb1", MountPath: "/mnt/bulk", DurableID: "uuid:abc-123"},
|
||||
}}
|
||||
h := newDiskServer(t, d, &fakeGate{}, sv, nil)
|
||||
|
||||
w := do(t, h, "GET", "/disks", "A", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("GET /disks: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
disks := decodeDisks(t, w.Body.Bytes())
|
||||
if len(disks) != 1 {
|
||||
t.Fatalf("want 1 disk, got %d", len(disks))
|
||||
}
|
||||
got := disks[0]
|
||||
// The stub in newDiskServer maps /dev/sdb1 → byid:wwn-sdb1 (the gate scheme).
|
||||
if got.WipeDurableID != "byid:wwn-sdb1" {
|
||||
t.Fatalf("WipeDurableID = %q, want byid:wwn-sdb1 (gate scheme)", got.WipeDurableID)
|
||||
}
|
||||
if got.DurableID != "uuid:abc-123" {
|
||||
t.Fatalf("DurableID = %q, want uuid:abc-123 (assign scheme, unchanged)", got.DurableID)
|
||||
}
|
||||
if got.WipeDurableID == got.DurableID {
|
||||
t.Fatal("wipe id must differ from the assign (uuid:) id")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDisks_WipeID_MatchesGateBinding asserts the BUG2 end-to-end property: when the customer confirms a
|
||||
// wipe with the wipe_durable_id surfaced by /disks, the handler derives the device's gate id through the
|
||||
// SAME seam — so the value the gate compares as DeviceDurableID equals the customer's ConfirmDurableID
|
||||
// (no binding_mismatch). Asserted via the request the handler forwards to the gate.
|
||||
func TestDisks_WipeID_MatchesGateBinding(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}}
|
||||
sv := fakeStorage{targets: []hub.StorageTarget{
|
||||
{Name: "bulk", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb1", MountPath: "/mnt/bulk", DurableID: "uuid:abc-123"},
|
||||
}}
|
||||
g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
|
||||
h := newDiskServer(t, d, g, sv, nil)
|
||||
|
||||
wipeID := decodeDisks(t, do(t, h, "GET", "/disks", "A", "").Body.Bytes())[0].WipeDurableID
|
||||
|
||||
body := `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"` + wipeID + `"}`
|
||||
w := do(t, h, "POST", "/disks/format", "A", body)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("confirmed wipe with the list's wipe id: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
reqs := g.requests()
|
||||
if len(reqs) != 1 {
|
||||
t.Fatalf("gate consulted %d times, want 1", len(reqs))
|
||||
}
|
||||
// The crux: the customer's confirm id (from the list) equals the device id the gate resolves.
|
||||
if reqs[0].ConfirmDurableID != wipeID {
|
||||
t.Fatalf("ConfirmDurableID forwarded = %q, want the list's wipe id %q", reqs[0].ConfirmDurableID, wipeID)
|
||||
}
|
||||
if reqs[0].DeviceDurableID != reqs[0].ConfirmDurableID {
|
||||
t.Fatalf("gate binding mismatch: device id %q != confirm id %q (BUG2 not fixed)", reqs[0].DeviceDurableID, reqs[0].ConfirmDurableID)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDisks_GuestAttached asserts F9 reporting: a drive bound into THIS guest's config (mp=<mount_path>)
|
||||
// reports guest_attached=true; a host-present-but-unbound drive reports false — the signal that was
|
||||
// missing when the HDD looked available but wasn't attached.
|
||||
func TestDisks_GuestAttached(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}}
|
||||
sv := fakeStorage{targets: []hub.StorageTarget{
|
||||
{Name: "usb", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb1", MountPath: "/mnt/felhom-usb"},
|
||||
{Name: "extra", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdc1", MountPath: "/mnt/extra"},
|
||||
}}
|
||||
// Guest 8200 (token "A") has /mnt/felhom-usb bound (Model-A bind: source .../felhom-data, mp=where),
|
||||
// but NOT /mnt/extra.
|
||||
h := newDiskServerWithGuestConfigs(t, d, sv, nil, map[int]map[string]string{
|
||||
8200: {"mp3": "/mnt/felhom-usb/felhom-data,mp=/mnt/felhom-usb"},
|
||||
})
|
||||
|
||||
disks := decodeDisks(t, do(t, h, "GET", "/disks", "A", "").Body.Bytes())
|
||||
got := map[string]bool{}
|
||||
for _, di := range disks {
|
||||
got[di.MountPath] = di.GuestAttached
|
||||
}
|
||||
if !got["/mnt/felhom-usb"] {
|
||||
t.Errorf("/mnt/felhom-usb should be guest_attached=true (bound into the guest)")
|
||||
}
|
||||
if got["/mnt/extra"] {
|
||||
t.Errorf("/mnt/extra should be guest_attached=false (host-present but not bound)")
|
||||
}
|
||||
}
|
||||
@@ -118,6 +118,9 @@ func newDiskServer(t *testing.T, d *fakeDiskOps, g *fakeGate, sv StorageView, gl
|
||||
// device the format tests use; antiRetargetResolve itself is covered directly
|
||||
// in wipe_reresolve_test.go.
|
||||
srv.reresolveWipe = func(_ context.Context, _ string) (string, error) { return "/dev/sdb", nil }
|
||||
// F20-BUG2: the wipe id derivation hits /dev/disk/by-* in production; stub it deterministically so
|
||||
// both the /disks list and the gate (which share this seam) resolve the same id in tests.
|
||||
srv.deviceDurableID = func(device string) (string, error) { return "byid:wwn-" + strings.TrimPrefix(device, "/dev/"), nil }
|
||||
return srv.Handler()
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// formatJob is the persisted record of the most recent / in-flight disk format (F20-BUG3). It exists so
|
||||
// that (a) an mkfs runs DETACHED from the HTTP request — a client/request deadline can no longer SIGKILL
|
||||
// an in-progress mkfs and corrupt the disk — and (b) an in-progress format SURVIVES an agent restart:
|
||||
// on boot, a record still in `running` is re-resolved by durable-id and re-run (mkfs is idempotent).
|
||||
type formatJob struct {
|
||||
JobID string `json:"job_id"`
|
||||
Device string `json:"device"`
|
||||
DurableID string `json:"durable_id"` // "" for a blank (benign) format — never auto-recovered
|
||||
FSType string `json:"fstype"`
|
||||
Phase string `json:"phase"` // running | done | failed
|
||||
Error string `json:"error,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
formatPhaseRunning = "running"
|
||||
formatPhaseDone = "done"
|
||||
formatPhaseFailed = "failed"
|
||||
)
|
||||
|
||||
// FormatJobStore persists the single most-recent format job (atomic tmp+rename, 0600). One slot: a host
|
||||
// formats one device at a time. Mirrors storage.IntentStore.
|
||||
type FormatJobStore struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
cur *formatJob
|
||||
}
|
||||
|
||||
// OpenFormatJobStore loads (or initializes) the store. Missing file = no job; corrupt file = error.
|
||||
func OpenFormatJobStore(path string) (*FormatJobStore, error) {
|
||||
s := &FormatJobStore{path: path}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return s, nil
|
||||
}
|
||||
return nil, fmt.Errorf("format-job store: read %s: %w", path, err)
|
||||
}
|
||||
if len(data) > 0 {
|
||||
var j formatJob
|
||||
if err := json.Unmarshal(data, &j); err != nil {
|
||||
return nil, fmt.Errorf("format-job store: parse %s: %w", path, err)
|
||||
}
|
||||
s.cur = &j
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *FormatJobStore) get() *formatJob {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.cur == nil {
|
||||
return nil
|
||||
}
|
||||
cp := *s.cur
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (s *FormatJobStore) save(j *formatJob) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cp := *j
|
||||
s.cur = &cp
|
||||
data, err := json.MarshalIndent(s.cur, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.path)
|
||||
}
|
||||
|
||||
// startFormatDetached persists a `running` record and runs mkfs in a goroutine off s.baseCtx (NOT the
|
||||
// request context), with a long bound. It returns a channel that yields the format error (nil on
|
||||
// success). The caller may stop waiting (client disconnect) without killing the mkfs — the goroutine
|
||||
// runs to completion and records the outcome. device is the ALREADY anti-retarget-resolved device; the
|
||||
// record carries durableID so a restart can re-resolve + re-run.
|
||||
func (s *Server) startFormatDetached(device, durableID, fstype string) <-chan error {
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
job := &formatJob{
|
||||
JobID: s.nowFn().UTC().Format("20060102T150405Z"), Device: device, DurableID: durableID,
|
||||
FSType: fstype, Phase: formatPhaseRunning,
|
||||
StartedAt: s.nowFn().UTC().Format(time.RFC3339), UpdatedAt: s.nowFn().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if s.formatJobs != nil {
|
||||
if err := s.formatJobs.save(job); err != nil {
|
||||
s.logger.Warn("format-job: could not persist running record (format will still run)", "device", device, "err", err)
|
||||
}
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(base, 60*time.Minute)
|
||||
defer cancel()
|
||||
err := s.disks.Format(ctx, device, fstype)
|
||||
s.finishFormatJob(job, err)
|
||||
done <- err
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
// finishFormatJob updates the persisted record to done/failed.
|
||||
func (s *Server) finishFormatJob(job *formatJob, err error) {
|
||||
if s.formatJobs == nil {
|
||||
return
|
||||
}
|
||||
job.UpdatedAt = s.nowFn().UTC().Format(time.RFC3339)
|
||||
if err != nil {
|
||||
job.Phase = formatPhaseFailed
|
||||
job.Error = err.Error()
|
||||
} else {
|
||||
job.Phase = formatPhaseDone
|
||||
}
|
||||
if serr := s.formatJobs.save(job); serr != nil {
|
||||
s.logger.Warn("format-job: could not persist final record", "device", job.Device, "phase", job.Phase, "err", serr)
|
||||
}
|
||||
}
|
||||
|
||||
// RecoverFormatJob, on agent startup, completes a format that an agent restart interrupted (F20-BUG3 +
|
||||
// the operator's "survive a restart" decision). A record still in `running` is re-resolved by its
|
||||
// durable-id (anti-retarget — a swapped/absent disk is NOT re-formatted) and the mkfs is re-run detached
|
||||
// (mkfs is idempotent). A blank (no-durable-id) format is NOT auto-re-run — it is marked failed for the
|
||||
// caller to retry, so recovery never formats a mutable /dev path.
|
||||
func (s *Server) RecoverFormatJob(ctx context.Context) {
|
||||
if s.formatJobs == nil {
|
||||
return
|
||||
}
|
||||
job := s.formatJobs.get()
|
||||
if job == nil || job.Phase != formatPhaseRunning {
|
||||
return
|
||||
}
|
||||
if job.DurableID == "" {
|
||||
s.logger.Warn("format-job recover: interrupted blank format — marking failed (retry needed; not auto-re-running a path-bound format)", "device", job.Device)
|
||||
s.finishFormatJob(job, fmt.Errorf("interrupted by agent restart; retry the format"))
|
||||
return
|
||||
}
|
||||
device, err := s.reresolveWipe(ctx, job.DurableID)
|
||||
if err != nil {
|
||||
s.logger.Warn("format-job recover: durable-id no longer resolves — NOT re-formatting (anti-retarget)", "durable_id", job.DurableID, "err", err)
|
||||
s.finishFormatJob(job, fmt.Errorf("durable-id %s did not re-resolve after restart: %w", job.DurableID, err))
|
||||
return
|
||||
}
|
||||
s.logger.Warn("format-job recover: re-running interrupted format detached", "durable_id", job.DurableID, "device", device, "fstype", job.FSType)
|
||||
_ = s.startFormatDetached(device, job.DurableID, job.FSType) // detached; updates the record on completion
|
||||
}
|
||||
|
||||
// nowFn returns the server clock (testable), defaulting to time.Now.
|
||||
func (s *Server) nowFn() time.Time {
|
||||
if s.now != nil {
|
||||
return s.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/storage"
|
||||
)
|
||||
|
||||
func tempFormatStore(t *testing.T) *FormatJobStore {
|
||||
t.Helper()
|
||||
fj, err := OpenFormatJobStore(filepath.Join(t.TempDir(), "format-job.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fj
|
||||
}
|
||||
|
||||
// formatServer builds a *Server with the destructive-format path wired (data-bearing probe, a gate, the
|
||||
// format-job store), plus the test stubs (reresolveWipe/deviceDurableID) so no real /dev is touched.
|
||||
func formatServer(t *testing.T, d *fakeDiskOps, g StorageGate, fj *FormatJobStore) *Server {
|
||||
t.Helper()
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0", Guests: &fakeGuests{}, Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||
Storage: fakeStorage{}, Tokens: staticTokens{"A": 8200},
|
||||
Disks: d, DiskGate: g, FormatJobs: fj, HostReader: sysOnSDA(),
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
srv.reresolveWipe = func(_ context.Context, _ string) (string, error) { return "/dev/sdb", nil }
|
||||
srv.deviceDurableID = func(device string) (string, error) { return "byid:wwn-" + device, nil }
|
||||
return srv
|
||||
}
|
||||
|
||||
func waitFormatPhase(t *testing.T, fj *FormatJobStore, want string) *formatJob {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if j := fj.get(); j != nil && j.Phase == want {
|
||||
return j
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
got := fj.get()
|
||||
t.Fatalf("format job did not reach phase %q in time (got %+v)", want, got)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestFormat_DetachedPersistsJobRecord asserts F20-BUG3: a customer-confirmed wipe runs through the
|
||||
// detached runner and a persisted job record reaches `done` with the device + durable-id bound (so it
|
||||
// can be polled / recovered). Also confirms backward-compat: the handler still returns 200 synchronously.
|
||||
func TestFormat_DetachedPersistsJobRecord(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: deviceProbeDataBearing()}
|
||||
g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
|
||||
fj := tempFormatStore(t)
|
||||
srv := formatServer(t, d, g, fj)
|
||||
h := srv.Handler()
|
||||
|
||||
w := do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("confirmed format: %d (%s)", w.Code, w.Body.String())
|
||||
}
|
||||
job := waitFormatPhase(t, fj, formatPhaseDone)
|
||||
if job.Device != "/dev/sdb" { // the anti-retarget re-resolved device (stub returns /dev/sdb)
|
||||
t.Fatalf("job device = %q, want /dev/sdb (re-resolved)", job.Device)
|
||||
}
|
||||
if job.DurableID == "" {
|
||||
t.Fatalf("job durable-id empty — recovery could not re-bind")
|
||||
}
|
||||
if len(d.formatted()) != 1 {
|
||||
t.Fatalf("mkfs called %d times, want 1", len(d.formatted()))
|
||||
}
|
||||
}
|
||||
|
||||
// TestFormatStatus reports the persisted job via GET /disks/format/status.
|
||||
func TestFormatStatus(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: deviceProbeDataBearing()}
|
||||
g := &fakeGate{decision: WipeDecision{Allowed: true, Tier: "customer_confirmable", Reason: "customer_confirmed"}}
|
||||
fj := tempFormatStore(t)
|
||||
srv := formatServer(t, d, g, fj)
|
||||
h := srv.Handler()
|
||||
|
||||
do(t, h, "POST", "/disks/format", "A", `{"device":"/dev/sdb1","fstype":"ext4","confirmed":true,"durable_id":"byid:wwn-/dev/sdb1"}`)
|
||||
waitFormatPhase(t, fj, formatPhaseDone)
|
||||
|
||||
w := do(t, h, "GET", "/disks/format/status", "A", "")
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status: %d", w.Code)
|
||||
}
|
||||
var resp struct {
|
||||
Data struct {
|
||||
Phase string `json:"phase"`
|
||||
Device string `json:"device"`
|
||||
} `json:"data"`
|
||||
}
|
||||
_ = json.Unmarshal(w.Body.Bytes(), &resp)
|
||||
if resp.Data.Phase != formatPhaseDone {
|
||||
t.Fatalf("status phase = %q, want done", resp.Data.Phase)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverFormatJob_ReRunsInterrupted asserts the operator's "survive an agent restart" decision: a
|
||||
// record left in `running` (agent died mid-format) is re-resolved by durable-id and the mkfs re-run.
|
||||
func TestRecoverFormatJob_ReRunsInterrupted(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: deviceProbeDataBearing()}
|
||||
fj := tempFormatStore(t)
|
||||
// Pre-seed an interrupted, durable-id-bound running job (as if the agent died mid-mkfs).
|
||||
if err := fj.save(&formatJob{JobID: "j1", Device: "/dev/sdb", DurableID: "byid:wwn-x", FSType: "ext4", Phase: formatPhaseRunning}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := formatServer(t, d, &fakeGate{}, fj)
|
||||
|
||||
srv.RecoverFormatJob(context.Background())
|
||||
waitFormatPhase(t, fj, formatPhaseDone)
|
||||
if got := d.formatted(); len(got) != 1 || got[0] != "/dev/sdb" {
|
||||
t.Fatalf("recovery should have re-run mkfs on /dev/sdb (re-resolved), got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverFormatJob_SkipsBlank: an interrupted BLANK (no durable-id) format is NOT auto-re-run (never
|
||||
// format a mutable /dev path on recovery) — it is marked failed for the caller to retry.
|
||||
func TestRecoverFormatJob_SkipsBlank(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: deviceProbeDataBearing()}
|
||||
fj := tempFormatStore(t)
|
||||
_ = fj.save(&formatJob{JobID: "j2", Device: "/dev/sdb", DurableID: "", FSType: "ext4", Phase: formatPhaseRunning})
|
||||
srv := formatServer(t, d, &fakeGate{}, fj)
|
||||
|
||||
srv.RecoverFormatJob(context.Background())
|
||||
if got := d.formatted(); len(got) != 0 {
|
||||
t.Fatalf("recovery must NOT re-run a path-bound (blank) format, but mkfs ran: %v", got)
|
||||
}
|
||||
if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
|
||||
t.Fatalf("blank interrupted job should be marked failed, got %+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecoverFormatJob_SkipsUnresolvable: if the durable-id no longer resolves (drive removed/replaced),
|
||||
// recovery must NOT format anything (anti-retarget) and marks the job failed.
|
||||
func TestRecoverFormatJob_SkipsUnresolvable(t *testing.T) {
|
||||
d := &fakeDiskOps{probe: deviceProbeDataBearing()}
|
||||
fj := tempFormatStore(t)
|
||||
_ = fj.save(&formatJob{JobID: "j3", Device: "/dev/sdb", DurableID: "byid:wwn-gone", FSType: "ext4", Phase: formatPhaseRunning})
|
||||
srv := formatServer(t, d, &fakeGate{}, fj)
|
||||
srv.reresolveWipe = func(_ context.Context, _ string) (string, error) {
|
||||
return "", context.DeadlineExceeded // simulate "durable-id no longer resolves"
|
||||
}
|
||||
|
||||
srv.RecoverFormatJob(context.Background())
|
||||
if got := d.formatted(); len(got) != 0 {
|
||||
t.Fatalf("recovery must NOT format when the durable-id is unresolvable: %v", got)
|
||||
}
|
||||
if j := fj.get(); j == nil || j.Phase != formatPhaseFailed {
|
||||
t.Fatalf("unresolvable interrupted job should be marked failed, got %+v", j)
|
||||
}
|
||||
}
|
||||
|
||||
func deviceProbeDataBearing() storage.DeviceProbe {
|
||||
return storage.DeviceProbe{Probed: true, HasFilesystem: true, FSType: "ext4"}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
)
|
||||
|
||||
func tempBindStore(t *testing.T) *GuestBindStore {
|
||||
t.Helper()
|
||||
gb, err := OpenGuestBindStore(filepath.Join(t.TempDir(), "guest-binds.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return gb
|
||||
}
|
||||
|
||||
func reassertServer(t *testing.T, ga GuestAttacher, sv StorageView, mounts map[int]map[string]string, gb *GuestBindStore) *Server {
|
||||
t.Helper()
|
||||
srv, err := NewServer(Options{
|
||||
ListenAddr: "127.0.0.1:0",
|
||||
Guests: &fakeGuestsCfg{mounts: mounts},
|
||||
Backups: &fakeBackups{}, Store: &fakeStore{},
|
||||
Storage: sv, Tokens: staticTokens{"A": 8200},
|
||||
GuestAttach: ga, GuestBinds: gb,
|
||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.baseCtx = context.Background()
|
||||
return srv
|
||||
}
|
||||
|
||||
// usbPresent is a storage view with felhom-usb present at /mnt/felhom-usb (durable uuid:usb-1).
|
||||
func usbPresent() fakeStorage {
|
||||
return fakeStorage{targets: []hub.StorageTarget{
|
||||
{Name: "usb", Type: hub.StorageTypeUSB, BackingDevice: "/dev/sdb1", MountPath: "/mnt/felhom-usb", DurableID: "uuid:usb-1"},
|
||||
}}
|
||||
}
|
||||
|
||||
// TestReassertGuestBinds_RestoresMissingBind is the F9 core proof (the operator's "fire the real
|
||||
// trigger" requirement): an enrolled drive that is present on the host but MISSING from the guest config
|
||||
// (the post-re-provision gap) is auto-re-attached on the startup re-assert — with NO manual guest-attach
|
||||
// call. Fails on the pre-fix code (no re-assert existed).
|
||||
func TestReassertGuestBinds_RestoresMissingBind(t *testing.T) {
|
||||
gb := tempBindStore(t)
|
||||
if err := gb.Record(8200, "uuid:usb-1"); err != nil { // enrolled at a prior boot
|
||||
t.Fatal(err)
|
||||
}
|
||||
ga := &fakeGuestAttacher{}
|
||||
// guest 8200 has docker-data only — the felhom-usb bind was dropped by the re-provision.
|
||||
srv := reassertServer(t, ga, usbPresent(), map[int]map[string]string{
|
||||
8200: {"mp0": "local-lvm:8,mp=/var/lib/docker"},
|
||||
}, gb)
|
||||
|
||||
srv.ReassertGuestBinds(context.Background())
|
||||
|
||||
if ga.count() != 1 {
|
||||
t.Fatalf("AttachBind called %d times, want 1 (auto-re-assert on startup)", ga.count())
|
||||
}
|
||||
if ga.calls[0].vmid != 8200 || ga.calls[0].where != "/mnt/felhom-usb" {
|
||||
t.Fatalf("re-asserted bind = %+v, want vmid 8200 where /mnt/felhom-usb", ga.calls[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestReassertGuestBinds_SkipsAbsentDurable: an enrolled drive whose durable-id is NOT currently present
|
||||
// (unplugged / swapped for a different disk) must NOT be auto-bound — the "on durable-id match" safety.
|
||||
func TestReassertGuestBinds_SkipsAbsentDurable(t *testing.T) {
|
||||
gb := tempBindStore(t)
|
||||
_ = gb.Record(8200, "uuid:usb-1")
|
||||
ga := &fakeGuestAttacher{}
|
||||
srv := reassertServer(t, ga, fakeStorage{}, map[int]map[string]string{ // empty storage view → absent
|
||||
8200: {"mp0": "local-lvm:8,mp=/var/lib/docker"},
|
||||
}, gb)
|
||||
|
||||
srv.ReassertGuestBinds(context.Background())
|
||||
if ga.count() != 0 {
|
||||
t.Fatalf("AttachBind called %d times — must NOT auto-bind an absent/swapped drive", ga.count())
|
||||
}
|
||||
}
|
||||
|
||||
// TestReassertGuestBinds_SkipsAlreadyBound: when the guest already has the bind, the re-assert is a no-op.
|
||||
func TestReassertGuestBinds_SkipsAlreadyBound(t *testing.T) {
|
||||
gb := tempBindStore(t)
|
||||
_ = gb.Record(8200, "uuid:usb-1")
|
||||
ga := &fakeGuestAttacher{}
|
||||
srv := reassertServer(t, ga, usbPresent(), map[int]map[string]string{
|
||||
8200: {"mp0": "local-lvm:8,mp=/var/lib/docker", "mp3": "/mnt/felhom-usb/felhom-data,mp=/mnt/felhom-usb"},
|
||||
}, gb)
|
||||
|
||||
srv.ReassertGuestBinds(context.Background())
|
||||
if ga.count() != 0 {
|
||||
t.Fatalf("AttachBind called %d times — already bound, must be a no-op", ga.count())
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuestBindStore_Persist round-trips the store across reopen (the record must survive an agent
|
||||
// restart, since that is exactly when the re-assert runs).
|
||||
func TestGuestBindStore_Persist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "guest-binds.json")
|
||||
gb, err := OpenGuestBindStore(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = gb.Record(8200, "uuid:usb-1")
|
||||
_ = gb.Record(8200, "uuid:usb-1") // idempotent
|
||||
_ = gb.Record(9300, "byid:wwn-x")
|
||||
|
||||
re, err := OpenGuestBindStore(path) // simulate restart
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := re.Guests()
|
||||
if len(g[8200]) != 1 || g[8200][0] != "uuid:usb-1" {
|
||||
t.Fatalf("vmid 8200 = %v, want [uuid:usb-1]", g[8200])
|
||||
}
|
||||
if len(g[9300]) != 1 || g[9300][0] != "byid:wwn-x" {
|
||||
t.Fatalf("vmid 9300 = %v, want [byid:wwn-x]", g[9300])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// GuestBindStore persists, per guest, the DURABLE-IDs of the user-data drives enrolled (guest-attached)
|
||||
// into it. F9: the in-guest bind (`pct set -mpN`) is config state that does NOT survive a destroy +
|
||||
// re-provision, and nothing re-asserted it — so a re-provisioned guest came up with the HDD unattached
|
||||
// even though it had been enrolled. This store is the record the startup re-assert (ReassertGuestBinds)
|
||||
// replays: for each enrolled durable-id still physically present, re-add the bind if the guest lacks it.
|
||||
//
|
||||
// Keyed by durable-id (NOT host path or sdX) so the re-assert is "on durable-id match" — a swapped or
|
||||
// absent drive is never auto-bound. Thread-safe; atomic file writes (tmp+rename), 0600. Mirrors
|
||||
// storage.IntentStore.
|
||||
type GuestBindStore struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
m map[int][]string // vmid -> sorted set of enrolled durable-ids
|
||||
}
|
||||
|
||||
// OpenGuestBindStore loads (or initializes) the store. Missing file = empty store; corrupt file = error
|
||||
// (fail loud — losing a bind record would silently drop the re-assert).
|
||||
func OpenGuestBindStore(path string) (*GuestBindStore, error) {
|
||||
s := &GuestBindStore{path: path, m: map[int][]string{}}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return s, nil
|
||||
}
|
||||
return nil, fmt.Errorf("guest-bind store: read %s: %w", path, err)
|
||||
}
|
||||
if len(data) > 0 {
|
||||
// stored as {"<vmid>": ["durable-id", ...]} (string keys — JSON object keys are strings)
|
||||
raw := map[string][]string{}
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return nil, fmt.Errorf("guest-bind store: parse %s: %w", path, err)
|
||||
}
|
||||
for k, v := range raw {
|
||||
vmid, err := strconv.Atoi(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("guest-bind store: bad vmid key %q: %w", k, err)
|
||||
}
|
||||
s.m[vmid] = v
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Record adds (vmid, durableID) to the enrolled set. Idempotent — no write if already present. A blank
|
||||
// durable-id is refused (the re-assert must never act on a drive whose identity it can't pin).
|
||||
func (s *GuestBindStore) Record(vmid int, durableID string) error {
|
||||
if durableID == "" {
|
||||
return fmt.Errorf("guest-bind store: refusing to record an empty durable-id for vmid %d", vmid)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, id := range s.m[vmid] {
|
||||
if id == durableID {
|
||||
return nil // idempotent
|
||||
}
|
||||
}
|
||||
s.m[vmid] = append(s.m[vmid], durableID)
|
||||
sort.Strings(s.m[vmid])
|
||||
return s.saveLocked()
|
||||
}
|
||||
|
||||
// Guests returns a copy of the vmid → enrolled-durable-ids map.
|
||||
func (s *GuestBindStore) Guests() map[int][]string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make(map[int][]string, len(s.m))
|
||||
for vmid, ids := range s.m {
|
||||
out[vmid] = append([]string(nil), ids...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *GuestBindStore) saveLocked() error {
|
||||
raw := make(map[string][]string, len(s.m))
|
||||
for vmid, ids := range s.m {
|
||||
raw[strconv.Itoa(vmid)] = ids
|
||||
}
|
||||
data, err := json.MarshalIndent(raw, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, s.path)
|
||||
}
|
||||
@@ -87,6 +87,14 @@ type Options struct {
|
||||
// Intent records drive enroll/eject intent for the self-heal watchdog (slice 10 P3). OPTIONAL —
|
||||
// when nil, no intent is recorded (self-heal runs ungated).
|
||||
Intent IntentRecorder
|
||||
// GuestBinds persists which user-data drives (by durable-id) are enrolled into each guest, so the
|
||||
// startup re-assert (ReassertGuestBinds) can restore a bind that a re-provision dropped (F9).
|
||||
// OPTIONAL — when nil, guest binds are not recorded and the startup re-assert is a no-op.
|
||||
GuestBinds *GuestBindStore
|
||||
// FormatJobs persists the in-flight/last disk-format job so mkfs runs detached from the request
|
||||
// (F20-BUG3: a request deadline can't kill it) and survives an agent restart (RecoverFormatJob).
|
||||
// OPTIONAL — when nil, formats still run detached but are not persisted/recovered.
|
||||
FormatJobs *FormatJobStore
|
||||
// HostReader is the root-free host topology reader used to classify a device/mount's protection
|
||||
// ROLE (it backs SystemDisks for the eject role-gate + the /disks role hints). OPTIONAL — when nil
|
||||
// it defaults to the production *storage.ProcHostReader. Injectable so the role-gate is testable.
|
||||
@@ -143,6 +151,8 @@ type Server struct {
|
||||
guestList GuestLister // slice 8C (optional)
|
||||
guestAttach GuestAttacher // slice 10 P2 (optional)
|
||||
intent IntentRecorder // slice 10 P3 (optional)
|
||||
guestBinds *GuestBindStore // F9 startup bind re-assert record (optional)
|
||||
formatJobs *FormatJobStore // F20-BUG3 detached-format job record (optional)
|
||||
host storage.HostReader // role classification source (optional; defaults to ProcHostReader)
|
||||
|
||||
hostMetrics HostMetricsProvider // slice 9 (optional)
|
||||
@@ -154,6 +164,13 @@ type Server struct {
|
||||
// override it to avoid touching real /dev.
|
||||
reresolveWipe func(ctx context.Context, durableID string) (string, error)
|
||||
|
||||
// deviceDurableID derives the WIPE-binding durable id of a block device (the byid:/byuuid: scheme
|
||||
// the wipe gate resolves against). F20-BUG2: BOTH the /disks list (DiskInfo.WipeDurableID) and the
|
||||
// format gate use this single seam, so the id the customer copies from the list is exactly the id
|
||||
// the gate accepts (no more uuid: vs byid: binding_mismatch). Defaults to storage.DeviceDurableID;
|
||||
// tests override it. (DiskInfo.DurableID stays the uuid: storage id — that one feeds /disks/assign.)
|
||||
deviceDurableID func(device string) (string, error)
|
||||
|
||||
jobsMu sync.Mutex
|
||||
jobs map[int]*backupJob // per-guest backup job state (slice 8B)
|
||||
|
||||
@@ -191,12 +208,15 @@ func NewServer(o Options) (*Server, error) {
|
||||
guestList: o.Guests2,
|
||||
guestAttach: o.GuestAttach,
|
||||
intent: o.Intent,
|
||||
guestBinds: o.GuestBinds,
|
||||
formatJobs: o.FormatJobs,
|
||||
host: o.HostReader,
|
||||
hostMetrics: o.HostMetrics,
|
||||
hostID: o.HostID,
|
||||
jobs: map[int]*backupJob{},
|
||||
}
|
||||
s.reresolveWipe = s.reresolveDurableForWipe
|
||||
s.deviceDurableID = storage.DeviceDurableID
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -218,6 +238,7 @@ func (s *Server) Handler() http.Handler {
|
||||
mux.HandleFunc("POST /disks/assign", s.withGuest(s.handleDiskAssign))
|
||||
mux.HandleFunc("POST /disks/eject", s.withGuest(s.handleDiskEject))
|
||||
mux.HandleFunc("POST /disks/format", s.withGuest(s.handleDiskFormat))
|
||||
mux.HandleFunc("GET /disks/format/status", s.withGuest(s.handleDiskFormatStatus))
|
||||
// Guest data-drive passthrough (slice 10 P2): bind an enrolled drive's felhom-data namespace in.
|
||||
mux.HandleFunc("POST /disks/guest-attach", s.withGuest(s.handleDiskGuestAttach))
|
||||
// Guest reboot (slice 10 P2 activation): user-triggered restart to activate pending drive binds.
|
||||
|
||||
Reference in New Issue
Block a user