agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/storage/netmount.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net"
 	"os"
@@ -246,8 +248,12 @@ func renderNetworkMountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): NO network-online.target ordering here. `_netdev` in Options is the
+	// correct + sufficient network ordering for the REAL mount — systemd classes a _netdev mount under
+	// remote-fs.target and orders it after the network without a hand-written After/Wants. A literal
+	// `After=network-online.target` on this unit (which the .automount pulls in via local-fs) closed the
+	// boot ordering cycle networking→local-fs→automount→network-online→networking; systemd broke it by
+	// DELETING an arbitrary job (one boot lost networking entirely, the next lost the automount).
 	b.WriteString("\n[Mount]\n")
 	fmt.Fprintf(&b, "What=%s\n", s.mountSource())
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
@@ -263,8 +269,12 @@ func renderNetworkAutomountUnit(s NetworkMountSpec) string {
 	b.WriteString("# " + netUnitMarker + " — do not edit by hand.\n")
 	b.WriteString("[Unit]\n")
 	fmt.Fprintf(&b, "Description=Felhom network storage automount %s (%s)\n", sanitizeDesc(s.Name), s.Protocol)
-	b.WriteString("After=network-online.target\n")
-	b.WriteString("Wants=network-online.target\n")
+	// F12 (CAMPAIGN-3, CRITICAL): the automount unit gets NO network relation of ANY kind. A trigger
+	// needs no network — it just watches the mountpoint and fires the .mount on first access (the
+	// .mount's `_netdev` then orders the real mount after the network). An automount is implicitly
+	// ordered Before=local-fs.target; adding After/Wants=network-online.target here created the boot
+	// ordering cycle that cost the host its network on one boot and its NAS on the next. Keep this unit
+	// orderable before local-fs WITHOUT dragging the network into that transaction.
 	b.WriteString("\n[Automount]\n")
 	fmt.Fprintf(&b, "Where=%s\n", s.Where())
 	fmt.Fprintf(&b, "TimeoutIdleSec=%d\n", s.idleTimeout())
@@ -347,6 +357,10 @@ func (h *SudoHostOps) EnsureNetworkMount(ctx context.Context, spec NetworkMountS
 	if err := ValidateNetworkMountSpec(spec); err != nil {
 		return err
 	}
+	// Template-drift reconcile: bring any already-installed units up to the current template before we
+	// touch the unit dir (F12 — a pre-0.85 unit still carrying the network-online ordering gets rewritten
+	// here even if the daemon-startup migration hasn't run in this process). Best-effort; never blocks add.
+	h.MigrateNetworkUnits(ctx)
 	// Defense in depth: never realise a network mount outside the user-data namespace.
 	if NetworkMountRole(spec.Where()) != RoleUserData {
 		return fmt.Errorf("netmount: refusing to mount outside the user-data namespace: %s", spec.Where())
@@ -427,6 +441,10 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 		h.logger.Debug("netmount: remove step", "verb", step[0], "unit", step[len(step)-1],
 			"ok", err == nil) // a "not loaded" failure here is expected + tolerated
 	}
+	// F2 (CAMPAIGN-3): clear any failed/start-limit runtime state on the pair BEFORE the files go, or
+	// systemd keeps them as `not-found failed` residue after daemon-reload. reset-failed while the units
+	// are still loaded; tolerate the not-failed case (nothing to reset).
+	h.resetNetworkUnitsIfFailed(ctx, automountUnit, mountUnit)
 
 	destAuto := filepath.Join(h.unitDir, automountUnit)
 	destMount := filepath.Join(h.unitDir, mountUnit)
@@ -441,10 +459,184 @@ func (h *SudoHostOps) RemoveNetworkMount(ctx context.Context, name string) error
 	if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
 		return fmt.Errorf("netmount: daemon-reload: %w", err)
 	}
+	// F1 (CAMPAIGN-3): remove the now-empty mountpoint directory (the campaign accumulated 10 stub-shaped
+	// leftovers). rmdir ONLY — a non-empty dir (unexpected data present) is left in place with a WARN, the
+	// fail-safe; `rm -rf` is forbidden here. The host removal propagates into running guests through the
+	// shared bind; a fresh guest re-binds cleanly on next start.
+	h.rmdirMountpoint(ctx, where)
 	h.logger.Info("netmount: removed network mount", "name", name, "where", where)
 	return nil
 }
 
+// resetNetworkUnitsIfFailed reset-failed's any of the given units that is in the failed state (F2 —
+// leave no `not-found failed`/start-limit residue behind a remove or a rolled-back add). Unprivileged
+// is-failed read + the FELHOM_NETMOUNT reset-failed grant; every step tolerated.
+func (h *SudoHostOps) resetNetworkUnitsIfFailed(ctx context.Context, units ...string) {
+	for _, unit := range units {
+		if h.unitFailed == nil || !h.unitFailed(ctx, unit) {
+			continue
+		}
+		if err := h.run(ctx, h.bins.Systemctl, "reset-failed", "--", unit); err != nil {
+			h.logger.Warn("netmount: reset-failed tolerated failure", "unit", unit, "err", err)
+		}
+	}
+}
+
+// rmdirMountpoint removes an empty network mountpoint dir under NetworkMountRoot. rmdir refuses a
+// non-empty dir (the fail-safe): unexpected data is preserved and flagged, never rm -rf'd. Best-effort.
+func (h *SudoHostOps) rmdirMountpoint(ctx context.Context, where string) {
+	if !strings.HasPrefix(where, NetworkMountRoot+"/") {
+		return // defense in depth: only ever under the bind root
+	}
+	if err := h.run(ctx, "/usr/bin/rmdir", where); err != nil {
+		// rmdir fails on a non-empty dir — leave it (fail-safe) and flag it for the operator.
+		h.logger.Warn("netmount: mountpoint dir not removed (non-empty or busy — left in place, fail-safe)",
+			"where", where, "err", err)
+	}
+}
+
+// MigrateNetworkUnits reconciles every marker-owned network-storage unit file on disk against a fresh
+// render of its own reconstructed spec — a general template-drift reconcile (the git-sync pattern:
+// content-hash compare, rewrite on drift, batched daemon-reload). It exists because a template change
+// must reach ALREADY-INSTALLED units, not only future adds: the F12 fix (CAMPAIGN-3) removed the
+// network-online ordering that turned every host boot with an enrolled share into a coin flip, and the
+// units installed before 0.85 still carry the ordering cycle until they are rewritten. Runs at agent
+// startup (before the reassert sweep) and at the head of EnsureNetworkMount. Idempotent: a unit already
+// byte-identical to its fresh render is left untouched (second pass rewrites nothing). Best-effort per
+// unit; one INFO line per migrated unit. Returns the count migrated.
+func (h *SudoHostOps) MigrateNetworkUnits(ctx context.Context) int {
+	entries, err := os.ReadDir(h.unitDir)
+	if err != nil {
+		h.logger.Warn("netmigrate: reading unit dir failed", "err", err)
+		return 0
+	}
+	migrated := 0
+	changed := false
+	for _, e := range entries {
+		if !strings.HasSuffix(e.Name(), ".mount") {
+			continue // the .mount carries What/Type; the paired .automount mirrors Where
+		}
+		mountPath := filepath.Join(h.unitDir, e.Name())
+		mountData, rerr := os.ReadFile(mountPath)
+		if rerr != nil || !strings.Contains(string(mountData), netUnitMarker) {
+			continue // unreadable or not one of ours
+		}
+		automountName := strings.TrimSuffix(e.Name(), ".mount") + ".automount"
+		automountData, aerr := os.ReadFile(filepath.Join(h.unitDir, automountName))
+		if aerr != nil {
+			continue // a .mount with no paired .automount is malformed — not ours to guess
+		}
+		spec, ok := specFromNetworkUnits(string(mountData), string(automountData))
+		if !ok {
+			continue
+		}
+		freshMount := renderNetworkMountUnit(spec)
+		freshAuto := renderNetworkAutomountUnit(spec)
+		if contentHash(string(mountData)) == contentHash(freshMount) &&
+			contentHash(string(automountData)) == contentHash(freshAuto) {
+			continue // already current — the idempotent no-op
+		}
+		if err := h.installUnit(ctx, e.Name(), freshMount); err != nil {
+			h.logger.Warn("netmigrate: rewriting mount unit failed", "unit", e.Name(), "err", err)
+			continue
+		}
+		if err := h.installUnit(ctx, automountName, freshAuto); err != nil {
+			h.logger.Warn("netmigrate: rewriting automount unit failed", "unit", automountName, "err", err)
+			continue
+		}
+		changed = true
+		migrated++
+		h.logger.Info("netmigrate: migrated network-storage unit to the current template (F12: dropped the boot ordering cycle)",
+			"name", spec.Name, "where", spec.Where())
+	}
+	if changed {
+		if err := h.run(ctx, h.bins.Systemctl, "daemon-reload"); err != nil {
+			h.logger.Warn("netmigrate: daemon-reload after migration failed", "err", err)
+		}
+	}
+	return migrated
+}
+
+// contentHash is the SHA-256 hex of a unit file's content — the drift comparator (git-sync pattern).
+func contentHash(s string) string {
+	sum := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(sum[:])
+}
+
+// specFromNetworkUnits reconstructs the NetworkMountSpec that renders EXACTLY the given installed unit
+// pair — the fresh-render input for the drift reconcile. Round-trips by construction: every field the
+// render templates read is recovered (proto/server/export/where from the .mount; the SMB uid/gid+creds
+// from its Options; the idle window from the .automount). A NFS spec's mapped uid never appears in a
+// rendered unit, so it is irrelevant to the render and left at the container default. ok=false for a
+// non-marker or unparseable pair.
+func specFromNetworkUnits(mountContent, automountContent string) (NetworkMountSpec, bool) {
+	proto, server, export, where, ok := parseNetworkMountUnit(mountContent)
+	if !ok {
+		return NetworkMountSpec{}, false
+	}
+	spec := NetworkMountSpec{
+		Name:      strings.TrimPrefix(where, NetworkMountRoot+"/"),
+		Protocol:  NetworkProtocol(proto),
+		Server:    server,
+		Export:    export,
+		MappedUID: 1000, // container default; unused by the NFS render, overwritten below for SMB
+		MappedGID: 1000,
+	}
+	if spec.Protocol == ProtocolSMB {
+		opts := unitLineValue(mountContent, "Options=")
+		if uid, ok := csvIntField(opts, "uid="); ok {
+			spec.MappedUID = uid - lxcUIDOffset
+		}
+		if gid, ok := csvIntField(opts, "gid="); ok {
+			spec.MappedGID = gid - lxcUIDOffset
+		}
+		spec.CredsRef = csvField(opts, "credentials=")
+	}
+	if idle, ok := csvIntField(unitLineValue(automountContent, "TimeoutIdleSec="), ""); ok && idle > 0 {
+		spec.IdleTimeoutSec = idle
+	}
+	return spec, true
+}
+
+// unitLineValue returns the value after the first line beginning with prefix (e.g. "Options="), trimmed.
+func unitLineValue(content, prefix string) string {
+	for _, line := range strings.Split(content, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, prefix) {
+			return strings.TrimPrefix(line, prefix)
+		}
+	}
+	return ""
+}
+
+// csvField finds the comma-separated token with the given key prefix (e.g. "credentials=") and returns
+// its value. "" if absent.
+func csvField(csv, key string) string {
+	for _, tok := range strings.Split(csv, ",") {
+		if strings.HasPrefix(tok, key) {
+			return strings.TrimPrefix(tok, key)
+		}
+	}
+	return ""
+}
+
+// csvIntField parses the int value of a comma-separated key (e.g. "uid=") — or, when key is "", parses
+// the whole string as an int (for a bare value like TimeoutIdleSec's already-extracted number).
+func csvIntField(csv, key string) (int, bool) {
+	val := csv
+	if key != "" {
+		val = csvField(csv, key)
+	}
+	if val == "" {
+		return 0, false
+	}
+	n, err := strconv.Atoi(strings.TrimSpace(val))
+	if err != nil {
+		return 0, false
+	}
+	return n, true
+}
+
 // ListNetworkMounts enumerates the installed network-storage units and reports per-share liveness. It
 // reads the (world-readable) unit dir + /proc/mounts and TCP-probes each NAS endpoint with a short
 // timeout — it NEVER stat()s the (possibly EIO/D-state) mountpoint, so a black-holed NAS cannot wedge