agent v0.85.0 WIP: F12/F11/F10/F9/F2/F1 boot-recovery plane + appliance self-heal (pre-build)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017CDMFpFx84pfviCTVuGGhf

internal/storage/hostops.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -98,12 +99,12 @@ type MountSpec struct {
 // Binaries holds the absolute paths of the allow-listed binaries (overridable from config
 // so the sudoers entries and the agent agree on exact paths).
 type Binaries struct {
-	Systemctl string
-	Install   string
-	Smartctl  string
-	Lvs       string
-	Blkid     string // device signature probe (8C data-bearing detection)
-	Lsblk     string // partition/mount topology (8C)
+	Systemctl   string
+	Install     string
+	Smartctl    string
+	Lvs         string
+	Blkid       string // device signature probe (8C data-bearing detection)
+	Lsblk       string // partition/mount topology (8C)
 	MkfsExt4    string // 8C format executor (ext4) — now invoked by the guarded wrapper, not the agent directly
 	MkfsXfs     string // 8C format executor (xfs) — now invoked by the guarded wrapper, not the agent directly
 	MkfsGuarded string // Impl-1 Part B: the guarded-mkfs wrapper the agent execs (device+fstype)
@@ -155,18 +156,22 @@ func (b Binaries) withDefaults() Binaries {
 type SudoHostOps struct {
 	runner   proxmox.Runner
 	bins     Binaries
-	unitDir  string // where enabled units live (e.g. /etc/systemd/system)
-	stageDir string // agent-owned staging dir for unit files before install
+	unitDir  string     // where enabled units live (e.g. /etc/systemd/system)
+	stageDir string     // agent-owned staging dir for unit files before install
 	host     HostReader // root-free reads (mount table) for the Impl-1 Format claim guard
 	logger   *slog.Logger
+	// unitFailed reports whether a systemd unit is in the failed state (incl. start-limit-hit). An
+	// UNPRIVILEGED read (`systemctl is-failed`) — seam-injected so the reassert's F10 reset-failed path
+	// is unit-testable without a real systemd. Default set in NewSudoHostOps.
+	unitFailed func(ctx context.Context, unit string) bool
 }
 
 // SudoHostOpsConfig configures a SudoHostOps.
 type SudoHostOpsConfig struct {
 	Runner   proxmox.Runner
 	Bins     Binaries
-	UnitDir  string // default /etc/systemd/system
-	StageDir string // default <dataDir>/units; must be agent-writable
+	UnitDir  string     // default /etc/systemd/system
+	StageDir string     // default <dataDir>/units; must be agent-writable
 	Host     HostReader // default NewProcHostReader(); the Format claim guard's mount-table read
 	Logger   *slog.Logger
 }
@@ -190,15 +195,24 @@ func NewSudoHostOps(cfg SudoHostOpsConfig) *SudoHostOps {
 		host = NewProcHostReader()
 	}
 	return &SudoHostOps{
-		runner:   cfg.Runner,
-		bins:     cfg.Bins.withDefaults(),
-		unitDir:  unitDir,
-		stageDir: stageDir,
-		host:     host,
-		logger:   logger,
+		runner:     cfg.Runner,
+		bins:       cfg.Bins.withDefaults(),
+		unitDir:    unitDir,
+		stageDir:   stageDir,
+		host:       host,
+		logger:     logger,
+		unitFailed: systemctlIsFailed,
 	}
 }
 
+// systemctlIsFailed is the production unitFailed: `systemctl is-failed <unit>` prints "failed" for a
+// failed/start-limit-hit unit and exits nonzero otherwise. UNPRIVILEGED (unit state is world-readable)
+// — deliberately NOT routed through the sudo runner, so it needs no sudoers grant.
+func systemctlIsFailed(ctx context.Context, unit string) bool {
+	out, _ := exec.CommandContext(ctx, "systemctl", "is-failed", "--", unit).Output()
+	return strings.TrimSpace(string(out)) == "failed"
+}
+
 // EnsureMount validates, renders, stages, installs and enables the .mount unit.
 func (h *SudoHostOps) EnsureMount(ctx context.Context, spec MountSpec) error {
 	// VALIDATE FIRST — refuse before constructing any command.