v0.189.0 — desired state + the app-stop crash marker (R-166 / D-b)

The box stops inferring the customer's intent from a container count and reads
what they actually asked for.

Part 1 — desired state. AppConfig gains a tri-state `desired_state`
(""/running/stopped), written ONLY by the customer's own action: the API action
switch, DeployStack, UpdateOptionalConfig's redeploy branch, and the .fab
import. Intent is written BEFORE the act and a failed write REFUSES the act.
StartStack/StopStack are deliberately not writers — 14 callers, only 2 are the
customer. bootrecon.isBootOrphan now reads intent instead of len(Containers)>0,
which closes R-157 mechanism B (a power cut or interrupted deploy left an app
with zero containers, read as a deliberate stop, and stranded silently).

ABSENT MEANS UNKNOWN, NEVER "running": every pre-v0.189.0 app.yaml reads absent,
so the legacy fallback is byte-identical to the old rule. A running-only startup
backfill converges the unambiguous cases; `stopped` is never inferred.

Part 2 — backup.AppStopGuard, a persisted marker over every stop→work→start
window (volume dump, offbox reconstitute, .fab export). Its own file, never
quiesce's. Written before the stop, cleared only after a restart that succeeded,
kept when one fails. Recover() completes before the boot reconciler is launched
and returns its outcome, which main.go reports on the existing backup_failed
event once the notifier exists. A defer is not the mechanism — a SIGKILL runs
none (Campaign 8 fault 10).

Also: SaveAppConfig rebuilt AppConfig field-by-field (the R-100 shape) and would
have dropped desired_state on every save across nine call sites. Replaced with
copy-and-overlay. Measured: app.yaml does not round-trip unknown YAML keys.

No hub change, no agent coupling, no user-visible string. 27/27 packages green;
7 red-proofs observed FAIL then restored.

controller/cmd/controller/appstop_wiring_test.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"testing"
+)
+
+// R-166 §10 seam discipline — the recovery and the backfill are seams, and a seam that is never
+// called is the defect class this project has shipped four times: a correct component, green unit
+// tests that inject it directly, and no production caller.
+//
+// These walk main.go's AST. NOT strings.Contains — the sibling bootrecon test records the reason at
+// first hand: a commented-out call still satisfies a substring match, so the text version passed the
+// very red-proof it existed to fail. Comments are not code.
+
+// mainBody returns func main()'s body from main.go, parsed.
+func mainBody(t *testing.T) *ast.BlockStmt {
+	t.Helper()
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "main.go", nil, 0)
+	if err != nil {
+		t.Fatalf("parse main.go: %v", err)
+	}
+	for _, decl := range f.Decls {
+		if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == "main" && fn.Body != nil {
+			return fn.Body
+		}
+	}
+	t.Fatal("func main() not found in main.go")
+	return nil
+}
+
+// callsInMain returns, in source order, the names of every call in func main() whose function
+// expression is `x.Sel(...)` or `Sel(...)` — enough to identify the wiring calls by name.
+func callsInMain(t *testing.T, body *ast.BlockStmt) []string {
+	t.Helper()
+	var names []string
+	ast.Inspect(body, func(n ast.Node) bool {
+		call, ok := n.(*ast.CallExpr)
+		if !ok {
+			return true
+		}
+		switch fun := call.Fun.(type) {
+		case *ast.SelectorExpr:
+			names = append(names, fun.Sel.Name)
+		case *ast.Ident:
+			names = append(names, fun.Name)
+		}
+		return true
+	})
+	return names
+}
+
+func indexOfCall(names []string, want string) int {
+	for i, n := range names {
+		if n == want {
+			return i
+		}
+	}
+	return -1
+}
+
+// TestMainWiresAppStopRecovery is the Group-I seam test. Comment out the `appStopGuard.Recover()`
+// line in main.go and this fails, where every behavioural test in internal/backup still passes.
+func TestMainWiresAppStopRecovery(t *testing.T) {
+	names := callsInMain(t, mainBody(t))
+
+	if indexOfCall(names, "NewAppStopGuard") < 0 {
+		t.Fatal("func main() no longer builds the R-166 app-stop guard — nothing writes or reads the marker")
+	}
+	if indexOfCall(names, "SetStarter") < 0 {
+		t.Fatal("func main() no longer calls SetStarter on the app-stop guard — Recover would find the " +
+			"marker and be unable to start anything, leaving every interrupted app down")
+	}
+	if indexOfCall(names, "Recover") < 0 {
+		t.Fatal("func main() no longer calls Recover() on the app-stop guard — apps left stopped by an " +
+			"interrupted backup stay down forever (the R-166 defect, un-fixed)")
+	}
+	if indexOfCall(names, "SetAppStopGuard") < 0 {
+		t.Fatal("func main() no longer hands the recovered guard to the backup manager — the manager " +
+			"would build a SECOND guard over the same file, i.e. one file with two owners")
+	}
+	if indexOfCall(names, "SetStopGuard") < 0 {
+		t.Fatal("func main() no longer wires the exporter's stop guard — the .fab export path would be " +
+			"the one uncovered stop-and-restart site, which is how a reader concludes the class is handled")
+	}
+}
+
+// TestMainWiresDesiredStateBackfill pins the Part-1.5 call.
+func TestMainWiresDesiredStateBackfill(t *testing.T) {
+	if indexOfCall(callsInMain(t, mainBody(t)), "BackfillDesiredState") < 0 {
+		t.Fatal("func main() no longer calls BackfillDesiredState — every existing app would stay on " +
+			"legacy inference until someone pressed a button on it")
+	}
+}
+
+// TestAppStopRecoveryPrecedesTheBootReconciler is §8.4's ORDERING requirement, and it is the reason
+// the recovery returns its result instead of pushing it through a notifier seam.
+//
+// The recovery must COMPLETE — not merely be reached — before `go runBootReconcile(...)` is
+// launched. If the boot reconciler ran first it would see an app the marker already explains, list
+// it as an unexplained boot orphan, and one fault would be reported as two.
+func TestAppStopRecoveryPrecedesTheBootReconciler(t *testing.T) {
+	names := callsInMain(t, mainBody(t))
+
+	recover := indexOfCall(names, "Recover")
+	bootrecon := indexOfCall(names, "runBootReconcile")
+	backfill := indexOfCall(names, "BackfillDesiredState")
+
+	if recover < 0 || bootrecon < 0 || backfill < 0 {
+		t.Fatalf("missing a call: Recover=%d runBootReconcile=%d BackfillDesiredState=%d", recover, bootrecon, backfill)
+	}
+	if recover >= bootrecon {
+		t.Fatal("the app-stop Recover no longer runs BEFORE the boot reconciler is launched — an app " +
+			"the marker explains would also be reported as an unexplained boot orphan (§8.4)")
+	}
+	if backfill >= bootrecon {
+		t.Fatal("the desired-state backfill no longer runs BEFORE the boot reconciler — the reconciler " +
+			"would decide from intent the backfill had not yet written")
+	}
+	if recover >= backfill {
+		t.Fatal("the backfill no longer runs AFTER the app-stop recovery — an app the recovery just " +
+			"restarted would still read as down and be left unrecorded")
+	}
+}
+
+// TestMainReportsTheInterruptedOperation pins §2.4: the recovery's outcome reaches the operator.
+//
+// The reporting call is deliberately far from the recovery (the notifier does not exist yet at
+// recovery time), which is exactly the distance across which a wiring gets dropped.
+func TestMainReportsTheInterruptedOperation(t *testing.T) {
+	body := mainBody(t)
+	names := callsInMain(t, body)
+
+	if indexOfCall(names, "NotifyBackupFailed") < 0 {
+		t.Fatal("func main() no longer reports an interrupted app-data operation to the operator — the " +
+			"controller died mid-backup and nobody is told (§2.4)")
+	}
+	// It must be guarded, not unconditional: a box with nothing to recover must not email an operator
+	// on every single boot.
+	guarded := false
+	ast.Inspect(body, func(n ast.Node) bool {
+		ifst, ok := n.(*ast.IfStmt)
+		if !ok || ifst.Cond == nil {
+			return true
+		}
+		bin, ok := ifst.Cond.(*ast.BinaryExpr)
+		if !ok {
+			return true
+		}
+		x, ok := bin.X.(*ast.Ident)
+		if !ok || x.Name != "appStopRecovery" {
+			return true
+		}
+		for _, name := range callsInMain(t, ifst.Body) {
+			if name == "NotifyBackupFailed" {
+				guarded = true
+			}
+		}
+		return true
+	})
+	if !guarded {
+		t.Fatal("the interrupted-operation alert is not guarded by `if appStopRecovery != nil` — every " +
+			"healthy boot would page the operator about a backup that was never interrupted")
+	}
+}

controller/cmd/controller/main.go

@@ -227,6 +227,31 @@ func main() {
 	// Recover FIRST (restart any stacks left stopped by a crash mid-quiesce), then start the loop.
 	quiesceLoop := startQuiesceLoop(ctx, cfg, sett, stackMgr, logger)
 
+	// --- R-166: recover apps left stopped by an interrupted app-data operation ---
+	// A volume dump, an offsite reconstitution or a `.fab` export stops the app, works on its data,
+	// and starts it again. A controller killed inside that window used to leave the app down with
+	// NOTHING on disk explaining it — and a stopped app has zero containers, which the boot
+	// reconciler below read as a deliberate customer stop and left alone, indefinitely.
+	//
+	// ORDERING IS LOAD-BEARING (§8.4) and this call must COMPLETE, not merely be reached, before the
+	// boot-reconcile goroutine is launched: an app the marker already explains must not also be
+	// reported as an unexplained boot orphan. Same position and same reason as the quiesce Recover
+	// immediately above.
+	// The guard is built HERE rather than taken from the backup manager because that manager is not
+	// constructed until ~40 lines below — and moving its construction up to suit this would be a far
+	// wider change than moving one object down. It is handed to the manager (SetAppStopGuard) and to
+	// the exporter later, so all three share ONE guard over ONE file.
+	appStopGuard := backup.NewAppStopGuard(filepath.Join(cfg.Paths.DataDir, "appstop-state.json"), logger)
+	appStopGuard.SetStarter(stackMgr)
+	appStopRecovery := appStopGuard.Recover()
+
+	// --- R-166: desired-state backfill (running-only) ---
+	// Converge the apps whose intent is unambiguous — deployed and observed UP — so the fleet stops
+	// depending on legacy inference without waiting for a button press. NEVER backfills "stopped":
+	// zero containers cannot distinguish a deliberate stop from a power cut, and that inference is
+	// the defect. Runs after the two recoveries so a just-restarted app is counted as running.
+	stackMgr.BackfillDesiredState()
+
 	// --- R-52: boot desired-state reconciliation ---
 	// A deployed app that missed its boot start used to stay down until a human noticed (F5: immich
 	// and calibre-web sat Exited for ~18 h while ten siblings came back). One bounded start-once
@@ -277,6 +302,9 @@ func main() {
 	}
 	if cfg.Backup.Enabled {
 		backupMgr = backup.NewManager(cfg, sett, logger)
+		// R-166: use the guard that already ran Recover at startup, not a second one over the same
+		// file (see SetAppStopGuard — one file, one owner).
+		backupMgr.SetAppStopGuard(appStopGuard)
 		backupMgr.SetStackProvider(stackProv)
 		backupMgr.SetVersion(Version)
 		// O4: restore-from-unit generates a replacement for an unrecoverable RESETTABLE secret
@@ -314,6 +342,19 @@ func main() {
 		quiesceLoop.SetTierNotifier(quiesceTierNotifier{n: notifier})
 	}
 
+	// R-166 §2.4: report an interrupted app-data operation to the operator, HERE, because the
+	// recovery itself had to run before the boot reconciler (line ~236) and the notifier does not
+	// exist until this line. An interrupted operation means the controller died mid-backup and that
+	// backup did not complete — operator-grade news even when every app came back.
+	//
+	// It rides the EXISTING `backup_failed` event type rather than a new one: a new type needs the
+	// hub's allowedEventTypes + customerMessages pair changed, which is a wire change, and this
+	// release ships no hub change. A controller emitting an unlisted type gets a flat 400 from
+	// POST /event. Reachability is covered by TestAppStopRecoveryIsWired.
+	if appStopRecovery != nil {
+		notifier.NotifyBackupFailed(appStopRecovery.Message(), appStopRecovery.Detail())
+	}
+
 	// --- Initialize the app-email SMTP shim (mailrelay) ---
 	// In-process shim: apps → shim → hub → Resend (the Resend key stays hub-side). It runs only
 	// when the controller has a hub (URL+key) AND the operational kill-switch is on; the runtime
@@ -975,6 +1016,11 @@ func main() {
 	exportProv := &exportAdapter{mgr: stackMgr, encKey: encKey}
 	appExporter := appexport.NewExporter(exportProv, logger, Version)
 	appExporter.SetDebug(cfg.Logging.Level == "debug")
+	// R-166: the exporter stops apps too (export with "stop the app first"), so it shares the backup
+	// manager's ONE marker file rather than opening a second one — one file, one recovery. Without
+	// this the export path would be the uncovered sibling of two covered ones, which is how a reader
+	// concludes the whole class is handled (§2.2).
+	appExporter.SetStopGuard(exportStopGuard{g: appStopGuard})
 	apiRouter.SetDebug(cfg.Logging.Level == "debug")
 
 	// --- Initialize web server ---
@@ -1738,6 +1784,17 @@ func (a *exportAdapter) GetStacksBaseDir() string {
 	return a.mgr.GetStacksBaseDir()
 }
 
+// exportStopGuard adapts *backup.AppStopGuard to the exporter's reason-free seam (R-166). The reason
+// is supplied HERE rather than passed in, so backup.ReasonAppExport's value exists in exactly one
+// place and the two packages cannot drift apart.
+type exportStopGuard struct{ g *backup.AppStopGuard }
+
+func (a exportStopGuard) Begin(opID string, stacks []string) error {
+	return a.g.Begin(opID, backup.ReasonAppExport, stacks)
+}
+
+func (a exportStopGuard) End() { a.g.End() }
+
 func (a *exportAdapter) SaveEncryptedAppConfig(stackDir string, env map[string]string) error {
 	meta := stacks.LoadMetadata(stackDir)
 	sensitiveVars := stacks.SensitiveEnvVars(&meta)
@@ -1745,6 +1802,12 @@ func (a *exportAdapter) SaveEncryptedAppConfig(stackDir string, env map[string]s
 		Deployed:   true,
 		DeployedAt: time.Now().Format(time.RFC3339),
 		Env:        env,
+		// R-166 — a CUSTOMER-INTENT POINT, and the one that is not the API action switch. Importing
+		// a `.fab` bundle is the customer installing that app on this box, and the import path starts
+		// it (appexport/restore.go). Without this the app would come back from a restore with NO
+		// recorded intent and fall to legacy boot behaviour — meaning a power cut days later would
+		// strand it, which is exactly the failure this release exists to remove.
+		DesiredState: stacks.DesiredStateRunning,
 	}
 	return stacks.SaveAppConfig(stackDir, cfg, a.encKey, sensitiveVars)
 }