D62: restore copies kept 30 days, listed with their date, deletable by hand (typed name, never a needed copy, operator event); D63: a test-build-only fault switch for the R-1256 H4/H5 live proofs — tests red first

controller/cmd/controller/d63_faultpoint_on_test.go

@@ -0,0 +1,34 @@
+//go:build faultinject
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// D63 (test build only): "<skip> <fail>" lets <skip> passes through, fails the next <fail>, then removes the file.
+func TestD63_TheTestBuildFailsTheArmedPointOnDemand(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("FELHOM_FAULT_DIR", dir)
+	p := filepath.Join(dir, "stop-radicale")
+	if err := os.WriteFile(p, []byte("1 1\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := faultPoint("stop-radicale"); err != nil {
+		t.Fatalf("the first pass must go through: %v", err)
+	}
+	if err := faultPoint("stop-other"); err != nil {
+		t.Fatalf("another point is not armed: %v", err)
+	}
+	if err := faultPoint("stop-radicale"); err == nil {
+		t.Fatal("the second pass must fail")
+	}
+	if _, err := os.Stat(p); !os.IsNotExist(err) {
+		t.Fatalf("the spent file must be gone: %v", err)
+	}
+	if err := faultPoint("stop-radicale"); err != nil {
+		t.Fatalf("a spent point fails nothing: %v", err)
+	}
+}

controller/cmd/controller/d63_faultpoint_test.go

@@ -0,0 +1,40 @@
+//go:build !faultinject
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// goBuildTag is a `go build -tags …` flag (build.sh's own `--print-tags` option is not one).
+var goBuildTag = regexp.MustCompile(`\s-tags[ =]`)
+
+// D63: the fault switch exists only in a `-tags faultinject` test build. In this (the release) build an armed fault
+// file injects nothing, and neither build.sh nor the Dockerfile passes a build tag.
+// RED-PROOF: build the release build with the switch (drop the !faultinject line from faultpoint_off.go and the
+// faultinject line from faultpoint_on.go) → the armed point fails → FAILS.
+func TestD63_TheReleaseBuildHasNoFaultSwitch(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv("FELHOM_FAULT_DIR", dir)
+	for _, p := range []string{"defwrite-after-compose", "stop-radicale"} {
+		if err := os.WriteFile(filepath.Join(dir, p), []byte("0 5\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		if err := faultPoint(p); err != nil {
+			t.Fatalf("the release build injected a fault at %s: %v", p, err)
+		}
+	}
+	for _, f := range []string{"../../build.sh", "../../Dockerfile"} {
+		b, err := os.ReadFile(f)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if goBuildTag.MatchString(string(b)) || strings.Contains(string(b), "faultinject") {
+			t.Fatalf("%s passes a build tag — the release image could carry the fault switch", f)
+		}
+	}
+}

controller/cmd/controller/faultpoint_off.go

@@ -0,0 +1,9 @@
+//go:build !faultinject
+
+package main
+
+// faultPoint is the D63 live-proof fault switch (operator ruling 2026-10-29: prove R-1256 H4 and H5's failed stop live,
+// with a fault switch in a TEST build only). In every build without the `faultinject` tag — the release image among
+// them: build.sh and the Dockerfile pass no build tags — it does nothing and reads nothing. Pinned by
+// TestD63_TheReleaseBuildHasNoFaultSwitch; the switch itself is faultpoint_on.go.
+func faultPoint(string) error { return nil }

controller/cmd/controller/faultpoint_on.go

@@ -0,0 +1,64 @@
+//go:build faultinject
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// faultPoint (TEST BUILDS ONLY — `go build -tags faultinject`; never the release image, faultpoint_off.go) fails a named
+// point on demand, for the D63 live proofs of R-1256 H4 (a definition write that fails after the compose was written)
+// and H5 (a put-back whose stop fails). Armed by a file <dir>/<point> holding "<skip> <fail>": the next <skip> passes go
+// through, the <fail> after them fail, and the file is removed when both are spent. <dir> is $FELHOM_FAULT_DIR, else
+// /opt/docker/felhom-controller/faults (the controller's data volume). A file that does not parse fails nothing.
+var faultMu sync.Mutex
+
+func faultPoint(point string) error {
+	faultMu.Lock()
+	defer faultMu.Unlock()
+	dir := os.Getenv("FELHOM_FAULT_DIR")
+	if dir == "" {
+		dir = "/opt/docker/felhom-controller/faults"
+	}
+	p := filepath.Join(dir, point)
+	data, err := os.ReadFile(p)
+	if err != nil {
+		return nil
+	}
+	f := strings.Fields(string(data))
+	if len(f) != 2 {
+		log.Printf("[WARN] [faultinject] %s: %q does not parse as \"<skip> <fail>\" — nothing injected", p, strings.TrimSpace(string(data)))
+		return nil
+	}
+	skip, e1 := strconv.Atoi(f[0])
+	fail, e2 := strconv.Atoi(f[1])
+	if e1 != nil || e2 != nil || skip < 0 || fail < 0 {
+		return nil
+	}
+	write := func() {
+		if skip == 0 && fail == 0 {
+			_ = os.Remove(p)
+			return
+		}
+		_ = os.WriteFile(p, []byte(fmt.Sprintf("%d %d\n", skip, fail)), 0o600)
+	}
+	if skip > 0 {
+		skip--
+		write()
+		log.Printf("[WARN] [faultinject] %s: passed (skip left %d, fail left %d)", point, skip, fail)
+		return nil
+	}
+	if fail > 0 {
+		fail--
+		write()
+		log.Printf("[ERROR] [faultinject] %s: FAULT INJECTED (fail left %d)", point, fail)
+		return fmt.Errorf("FAULT INJECTED at %s (D63 test build)", point)
+	}
+	return nil
+}

controller/cmd/controller/main.go

@@ -3191,6 +3191,9 @@ func (a *stackAdapter) ListDeployedStacks() []backup.StackSummary {
 }
 
 func (a *stackAdapter) StopStack(name string) error {
+	if err := faultPoint("stop-" + name); err != nil { // D63: a no-op outside a -tags faultinject test build
+		return err
+	}
 	return a.mgr.StopStack(name)
 }
 
@@ -3435,6 +3438,9 @@ func (a *stackAdapter) RecreateStackDefinitionFromUnit(name, composeSrcDir strin
 			return fmt.Errorf("restoring %s from unit: %w", fname, err)
 		}
 	}
+	if err := faultPoint("defwrite-after-compose"); err != nil { // D63: a no-op outside a -tags faultinject test build
+		return err
+	}
 	// R-1057 (v0.314.0): the gate as the backup recorded it — the unit's app.yaml, read BEFORE the write.
 	gate := stacks.ReadGateAtBackup(filepath.Join(composeSrcDir, "app.yaml"))
 	if err := a.mgr.PersistUnitRedeployConfigGate(name, fullEnv, gate); err != nil {