clear_hold (D64 = B) + the box command through the running controller (R-1343), R-1340, R-1341 — tests red first; commit only

- clear_hold: a fifth verb in the closed operator-action list; lifts a hold of any reason in the RUNNING controller (memory + settings.json together), forgets the update's held flag, drops a waiting put-back from the app-stop marker (copy kept). Refused on a bad name or no hold. No new event type. - --clear-restore-hold asks the running controller over <data dir>/ctl/hold.sock (unix, 0600, peer uid checked, no TCP); edits settings.json only when no controller answers, and says so (in-guest restart). - R-1340: the operator's HELD STOPPED mail is built from the hold record (restore kind, mixed cause, failed put-back step, database replay or not, kept copy) — never the option-C database story by default. - R-1341: the OOM WARN line once per new kill (container, start time, kill count); event unchanged.
2026-10-29 12:49:46 +01:00
1 parent c7bb776e00
commit 5b3b541602
@@ -0,0 +1,140 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// R-1340: the operator's „HELD STOPPED" mail names the hold's real cause — one test per reason. Each asserts the words
+// that MUST be there and the words that must NOT (the option-C database story was the bug).
+//
+// RED-PROOF (2026-10-29): with mixedHoldText's body replaced by the pre-R-1340 sentence („its off-site database restore
+// failed. The database was rolled back to the pre-restore copy and the live definition written back, but its files
+// and named volumes are the snapshot's"), TestR1340_PutBackWithoutADatabase, _DefinitionWrite, _LocalRestore and
+// _InterruptedPutBack fail on „database" / „off-site" being present and their own step missing.
+
+const optionCStory = "files and named volumes are the snapshot's"
+
+func mustHave(t *testing.T, msg string, want ...string) {
+	t.Helper()
+	for _, w := range want {
+		if !strings.Contains(msg, w) {
+			t.Errorf("the mail must say %q:\n%s", w, msg)
+		}
+	}
+}
+
+func mustNotHave(t *testing.T, msg string, bad ...string) {
+	t.Helper()
+	for _, b := range bad {
+		if strings.Contains(strings.ToLower(msg), strings.ToLower(b)) {
+			t.Errorf("the mail must NOT say %q:\n%s", b, msg)
+		}
+	}
+}
+
+func TestR1340_UnhealthyStop(t *testing.T) {
+	msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "romm", At: "2026-10-29T08:00:00Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"})
+	mustHave(t, msg, `"romm"`, "crash loop", "no data was touched", "Start gives it one more try")
+	mustNotHave(t, msg, "database", "restore failed", optionCStory)
+	if leg != "unhealthy-stop" {
+		t.Errorf("leg %q", leg)
+	}
+	oom, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "romm", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "oom_storm", Trip: 2})
+	mustHave(t, oom, "out-of-memory storm", "stop 2 within 24 hours")
+}
+
+func TestR1340_UpdateFailed(t *testing.T) {
+	msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "kimai", At: "2026-10-29T03:10:00Z", Reason: settings.HoldReasonUpdateFailed,
+		UndoState: "half", CopyTier: 1, CopyDate: "2026-10-29T02:00:00Z"})
+	mustHave(t, msg, "update at 2026-10-29T03:10:00Z did not come up healthy", "undo failed half-way", "tier 1, 2026-10-29T02:00:00Z", "--clear-restore-hold")
+	mustNotHave(t, msg, "database", optionCStory)
+	if leg != "update-hold" {
+		t.Errorf("leg %q", leg)
+	}
+}
+
+func TestR1340_RestoreFailedWithADatabase(t *testing.T) {
+	msg, leg, reason := heldStoppedOperatorText(settings.RestoreHold{Stack: "paperless", Reason: settings.HoldReasonRestoreFailed, RestoreKind: settings.RestoreKindOffsite,
+		DBReplay: true, ReplayError: "ERROR: relation exists", RollbackErr: "psql: connection refused"})
+	mustHave(t, msg, "off-site restore failed at the database replay", "rollback to the pre-restore database copy also failed", "ERROR: relation exists", "psql: connection refused")
+	mustNotHave(t, msg, optionCStory)
+	if leg != "restore-hold" || reason != "psql: connection refused" {
+		t.Errorf("leg %q reason %q", leg, reason)
+	}
+}
+
+// The measured case: Radicale has no database; the volume could not be removed and the put-back's stop failed.
+func TestR1340_PutBackWithoutADatabase(t *testing.T) {
+	msg, leg, reason := heldStoppedOperatorText(settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
+		MixedCause: settings.MixedCausePutBack, FailedStep: "stopping the app before the put-back failed, so nothing was put back: compose down: timeout",
+		ReplayError: "volume rm -f radicale_data: volume is in use", SafetyDump: "/mnt/hdd/felhom-data/backups/restore-undo/radicale/20261029T081800Z"})
+	mustHave(t, msg, `"radicale"`, "off-site restore failed, and putting the live data back FAILED", "stopping the app before the put-back failed",
+		"compose down: timeout", "kept at /mnt/hdd/felhom-data/backups/restore-undo/radicale/20261029T081800Z", "volume is in use")
+	mustNotHave(t, msg, "database", "rolled back", optionCStory)
+	if leg != "restore-hold-mixed" || reason != "put-back failed: stopping the app before the put-back failed, so nothing was put back" {
+		t.Errorf("leg %q reason %q", leg, reason)
+	}
+}
+
+func TestR1340_PutBackAfterADatabaseReplay(t *testing.T) {
+	msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
+		MixedCause: settings.MixedCausePutBack, DBReplay: true, FailedStep: "putting the live volumes back failed: tar xf failed"})
+	mustHave(t, msg, "failed at the database replay", "putting the live volumes back failed: tar xf failed", "There is no copy of the live volumes")
+}
+
+func TestR1340_DefinitionWrite(t *testing.T) {
+	msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "gokapi", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindOffsite,
+		MixedCause: settings.MixedCauseDefinitionWrite, FailedStep: "writing the live app definition back failed: app.yaml write failed", ReplayError: "app.yaml write failed"})
+	mustHave(t, msg, "while writing the backup's app definition, before any data was touched", "writing the live app definition back failed", "data is the live data")
+	mustNotHave(t, msg, "database", optionCStory)
+}
+
+func TestR1340_LocalRestore(t *testing.T) {
+	msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "app", Reason: settings.HoldReasonRestoreMixed, RestoreKind: settings.RestoreKindLocal,
+		MixedCause: settings.MixedCausePutBack, FailedStep: "putting the live volumes back failed: tar xf failed", SafetyDump: "/srv/undo/app/1"})
+	mustHave(t, msg, "restore from its own backup failed", "putting the live volumes back failed")
+	mustNotHave(t, msg, "off-site", "database", optionCStory)
+}
+
+func TestR1340_PutBackPending(t *testing.T) {
+	msg, leg, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonPutBackPending, SafetyDump: "/mnt/hdd/undo/immich/1"})
+	mustHave(t, msg, "copy (/mnt/hdd/undo/immich/1) is on a drive that is not bound", "poured back and the app started when the drive is bound", "DROPS that put-back")
+	mustNotHave(t, msg, "database", optionCStory)
+	if leg != "putback-pending" {
+		t.Errorf("leg %q", leg)
+	}
+}
+
+func TestR1340_InterruptedPutBack(t *testing.T) {
+	msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "immich", Reason: settings.HoldReasonRestoreMixed, MixedCause: settings.MixedCauseInterruptedPutBack,
+		FailedStep: "putting the interrupted restore's copy back failed: compose down: timeout", SafetyDump: "/mnt/hdd/undo/immich/1"})
+	mustHave(t, msg, "cut by a stop could not be put back", "compose down: timeout", "kept at /mnt/hdd/undo/immich/1")
+	mustNotHave(t, msg, "database", "off-site", optionCStory)
+}
+
+// A mixed hold written before R-1340 names only what its record says.
+func TestR1340_LegacyMixedHoldAssumesNothing(t *testing.T) {
+	msg, _, _ := heldStoppedOperatorText(settings.RestoreHold{Stack: "x", At: "2026-10-01T00:00:00Z", Reason: settings.HoldReasonRestoreMixed, ReplayError: "boom"})
+	mustHave(t, msg, "stopped half-way", "The hold record says: boom")
+	mustNotHave(t, msg, "database", optionCStory)
+}
+
+// The notification reads the hold the writer persisted; when that write failed, it rebuilds one from the errors.
+func TestR1340_NotifyReadsTheHoldRecord(t *testing.T) {
+	rec := settings.RestoreHold{Stack: "radicale", Reason: settings.HoldReasonRestoreMixed, MixedCause: settings.MixedCausePutBack, FailedStep: "x"}
+	get := func(s string) (settings.RestoreHold, bool) { return rec, s == "radicale" }
+	if h := heldHoldFromNotify(get, "radicale", errors.New("e"), nil); h.MixedCause != settings.MixedCausePutBack {
+		t.Fatalf("the persisted record was not used: %+v", h)
+	}
+	h := heldHoldFromNotify(get, "other", errors.New("replay"), errors.New("rollback"))
+	if h.Reason != settings.HoldReasonRestoreFailed || h.RollbackErr != "rollback" || h.ReplayError != "replay" {
+		t.Fatalf("fallback hold %+v", h)
+	}
+	if h := heldHoldFromNotify(nil, "other", errors.New("replay"), nil); h.Reason != settings.HoldReasonRestoreMixed {
+		t.Fatalf("no rollback error = mixed, got %+v", h)
+	}
+}