hub v0.150.0: app_repaired_after_crash — allow-listed + operator-only, counts line in the operator mail (R-1007); decision 236 recorded
gates / gates (push) Successful in 6m40s

The operator's half of the repair after an unclean boot (controller 0.311.0): counts only, never a
file name; warning mails, info is recorded. Registered in allowedEventTypes and operatorOnlyEvents in
the same commit, plus the per-app cooldown list; tests pin the pair, a household that enabled the type
gets nothing, and the counts line (red-proofed). Decision 236 (operator, 2026-10-14): the box asks
Immich through Immich's own job queue. R-1007 updated; the scripts' run on 9202 added as evidence.

Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
This commit is contained in:
2026-10-14 16:58:35 +02:00
parent 5d39171e90
commit 822d039344
10 changed files with 155 additions and 3 deletions
+15
View File
@@ -1,3 +1,18 @@
## v0.150.0 — an app's repair after a power cut reaches the operator, counts only (R-1007) (2026-10-14)
Needs **controller ≥ 0.311.0** to see anything (it sends the event). Older boxes never send it — nothing changes for them.
- **`app_repaired_after_crash`** — what an app's `after_unclean_boot` run found and did after an unclean host boot:
empty files found / repaired / still broken, originals lost, the boot, the duration, or why it was skipped. Allow-listed
AND operator-only, registered in the same commit (`internal/api/handler.go`, `notify.operatorOnlyEvents`); a
per-app cooldown member (one app's event, no digest). The household's side is the lost-files note on its own app
page — never a mail with its file names. warning mails; info (all repaired) is recorded, not mailed.
- The operator mail carries one readable line: „Repair: Immich — after the unclean boot at …: 81 empty file(s) found,
81 repaired, 0 still broken; originals lost: 1; 4 s".
- Tests: `internal/api/chaosnight_events_test.go` (the pair), `internal/notify/app_repaired_after_crash_test.go` (a
household that enabled the type by name gets nothing; info is not mailed; the counts line), `r389` (the cooldown
list). Red-proofed: each of the three guards removed convicted its test.
## v0.149.0 — a box the household switched off is not an outage, for 14 days (R-998) (2026-10-13)
Needs **agent ≥ 0.155.0** to see anything: the agent's new `deliberate_off.requested_at` report field (household
@@ -111,3 +111,15 @@ func TestAppStoppedUnhealthyIsAHouseholdEvent(t *testing.T) {
}
}
}
// R-1007 (v0.150.0) — app_repaired_after_crash is allow-listed AND operator-only, pinned together.
// RED-PROOF: drop it from allowedEventTypes → "the controller's push would 400".
func TestAppRepairedAfterCrashIsAllowlistedAndOperatorOnly(t *testing.T) {
et := "app_repaired_after_crash"
if !allowedEventTypes[et] {
t.Fatalf("%s must be in allowedEventTypes — the controller's push would 400", et)
}
if !notify.IsOperatorOnly(et) {
t.Fatalf("%s must be operator-only — counts of a household's damaged files are support's", et)
}
}
+4
View File
@@ -2318,6 +2318,10 @@ var allowedEventTypes = map[string]bool{
// with NO copy on its box that brings it back whole — support must act. Operator-only
// (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
"app_hold_no_whole_copy": true,
// R-1007 (hub v0.150.0, controller v0.311.0, `09` §3 decision 236): what an app's after_unclean_boot run found
// and repaired after an unclean host boot — COUNTS ONLY. Operator-only (notify.operatorOnlyEvents) — registered in
// BOTH in the same commit; the household's side is the app page's lost-files note on its own box.
"app_repaired_after_crash": true,
// Special
"test": true,
}
@@ -0,0 +1,49 @@
package notify
import (
"io"
"log"
"strings"
"testing"
)
// R-1007 (v0.150.0, controller v0.311.0, `09` §3 decision 236) — app_repaired_after_crash reaches the OPERATOR and
// never the household, even a household that enabled it by name; warning mails, info is recorded only; the mail
// carries the counts as one line.
// RED-PROOF: drop it from operatorOnlyEvents → the customer gets a mail → fails.
func TestAppRepairedAfterCrash_OperatorOnlyEvenWhenTheHouseholdEnabledIt(t *testing.T) {
st := newDispStore(t)
if err := st.SaveNotificationPrefs("c1", "cust@example.com", []string{"app_repaired_after_crash", "node_down"}, 6); err != nil {
t.Fatal(err)
}
d := NewDispatcher(st, "test-key", "from@felhom.eu", "op@felhom.eu", true, log.New(io.Discard, "", 0))
sent := captureSeam(d)
details := `{"app":"Immich","stack_name":"immich","boot_at":"2026-10-14T12:07:26Z","found":81,"repaired":81,"still_broken":0,"lost":1,"lost_checked":true,"duration_s":4}`
d.ProcessEvent("c1", "app_repaired_after_crash", "warning", "Immich: ...", details, "controller")
if got := mailsFor(*sent, "cust@example.com"); len(got) != 0 {
t.Fatalf("the household got %d mail(s) — the counts are the operator's; its side is the app page's note", len(got))
}
if got := len(mailsFor(*sent, "op@felhom.eu")); got != 1 {
t.Fatalf("operator mails = %d, want 1", got)
}
// info (everything repaired): recorded, not mailed.
*sent = nil
d.ProcessEvent("c1", "app_repaired_after_crash", "info", "Immich: ...", strings.Replace(details, `"lost":1`, `"lost":0`, 1), "controller")
if got := len(*sent); got != 0 {
t.Fatalf("an all-repaired (info) run mailed %d time(s)", got)
}
}
// RED-PROOF: drop the renderAppRepair call → the body lacks the counts line → fails.
func TestAppRepairedAfterCrash_OperatorMailHasTheCountsLine(t *testing.T) {
_, body := FormatOperatorEmail("c1", "app_repaired_after_crash", "warning", "msg",
`{"app":"Immich","boot_at":"2026-10-14T12:07:26Z","found":81,"repaired":80,"still_broken":1,"lost":1,"lost_checked":true,"duration_s":4}`)
want := "Repair: Immich — after the unclean boot at 2026-10-14T12:07:26Z: 81 empty file(s) found, 80 repaired, 1 still broken; originals lost: 1; 4 s"
if !strings.Contains(body, want) {
t.Fatalf("body lacks %q:\n%s", want, body)
}
_, skipped := FormatOperatorEmail("c1", "app_repaired_after_crash", "warning", "msg", `{"app":"Immich","boot_at":"b","skipped":"not proven"}`)
if !strings.Contains(skipped, "SKIPPED after the unclean boot at b: not proven") {
t.Fatalf("skipped body:\n%s", skipped)
}
}
+7
View File
@@ -480,6 +480,9 @@ var perAppCooldownEvents = map[string]bool{
"app_hold_no_whole_copy": true,
// v0.123.0 (decision 28): one app's stop, no digest behind it.
"app_stopped_unhealthy": true,
// v0.150.0 (R-1007): one app's repair after an unclean boot, no digest behind it — two apps on one boot are two
// records, and a lost photo on the second must not be swallowed by the first's cooldown.
"app_repaired_after_crash": true,
}
// perAppCustomerCooldownEvents is the CUSTOMER-leg sibling of perAppCooldownEvents (v0.120.0). The
@@ -857,6 +860,10 @@ var operatorOnlyEvents = map[string]bool{
// operator-grade (the copies seen, per tier, with dates), and the act it calls for is support's.
// Registered in the same commit that mints it.
"app_hold_no_whole_copy": true,
// R-1007 (v0.150.0, controller v0.311.0, `09` §3 decision 236). An app's repair after an unclean host boot: counts
// of empty files found / repaired / still broken / originals lost. The household learns of a lost photo on its own
// app page (name and date, on its box); THESE counts are the operator's. Registered in the same commit that mints it.
"app_repaired_after_crash": true,
}
// IsOperatorOnly reports whether an event type is barred from customer dispatch. Exported so the
@@ -71,7 +71,8 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
// v0.121.0 widened it again, on purpose (R-636): an OOM storm is one app's event with no digest.
// v0.122.0 once more (R-659): a stranded held app is one app's event with no digest.
// v0.123.0 once more (decision 28): an unhealthy stop is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"}
// v0.150.0 once more (R-1007): an app's repair after an unclean boot is one app's event with no digest.
want := []string{"app_hold_no_whole_copy", "app_oom_storm", "app_repaired_after_crash", "app_start_failed", "app_stopped_unhealthy", "app_update_held", "app_update_undone"}
ok := len(perAppCooldownEvents) == len(want)
for _, w := range want {
ok = ok && perAppCooldownEvents[w]
@@ -81,7 +82,7 @@ func TestR389_TheAllowListHasExactlyOneMember(t *testing.T) {
for k := range perAppCooldownEvents {
got = append(got, k)
}
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+
t.Fatalf("perAppCooldownEvents = %v, want exactly [app_hold_no_whole_copy app_oom_storm app_repaired_after_crash app_start_failed app_stopped_unhealthy app_update_held app_update_undone]. Adding a member is the "+
"fenced act: the backup family's cooldown is coarse ON PURPOSE (R-97a, R-182) so one full "+
"disk sends one digest, not one mail per app. Read the fence before widening this.", got)
}
+39
View File
@@ -63,6 +63,13 @@ Message: %s`, customerID, eventType, severity, now, message)
// digest that renders badly still tells the operator something; a swallowed one does not.
}
// R-1007: the repair after an unclean boot, as one readable line of counts (the payload carries nothing else).
if eventType == "app_repaired_after_crash" {
if line, ok := renderAppRepair(detailsJSON); ok {
body += "\n" + line
}
}
if detailsJSON != "" && detailsJSON != "{}" {
body += fmt.Sprintf("\nDetails: %s", detailsJSON)
}
@@ -424,3 +431,35 @@ func stackNameOf(detailsJSON string) string {
}
return d.StackName
}
// renderAppRepair is app_repaired_after_crash's operator line (R-1007): the counts, the boot and the duration.
func renderAppRepair(detailsJSON string) (string, bool) {
var d struct {
App string `json:"app"`
BootAt string `json:"boot_at"`
Found int `json:"found"`
Repaired int `json:"repaired"`
StillBroken int `json:"still_broken"`
Lost int `json:"lost"`
LostChecked bool `json:"lost_checked"`
DurationS int64 `json:"duration_s"`
Skipped string `json:"skipped"`
Problem string `json:"problem"`
}
if json.Unmarshal([]byte(detailsJSON), &d) != nil || d.App == "" {
return "", false
}
if d.Skipped != "" {
return fmt.Sprintf("Repair: %s — SKIPPED after the unclean boot at %s: %s", d.App, d.BootAt, d.Skipped), true
}
lost := "not checked"
if d.LostChecked {
lost = fmt.Sprintf("%d", d.Lost)
}
line := fmt.Sprintf("Repair: %s — after the unclean boot at %s: %d empty file(s) found, %d repaired, %d still broken; originals lost: %s; %d s",
d.App, d.BootAt, d.Found, d.Repaired, d.StillBroken, lost, d.DurationS)
if d.Problem != "" {
line += "\nProblem: " + d.Problem
}
return line, true
}