v0.295.0: a box that was off at its backup time catches up once (R-871, decision 109); the missed-backup banner (decision 110); a late daily timer after a host suspend is skipped

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/nightchain/banner.go

@@ -0,0 +1,125 @@
+package nightchain
+
+import (
+	"fmt"
+	"time"
+)
+
+// ── The missed-backup banner (R-871, `09` decision 110 — the operator's idea) ──────────────────────────────────
+//
+// The household's dashboard says it plainly when a daily backup is missing: when the last one was, that the box
+// was off at the backup time (when the box's own record says so), and a suggested different time. It NEVER changes
+// the time by itself. The household can close it; it stays closed until the NEXT missed backup time, and it goes
+// away by itself after a successful night.
+//
+// "The box's own record of when it was on" is the controller's system-metrics table: one sample a minute, kept 30
+// days (internal/metrics). An hour with at least 30 samples counts as "on" in that hour.
+//
+// Pinned by banner_test.go (TestBanner_*).
+
+// StaleAfter: a daily tier with no success for this long shows the banner. 26 h = one night (24 h) plus the chain's
+// own two hours, the same line as the hub's backupStaleAfter — so the household and the operator see the same
+// fact at the same time. Decided by CC — operator may reverse.
+const StaleAfter = 26 * time.Hour
+
+// suggestMinDays: an hour is "usually on" when the box was on in it on at least this many of the last 7 days.
+const suggestMinDays = 5
+
+// HourDays counts, per Budapest hour of day, on how many of the last 7 days the box was on in that hour.
+type HourDays [24]int
+
+// BannerInput is everything the banner rule reads.
+type BannerInput struct {
+	Now               time.Time
+	Window            string // W, Budapest HH:MM
+	Loc               *time.Location
+	SeededAt          time.Time // the ledger's seed: before it nothing is known
+	DBDumpOK          time.Time // the last successful database dump
+	OffsiteConfigured bool
+	OffsiteOK         time.Time                               // the off-site tier's LastSuccess
+	Dismissed         time.Time                               // the missed W instant the household closed
+	OnAtW             func(t time.Time) (on bool, known bool) // was the box on at t (from the metrics record)
+	Hours             *HourDays                               // nil = no record
+}
+
+// Banner is what the dashboard shows.
+type Banner struct {
+	Show       bool
+	LastBackup time.Time // zero = never
+	DaysAgo    int
+	OffAt      string    // "02:30" when the record says the box was off at the last backup time; "" = unknown/was on
+	Suggest    string    // "21:00", or "" (no clear pattern — say only "choose a time when the box is usually on")
+	MissedAt   time.Time // the last backup time that was missed (what a dismissal records)
+}
+
+// ComputeBanner is the PURE rule.
+func ComputeBanner(in BannerInput) Banner {
+	last := in.DBDumpOK
+	if in.OffsiteConfigured && (in.OffsiteOK.IsZero() || in.OffsiteOK.Before(last)) {
+		last = in.OffsiteOK
+	}
+	known := last
+	if known.Before(in.SeededAt) {
+		known = in.SeededAt // nothing before the seed is judged (an upgrade must not show a banner at once)
+	}
+	if in.Now.Sub(known) <= StaleAfter {
+		return Banner{}
+	}
+	w, ok := LastInstant(in.Now, in.Window, in.Loc)
+	if !ok {
+		return Banner{}
+	}
+	if !w.After(in.Dismissed) {
+		return Banner{} // closed by the household, and no backup time has been missed since
+	}
+	b := Banner{Show: true, LastBackup: last, MissedAt: w}
+	if !last.IsZero() {
+		b.DaysAgo = int(in.Now.Sub(last).Hours() / 24)
+	}
+	if in.OnAtW != nil {
+		if on, known := in.OnAtW(w); known && !on {
+			b.OffAt = w.In(in.Loc).Format("15:04")
+		}
+	}
+	if in.Hours != nil {
+		b.Suggest = Suggest(*in.Hours, w.In(in.Loc).Hour())
+	}
+	return b
+}
+
+// Suggest proposes a window start: the LATEST hour H such that H, H+1 and H+2 were all "usually on" (the chain
+// needs about two hours: the dump at W, the second copy at W+1h, the off-site copy at W+1h45m). Latest, because a
+// later evening disturbs the household least. "" when there is no such hour, and "" when the CURRENT window's
+// span is already usually on (a new time would not help; the miss has another cause). Decided by CC — operator
+// may reverse.
+func Suggest(h HourDays, currentHour int) string {
+	usual := func(start int) bool {
+		for k := 0; k < 3; k++ {
+			if h[(start+k)%24] < suggestMinDays {
+				return false
+			}
+		}
+		return true
+	}
+	if usual(currentHour) {
+		return ""
+	}
+	for start := 23; start >= 0; start-- {
+		if usual(start) {
+			return fmt.Sprintf("%02d:00", start)
+		}
+	}
+	return ""
+}
+
+// HourDaysFrom folds per-UTC-hour sample counts (unix hour start → samples) into "on how many of the days was the
+// box on in this Budapest hour": an hour with at least 30 one-minute samples counts as on.
+func HourDaysFrom(counts map[int64]int, loc *time.Location) HourDays {
+	var h HourDays
+	for start, n := range counts {
+		if n >= 30 {
+			h[time.Unix(start, 0).In(loc).Hour()]++
+		}
+	}
+	return h
+}

controller/internal/nightchain/banner_test.go

@@ -0,0 +1,123 @@
+package nightchain
+
+import (
+	"testing"
+	"time"
+)
+
+// R-871 / decision 110 — the banner's rules. Red-proofs: catchup-2026-10-05/partB/red-proofs.txt.
+
+func laptop() *HourDays { // on 18:00–23:59 every day of the week (Tester 2's shape)
+	var h HourDays
+	for k := 18; k <= 23; k++ {
+		h[k] = 7
+	}
+	return &h
+}
+
+func offAt(on bool) func(time.Time) (bool, bool) {
+	return func(time.Time) (bool, bool) { return on, true }
+}
+
+func base(now time.Time) BannerInput {
+	return BannerInput{Now: now, Window: "02:30", Loc: bud, SeededAt: at(1, 12, 0), DBDumpOK: at(3, 2, 31),
+		OnAtW: offAt(false), Hours: laptop()}
+}
+
+// Shown: the last dump is 2+ days old, the box was off at 02:30, and the record suggests 21:00.
+// RED-PROOF: return Banner{} when stale (the pre-v0.295.0 silence) → "not shown".
+func TestBanner_ShownWithReasonAndSuggestion(t *testing.T) {
+	b := ComputeBanner(base(at(5, 19, 0)))
+	if !b.Show {
+		t.Fatal("not shown although the last backup is 2 days old")
+	}
+	if b.OffAt != "02:30" || b.Suggest != "21:00" || b.DaysAgo != 2 {
+		t.Fatalf("banner = %+v, want off at 02:30, suggest 21:00, 2 days ago", b)
+	}
+}
+
+// Not shown: a backup within the last 26 h.
+func TestBanner_NotShownWhenFresh(t *testing.T) {
+	in := base(at(5, 19, 0))
+	in.DBDumpOK = at(5, 2, 31)
+	if b := ComputeBanner(in); b.Show {
+		t.Fatalf("shown although the last dump is 16 h old: %+v", b)
+	}
+}
+
+// Not shown right after an upgrade (the ledger was seeded less than 26 h ago and knows no dump).
+func TestBanner_NotShownBeforeTheRecordKnows(t *testing.T) {
+	in := base(at(5, 19, 0))
+	in.SeededAt, in.DBDumpOK = at(5, 9, 0), time.Time{}
+	if b := ComputeBanner(in); b.Show {
+		t.Fatalf("shown on the day of the upgrade: %+v", b)
+	}
+}
+
+// Dismissed: closed until the NEXT missed backup time, then back.
+// RED-PROOF: drop the Dismissed check → "a closed banner came back without a new miss".
+func TestBanner_DismissedThenBackAfterANewMiss(t *testing.T) {
+	in := base(at(5, 19, 0))
+	b := ComputeBanner(in)
+	in.Dismissed = b.MissedAt // the household closed it (02:30 on the 5th)
+	if ComputeBanner(in).Show {
+		t.Fatal("a closed banner came back without a new miss")
+	}
+	in.Now = at(5, 23, 0) // same evening — still the same miss
+	if ComputeBanner(in).Show {
+		t.Fatal("a closed banner came back the same evening")
+	}
+	in.Now = at(6, 19, 0) // off again at 02:30 on the 6th
+	if !ComputeBanner(in).Show {
+		t.Fatal("the banner did not come back after the next missed backup time")
+	}
+}
+
+// Gone by itself after a successful night (e.g. the catch-up ran).
+func TestBanner_GoneAfterASuccess(t *testing.T) {
+	in := base(at(5, 19, 0))
+	in.DBDumpOK = at(5, 18, 20) // the catch-up's dump
+	if ComputeBanner(in).Show {
+		t.Fatal("still shown after a successful backup")
+	}
+}
+
+// The off-site tier counts when configured: a fresh dump with a 3-day-old off-site copy still shows.
+func TestBanner_OffsiteTierCounts(t *testing.T) {
+	in := base(at(5, 19, 0))
+	in.DBDumpOK = at(5, 18, 20)
+	in.OffsiteConfigured, in.OffsiteOK = true, at(2, 4, 20)
+	b := ComputeBanner(in)
+	if !b.Show || b.DaysAgo != 3 {
+		t.Fatalf("banner = %+v, want shown with the off-site copy's 3 days", b)
+	}
+}
+
+// No clear pattern → no suggestion ("choose a time when the box is usually on"); box ON at W → no "off at".
+// RED-PROOF: let Suggest return the first start without checking the days → a time is suggested from noise.
+func TestBanner_NoPatternNoSuggestion(t *testing.T) {
+	in := base(at(5, 19, 0))
+	var noisy HourDays
+	for k := 18; k <= 23; k++ {
+		noisy[k] = 2 // on some evenings — 2 of 7 days is not "usually"
+	}
+	in.Hours, in.OnAtW = &noisy, offAt(true)
+	b := ComputeBanner(in)
+	if !b.Show || b.Suggest != "" || b.OffAt != "" {
+		t.Fatalf("banner = %+v, want shown, no suggestion, no 'off at'", b)
+	}
+}
+
+// A box usually ON at its window gets no new time (the miss has another cause).
+func TestBanner_UsuallyOnAtWNoSuggestion(t *testing.T) {
+	var always HourDays
+	for k := range always {
+		always[k] = 7
+	}
+	if s := Suggest(always, 2); s != "" {
+		t.Fatalf("suggested %q for a box that is on at its window", s)
+	}
+	if s := Suggest(*laptop(), 2); s != "21:00" {
+		t.Fatalf("laptop suggestion %q, want 21:00", s)
+	}
+}

controller/internal/nightchain/nightchain.go

@@ -0,0 +1,365 @@
+// Package nightchain makes a missed night's backups up once, when the box comes back (R-871, `09` §3 decision 109,
+// design `07` §6.1.1).
+//
+// THE PROBLEM, MEASURED (2026-10-05 Part F spike, Tester 2 — a laptop switched off at night): the controller's
+// daily jobs always schedule the NEXT future time (scheduler.nextDailyRun), and nothing remembers that a night was
+// missed. A box that is off at its window W never gets its database dumps, its second copy or its off-site copy —
+// for ever, with no alarm.
+//
+// THE MECHANISM:
+//   - A LEDGER (`night-ledger.json` in the data dir) records when each of the three nightly backup legs last RAN TO
+//     ITS END. It is an attempt record and is used for ONE question only — "was this leg's last scheduled time
+//     missed?" — never as evidence that a backup exists (that is LastSuccess's job, R-100). A leg that ran and
+//     failed was NOT missed: failures have their own alarms.
+//   - On a controller START and on a host RESUME (a suspended laptop: Go timers run on CLOCK_MONOTONIC, which does
+//     not count suspended time), Evaluate asks the ledger which legs missed their last scheduled time. If any did,
+//     ONE catch-up is scheduled 15 minutes later (apps settle; a box switched on and off again at once does nothing).
+//   - The catch-up runs ONLY the backup legs, in their normal order (database dump, second copy, off-site copy).
+//     It never runs app updates or Docker steps — they restart apps and wait for a real night.
+//   - Several missed nights are ONE catch-up (the question is about the LAST scheduled time only). A normal night
+//     followed by a daytime restart is none. A power cut in the middle of the chain makes up only the legs that did
+//     not end.
+//   - A leg whose own scheduled time is less than 30 minutes away is left to its normal run.
+//   - Every leg — scheduled or catch-up — holds one mutex (`Ledger.LegLock`), so a catch-up and a scheduled leg
+//     never run at the same time. The whole-guest backup (agent-side, quiesce) waits while a catch-up runs
+//     (quiesce.Options.CatchUpFn) and the catch-up waits while a quiesce holds the apps (QuiesceBusy).
+//
+// A ledger that does not exist yet (the first start on this release) is SEEDED at that moment and nothing before
+// it counts as missed — an upgrade must not set off a surprise catch-up on every box.
+//
+// Pinned by nightchain_test.go (TestCatchUp_*), each rule with a red-proof.
+package nightchain
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
+)
+
+// Leg is one nightly backup leg.
+type Leg string
+
+const (
+	LegDBDump  Leg = "db-dump"
+	LegTier2   Leg = "tier2"
+	LegOffsite Leg = "offsite"
+)
+
+// Order is the night chain's own order (07 §6.1).
+var Order = []Leg{LegDBDump, LegTier2, LegOffsite}
+
+const (
+	// DefaultDelay: the catch-up waits this long after the trigger (decision 109's "soon after it is switched on").
+	DefaultDelay = 15 * time.Minute
+	// leaveToNormal: a leg whose own scheduled time is closer than this runs normally, not in the catch-up.
+	leaveToNormal = 30 * time.Minute
+	// quiesceWaitMax: how long a catch-up waits for a whole-guest backup that holds the apps.
+	quiesceWaitMax = 2 * time.Hour
+)
+
+type ledgerData struct {
+	SeededAt time.Time         `json:"seeded_at"`
+	Ended    map[Leg]time.Time `json:"ended"` // ATTEMPT record: the leg ran to its end (success or failure)
+	// DBDumpOK is the last SUCCESSFUL database dump (the banner's evidence; the off-site tier has its own
+	// LastSuccess in settings). Written only on success (R-100's rule).
+	DBDumpOK        time.Time `json:"db_dump_ok,omitempty"`
+	LastCatchUp     time.Time `json:"last_catch_up,omitempty"`
+	BannerDismissed time.Time `json:"banner_dismissed_through,omitempty"` // the missed W instant the household closed
+}
+
+// Ledger is the persisted night record.
+type Ledger struct {
+	mu   sync.Mutex
+	path string
+	d    ledgerData
+	// LegLock is held by every leg run, scheduled or catch-up.
+	LegLock sync.Mutex
+}
+
+// Open reads the ledger, or seeds a new one at now.
+func Open(path string, now time.Time) (*Ledger, error) {
+	l := &Ledger{path: path, d: ledgerData{Ended: map[Leg]time.Time{}}}
+	b, err := os.ReadFile(path)
+	switch {
+	case err == nil:
+		if jerr := json.Unmarshal(b, &l.d); jerr != nil {
+			return nil, fmt.Errorf("nightchain: ledger unreadable: %w", jerr)
+		}
+		if l.d.Ended == nil {
+			l.d.Ended = map[Leg]time.Time{}
+		}
+	case os.IsNotExist(err):
+		l.d.SeededAt = now.UTC()
+		if serr := l.saveLocked(); serr != nil {
+			return nil, serr
+		}
+	default:
+		return nil, fmt.Errorf("nightchain: ledger: %w", err)
+	}
+	return l, nil
+}
+
+func (l *Ledger) saveLocked() error {
+	b, _ := json.MarshalIndent(l.d, "", "  ")
+	if err := os.MkdirAll(filepath.Dir(l.path), 0o755); err != nil {
+		return err
+	}
+	tmp := l.path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o644); err != nil {
+		return err
+	}
+	return os.Rename(tmp, l.path)
+}
+
+// MarkEnded records that a leg ran to its end (success or failure).
+func (l *Ledger) MarkEnded(leg Leg, at time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.d.Ended[leg] = at.UTC()
+	_ = l.saveLocked()
+}
+
+// MarkDBDumpOK records a successful database dump.
+func (l *Ledger) MarkDBDumpOK(at time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.d.DBDumpOK = at.UTC()
+	_ = l.saveLocked()
+}
+
+func (l *Ledger) markCatchUp(at time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.d.LastCatchUp = at.UTC()
+	_ = l.saveLocked()
+}
+
+// DismissBanner records that the household closed the banner for every miss up to `through`.
+func (l *Ledger) DismissBanner(through time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.d.BannerDismissed = through.UTC()
+	_ = l.saveLocked()
+}
+
+// Snapshot returns a copy of the record (for the banner and the debug page).
+func (l *Ledger) Snapshot() (seeded, dbDumpOK, lastCatchUp, dismissed time.Time, ended map[Leg]time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	ended = map[Leg]time.Time{}
+	for k, v := range l.d.Ended {
+		ended[k] = v
+	}
+	return l.d.SeededAt, l.d.DBDumpOK, l.d.LastCatchUp, l.d.BannerDismissed, ended
+}
+
+// legOffsets: each leg's time relative to W — taken from backupwindow.LegTimes so the two can never disagree.
+func legTimes(window string) map[Leg]string {
+	db, t2, off := backupwindow.LegTimes(window)
+	return map[Leg]string{LegDBDump: db, LegTier2: t2, LegOffsite: off}
+}
+
+// LastInstant is the most recent occurrence of the Budapest clock time hhmm at or before now.
+func LastInstant(now time.Time, hhmm string, loc *time.Location) (time.Time, bool) {
+	min, err := backupwindow.ParseHHMM(hhmm)
+	if err != nil {
+		return time.Time{}, false
+	}
+	n := now.In(loc)
+	t := time.Date(n.Year(), n.Month(), n.Day(), min/60, min%60, 0, 0, loc)
+	if t.After(n) {
+		t = time.Date(n.Year(), n.Month(), n.Day()-1, min/60, min%60, 0, 0, loc)
+	}
+	return t, true
+}
+
+// NextInstant is the next occurrence of hhmm strictly after now.
+func NextInstant(now time.Time, hhmm string, loc *time.Location) (time.Time, bool) {
+	last, ok := LastInstant(now, hhmm, loc)
+	if !ok {
+		return time.Time{}, false
+	}
+	return time.Date(last.Year(), last.Month(), last.Day()+1, last.Hour(), last.Minute(), 0, 0, loc), true
+}
+
+// Missed is the PURE rule: the legs (in chain order) whose last scheduled time passed after the ledger was seeded
+// and that have not run to their end since — minus a leg whose next scheduled time is under 30 minutes away.
+// lastW is the most recent missed leg instant (the banner's "the box was off at …").
+func (l *Ledger) Missed(now time.Time, window string, loc *time.Location) (legs []Leg, lastW time.Time) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	lt := legTimes(window)
+	for _, leg := range Order {
+		inst, ok := LastInstant(now, lt[leg], loc)
+		if !ok || !inst.After(l.d.SeededAt) {
+			continue
+		}
+		if !l.d.Ended[leg].Before(inst) {
+			continue // ran to its end at or after its last scheduled time
+		}
+		if next, ok := NextInstant(now, lt[leg], loc); ok && next.Sub(now) < leaveToNormal {
+			continue // its normal run is about to happen
+		}
+		legs = append(legs, leg)
+		if inst.After(lastW) {
+			lastW = inst
+		}
+	}
+	return legs, lastW
+}
+
+// CatchUp runs at most one catch-up at a time.
+type CatchUp struct {
+	Ledger      *Ledger
+	Window      func() string                       // the household's backup window W (settings > config > 02:30)
+	Legs        map[Leg]func(context.Context) error // the leg bodies — WITHOUT the update leg
+	QuiesceBusy func() bool                         // a whole-guest backup holds the apps now
+	Event       func(missedAt time.Time)            // the household's timeline line
+	Logger      *log.Logger
+	Delay       time.Duration
+	Loc         *time.Location
+	Now         func() time.Time
+	// Sleep waits d or until ctx ends (false). A seam for tests.
+	Sleep func(ctx context.Context, d time.Duration) bool
+
+	mu      sync.Mutex
+	pending bool
+	running bool
+}
+
+func (c *CatchUp) now() time.Time {
+	if c.Now != nil {
+		return c.Now()
+	}
+	return time.Now()
+}
+
+func (c *CatchUp) sleep(ctx context.Context, d time.Duration) bool {
+	if c.Sleep != nil {
+		return c.Sleep(ctx, d)
+	}
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-t.C:
+		return true
+	}
+}
+
+func (c *CatchUp) delay() time.Duration {
+	if c.Delay > 0 {
+		return c.Delay
+	}
+	return DefaultDelay
+}
+
+// Running reports whether a catch-up is running its legs now (the whole-guest backup waits for it).
+func (c *CatchUp) Running() bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.running
+}
+
+// Evaluate is the trigger (controller start, host resume). It schedules ONE catch-up when a leg was missed and
+// none is pending; it returns what it scheduled (nil = nothing missed or one already pending). The work runs in
+// its own goroutine; Wait-style callers use EvaluateSync.
+func (c *CatchUp) Evaluate(ctx context.Context, why string) []Leg {
+	legs, lastW := c.Ledger.Missed(c.now(), c.Window(), c.Loc)
+	if len(legs) == 0 {
+		c.Logger.Printf("[INFO] [catch-up] %s: no backup leg missed its last scheduled time — nothing to make up", why)
+		return nil
+	}
+	c.mu.Lock()
+	if c.pending || c.running {
+		c.mu.Unlock()
+		c.Logger.Printf("[INFO] [catch-up] %s: missed %v, but a catch-up is already scheduled", why, legs)
+		return nil
+	}
+	c.pending = true
+	c.mu.Unlock()
+	c.Logger.Printf("[INFO] [catch-up] %s: the box missed %v (last scheduled %s) — ONE catch-up in %s (backup legs only; app updates wait for a real night)",
+		why, legs, lastW.In(c.Loc).Format("2006-01-02 15:04"), c.delay())
+	go c.run(ctx, why)
+	return legs
+}
+
+// run waits, re-reads the ledger (a normal run may have happened meanwhile), waits out a whole-guest backup, then
+// runs the still-missed legs in chain order.
+func (c *CatchUp) run(ctx context.Context, why string) {
+	defer func() {
+		c.mu.Lock()
+		c.pending, c.running = false, false
+		c.mu.Unlock()
+	}()
+	if !c.sleep(ctx, c.delay()) {
+		return
+	}
+	legs, lastW := c.Ledger.Missed(c.now(), c.Window(), c.Loc)
+	if len(legs) == 0 {
+		c.Logger.Printf("[INFO] [catch-up] (%s) nothing is missed any more — not run", why)
+		return
+	}
+	for waited := time.Duration(0); c.QuiesceBusy != nil && c.QuiesceBusy(); waited += time.Minute {
+		if waited >= quiesceWaitMax {
+			c.Logger.Printf("[WARN] [catch-up] (%s) a whole-guest backup held the apps for %s — catch-up dropped; the next start or resume tries again", why, waited)
+			return
+		}
+		if !c.sleep(ctx, time.Minute) {
+			return
+		}
+	}
+	c.mu.Lock()
+	c.running = true
+	c.mu.Unlock()
+	start := c.now()
+	var names []string
+	for _, leg := range legs {
+		fn := c.Legs[leg]
+		if fn == nil {
+			continue
+		}
+		c.Logger.Printf("[INFO] [catch-up] running the missed %s leg", leg)
+		if err := fn(ctx); err != nil {
+			c.Logger.Printf("[WARN] [catch-up] the %s leg ended with an error (its own alarm reports it): %v", leg, err)
+		}
+		names = append(names, string(leg))
+	}
+	c.Ledger.markCatchUp(c.now())
+	sort.Strings(names)
+	c.Logger.Printf("[INFO] [catch-up] done: %s in %s (missed at %s)", strings.Join(names, ", "),
+		c.now().Sub(start).Round(time.Second), lastW.In(c.Loc).Format("2006-01-02 15:04"))
+	if c.Event != nil {
+		c.Event(lastW)
+	}
+}
+
+// ResumeWatch calls onResume when, between two ticks, the WALL clock advanced more than gap beyond the MONOTONIC
+// clock — the shape of a host suspend/resume: Go's timers and monotonic readings run on CLOCK_MONOTONIC, which does
+// not count suspended time, while the wall clock does. Seams: wall() is the wall clock, mono() a monotonic elapsed
+// duration (production: time.Now().Round(0) and time.Since(a fixed start)). Pinned by TestResumeWatch_*.
+func ResumeWatch(ctx context.Context, tick <-chan time.Time, wall func() time.Time, mono func() time.Duration, gap time.Duration, onResume func(slept time.Duration)) {
+	pw, pm := wall(), mono()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick:
+		}
+		cw, cm := wall(), mono()
+		if d := cw.Sub(pw) - (cm - pm); d > gap {
+			onResume(d)
+		}
+		pw, pm = cw, cm
+	}
+}

controller/internal/nightchain/nightchain_test.go

@@ -0,0 +1,248 @@
+package nightchain
+
+import (
+	"context"
+	"io"
+	"log"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+)
+
+// R-871 (`09` decision 109). Every rule of the catch-up, each with a COMPANION RED-PROOF recorded in
+// felhom.eu/documentation/audits/catchup-2026-10-05/partA/red-proofs.txt.
+
+var bud = func() *time.Location { l, _ := time.LoadLocation("Europe/Budapest"); return l }()
+
+func at(day, hh, mm int) time.Time { return time.Date(2026, 10, day, hh, mm, 0, 0, bud) }
+
+type harness struct {
+	l      *Ledger
+	c      *CatchUp
+	mu     sync.Mutex
+	ran    []Leg
+	events []time.Time
+	slept  []time.Duration
+	now    time.Time
+	busy   int // QuiesceBusy answers true this many times
+	done   chan struct{}
+}
+
+func newHarness(t *testing.T, seeded, now time.Time) *harness {
+	t.Helper()
+	l, err := Open(filepath.Join(t.TempDir(), "night-ledger.json"), seeded)
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := &harness{l: l, now: now, done: make(chan struct{}, 4)}
+	leg := func(g Leg) func(context.Context) error {
+		return func(context.Context) error {
+			h.mu.Lock()
+			h.ran = append(h.ran, g)
+			h.mu.Unlock()
+			l.MarkEnded(g, h.clock())
+			return nil
+		}
+	}
+	h.c = &CatchUp{
+		Ledger: l, Window: func() string { return "02:30" }, Loc: bud,
+		Legs:   map[Leg]func(context.Context) error{LegDBDump: leg(LegDBDump), LegTier2: leg(LegTier2), LegOffsite: leg(LegOffsite)},
+		Logger: log.New(io.Discard, "", 0),
+		Now:    h.clock,
+		Sleep: func(_ context.Context, d time.Duration) bool {
+			h.mu.Lock()
+			h.slept = append(h.slept, d)
+			h.now = h.now.Add(d)
+			h.mu.Unlock()
+			return true
+		},
+		QuiesceBusy: func() bool {
+			h.mu.Lock()
+			defer h.mu.Unlock()
+			if h.busy > 0 {
+				h.busy--
+				return true
+			}
+			return false
+		},
+		Event: func(w time.Time) {
+			h.mu.Lock()
+			h.events = append(h.events, w)
+			h.mu.Unlock()
+			h.done <- struct{}{}
+		},
+	}
+	return h
+}
+
+func (h *harness) clock() time.Time { h.mu.Lock(); defer h.mu.Unlock(); return h.now }
+
+func (h *harness) wait(t *testing.T) {
+	t.Helper()
+	select {
+	case <-h.done:
+	case <-time.After(3 * time.Second):
+		t.Fatal("the catch-up never finished")
+	}
+}
+
+// Off at W (02:30 on the 5th), switched on at 09:00: ONE catch-up, 15 minutes later, all three legs in chain order.
+// RED-PROOF: make Missed return nothing (the pre-v0.295.0 behaviour) → "no catch-up was scheduled".
+func TestCatchUp_OffAtWThenOn_OneCatchUpAfter15Min(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
+	h.l.MarkEnded(LegDBDump, at(4, 2, 31)) // the night of the 4th ran
+	h.l.MarkEnded(LegTier2, at(4, 3, 31))
+	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
+	legs := h.c.Evaluate(context.Background(), "start")
+	if len(legs) != 3 {
+		t.Fatalf("no catch-up was scheduled for a box off across W (missed %v)", legs)
+	}
+	h.wait(t)
+	if len(h.slept) == 0 || h.slept[0] != DefaultDelay {
+		t.Fatalf("the catch-up must wait %s first, slept %v", DefaultDelay, h.slept)
+	}
+	if got := []Leg{LegDBDump, LegTier2, LegOffsite}; len(h.ran) != 3 || h.ran[0] != got[0] || h.ran[1] != got[1] || h.ran[2] != got[2] {
+		t.Fatalf("legs ran %v, want the chain order %v", h.ran, got)
+	}
+	if len(h.events) != 1 || !h.events[0].Equal(at(5, 4, 15)) {
+		t.Fatalf("household line %v, want one for the missed 04:15 leg", h.events)
+	}
+}
+
+// On at W (the night ran) then a daytime restart: no catch-up.
+// RED-PROOF: drop the `!Ended.Before(inst)` check → the restart makes up a night that ran.
+func TestCatchUp_NightRan_DaytimeRestart_None(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 11, 0))
+	h.l.MarkEnded(LegDBDump, at(5, 2, 31))
+	h.l.MarkEnded(LegTier2, at(5, 3, 31))
+	h.l.MarkEnded(LegOffsite, at(5, 4, 20))
+	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
+		t.Fatalf("a normal night was made up again: %v", legs)
+	}
+}
+
+// Two missed nights: ONE catch-up, each leg once.
+func TestCatchUp_TwoMissedNights_One(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(6, 18, 0))
+	h.l.MarkEnded(LegDBDump, at(4, 2, 31))
+	h.l.MarkEnded(LegTier2, at(4, 3, 31))
+	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
+	h.c.Evaluate(context.Background(), "start")
+	if again := h.c.Evaluate(context.Background(), "resume"); again != nil {
+		t.Fatalf("a second catch-up was scheduled while one was pending: %v", again)
+	}
+	h.wait(t)
+	if len(h.ran) != 3 {
+		t.Fatalf("two missed nights ran %v — want each leg once", h.ran)
+	}
+	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
+		t.Fatalf("after the catch-up nothing is missed any more, got %v", legs)
+	}
+}
+
+// A power cut in the middle of the chain: only the legs that did not end.
+func TestCatchUp_PowerCutMidChain_FinishesTheRest(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 10, 0))
+	h.l.MarkEnded(LegDBDump, at(5, 2, 31)) // the dump ended, then the power went
+	h.l.MarkEnded(LegTier2, at(4, 3, 31))
+	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
+	h.c.Evaluate(context.Background(), "start")
+	h.wait(t)
+	if len(h.ran) != 2 || h.ran[0] != LegTier2 || h.ran[1] != LegOffsite {
+		t.Fatalf("ran %v, want only tier2 + offsite", h.ran)
+	}
+}
+
+// The first start on this release seeds the ledger: nothing before it is "missed" (no surprise catch-up on upgrade).
+func TestCatchUp_FreshLedger_None(t *testing.T) {
+	h := newHarness(t, at(5, 9, 0), at(5, 9, 0))
+	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
+		t.Fatalf("a freshly seeded ledger made up %v", legs)
+	}
+}
+
+// Started at 02:10 (the dump is due at 02:30): the dump is left to its normal run.
+func TestCatchUp_LegAboutToRun_LeftToNormal(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 2, 10))
+	h.l.MarkEnded(LegDBDump, at(3, 2, 31))
+	h.l.MarkEnded(LegTier2, at(3, 3, 31))
+	h.l.MarkEnded(LegOffsite, at(3, 4, 20))
+	legs, _ := h.l.Missed(at(5, 2, 10), "02:30", bud)
+	for _, l := range legs {
+		if l == LegDBDump {
+			t.Fatalf("the dump due in 20 minutes was put in the catch-up: %v", legs)
+		}
+	}
+	if len(legs) != 2 {
+		t.Fatalf("missed %v, want tier2 + offsite of the night of the 4th", legs)
+	}
+}
+
+// App updates are never in a catch-up: only the chain's backup legs are run, whatever else Legs holds.
+// RED-PROOF: iterate c.Legs instead of the missed chain legs → "update" runs.
+func TestCatchUp_NeverRunsAnythingButBackupLegs(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
+	updateRan := false
+	h.c.Legs["update"] = func(context.Context) error { updateRan = true; return nil }
+	h.c.Evaluate(context.Background(), "start")
+	h.wait(t)
+	if updateRan {
+		t.Fatal("an app update ran inside a catch-up")
+	}
+	for _, l := range Order {
+		if l == "update" || l == "docker" {
+			t.Fatalf("the chain order names a non-backup leg %q", l)
+		}
+	}
+}
+
+// A whole-guest backup holding the apps: the catch-up waits for it.
+// RED-PROOF: drop the QuiesceBusy loop → the legs run while busy (ran before the waits).
+func TestCatchUp_WaitsForAWholeGuestBackup(t *testing.T) {
+	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
+	h.busy = 3
+	h.c.Evaluate(context.Background(), "start")
+	h.wait(t)
+	if len(h.slept) != 4 || h.slept[1] != time.Minute {
+		t.Fatalf("slept %v — want the 15 min delay then 3 one-minute waits for the quiesce", h.slept)
+	}
+	if len(h.ran) != 3 {
+		t.Fatalf("ran %v", h.ran)
+	}
+}
+
+// A host resume is a trigger too: the wall clock jumps ahead of the monotonic clock.
+// RED-PROOF: compare wall with wall → the suspend is never seen.
+func TestResumeWatch_SeesASuspend(t *testing.T) {
+	wall := at(5, 1, 0)
+	var mono time.Duration
+	tick := make(chan time.Time)
+	got := make(chan time.Duration, 2)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	var mu sync.Mutex
+	go ResumeWatch(ctx, tick, func() time.Time { mu.Lock(); defer mu.Unlock(); return wall },
+		func() time.Duration { mu.Lock(); defer mu.Unlock(); return mono }, 5*time.Minute, func(d time.Duration) { got <- d })
+	step := func(w, m time.Duration) {
+		mu.Lock()
+		wall, mono = wall.Add(w), mono+m
+		mu.Unlock()
+		tick <- time.Time{}
+	}
+	step(time.Minute, time.Minute) // a normal minute
+	step(7*time.Hour, time.Minute) // suspended 02:00 → 09:00: wall +7h, monotonic +1 min
+	select {
+	case d := <-got:
+		if d < 6*time.Hour {
+			t.Fatalf("slept %s, want ~7h", d)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("a 7-hour suspend was not seen")
+	}
+	select {
+	case d := <-got:
+		t.Fatalf("a normal minute was read as a resume (%s)", d)
+	default:
+	}
+}