felhom-controller/controller/internal/nightchain/nightchain_test.go

package nightchain

import (
	"context"
	"io"
	"log"
	"path/filepath"
	"sync"
	"testing"
	"time"
)

// R-871 (`09` decision 109). Every rule of the catch-up, each with a COMPANION RED-PROOF recorded in
// felhom.eu/documentation/audits/catchup-2026-10-05/partA/red-proofs.txt.

var bud = func() *time.Location { l, _ := time.LoadLocation("Europe/Budapest"); return l }()

func at(day, hh, mm int) time.Time { return time.Date(2026, 10, day, hh, mm, 0, 0, bud) }

type harness struct {
	l      *Ledger
	c      *CatchUp
	mu     sync.Mutex
	ran    []Leg
	events []time.Time
	slept  []time.Duration
	now    time.Time
	busy   int // QuiesceBusy answers true this many times
	done   chan struct{}
}

func newHarness(t *testing.T, seeded, now time.Time) *harness {
	t.Helper()
	l, err := Open(filepath.Join(t.TempDir(), "night-ledger.json"), seeded)
	if err != nil {
		t.Fatal(err)
	}
	h := &harness{l: l, now: now, done: make(chan struct{}, 4)}
	leg := func(g Leg) func(context.Context) error {
		return func(context.Context) error {
			h.mu.Lock()
			h.ran = append(h.ran, g)
			h.mu.Unlock()
			l.MarkEnded(g, h.clock())
			return nil
		}
	}
	h.c = &CatchUp{
		Ledger: l, Window: func() string { return "02:30" }, Loc: bud,
		Legs:   map[Leg]func(context.Context) error{LegDBDump: leg(LegDBDump), LegTier2: leg(LegTier2), LegOffsite: leg(LegOffsite)},
		Logger: log.New(io.Discard, "", 0),
		Now:    h.clock,
		Sleep: func(_ context.Context, d time.Duration) bool {
			h.mu.Lock()
			h.slept = append(h.slept, d)
			h.now = h.now.Add(d)
			h.mu.Unlock()
			return true
		},
		QuiesceBusy: func() bool {
			h.mu.Lock()
			defer h.mu.Unlock()
			if h.busy > 0 {
				h.busy--
				return true
			}
			return false
		},
		Event: func(w time.Time) {
			h.mu.Lock()
			h.events = append(h.events, w)
			h.mu.Unlock()
			h.done <- struct{}{}
		},
	}
	return h
}

func (h *harness) clock() time.Time { h.mu.Lock(); defer h.mu.Unlock(); return h.now }

func (h *harness) wait(t *testing.T) {
	t.Helper()
	select {
	case <-h.done:
	case <-time.After(3 * time.Second):
		t.Fatal("the catch-up never finished")
	}
}

// Off at W (02:30 on the 5th), switched on at 09:00: ONE catch-up, 15 minutes later, all three legs in chain order.
// RED-PROOF: make Missed return nothing (the pre-v0.295.0 behaviour) → "no catch-up was scheduled".
func TestCatchUp_OffAtWThenOn_OneCatchUpAfter15Min(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
	h.l.MarkEnded(LegDBDump, at(4, 2, 31)) // the night of the 4th ran
	h.l.MarkEnded(LegTier2, at(4, 3, 31))
	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
	legs := h.c.Evaluate(context.Background(), "start")
	if len(legs) != 3 {
		t.Fatalf("no catch-up was scheduled for a box off across W (missed %v)", legs)
	}
	h.wait(t)
	if len(h.slept) == 0 || h.slept[0] != DefaultDelay {
		t.Fatalf("the catch-up must wait %s first, slept %v", DefaultDelay, h.slept)
	}
	if got := []Leg{LegDBDump, LegTier2, LegOffsite}; len(h.ran) != 3 || h.ran[0] != got[0] || h.ran[1] != got[1] || h.ran[2] != got[2] {
		t.Fatalf("legs ran %v, want the chain order %v", h.ran, got)
	}
	if len(h.events) != 1 || !h.events[0].Equal(at(5, 4, 15)) {
		t.Fatalf("household line %v, want one for the missed 04:15 leg", h.events)
	}
}

// On at W (the night ran) then a daytime restart: no catch-up.
// RED-PROOF: drop the `!Ended.Before(inst)` check → the restart makes up a night that ran.
func TestCatchUp_NightRan_DaytimeRestart_None(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 11, 0))
	h.l.MarkEnded(LegDBDump, at(5, 2, 31))
	h.l.MarkEnded(LegTier2, at(5, 3, 31))
	h.l.MarkEnded(LegOffsite, at(5, 4, 20))
	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
		t.Fatalf("a normal night was made up again: %v", legs)
	}
}

// Two missed nights: ONE catch-up, each leg once.
func TestCatchUp_TwoMissedNights_One(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(6, 18, 0))
	h.l.MarkEnded(LegDBDump, at(4, 2, 31))
	h.l.MarkEnded(LegTier2, at(4, 3, 31))
	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
	h.c.Evaluate(context.Background(), "start")
	if again := h.c.Evaluate(context.Background(), "resume"); again != nil {
		t.Fatalf("a second catch-up was scheduled while one was pending: %v", again)
	}
	h.wait(t)
	if len(h.ran) != 3 {
		t.Fatalf("two missed nights ran %v — want each leg once", h.ran)
	}
	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
		t.Fatalf("after the catch-up nothing is missed any more, got %v", legs)
	}
}

// A power cut in the middle of the chain: only the legs that did not end.
func TestCatchUp_PowerCutMidChain_FinishesTheRest(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 10, 0))
	h.l.MarkEnded(LegDBDump, at(5, 2, 31)) // the dump ended, then the power went
	h.l.MarkEnded(LegTier2, at(4, 3, 31))
	h.l.MarkEnded(LegOffsite, at(4, 4, 20))
	h.c.Evaluate(context.Background(), "start")
	h.wait(t)
	if len(h.ran) != 2 || h.ran[0] != LegTier2 || h.ran[1] != LegOffsite {
		t.Fatalf("ran %v, want only tier2 + offsite", h.ran)
	}
}

// The first start on this release seeds the ledger: nothing before it is "missed" (no surprise catch-up on upgrade).
func TestCatchUp_FreshLedger_None(t *testing.T) {
	h := newHarness(t, at(5, 9, 0), at(5, 9, 0))
	if legs := h.c.Evaluate(context.Background(), "start"); legs != nil {
		t.Fatalf("a freshly seeded ledger made up %v", legs)
	}
}

// Started at 02:10 (the dump is due at 02:30): the dump is left to its normal run.
func TestCatchUp_LegAboutToRun_LeftToNormal(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 2, 10))
	h.l.MarkEnded(LegDBDump, at(3, 2, 31))
	h.l.MarkEnded(LegTier2, at(3, 3, 31))
	h.l.MarkEnded(LegOffsite, at(3, 4, 20))
	legs, _ := h.l.Missed(at(5, 2, 10), "02:30", bud)
	for _, l := range legs {
		if l == LegDBDump {
			t.Fatalf("the dump due in 20 minutes was put in the catch-up: %v", legs)
		}
	}
	if len(legs) != 2 {
		t.Fatalf("missed %v, want tier2 + offsite of the night of the 4th", legs)
	}
}

// App updates are never in a catch-up: only the chain's backup legs are run, whatever else Legs holds.
// RED-PROOF: iterate c.Legs instead of the missed chain legs → "update" runs.
func TestCatchUp_NeverRunsAnythingButBackupLegs(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
	updateRan := false
	h.c.Legs["update"] = func(context.Context) error { updateRan = true; return nil }
	h.c.Evaluate(context.Background(), "start")
	h.wait(t)
	if updateRan {
		t.Fatal("an app update ran inside a catch-up")
	}
	for _, l := range Order {
		if l == "update" || l == "docker" {
			t.Fatalf("the chain order names a non-backup leg %q", l)
		}
	}
}

// A whole-guest backup holding the apps: the catch-up waits for it.
// RED-PROOF: drop the QuiesceBusy loop → the legs run while busy (ran before the waits).
func TestCatchUp_WaitsForAWholeGuestBackup(t *testing.T) {
	h := newHarness(t, at(1, 12, 0), at(5, 9, 0))
	h.busy = 3
	h.c.Evaluate(context.Background(), "start")
	h.wait(t)
	if len(h.slept) != 4 || h.slept[1] != time.Minute {
		t.Fatalf("slept %v — want the 15 min delay then 3 one-minute waits for the quiesce", h.slept)
	}
	if len(h.ran) != 3 {
		t.Fatalf("ran %v", h.ran)
	}
}

// A host resume is a trigger too: the wall clock jumps ahead of the monotonic clock.
// RED-PROOF: compare wall with wall → the suspend is never seen.
func TestResumeWatch_SeesASuspend(t *testing.T) {
	wall := at(5, 1, 0)
	var mono time.Duration
	tick := make(chan time.Time)
	got := make(chan time.Duration, 2)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	var mu sync.Mutex
	go ResumeWatch(ctx, tick, func() time.Time { mu.Lock(); defer mu.Unlock(); return wall },
		func() time.Duration { mu.Lock(); defer mu.Unlock(); return mono }, 5*time.Minute, func(d time.Duration) { got <- d })
	step := func(w, m time.Duration) {
		mu.Lock()
		wall, mono = wall.Add(w), mono+m
		mu.Unlock()
		tick <- time.Time{}
	}
	step(time.Minute, time.Minute) // a normal minute
	step(7*time.Hour, time.Minute) // suspended 02:00 → 09:00: wall +7h, monotonic +1 min
	select {
	case d := <-got:
		if d < 6*time.Hour {
			t.Fatalf("slept %s, want ~7h", d)
		}
	case <-time.After(2 * time.Second):
		t.Fatal("a 7-hour suspend was not seen")
	}
	select {
	case d := <-got:
		t.Fatalf("a normal minute was read as a resume (%s)", d)
	default:
	}
}