felhom-controller/controller/internal/nightchain/nightchain.go

// Package nightchain makes a missed night's backups up once, when the box comes back (R-871, `09` §3 decision 109,
// design `07` §6.1.1).
//
// THE PROBLEM, MEASURED (2026-10-05 Part F spike, Tester 2 — a laptop switched off at night): the controller's
// daily jobs always schedule the NEXT future time (scheduler.nextDailyRun), and nothing remembers that a night was
// missed. A box that is off at its window W never gets its database dumps, its second copy or its off-site copy —
// for ever, with no alarm.
//
// THE MECHANISM:
//   - A LEDGER (`night-ledger.json` in the data dir) records when each of the three nightly backup legs last RAN TO
//     ITS END. It is an attempt record and is used for ONE question only — "was this leg's last scheduled time
//     missed?" — never as evidence that a backup exists (that is LastSuccess's job, R-100). A leg that ran and
//     failed was NOT missed: failures have their own alarms.
//   - On a controller START and on a host RESUME (a suspended laptop: Go timers run on CLOCK_MONOTONIC, which does
//     not count suspended time), Evaluate asks the ledger which legs missed their last scheduled time. If any did,
//     ONE catch-up is scheduled 15 minutes later (apps settle; a box switched on and off again at once does nothing).
//   - The catch-up runs ONLY the backup legs, in their normal order (database dump, second copy, off-site copy).
//     It never runs app updates or Docker steps — they restart apps and wait for a real night.
//   - Several missed nights are ONE catch-up (the question is about the LAST scheduled time only). A normal night
//     followed by a daytime restart is none. A power cut in the middle of the chain makes up only the legs that did
//     not end.
//   - A leg whose own scheduled time is less than 30 minutes away is left to its normal run.
//   - Every leg — scheduled or catch-up — holds one mutex (`Ledger.LegLock`), so a catch-up and a scheduled leg
//     never run at the same time. The whole-guest backup (agent-side, quiesce) waits while a catch-up runs
//     (quiesce.Options.CatchUpFn) and the catch-up waits while a quiesce holds the apps (QuiesceBusy).
//
// A ledger that does not exist yet (the first start on this release) is SEEDED at that moment and nothing before
// it counts as missed — an upgrade must not set off a surprise catch-up on every box.
//
// Pinned by nightchain_test.go (TestCatchUp_*), each rule with a red-proof.
package nightchain

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
)

// Leg is one nightly backup leg.
type Leg string

const (
	LegDBDump  Leg = "db-dump"
	LegTier2   Leg = "tier2"
	LegOffsite Leg = "offsite"
)

// Order is the night chain's own order (07 §6.1).
var Order = []Leg{LegDBDump, LegTier2, LegOffsite}

const (
	// DefaultDelay: the catch-up waits this long after the trigger (decision 109's "soon after it is switched on").
	DefaultDelay = 15 * time.Minute
	// leaveToNormal: a leg whose own scheduled time is closer than this runs normally, not in the catch-up.
	leaveToNormal = 30 * time.Minute
	// quiesceWaitMax: how long a catch-up waits for a whole-guest backup that holds the apps.
	quiesceWaitMax = 2 * time.Hour
)

type ledgerData struct {
	SeededAt time.Time         `json:"seeded_at"`
	Ended    map[Leg]time.Time `json:"ended"` // ATTEMPT record: the leg ran to its end (success or failure)
	// DBDumpOK is the last SUCCESSFUL database dump (the banner's evidence; the off-site tier has its own
	// LastSuccess in settings). Written only on success (R-100's rule).
	DBDumpOK        time.Time `json:"db_dump_ok,omitempty"`
	LastCatchUp     time.Time `json:"last_catch_up,omitempty"`
	BannerDismissed time.Time `json:"banner_dismissed_through,omitempty"` // the missed W instant the household closed
}

// Ledger is the persisted night record.
type Ledger struct {
	mu   sync.Mutex
	path string
	d    ledgerData
	// LegLock is held by every leg run, scheduled or catch-up.
	LegLock sync.Mutex
}

// Open reads the ledger, or seeds a new one at now.
func Open(path string, now time.Time) (*Ledger, error) {
	l := &Ledger{path: path, d: ledgerData{Ended: map[Leg]time.Time{}}}
	b, err := os.ReadFile(path)
	switch {
	case err == nil:
		if jerr := json.Unmarshal(b, &l.d); jerr != nil {
			return nil, fmt.Errorf("nightchain: ledger unreadable: %w", jerr)
		}
		if l.d.Ended == nil {
			l.d.Ended = map[Leg]time.Time{}
		}
	case os.IsNotExist(err):
		l.d.SeededAt = now.UTC()
		if serr := l.saveLocked(); serr != nil {
			return nil, serr
		}
	default:
		return nil, fmt.Errorf("nightchain: ledger: %w", err)
	}
	return l, nil
}

func (l *Ledger) saveLocked() error {
	b, _ := json.MarshalIndent(l.d, "", "  ")
	if err := os.MkdirAll(filepath.Dir(l.path), 0o755); err != nil {
		return err
	}
	tmp := l.path + ".tmp"
	if err := os.WriteFile(tmp, b, 0o644); err != nil {
		return err
	}
	return os.Rename(tmp, l.path)
}

// MarkEnded records that a leg ran to its end (success or failure).
func (l *Ledger) MarkEnded(leg Leg, at time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.d.Ended[leg] = at.UTC()
	_ = l.saveLocked()
}

// MarkDBDumpOK records a successful database dump.
func (l *Ledger) MarkDBDumpOK(at time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.d.DBDumpOK = at.UTC()
	_ = l.saveLocked()
}

func (l *Ledger) markCatchUp(at time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.d.LastCatchUp = at.UTC()
	_ = l.saveLocked()
}

// DismissBanner records that the household closed the banner for every miss up to `through`.
func (l *Ledger) DismissBanner(through time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.d.BannerDismissed = through.UTC()
	_ = l.saveLocked()
}

// Snapshot returns a copy of the record (for the banner and the debug page).
func (l *Ledger) Snapshot() (seeded, dbDumpOK, lastCatchUp, dismissed time.Time, ended map[Leg]time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	ended = map[Leg]time.Time{}
	for k, v := range l.d.Ended {
		ended[k] = v
	}
	return l.d.SeededAt, l.d.DBDumpOK, l.d.LastCatchUp, l.d.BannerDismissed, ended
}

// legOffsets: each leg's time relative to W — taken from backupwindow.LegTimes so the two can never disagree.
func legTimes(window string) map[Leg]string {
	db, t2, off := backupwindow.LegTimes(window)
	return map[Leg]string{LegDBDump: db, LegTier2: t2, LegOffsite: off}
}

// LastInstant is the most recent occurrence of the Budapest clock time hhmm at or before now.
func LastInstant(now time.Time, hhmm string, loc *time.Location) (time.Time, bool) {
	min, err := backupwindow.ParseHHMM(hhmm)
	if err != nil {
		return time.Time{}, false
	}
	n := now.In(loc)
	t := time.Date(n.Year(), n.Month(), n.Day(), min/60, min%60, 0, 0, loc)
	if t.After(n) {
		t = time.Date(n.Year(), n.Month(), n.Day()-1, min/60, min%60, 0, 0, loc)
	}
	return t, true
}

// NextInstant is the next occurrence of hhmm strictly after now.
func NextInstant(now time.Time, hhmm string, loc *time.Location) (time.Time, bool) {
	last, ok := LastInstant(now, hhmm, loc)
	if !ok {
		return time.Time{}, false
	}
	return time.Date(last.Year(), last.Month(), last.Day()+1, last.Hour(), last.Minute(), 0, 0, loc), true
}

// Missed is the PURE rule: the legs (in chain order) whose last scheduled time passed after the ledger was seeded
// and that have not run to their end since — minus a leg whose next scheduled time is under 30 minutes away.
// lastW is the most recent missed leg instant (the banner's "the box was off at …").
func (l *Ledger) Missed(now time.Time, window string, loc *time.Location) (legs []Leg, lastW time.Time) {
	l.mu.Lock()
	defer l.mu.Unlock()
	lt := legTimes(window)
	for _, leg := range Order {
		inst, ok := LastInstant(now, lt[leg], loc)
		if !ok || !inst.After(l.d.SeededAt) {
			continue
		}
		if !l.d.Ended[leg].Before(inst) {
			continue // ran to its end at or after its last scheduled time
		}
		if next, ok := NextInstant(now, lt[leg], loc); ok && next.Sub(now) < leaveToNormal {
			continue // its normal run is about to happen
		}
		legs = append(legs, leg)
		if inst.After(lastW) {
			lastW = inst
		}
	}
	return legs, lastW
}

// CatchUp runs at most one catch-up at a time.
type CatchUp struct {
	Ledger      *Ledger
	Window      func() string                       // the household's backup window W (settings > config > 02:30)
	Legs        map[Leg]func(context.Context) error // the leg bodies — WITHOUT the update leg
	QuiesceBusy func() bool                         // a whole-guest backup holds the apps now
	Event       func(missedAt time.Time)            // the household's timeline line
	Logger      *log.Logger
	Delay       time.Duration
	Loc         *time.Location
	Now         func() time.Time
	// Sleep waits d or until ctx ends (false). A seam for tests.
	Sleep func(ctx context.Context, d time.Duration) bool

	mu      sync.Mutex
	pending bool
	running bool
}

func (c *CatchUp) now() time.Time {
	if c.Now != nil {
		return c.Now()
	}
	return time.Now()
}

func (c *CatchUp) sleep(ctx context.Context, d time.Duration) bool {
	if c.Sleep != nil {
		return c.Sleep(ctx, d)
	}
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-ctx.Done():
		return false
	case <-t.C:
		return true
	}
}

func (c *CatchUp) delay() time.Duration {
	if c.Delay > 0 {
		return c.Delay
	}
	return DefaultDelay
}

// Running reports whether a catch-up is running its legs now (the whole-guest backup waits for it).
func (c *CatchUp) Running() bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.running
}

// Evaluate is the trigger (controller start, host resume). It schedules ONE catch-up when a leg was missed and
// none is pending; it returns what it scheduled (nil = nothing missed or one already pending). The work runs in
// its own goroutine; Wait-style callers use EvaluateSync.
func (c *CatchUp) Evaluate(ctx context.Context, why string) []Leg {
	legs, lastW := c.Ledger.Missed(c.now(), c.Window(), c.Loc)
	if len(legs) == 0 {
		c.Logger.Printf("[INFO] [catch-up] %s: no backup leg missed its last scheduled time — nothing to make up", why)
		return nil
	}
	c.mu.Lock()
	if c.pending || c.running {
		c.mu.Unlock()
		c.Logger.Printf("[INFO] [catch-up] %s: missed %v, but a catch-up is already scheduled", why, legs)
		return nil
	}
	c.pending = true
	c.mu.Unlock()
	c.Logger.Printf("[INFO] [catch-up] %s: the box missed %v (last scheduled %s) — ONE catch-up in %s (backup legs only; app updates wait for a real night)",
		why, legs, lastW.In(c.Loc).Format("2006-01-02 15:04"), c.delay())
	go c.run(ctx, why)
	return legs
}

// run waits, re-reads the ledger (a normal run may have happened meanwhile), waits out a whole-guest backup, then
// runs the still-missed legs in chain order.
func (c *CatchUp) run(ctx context.Context, why string) {
	defer func() {
		c.mu.Lock()
		c.pending, c.running = false, false
		c.mu.Unlock()
	}()
	if !c.sleep(ctx, c.delay()) {
		return
	}
	legs, lastW := c.Ledger.Missed(c.now(), c.Window(), c.Loc)
	if len(legs) == 0 {
		c.Logger.Printf("[INFO] [catch-up] (%s) nothing is missed any more — not run", why)
		return
	}
	for waited := time.Duration(0); c.QuiesceBusy != nil && c.QuiesceBusy(); waited += time.Minute {
		if waited >= quiesceWaitMax {
			c.Logger.Printf("[WARN] [catch-up] (%s) a whole-guest backup held the apps for %s — catch-up dropped; the next start or resume tries again", why, waited)
			return
		}
		if !c.sleep(ctx, time.Minute) {
			return
		}
	}
	c.mu.Lock()
	c.running = true
	c.mu.Unlock()
	start := c.now()
	var names []string
	for _, leg := range legs {
		fn := c.Legs[leg]
		if fn == nil {
			continue
		}
		c.Logger.Printf("[INFO] [catch-up] running the missed %s leg", leg)
		if err := fn(ctx); err != nil {
			c.Logger.Printf("[WARN] [catch-up] the %s leg ended with an error (its own alarm reports it): %v", leg, err)
		}
		names = append(names, string(leg))
	}
	c.Ledger.markCatchUp(c.now())
	sort.Strings(names)
	c.Logger.Printf("[INFO] [catch-up] done: %s in %s (missed at %s)", strings.Join(names, ", "),
		c.now().Sub(start).Round(time.Second), lastW.In(c.Loc).Format("2006-01-02 15:04"))
	if c.Event != nil {
		c.Event(lastW)
	}
}

// ResumeWatch calls onResume when, between two ticks, the WALL clock advanced more than gap beyond the MONOTONIC
// clock — the shape of a host suspend/resume: Go's timers and monotonic readings run on CLOCK_MONOTONIC, which does
// not count suspended time, while the wall clock does. Seams: wall() is the wall clock, mono() a monotonic elapsed
// duration (production: time.Now().Round(0) and time.Since(a fixed start)). Pinned by TestResumeWatch_*.
func ResumeWatch(ctx context.Context, tick <-chan time.Time, wall func() time.Time, mono func() time.Duration, gap time.Duration, onResume func(slept time.Duration)) {
	pw, pm := wall(), mono()
	for {
		select {
		case <-ctx.Done():
			return
		case <-tick:
		}
		cw, cm := wall(), mono()
		if d := cw.Sub(pw) - (cm - pm); d > gap {
			onResume(d)
		}
		pw, pm = cw, cm
	}
}