Files
felhom.eu/hub/internal/monitor/daily_ledger.go
T

110 lines
4.6 KiB
Go

package monitor
import (
"encoding/json"
"fmt"
"log"
"time"
)
// DailyLedger (R-1383, D78, `09` §3 decision 299; `05-hub-architecture.md` §16.5) remembers ACROSS A RESTART when a
// hub daily check last ran, and tells the operator when a slot passed while the hub was down.
//
// Before it, „when did this job last run” lived in the scheduler's memory: a hub that was down at 05:00 skipped that
// day's missed-backup check with no trace, and a missed backup was then seen one day late (clock list CH02-4, CH11-4,
// CH16-4). The ledger does NOT run the missed check late — it tells. (The database snapshot and the prune make up for
// a missed slot by themselves and are not watched.)
//
// The state is two times per job in the hub's settings (no new table): `daily_last:<name>` = the slot that last RAN
// to its end, `daily_skip_told:<name>` = the newest missed slot that was told. Pinned by TestDailyLedger_*.
type DailyLedger struct {
get func(key string) time.Time
set func(key string, t time.Time) error
onEvent EventNotifyFunc
logger *log.Logger
watched map[string]bool
}
// EventDailyJobSkipped is operator-only (notify.operatorOnlyEvents).
const EventDailyJobSkipped = "hub_daily_job_skipped"
// NewDailyLedger: get/set are the hub's small time-by-key state (store.OSAlarmRaised / SetOSAlarmRaised); watched is
// the list of job names the ledger speaks for.
func NewDailyLedger(get func(string) time.Time, set func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger, watched ...string) *DailyLedger {
w := map[string]bool{}
for _, n := range watched {
w[n] = true
}
return &DailyLedger{get: get, set: set, onEvent: onEvent, logger: logger, watched: w}
}
// PrevDailySlot returns the newest HH:MM in loc that is not after now.
func PrevDailySlot(now time.Time, hour, min int, loc *time.Location) time.Time {
now = now.In(loc)
for d := 0; ; d-- {
p := time.Date(now.Year(), now.Month(), now.Day()+d, hour, min, 0, 0, loc)
if !p.After(now) {
return p
}
}
}
// Started is called once when the scheduler of `name` starts (a hub start). It tells the operator, once per missed
// slot, when the newest slot before now did not run. A job the ledger has never seen is not judged: its watch begins
// now.
func (l *DailyLedger) Started(name string, now time.Time, hour, min int, loc *time.Location) {
if l == nil || !l.watched[name] {
return
}
last := l.get("daily_last:" + name)
prev := PrevDailySlot(now, hour, min, loc)
if last.IsZero() {
// the first start with the ledger: nothing to compare with. The watch begins at the newest slot, so the next
// missed one is seen.
_ = l.set("daily_last:"+name, prev)
l.logger.Printf("[INFO] daily ledger: %s is watched from now (no earlier record)", name)
return
}
if !last.Before(prev) {
return // the newest slot ran
}
if told := l.get("daily_skip_told:" + name); !told.Before(prev) {
return // this missed slot was told at an earlier start
}
_ = l.set("daily_skip_told:"+name, prev)
// how many slots passed: whole calendar days between the two slots
missed := 0
for s := prev; s.After(last) && missed < 400; s = time.Date(s.Year(), s.Month(), s.Day()-1, hour, min, 0, 0, loc) {
missed++
}
l.logger.Printf("[WARN] daily ledger: %s did not run at %s — the hub was not running then (last run %s; %d slot(s) missed)",
name, prev.Format("2006-01-02 15:04 MST"), last.In(loc).Format("2006-01-02 15:04 MST"), missed)
msg := fmt.Sprintf("The hub's daily check „%s” did NOT run at %s: the hub was not running at that minute. It last ran at %s (%d slot(s) missed). The hub does not run it late. %s",
name, prev.Format("2006-01-02 15:04 MST"), last.In(loc).Format("2006-01-02 15:04 MST"), missed, dailySkipAdvice(name))
details, _ := json.Marshal(map[string]any{"job": name, "missed_slot": prev.UTC().Format(time.RFC3339),
"last_run": last.UTC().Format(time.RFC3339), "missed": missed, "subject": name})
if l.onEvent != nil {
l.onEvent("", EventDailyJobSkipped, "warning", msg, string(details), "hub")
}
}
// Ran records that the slot ran to its end.
func (l *DailyLedger) Ran(name string, slot time.Time) {
if l == nil || !l.watched[name] {
return
}
if err := l.set("daily_last:"+name, slot); err != nil {
l.logger.Printf("[WARN] daily ledger: could not record the run of %s: %v", name, err)
}
}
func dailySkipAdvice(name string) string {
switch name {
case "deadline-check":
return "What it means: a backup that was missed last night is told one day late. Do: read the Backup card of each box today."
case "offsite-key-audit":
return "What it means: the off-site keys were not audited that day. The next slot audits them; no hand step."
}
return "The next slot runs it."
}