110 lines
4.6 KiB
Go
110 lines
4.6 KiB
Go
package monitor
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"time"
|
|
)
|
|
|
|
// DailyLedger (R-1383, D78, `09` §3 decision 299; `05-hub-architecture.md` §16.5) remembers ACROSS A RESTART when a
|
|
// hub daily check last ran, and tells the operator when a slot passed while the hub was down.
|
|
//
|
|
// Before it, „when did this job last run” lived in the scheduler's memory: a hub that was down at 05:00 skipped that
|
|
// day's missed-backup check with no trace, and a missed backup was then seen one day late (clock list CH02-4, CH11-4,
|
|
// CH16-4). The ledger does NOT run the missed check late — it tells. (The database snapshot and the prune make up for
|
|
// a missed slot by themselves and are not watched.)
|
|
//
|
|
// The state is two times per job in the hub's settings (no new table): `daily_last:<name>` = the slot that last RAN
|
|
// to its end, `daily_skip_told:<name>` = the newest missed slot that was told. Pinned by TestDailyLedger_*.
|
|
type DailyLedger struct {
|
|
get func(key string) time.Time
|
|
set func(key string, t time.Time) error
|
|
onEvent EventNotifyFunc
|
|
logger *log.Logger
|
|
watched map[string]bool
|
|
}
|
|
|
|
// EventDailyJobSkipped is operator-only (notify.operatorOnlyEvents).
|
|
const EventDailyJobSkipped = "hub_daily_job_skipped"
|
|
|
|
// NewDailyLedger: get/set are the hub's small time-by-key state (store.OSAlarmRaised / SetOSAlarmRaised); watched is
|
|
// the list of job names the ledger speaks for.
|
|
func NewDailyLedger(get func(string) time.Time, set func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger, watched ...string) *DailyLedger {
|
|
w := map[string]bool{}
|
|
for _, n := range watched {
|
|
w[n] = true
|
|
}
|
|
return &DailyLedger{get: get, set: set, onEvent: onEvent, logger: logger, watched: w}
|
|
}
|
|
|
|
// PrevDailySlot returns the newest HH:MM in loc that is not after now.
|
|
func PrevDailySlot(now time.Time, hour, min int, loc *time.Location) time.Time {
|
|
now = now.In(loc)
|
|
for d := 0; ; d-- {
|
|
p := time.Date(now.Year(), now.Month(), now.Day()+d, hour, min, 0, 0, loc)
|
|
if !p.After(now) {
|
|
return p
|
|
}
|
|
}
|
|
}
|
|
|
|
// Started is called once when the scheduler of `name` starts (a hub start). It tells the operator, once per missed
|
|
// slot, when the newest slot before now did not run. A job the ledger has never seen is not judged: its watch begins
|
|
// now.
|
|
func (l *DailyLedger) Started(name string, now time.Time, hour, min int, loc *time.Location) {
|
|
if l == nil || !l.watched[name] {
|
|
return
|
|
}
|
|
last := l.get("daily_last:" + name)
|
|
prev := PrevDailySlot(now, hour, min, loc)
|
|
if last.IsZero() {
|
|
// the first start with the ledger: nothing to compare with. The watch begins at the newest slot, so the next
|
|
// missed one is seen.
|
|
_ = l.set("daily_last:"+name, prev)
|
|
l.logger.Printf("[INFO] daily ledger: %s is watched from now (no earlier record)", name)
|
|
return
|
|
}
|
|
if !last.Before(prev) {
|
|
return // the newest slot ran
|
|
}
|
|
if told := l.get("daily_skip_told:" + name); !told.Before(prev) {
|
|
return // this missed slot was told at an earlier start
|
|
}
|
|
_ = l.set("daily_skip_told:"+name, prev)
|
|
// how many slots passed: whole calendar days between the two slots
|
|
missed := 0
|
|
for s := prev; s.After(last) && missed < 400; s = time.Date(s.Year(), s.Month(), s.Day()-1, hour, min, 0, 0, loc) {
|
|
missed++
|
|
}
|
|
l.logger.Printf("[WARN] daily ledger: %s did not run at %s — the hub was not running then (last run %s; %d slot(s) missed)",
|
|
name, prev.Format("2006-01-02 15:04 MST"), last.In(loc).Format("2006-01-02 15:04 MST"), missed)
|
|
msg := fmt.Sprintf("The hub's daily check „%s” did NOT run at %s: the hub was not running at that minute. It last ran at %s (%d slot(s) missed). The hub does not run it late. %s",
|
|
name, prev.Format("2006-01-02 15:04 MST"), last.In(loc).Format("2006-01-02 15:04 MST"), missed, dailySkipAdvice(name))
|
|
details, _ := json.Marshal(map[string]any{"job": name, "missed_slot": prev.UTC().Format(time.RFC3339),
|
|
"last_run": last.UTC().Format(time.RFC3339), "missed": missed, "subject": name})
|
|
if l.onEvent != nil {
|
|
l.onEvent("", EventDailyJobSkipped, "warning", msg, string(details), "hub")
|
|
}
|
|
}
|
|
|
|
// Ran records that the slot ran to its end.
|
|
func (l *DailyLedger) Ran(name string, slot time.Time) {
|
|
if l == nil || !l.watched[name] {
|
|
return
|
|
}
|
|
if err := l.set("daily_last:"+name, slot); err != nil {
|
|
l.logger.Printf("[WARN] daily ledger: could not record the run of %s: %v", name, err)
|
|
}
|
|
}
|
|
|
|
func dailySkipAdvice(name string) string {
|
|
switch name {
|
|
case "deadline-check":
|
|
return "What it means: a backup that was missed last night is told one day late. Do: read the Backup card of each box today."
|
|
case "offsite-key-audit":
|
|
return "What it means: the off-site keys were not audited that day. The next slot audits them; no hand step."
|
|
}
|
|
return "The next slot runs it."
|
|
}
|