// Package holdreport tells the hub which apps are held (D53, felhom.eu `08` §10). // // A held app is stopped on purpose and stays stopped until a person acts. Until D53 the hub knew nothing about it: // the crash-loop stop sent one event, the other holds sent none, and demo-hp's Kimai sat held for 11 days unseen // (R-1291). The legacy report is frozen, so the event channel carries the holds: // // - app_held when a hold appears or changes (stack, reason, since, the hold's sentence); // - app_hold_cleared when it is gone — also when it went while the controller was down, because the set last // reported is kept in settings (ReportedHolds), not in memory; // - every hold still in place is sent again once a day and at every controller start, because an event can be // lost. The hub treats a repeat as idempotent. // // ONE OBSERVER, NOT A HOOK PER WRITER. Seven code paths write a hold and four clear one. A send in each of them is // the shape where the one path nobody found is the next silent hold; this package reads the hold store itself, so a // new writer is covered the day it is added. Both events are severity info: the hub mails nothing on receipt. Its // reminder after the grace per reason is the operator's mail, and it never goes to the household (D53). // // Pinned by holdreport_test.go. package holdreport import ( "log" "sort" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // Event types — the hub's allowedEventTypes carries both (the coupling rule). const ( EventHeld = "app_held" EventCleared = "app_hold_cleared" ) // ResendEvery is how often every hold still in place is sent again. const ResendEvery = 24 * time.Hour // Store is what the reporter needs of the settings. type Store interface { ListRestoreHolds() []settings.RestoreHold GetReportedHolds() map[string]string SetReportedHolds(map[string]string) error } // Push sends one event to the hub (notify.Notifier.PushEvent). type Push func(eventType, severity, message string, details interface{}) // Details is the payload of both events. type Details struct { StackName string `json:"stack_name"` Reason string `json:"reason,omitempty"` Since string `json:"since,omitempty"` Why string `json:"why,omitempty"` Repeat bool `json:"repeat,omitempty"` // a daily or start-up re-send, not a new hold } // Reporter compares the hold store with what was last reported. Tick is called by the scheduler. type Reporter struct { store Store push Push logger *log.Logger lastFull time.Time // installed (R-1298) — nil = every app counts as installed. installed func(stack string) bool skipLogged map[string]bool } // New builds a reporter. The first Tick re-sends every hold (a controller start). func New(store Store, push Push, logger *log.Logger) *Reporter { return &Reporter{store: store, push: push, logger: logger} } func key(h settings.RestoreHold) string { return h.WireReason() + "|" + h.At } // SetInstalled wires the „is this app installed?" check (R-1298). nil = every app counts as installed. func (r *Reporter) SetInstalled(f func(stack string) bool) { r.installed = f } // installHold is a hold that belongs to an app's INSTALL (R-704): a removal ends it. A restore hold (restore_failed, // restore_mixed) does not — a person clears it, so it stays reported whatever the app's state (R-379). func installHold(h settings.RestoreHold) bool { switch h.Reason { case settings.HoldReasonUnhealthyStop, settings.HoldReasonUpdateFailed, settings.HoldReasonPutBackPending: return true } return false } // Tick sends what changed since the last tick, and every hold again when a day has passed. func (r *Reporter) Tick(now time.Time) { if r.store == nil || r.push == nil { return } all := r.store.ListRestoreHolds() sort.Slice(all, func(i, j int) bool { return all[i].Stack < all[j].Stack }) // R-1298: an install's hold of an app that is not installed any more (removed outside the product) is not a held // app — it is skipped, and reported cleared if it was reported before. Logged once per stack per process. holds := all[:0:0] for _, h := range all { if r.installed != nil && installHold(h) && !r.installed(h.Stack) { if r.skipLogged == nil { r.skipLogged = map[string]bool{} } if !r.skipLogged[h.Stack] { r.skipLogged[h.Stack] = true r.logf("[INFO] [holdreport] %s: a %s hold of an app that is not installed — not reported (R-1298)", h.Stack, h.WireReason()) } continue } holds = append(holds, h) } prev := r.store.GetReportedHolds() full := r.lastFull.IsZero() || now.Sub(r.lastFull) >= ResendEvery cur := make(map[string]string, len(holds)) sent, cleared := 0, 0 for _, h := range holds { k := key(h) cur[h.Stack] = k known, seen := prev[h.Stack] if seen && known == k && !full { continue } repeat := seen && known == k r.push(EventHeld, "info", "App "+h.Stack+" is held: "+h.Why(), Details{StackName: h.Stack, Reason: h.WireReason(), Since: h.At, Why: h.Why(), Repeat: repeat}) sent++ if !repeat { r.logf("[INFO] [holdreport] %s held (%s since %s) — reported to the hub", h.Stack, h.WireReason(), h.At) } } gone := make([]string, 0) for stack := range prev { if _, ok := cur[stack]; !ok { gone = append(gone, stack) } } sort.Strings(gone) for _, stack := range gone { r.push(EventCleared, "info", "App "+stack+" is no longer held", Details{StackName: stack}) cleared++ r.logf("[INFO] [holdreport] %s hold cleared — reported to the hub", stack) } if full { r.lastFull = now r.logf("[DEBUG] [holdreport] full re-send: %d hold(s) sent, %d cleared", sent, cleared) } if !sameMap(prev, cur) { if err := r.store.SetReportedHolds(cur); err != nil { // Not fatal: the next tick sends the same difference again, which the hub reads as idempotent. r.logf("[WARN] [holdreport] could not save the reported holds: %v — the next tick repeats the difference", err) } } } func sameMap(a, b map[string]string) bool { if len(a) != len(b) { return false } for k, v := range a { if b[k] != v { return false } } return true } func (r *Reporter) logf(format string, args ...interface{}) { if r.logger != nil { r.logger.Printf(format, args...) } }