gates / gates (push) Successful in 1m58s
app_held / app_hold_cleared allow-listed and operator-only; handleEvent mirrors them in held_apps; HeldAppsChecker sends ONE app_held_reminder (warning, operator-only, per-app cooldown) after the grace per reason (6 h / 1 d / 3 d), then weekly; recorded before sending; stale rows not reminded; the hub never clears a hold. Test-only alerting.held_app_grace_override. TestD53_* in api/monitor/notify/web, red-proved by six mutations. Commit only.
125 lines
4.7 KiB
Go
125 lines
4.7 KiB
Go
package store
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// ── D53 (`08` §10, R-1291): the held apps a box reports ───────────────────────────────────────────
|
|
//
|
|
// A box (controller ≥ the D53 release) sends app_held for every hold in place — at once, then once a day — and
|
|
// app_hold_cleared when one is gone. This table is the hub's mirror of that: one row per (customer, stack). The hub
|
|
// NEVER clears a hold on a box; a row goes only when the box says the hold is gone. The reminder checker
|
|
// (monitor.HeldAppsChecker) reads it and records here when it mailed, so a hub restart neither re-mails nor forgets.
|
|
// All times are stored as RFC3339 UTC text.
|
|
|
|
// HeldApp is one held app as the hub knows it.
|
|
type HeldApp struct {
|
|
CustomerID string
|
|
Stack string
|
|
Reason string // restore_failed, restore_mixed, update_failed, unhealthy_stop, putback_pending
|
|
Since time.Time // the hold's own time (the box's)
|
|
Why string // the box's one-line sentence: what put it there and what lifts it
|
|
FirstSeen time.Time // when the hub first heard of this hold
|
|
LastSeen time.Time // the newest app_held for it — the box repeats it daily
|
|
RemindedAt time.Time // the last operator reminder; zero = none yet
|
|
}
|
|
|
|
func (s *Store) migrateHeldApps() error {
|
|
_, err := s.db.Exec(`
|
|
CREATE TABLE IF NOT EXISTS held_apps (
|
|
customer_id TEXT NOT NULL,
|
|
stack TEXT NOT NULL,
|
|
reason TEXT NOT NULL DEFAULT '',
|
|
since TEXT NOT NULL DEFAULT '',
|
|
why TEXT NOT NULL DEFAULT '',
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
reminded_at TEXT NOT NULL DEFAULT '',
|
|
PRIMARY KEY (customer_id, stack)
|
|
);`)
|
|
return err
|
|
}
|
|
|
|
func fmtT(t time.Time) string {
|
|
if t.IsZero() {
|
|
return ""
|
|
}
|
|
return t.UTC().Format(time.RFC3339)
|
|
}
|
|
|
|
func parseT(s string) time.Time {
|
|
if s == "" {
|
|
return time.Time{}
|
|
}
|
|
return parseSQLiteTime(s)
|
|
}
|
|
|
|
// UpsertHeldApp records an app_held. A repeat of the same hold (same reason and since) only moves last_seen; a NEW
|
|
// hold on the stack (another reason or another since) starts over — first_seen now and no reminder yet — so its own
|
|
// grace runs from the start.
|
|
func (s *Store) UpsertHeldApp(customerID, stack, reason string, since time.Time, why string, now time.Time) error {
|
|
if customerID == "" || stack == "" {
|
|
return fmt.Errorf("held app: empty customer or stack")
|
|
}
|
|
res, err := s.db.Exec(`UPDATE held_apps SET last_seen = ?, why = ?
|
|
WHERE customer_id = ? AND stack = ? AND reason = ? AND since = ?`,
|
|
fmtT(now), why, customerID, stack, reason, fmtT(since))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n > 0 {
|
|
return nil
|
|
}
|
|
_, err = s.db.Exec(`INSERT INTO held_apps (customer_id, stack, reason, since, why, first_seen, last_seen, reminded_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, '')
|
|
ON CONFLICT(customer_id, stack) DO UPDATE SET reason = excluded.reason, since = excluded.since,
|
|
why = excluded.why, first_seen = excluded.first_seen, last_seen = excluded.last_seen, reminded_at = ''`,
|
|
customerID, stack, reason, fmtT(since), why, fmtT(now), fmtT(now))
|
|
return err
|
|
}
|
|
|
|
// ClearHeldApp removes a hold the box says is gone. Returns whether a row existed.
|
|
func (s *Store) ClearHeldApp(customerID, stack string) (bool, error) {
|
|
res, err := s.db.Exec(`DELETE FROM held_apps WHERE customer_id = ? AND stack = ?`, customerID, stack)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
return n > 0, nil
|
|
}
|
|
|
|
// MarkHeldAppReminded records the operator reminder for the hold as it is now (same reason and since).
|
|
func (s *Store) MarkHeldAppReminded(customerID, stack, reason string, since, at time.Time) error {
|
|
_, err := s.db.Exec(`UPDATE held_apps SET reminded_at = ? WHERE customer_id = ? AND stack = ? AND reason = ? AND since = ?`,
|
|
fmtT(at), customerID, stack, reason, fmtT(since))
|
|
return err
|
|
}
|
|
|
|
// ListHeldApps returns every held app, or one customer's when customerID is not empty, oldest hold first.
|
|
func (s *Store) ListHeldApps(customerID string) ([]HeldApp, error) {
|
|
q := `SELECT customer_id, stack, reason, since, why, first_seen, last_seen, reminded_at FROM held_apps`
|
|
args := []interface{}{}
|
|
if customerID != "" {
|
|
q += ` WHERE customer_id = ?`
|
|
args = append(args, customerID)
|
|
}
|
|
q += ` ORDER BY since, customer_id, stack`
|
|
rows, err := s.db.Query(q, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []HeldApp
|
|
for rows.Next() {
|
|
var h HeldApp
|
|
var since, first, last, rem string
|
|
if err := rows.Scan(&h.CustomerID, &h.Stack, &h.Reason, &since, &h.Why, &first, &last, &rem); err != nil {
|
|
return nil, err
|
|
}
|
|
h.Since, h.FirstSeen, h.LastSeen, h.RemindedAt = parseT(since), parseT(first), parseT(last), parseT(rem)
|
|
out = append(out, h)
|
|
}
|
|
return out, rows.Err()
|
|
}
|