hub (unreleased): two end dates warn the operator 14 days before — an operator token and a box's web certificate (R-1379, D77); the other sixteen as a design table
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
68292e40f6
commit
86bb912a4b
11 files changed
+527
-5
No files matched your search
@@ -0,0 +1,190 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// EndDateChecker (R-1379, D77, `09` §3 decision 298; `08-alarm-ladder.md`) warns the operator 14 days before two end
|
||||
// dates the product itself can read. Until it, no part of the product read either, and the first sign was the
|
||||
// failure on the day:
|
||||
//
|
||||
// - an OPERATOR ACCESS TOKEN (hub §17): 90 days by default; on its end date the release step that raises a floor
|
||||
// answers 401. The hub knows the date (api_tokens.expires_at).
|
||||
// - a box's WEB CERTIFICATE: only the box knows the date (it is in Traefik's store in the guest; the household's
|
||||
// public names end at Cloudflare). The controller carries it in its report (`web_certs`, controllers after
|
||||
// 0.352.1). Traefik renews 30 days before the end by itself — so 14 days left means the renewal has failed for
|
||||
// two weeks (a Cloudflare token of the household that ended, a name check that fails).
|
||||
//
|
||||
// One event per (token) and per (household, name, end date): told once at 14 days, and once more — as an ERROR — if
|
||||
// the date passes with the same certificate or token still there. A renewed certificate has a new end date, so
|
||||
// nothing has to clear. State: the hub's time-by-key settings, no new table. Operator-only. Pinned by TestEndDates_*.
|
||||
type EndDateChecker struct {
|
||||
tokens func() ([]EndingToken, error)
|
||||
reports func() ([]BoxCertReport, error)
|
||||
told func(key string) time.Time
|
||||
setTold func(key string, t time.Time) error
|
||||
onEvent EventNotifyFunc
|
||||
logger *log.Logger
|
||||
now func() time.Time
|
||||
every time.Duration
|
||||
warnDays int
|
||||
|
||||
mu sync.Mutex
|
||||
lastRun time.Time
|
||||
}
|
||||
|
||||
// EndingToken is one operator access token that is still valid to be used (not revoked).
|
||||
type EndingToken struct {
|
||||
ID int64
|
||||
Name string
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// BoxCertReport is one household's newest controller report (raw), from which `web_certs` is read.
|
||||
type BoxCertReport struct {
|
||||
CustomerID string
|
||||
ReportJSON string
|
||||
}
|
||||
|
||||
const (
|
||||
EventTokenEndsSoon = "operator_token_ends_soon"
|
||||
EventWebCertEndsSoon = "web_cert_ends_soon"
|
||||
endDateWarnDays = 14
|
||||
endDateCheckEvery = time.Hour
|
||||
webCertsReadMax = 8
|
||||
)
|
||||
|
||||
var endDateNameRe = regexp.MustCompile(`^[A-Za-z0-9*._-]{1,253}$`)
|
||||
|
||||
// a token's name is the operator's own text (hub Tokens page); anything else is shown as „token #id”.
|
||||
var endDateTokenNameRe = regexp.MustCompile(`^[A-Za-z0-9 ._-]{1,64}$`)
|
||||
|
||||
func NewEndDateChecker(tokens func() ([]EndingToken, error), reports func() ([]BoxCertReport, error),
|
||||
told func(string) time.Time, setTold func(string, time.Time) error, onEvent EventNotifyFunc, logger *log.Logger) *EndDateChecker {
|
||||
return &EndDateChecker{tokens: tokens, reports: reports, told: told, setTold: setTold, onEvent: onEvent, logger: logger,
|
||||
now: time.Now, every: endDateCheckEvery, warnDays: endDateWarnDays}
|
||||
}
|
||||
|
||||
// Check runs on the hub's 60 s sweep and does its work once an hour.
|
||||
func (c *EndDateChecker) Check() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
now := c.now()
|
||||
if !c.lastRun.IsZero() && now.Sub(c.lastRun) < c.every {
|
||||
return
|
||||
}
|
||||
c.lastRun = now
|
||||
warn := time.Duration(c.warnDays) * 24 * time.Hour
|
||||
|
||||
// stage: "soon" inside the 14 days, "ended" from the end date on. Each stage is told once per key.
|
||||
tell := func(key string, end time.Time) (stage string, ok bool) {
|
||||
left := end.Sub(now)
|
||||
switch {
|
||||
case left > warn || left < -warn:
|
||||
return "", false // not yet — or ended so long ago that it is no news (an old token nobody replaced)
|
||||
case left > 0:
|
||||
stage = "soon"
|
||||
default:
|
||||
stage = "ended"
|
||||
}
|
||||
k := key + ":" + stage
|
||||
if !c.told(k).IsZero() {
|
||||
return stage, false
|
||||
}
|
||||
_ = c.setTold(k, now)
|
||||
return stage, true
|
||||
}
|
||||
daysLeft := func(end time.Time) int { return int(end.Sub(now) / (24 * time.Hour)) }
|
||||
|
||||
if toks, err := c.tokens(); err != nil {
|
||||
c.logger.Printf("[WARN] end-date check: cannot list the operator tokens: %v", err)
|
||||
} else {
|
||||
for _, t := range toks {
|
||||
stage, ok := tell(fmt.Sprintf("end_date:token:%d", t.ID), t.ExpiresAt)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := t.Name
|
||||
if !endDateTokenNameRe.MatchString(name) {
|
||||
name = fmt.Sprintf("token #%d", t.ID)
|
||||
}
|
||||
sev, msg := "warning", fmt.Sprintf("The operator access token „%s” ends in %d day(s), on %s. On that day every script that uses it gets 401 (for one: the release step that raises the floor). Do: make a new token on the hub's Tokens page and put it in the store of the program that uses it.",
|
||||
name, daysLeft(t.ExpiresAt), t.ExpiresAt.UTC().Format("2006-01-02"))
|
||||
if stage == "ended" {
|
||||
sev, msg = "error", fmt.Sprintf("The operator access token „%s” ENDED on %s and is not replaced: every script that uses it gets 401. Do: make a new token on the hub's Tokens page.",
|
||||
name, t.ExpiresAt.UTC().Format("2006-01-02"))
|
||||
}
|
||||
c.logger.Printf("[WARN] end-date check: operator token %d (%s) %s — ends %s", t.ID, name, stage, t.ExpiresAt.UTC().Format(time.RFC3339))
|
||||
details, _ := json.Marshal(map[string]any{"token_id": t.ID, "name": name, "expires_at": t.ExpiresAt.UTC().Format(time.RFC3339), "stage": stage})
|
||||
c.onEvent("", EventTokenEndsSoon, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
|
||||
reps, err := c.reports()
|
||||
if err != nil {
|
||||
c.logger.Printf("[WARN] end-date check: cannot read the reports: %v", err)
|
||||
return
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, r := range reps {
|
||||
if seen[r.CustomerID] {
|
||||
continue
|
||||
}
|
||||
seen[r.CustomerID] = true
|
||||
for _, wc := range parseWebCerts(r.ReportJSON) {
|
||||
stage, ok := tell("end_date:web_cert:"+r.CustomerID+":"+wc.Domain+":"+wc.NotAfter.UTC().Format("20060102T150405Z"), wc.NotAfter)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
sev, msg := "warning", fmt.Sprintf("The web certificate of %s for %s ends in %d day(s), on %s. The box renews it by itself 30 days before the end, so the renewal has FAILED for about two weeks. On the end date every app address of the household shows a browser warning. Do: read the Traefik log on the box for „acme”; the usual cause is the household's Cloudflare token.",
|
||||
r.CustomerID, wc.Domain, daysLeft(wc.NotAfter), wc.NotAfter.UTC().Format("2006-01-02"))
|
||||
if stage == "ended" {
|
||||
sev, msg = "error", fmt.Sprintf("The web certificate of %s for %s ENDED on %s and the box still reports it: every app address of the household shows a browser warning. Do: read the Traefik log on the box for „acme”.",
|
||||
r.CustomerID, wc.Domain, wc.NotAfter.UTC().Format("2006-01-02"))
|
||||
}
|
||||
c.logger.Printf("[WARN] end-date check: web certificate of %s for %s %s — ends %s", r.CustomerID, wc.Domain, stage, wc.NotAfter.UTC().Format(time.RFC3339))
|
||||
details, _ := json.Marshal(map[string]any{"domain": wc.Domain, "not_after": wc.NotAfter.UTC().Format(time.RFC3339), "stage": stage})
|
||||
c.onEvent(r.CustomerID, EventWebCertEndsSoon, sev, msg, string(details), "hub")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type webCert struct {
|
||||
Domain string
|
||||
NotAfter time.Time
|
||||
}
|
||||
|
||||
// parseWebCerts reads `web_certs` from a controller report: at most webCertsReadMax entries, each with a plain name
|
||||
// and a date. The list is the box's: a name that is not a host name, or a date that is not one, is dropped — the
|
||||
// box's words never reach a mail unchecked. An older controller sends none.
|
||||
func parseWebCerts(reportJSON string) []webCert {
|
||||
var w struct {
|
||||
WebCerts []struct {
|
||||
Domain string `json:"domain"`
|
||||
NotAfter string `json:"not_after"`
|
||||
} `json:"web_certs"`
|
||||
}
|
||||
if json.Unmarshal([]byte(reportJSON), &w) != nil {
|
||||
return nil
|
||||
}
|
||||
var out []webCert
|
||||
for _, c := range w.WebCerts {
|
||||
if len(out) == webCertsReadMax {
|
||||
break
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339, c.NotAfter)
|
||||
if err != nil || !endDateNameRe.MatchString(c.Domain) || strings.Contains(c.Domain, "..") {
|
||||
continue
|
||||
}
|
||||
out = append(out, webCert{Domain: c.Domain, NotAfter: at})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].NotAfter.Before(out[j].NotAfter) })
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type endDateFix struct {
|
||||
now time.Time
|
||||
tokens []EndingToken
|
||||
reps []BoxCertReport
|
||||
state map[string]time.Time
|
||||
events []string
|
||||
c *EndDateChecker
|
||||
}
|
||||
|
||||
func newEndDateFix() *endDateFix {
|
||||
f := &endDateFix{now: time.Date(2026, 11, 14, 12, 0, 0, 0, time.UTC), state: map[string]time.Time{}}
|
||||
f.c = NewEndDateChecker(
|
||||
func() ([]EndingToken, error) { return f.tokens, nil },
|
||||
func() ([]BoxCertReport, error) { return f.reps, nil },
|
||||
func(k string) time.Time { return f.state[k] },
|
||||
func(k string, t time.Time) error { f.state[k] = t; return nil },
|
||||
func(cust, typ, sev, msg, details, src string) {
|
||||
f.events = append(f.events, cust+"|"+typ+"|"+sev+"|"+msg+"|"+details)
|
||||
}, log.New(io.Discard, "", 0))
|
||||
f.c.now = func() time.Time { return f.now }
|
||||
return f
|
||||
}
|
||||
|
||||
// step moves the clock (past the hourly throttle) and runs the check.
|
||||
func (f *endDateFix) step(d time.Duration) { f.now = f.now.Add(d); f.c.Check() }
|
||||
|
||||
func certReport(certs ...string) string { return `{"web_certs":[` + strings.Join(certs, ",") + `]}` }
|
||||
func cert(domain string, at time.Time) string {
|
||||
return fmt.Sprintf(`{"domain":%q,"not_after":%q,"issuer":"YR2"}`, domain, at.UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// R-1379 (D77): the operator is warned 14 days before an end date — not at 15, once at 14, not again every hour, once
|
||||
// more (as an error) when the date passes with the same thing still there, and never for a token or certificate that
|
||||
// was renewed. The stored effect asserted is the event. Both kinds, one test each way.
|
||||
// COMPANION RED-PROOFS (observed, audits/night-build-2026-11-14/B/redproof-hub-end-dates.txt).
|
||||
func TestEndDates_FourteenDaysOnceThenEndedOnce(t *testing.T) {
|
||||
f := newEndDateFix()
|
||||
day := 24 * time.Hour
|
||||
tokEnd := f.now.Add(16*day + time.Hour)
|
||||
certEnd := f.now.Add(16*day + 2*time.Hour)
|
||||
f.tokens = []EndingToken{{ID: 7, Name: "CC hub token", ExpiresAt: tokEnd}, {ID: 8, Name: "fresh", ExpiresAt: f.now.Add(80 * day)}}
|
||||
f.reps = []BoxCertReport{{CustomerID: "demo-hp", ReportJSON: certReport(cert("*.enkisfelhom.hu", certEnd))},
|
||||
{CustomerID: "old-box", ReportJSON: `{"health":{"status":"ok"}}`}} // a controller that sends no web_certs
|
||||
f.step(0)
|
||||
f.step(day) // 15 days and a bit left
|
||||
if len(f.events) != 0 {
|
||||
t.Fatalf("at more than 14 days: %v", f.events)
|
||||
}
|
||||
f.step(2 * day) // 13 days and a bit left
|
||||
for i := 0; i < 30; i++ {
|
||||
f.step(2 * time.Hour)
|
||||
}
|
||||
if len(f.events) != 2 {
|
||||
t.Fatalf("inside the 14 days, 31 hourly checks: %d event(s), want ONE per end date: %v", len(f.events), f.events)
|
||||
}
|
||||
all := strings.Join(f.events, "\n")
|
||||
for _, want := range []string{"|operator_token_ends_soon|warning|", "„CC hub token”", tokEnd.Format("2006-01-02"), "401",
|
||||
"demo-hp|web_cert_ends_soon|warning|", "*.enkisfelhom.hu", certEnd.Format("2006-01-02"), "renewal has FAILED"} {
|
||||
if !strings.Contains(all, want) {
|
||||
t.Errorf("the events lack %q:\n%s", want, all)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(f.events[0], "|") {
|
||||
t.Errorf("the token's event must carry no household: %s", f.events[0])
|
||||
}
|
||||
// the end dates pass, nothing was renewed: one ERROR each, once
|
||||
f.now = certEnd.Add(time.Hour)
|
||||
f.c.Check()
|
||||
f.step(2 * time.Hour)
|
||||
if len(f.events) != 4 || !strings.Contains(f.events[2]+f.events[3], "ENDED on") || strings.Count(f.events[2]+f.events[3], "|error|") != 2 {
|
||||
t.Fatalf("after the end date: %v", f.events[2:])
|
||||
}
|
||||
}
|
||||
|
||||
// A renewed certificate has a new end date: nothing is told. A certificate that is renewed AFTER the warning needs no
|
||||
// „all clear” — and a later failure of the new one is told again. Old things are no news: a token that ended months
|
||||
// ago, and nobody replaced, raises nothing when the checker first runs.
|
||||
func TestEndDates_RenewedAndLongGoneAreQuiet(t *testing.T) {
|
||||
f := newEndDateFix()
|
||||
day := 24 * time.Hour
|
||||
f.tokens = []EndingToken{{ID: 1, Name: "old", ExpiresAt: f.now.Add(-120 * day)}}
|
||||
f.reps = []BoxCertReport{{CustomerID: "n100", ReportJSON: certReport(cert("*.example.hu", f.now.Add(40*day)))}}
|
||||
f.step(0)
|
||||
f.step(2 * time.Hour)
|
||||
if len(f.events) != 0 {
|
||||
t.Fatalf("a token that ended 120 days ago and a certificate 40 days out: %v", f.events)
|
||||
}
|
||||
f.reps[0].ReportJSON = certReport(cert("*.example.hu", f.now.Add(10*day)))
|
||||
f.step(2 * time.Hour)
|
||||
if len(f.events) != 1 {
|
||||
t.Fatalf("10 days left: %v", f.events)
|
||||
}
|
||||
f.reps[0].ReportJSON = certReport(cert("*.example.hu", f.now.Add(89*day))) // renewed
|
||||
f.step(2 * time.Hour)
|
||||
f.step(11 * day)
|
||||
if len(f.events) != 1 {
|
||||
t.Fatalf("after the renewal: %v", f.events[1:])
|
||||
}
|
||||
f.step(70 * day) // the NEW certificate is now 8 days from its end
|
||||
if len(f.events) != 2 {
|
||||
t.Fatalf("the new certificate's own warning: %d event(s)", len(f.events))
|
||||
}
|
||||
}
|
||||
|
||||
// The list is the box's. A name that is not a host name and a date that is not a date are dropped, more than eight
|
||||
// entries are cut, and a token whose name is not plain text is named by its number.
|
||||
func TestEndDates_TheBoxesWordsAreChecked(t *testing.T) {
|
||||
f := newEndDateFix()
|
||||
soon := f.now.Add(5 * 24 * time.Hour)
|
||||
var many []string
|
||||
for i := 0; i < 20; i++ {
|
||||
many = append(many, cert(fmt.Sprintf("n%02d.example.hu", i), soon.Add(time.Duration(i)*time.Minute)))
|
||||
}
|
||||
f.reps = []BoxCertReport{
|
||||
{CustomerID: "evil", ReportJSON: certReport(cert(`<script>alert(1)</script>`, soon), cert("a b.example.hu", soon),
|
||||
`{"domain":"ok.example.hu","not_after":"tomorrow"}`, cert("fine.example.hu", soon))},
|
||||
{CustomerID: "many", ReportJSON: certReport(many...)},
|
||||
{CustomerID: "evil", ReportJSON: certReport(cert("second-report.example.hu", soon))}, // a duplicate row of the same household
|
||||
}
|
||||
f.tokens = []EndingToken{{ID: 9, Name: "<b>x</b>", ExpiresAt: soon}}
|
||||
f.step(0)
|
||||
all := strings.Join(f.events, "\n")
|
||||
if strings.Contains(all, "alert(1)") || strings.Contains(all, "a b.example") || strings.Contains(all, "<b>") || strings.Contains(all, "second-report") {
|
||||
t.Fatalf("an unchecked word of a box or a token name reached an event:\n%s", all)
|
||||
}
|
||||
if n := strings.Count(all, "evil|web_cert_ends_soon"); n != 1 || !strings.Contains(all, "fine.example.hu") {
|
||||
t.Errorf("household „evil”: %d event(s), want one for fine.example.hu", n)
|
||||
}
|
||||
if n := strings.Count(all, "many|web_cert_ends_soon"); n != webCertsReadMax {
|
||||
t.Errorf("household „many”: %d event(s), want the cap %d", n, webCertsReadMax)
|
||||
}
|
||||
if !strings.Contains(all, "„token #9”") {
|
||||
t.Errorf("a token with a name that is not plain text must be named by its number:\n%s", all)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user