212 lines
8.1 KiB
Go
212 lines
8.1 KiB
Go
package holdreport
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
type fakeStore struct {
|
|
holds map[string]settings.RestoreHold
|
|
reported map[string]string
|
|
}
|
|
|
|
func (f *fakeStore) ListRestoreHolds() []settings.RestoreHold {
|
|
out := []settings.RestoreHold{}
|
|
for _, h := range f.holds {
|
|
out = append(out, h)
|
|
}
|
|
return out
|
|
}
|
|
func (f *fakeStore) GetReportedHolds() map[string]string {
|
|
out := map[string]string{}
|
|
for k, v := range f.reported {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
func (f *fakeStore) SetReportedHolds(m map[string]string) error { f.reported = m; return nil }
|
|
|
|
type sent struct {
|
|
typ, sev string
|
|
d Details
|
|
}
|
|
|
|
func rec(out *[]sent) Push {
|
|
return func(t, sev, msg string, d interface{}) { *out = append(*out, sent{t, sev, d.(Details)}) }
|
|
}
|
|
|
|
var t0 = time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC)
|
|
|
|
// The report field (D53): a new hold is sent once as app_held with its stack, reason and since, severity info (the hub
|
|
// mails nothing on receipt; its reminder after the grace is the mail).
|
|
func TestD53_ANewHoldIsReportedWithStackReasonAndSince(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{
|
|
"kimai": {Stack: "kimai", At: "2026-10-14T08:00:00Z", Reason: settings.HoldReasonUnhealthyStop, UnhealthyKind: "crash_loop"},
|
|
}}
|
|
var out []sent
|
|
r := New(st, rec(&out), nil)
|
|
r.Tick(t0)
|
|
if len(out) != 1 || out[0].typ != EventHeld || out[0].sev != "info" {
|
|
t.Fatalf("want one info app_held, got %+v", out)
|
|
}
|
|
d := out[0].d
|
|
if d.StackName != "kimai" || d.Reason != "unhealthy_stop" || d.Since != "2026-10-14T08:00:00Z" || d.Why == "" {
|
|
t.Fatalf("payload: %+v", d)
|
|
}
|
|
if st.reported["kimai"] != "unhealthy_stop|2026-10-14T08:00:00Z" {
|
|
t.Fatalf("reported record: %v", st.reported)
|
|
}
|
|
// The next tick inside a day sends nothing.
|
|
out = nil
|
|
r.Tick(t0.Add(time.Minute))
|
|
if len(out) != 0 {
|
|
t.Fatalf("an unchanged hold was sent again inside a day: %+v", out)
|
|
}
|
|
}
|
|
|
|
// An event can be lost: every hold still in place is sent again after a day, marked as a repeat.
|
|
func TestD53_AHoldIsSentAgainEveryDay(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{"a": {Stack: "a", At: "2026-10-20T00:00:00Z", Reason: settings.HoldReasonUpdateFailed}}}
|
|
var out []sent
|
|
r := New(st, rec(&out), nil)
|
|
r.Tick(t0)
|
|
out = nil
|
|
r.Tick(t0.Add(ResendEvery))
|
|
if len(out) != 1 || out[0].typ != EventHeld || !out[0].d.Repeat {
|
|
t.Fatalf("want one repeat app_held after a day, got %+v", out)
|
|
}
|
|
}
|
|
|
|
// A hold cleared by a person ends it: one app_hold_cleared, then nothing.
|
|
func TestD53_AClearedHoldSendsClearedOnce(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{"a": {Stack: "a", At: "2026-10-20T00:00:00Z", Reason: settings.HoldReasonRestoreMixed}}}
|
|
var out []sent
|
|
r := New(st, rec(&out), nil)
|
|
r.Tick(t0)
|
|
delete(st.holds, "a")
|
|
out = nil
|
|
r.Tick(t0.Add(time.Minute))
|
|
if len(out) != 1 || out[0].typ != EventCleared || out[0].d.StackName != "a" || out[0].sev != "info" {
|
|
t.Fatalf("want one info app_hold_cleared for a, got %+v", out)
|
|
}
|
|
if len(st.reported) != 0 {
|
|
t.Fatalf("the reported record still names the cleared hold: %v", st.reported)
|
|
}
|
|
out = nil
|
|
r.Tick(t0.Add(2 * time.Minute))
|
|
r.Tick(t0.Add(ResendEvery + time.Hour))
|
|
if len(out) != 0 {
|
|
t.Fatalf("a cleared hold was reported again: %+v", out)
|
|
}
|
|
}
|
|
|
|
// A hold cleared while the controller was down is still reported cleared: the record lives in settings.
|
|
func TestD53_AHoldClearedWhileDownIsReportedAtStart(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{}, reported: map[string]string{"gone": "update_failed|2026-10-01T00:00:00Z"}}
|
|
var out []sent
|
|
New(st, rec(&out), nil).Tick(t0)
|
|
if len(out) != 1 || out[0].typ != EventCleared || out[0].d.StackName != "gone" {
|
|
t.Fatalf("want app_hold_cleared for gone at start, got %+v", out)
|
|
}
|
|
}
|
|
|
|
// A hold whose reason changes (a put-back that failed becomes a mixed restore) is a new hold for the hub.
|
|
func TestD53_AChangedReasonIsReportedAgain(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{"a": {Stack: "a", At: "2026-10-20T00:00:00Z", Reason: settings.HoldReasonPutBackPending}}}
|
|
var out []sent
|
|
r := New(st, rec(&out), nil)
|
|
r.Tick(t0)
|
|
st.holds["a"] = settings.RestoreHold{Stack: "a", At: "2026-10-20T01:00:00Z", Reason: settings.HoldReasonRestoreMixed}
|
|
out = nil
|
|
r.Tick(t0.Add(time.Minute))
|
|
if len(out) != 1 || out[0].typ != EventHeld || out[0].d.Reason != "restore_mixed" || out[0].d.Repeat {
|
|
t.Fatalf("want a new (not repeat) app_held restore_mixed, got %+v", out)
|
|
}
|
|
}
|
|
|
|
// The legacy empty reason is a restore that failed with its rollback: the hub reads it as restore_failed.
|
|
func TestD53_TheEmptyReasonIsRestoreFailedOnTheWire(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{"db": {Stack: "db", At: "2026-10-20T00:00:00Z"}}}
|
|
var out []sent
|
|
New(st, rec(&out), nil).Tick(t0)
|
|
if len(out) != 1 || out[0].d.Reason != "restore_failed" {
|
|
t.Fatalf("want reason restore_failed, got %+v", out)
|
|
}
|
|
}
|
|
|
|
// The seam is wired: main builds the reporter on the real settings and notifier and ticks it from the scheduler. A
|
|
// reporter nobody ticks is the inert-seam shape (four instances in this project). A commented-out line does not count.
|
|
func TestD53_TheReporterIsWiredInMain(t *testing.T) {
|
|
raw, err := os.ReadFile(filepath.Join("..", "..", "cmd", "controller", "main.go"))
|
|
if err != nil {
|
|
t.Fatalf("cannot read main.go: %v", err)
|
|
}
|
|
live := []string{}
|
|
for _, l := range strings.Split(string(raw), "\n") {
|
|
if !strings.HasPrefix(strings.TrimSpace(l), "//") {
|
|
live = append(live, l)
|
|
}
|
|
}
|
|
src := strings.Join(live, "\n")
|
|
for _, want := range []string{"holdreport.New(sett, notifier.PushEvent, logger)", `sched.Every("hold-report"`, "holdReporter.Tick(", "holdReporter.SetInstalled("} {
|
|
if !strings.Contains(src, want) {
|
|
t.Errorf("main.go lacks %q — the hold report is not wired", want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The reported record survives a restart: it is what lets a hold cleared while the controller was down be reported.
|
|
func TestD53_TheReportedRecordSurvivesAReload(t *testing.T) {
|
|
p := filepath.Join(t.TempDir(), "settings.json")
|
|
s, err := settings.Load(p, log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := s.SetReportedHolds(map[string]string{"kimai": "unhealthy_stop|2026-10-14T08:00:00Z"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s2, err := settings.Load(p, log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := s2.GetReportedHolds()["kimai"]; got != "unhealthy_stop|2026-10-14T08:00:00Z" {
|
|
t.Fatalf("after a reload the record reads %q", got)
|
|
}
|
|
}
|
|
|
|
// R-1298: an install's hold (crash-loop stop, failed update, waiting put-back) of an app that is NOT installed any more is
|
|
// not reported — on 9202 four such holds from a hand removal would have mailed the operator weekly about apps that do not
|
|
// exist. A restore hold stays reported whatever the app's state (R-379: a person clears it). A hold reported earlier and
|
|
// now skipped is reported cleared. COMPANION RED-PROOF: without the installed check all three holds are reported.
|
|
func TestR1298_OnlyHoldsOfInstalledAppsAreReported(t *testing.T) {
|
|
st := &fakeStore{holds: map[string]settings.RestoreHold{
|
|
"gone": {Stack: "gone", At: "2026-09-24T12:16:24Z", Reason: settings.HoldReasonUnhealthyStop},
|
|
"gonedb": {Stack: "gonedb", At: "2026-09-24T12:16:24Z", Reason: settings.HoldReasonRestoreFailed},
|
|
"present": {Stack: "present", At: "2026-10-26T17:37:52Z", Reason: settings.HoldReasonUpdateFailed},
|
|
}, reported: map[string]string{"gone": "unhealthy_stop|2026-09-24T12:16:24Z"}}
|
|
var out []sent
|
|
r := New(st, rec(&out), nil)
|
|
r.SetInstalled(func(stack string) bool { return stack == "present" })
|
|
r.Tick(t0)
|
|
got := map[string]string{}
|
|
for _, s := range out {
|
|
got[s.d.StackName] = s.typ
|
|
}
|
|
if got["gone"] != EventCleared {
|
|
t.Errorf("the hold of an app that is not installed must be reported cleared, got %q", got["gone"])
|
|
}
|
|
if got["gonedb"] != EventHeld {
|
|
t.Errorf("a restore hold must stay reported even when the app is gone, got %q", got["gonedb"])
|
|
}
|
|
if got["present"] != EventHeld {
|
|
t.Errorf("an installed app's hold must be reported, got %q", got["present"])
|
|
}
|
|
}
|