felhom.eu/hub/internal/monitor/held_apps_test.go

package monitor

import (
	"io"
	"log"
	"path/filepath"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

type heldEv struct{ customer, typ, sev, details string }

func heldSetup(t *testing.T) (*store.Store, *HeldAppsChecker, *[]heldEv, *time.Time) {
	t.Helper()
	st, err := store.New(filepath.Join(t.TempDir(), "h.db"), log.New(io.Discard, "", 0))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { st.Close() })
	var evs []heldEv
	now := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
	hc := NewHeldAppsChecker(st, 0, func(c, typ, sev, msg, details, src string) {
		evs = append(evs, heldEv{c, typ, sev, details})
	}, log.New(io.Discard, "", 0))
	hc.now = func() time.Time { return now }
	hc.every = 0
	return st, hc, &evs, &now
}

// The reminder after the grace (D53): a crash-loop hold waits 3 days, then ONE warning reminder to the operator, then
// nothing until a week has passed, then one more.
func TestD53_OneReminderAfterTheGraceThenWeekly(t *testing.T) {
	st, hc, evs, now := heldSetup(t)
	since := now.Add(-time.Hour)
	if err := st.UpsertHeldApp("c1", "kimai", "unhealthy_stop", since, "stopped by the box", *now); err != nil {
		t.Fatal(err)
	}
	hc.Check()
	if len(*evs) != 0 {
		t.Fatalf("a reminder inside the 3-day grace: %+v", *evs)
	}
	for _, step := range []time.Duration{72 * time.Hour, time.Hour, 24 * time.Hour} {
		*now = now.Add(step)
		_ = st.UpsertHeldApp("c1", "kimai", "unhealthy_stop", since, "stopped by the box", *now) // the box's daily repeat
		hc.Check()
	}
	if len(*evs) != 1 || (*evs)[0].typ != "app_held_reminder" || (*evs)[0].sev != "warning" || (*evs)[0].customer != "c1" {
		t.Fatalf("want exactly one warning app_held_reminder after the grace, got %+v", *evs)
	}
	*now = now.Add(7 * 24 * time.Hour)
	_ = st.UpsertHeldApp("c1", "kimai", "unhealthy_stop", since, "stopped by the box", *now)
	hc.Check()
	if len(*evs) != 2 {
		t.Fatalf("want the weekly repeat, got %d events", len(*evs))
	}
}

// No reminder after a clear: the box's app_hold_cleared removes the row, and nothing is mailed for it.
func TestD53_NoReminderAfterAClear(t *testing.T) {
	st, hc, evs, now := heldSetup(t)
	_ = st.UpsertHeldApp("c1", "db", "restore_failed", now.Add(-2*time.Hour), "held after a failed restore", *now)
	if ok, err := st.ClearHeldApp("c1", "db"); err != nil || !ok {
		t.Fatalf("clear: %v %v", ok, err)
	}
	*now = now.Add(30 * 24 * time.Hour)
	hc.Check()
	if len(*evs) != 0 {
		t.Fatalf("a cleared hold was reminded: %+v", *evs)
	}
	if rows, _ := st.ListHeldApps(""); len(rows) != 0 {
		t.Fatalf("the list still holds a cleared app: %+v", rows)
	}
}

// The grace follows the reason (`08` §10): put-back 6 h, restore 1 day, update / crash loop 3 days.
func TestD53_TheGraceFollowsTheReason(t *testing.T) {
	cases := map[string]time.Duration{"putback_pending": 6 * time.Hour, "restore_failed": 24 * time.Hour,
		"restore_mixed": 24 * time.Hour, "update_failed": 72 * time.Hour, "unhealthy_stop": 72 * time.Hour}
	for reason, grace := range cases {
		st, hc, evs, now := heldSetup(t)
		_ = st.UpsertHeldApp("c1", "a", reason, *now, "x", *now)
		*now = now.Add(grace - time.Minute)
		_ = st.UpsertHeldApp("c1", "a", reason, now.Add(-(grace - time.Minute)), "x", *now)
		hc.Check()
		if len(*evs) != 0 {
			t.Errorf("%s: reminded before its grace %s", reason, grace)
		}
		*now = now.Add(2 * time.Minute)
		hc.Check()
		if len(*evs) != 1 {
			t.Errorf("%s: no reminder just after its grace %s (%d events)", reason, grace, len(*evs))
		}
	}
}

// A box that stopped confirming its hold (down or gone) is not reminded about it; its own staleness alarms speak.
func TestD53_AStaleRowIsNotReminded(t *testing.T) {
	st, hc, evs, now := heldSetup(t)
	_ = st.UpsertHeldApp("c1", "a", "restore_failed", now.Add(-10*24*time.Hour), "x", now.Add(-3*24*time.Hour))
	hc.Check()
	if len(*evs) != 0 {
		t.Fatalf("a row the box stopped confirming was reminded: %+v", *evs)
	}
}

// A new hold on the same app (another reason) starts its own grace: the old reminder does not carry over.
func TestD53_ANewHoldOnTheSameAppStartsOver(t *testing.T) {
	st, hc, evs, now := heldSetup(t)
	_ = st.UpsertHeldApp("c1", "a", "putback_pending", now.Add(-7*time.Hour), "x", *now)
	hc.Check()
	if len(*evs) != 1 {
		t.Fatalf("want the put-back reminder, got %d", len(*evs))
	}
	_ = st.UpsertHeldApp("c1", "a", "restore_mixed", *now, "y", *now)
	*now = now.Add(25 * time.Hour)
	_ = st.UpsertHeldApp("c1", "a", "restore_mixed", now.Add(-25*time.Hour), "y", *now)
	hc.Check()
	if len(*evs) != 2 {
		t.Fatalf("the new hold's reminder was swallowed by the old one's week: %d events", len(*evs))
	}
}