7fff45d688
gates / gates (push) Successful in 7s
Part 4 (ships): `david` — a prospective customer with hosts=0, host_deletions=0, reports=0 — e-mailed an expected_dbdump_missed ERROR at 03:00 UTC three mornings running. The existing down-skip could never cover it: it reads the staleness checker's state, which is seeded from a query over the `reports` table, so a customer that never reported has no state at all and GetState() returns "" rather than "down". store.HasEverBoundHost (hosts row OR host_deletions tombstone) is consulted once per customer at the top of the deadline loop. The discriminator is "was a host EVER bound", never "has a report arrived" — a box installed and never heard from is a real fault and keeps alarming. Fail-OPEN on a read error. Red-proof observed: removing the guard fails with `got [expected_dbdump_missed]`, verbatim the event david sent. Parts 0-3 (spike, NO production code for R-193/R-192): audits/SPIKE-offsite-credential-recovery-2026-08-04.md establishes that the one-shot provider password is the RECOVERABLE secret and the restic repository password is the irreplaceable one — and that a guest rebuild mints a fresh one, orphaning the previous off-site history. Measured without touching a box, by comparing host_escrow.restic_pw_sha256 against host_escrow_superseded: BOTH demo boxes changed (demo-hp 15 snapshots / 40.9 MB, demo-felhom 36 snapshots / 1.14 GB). demo-felhom's "lucky" 76-second recovery restored delivery and not the repository, silently, for 13h. ReissueCredentials does NOT rotate the restic password (R-39's record and two hub comments are wrong -> R-196); candidate (b) is not implementable against a zero-knowledge escrow; candidate (a) already exists as F3 and is wired to the wrong event. Ends in ranked options and an unanswered question for the operator. R-195 SHIPPED; R-196 + R-197 filed; R-192 + R-193 updated, neither closed.
149 lines
6.9 KiB
Go
149 lines
6.9 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// R-195 — a customer with NO machine EVER bound must not alarm; a customer WITH one must.
|
|
//
|
|
// Origin, measured on the live hub 2026-08-04: `david` is a prospective customer whose record was
|
|
// created 2026-08-01 16:51:49 with no host ever bound (hosts=0, host_deletions=0, host_reports=0,
|
|
// reports=0). It e-mailed an `expected_dbdump_missed` ERROR at 03:00 UTC on 08-02, 08-03 and 08-04.
|
|
//
|
|
// The mechanism, established at source: the down-skip in CheckBackupDeadlines reads
|
|
// StalenessChecker.GetState(), whose map is seeded from store.GetCustomers() — a query over the
|
|
// `reports` table. A customer with zero reports is in no row, so it has no state, and GetState()
|
|
// returns "" rather than "down". The skip that protects every other silent customer misses the one
|
|
// that never reported at all.
|
|
//
|
|
// These tests pin BOTH halves. A suite that only proved the silence would pass against an
|
|
// implementation that never alarms, which is strictly worse than the defect it replaces.
|
|
|
|
// newUnboundStore creates a store holding ONE active customer and NO host row at all.
|
|
func newUnboundStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatalf("store.New: %v", err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"}); err != nil {
|
|
t.Fatalf("SaveCustomerConfig: %v", err)
|
|
}
|
|
return st
|
|
}
|
|
|
|
// ── Half 1: the silence ────────────────────────────────────────────────────────────────────────
|
|
|
|
// TestCheckBackupDeadlines_NeverBoundHost_Silent is the david case.
|
|
//
|
|
// COMPANION RED-PROOF (observed): deleting the HasEverBoundHost guard from CheckBackupDeadlines
|
|
// makes this test fail with
|
|
//
|
|
// deadline_unbound_test.go: a customer with NO host ever bound must raise NOTHING;
|
|
// got [expected_dbdump_missed]
|
|
//
|
|
// which is verbatim the event `david` e-mailed three mornings running. Restored after.
|
|
func TestCheckBackupDeadlines_NeverBoundHost_Silent(t *testing.T) {
|
|
st := newUnboundStore(t)
|
|
// No UpsertHost, no host-report, no db_dump_completed event — nothing has ever been expected.
|
|
got := runDeadline(t, st)
|
|
if len(got) != 0 {
|
|
t.Fatalf("a customer with NO host ever bound must raise NOTHING; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestHasEverBoundHost pins the predicate itself across its three inputs, because the whole
|
|
// behaviour above turns on it and a predicate that answered `false` for everything would make the
|
|
// test above pass while silencing the entire fleet.
|
|
func TestHasEverBoundHost(t *testing.T) {
|
|
st := newUnboundStore(t)
|
|
|
|
if bound, err := st.HasEverBoundHost("c1"); err != nil || bound {
|
|
t.Fatalf("no host rows → want (false,nil); got (%v,%v)", bound, err)
|
|
}
|
|
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
if bound, err := st.HasEverBoundHost("c1"); err != nil || !bound {
|
|
t.Fatalf("live host row → want (true,nil); got (%v,%v)", bound, err)
|
|
}
|
|
|
|
// An unknown customer is never bound — the predicate must not answer from another customer's rows.
|
|
if bound, err := st.HasEverBoundHost("nobody"); err != nil || bound {
|
|
t.Fatalf("unknown customer → want (false,nil); got (%v,%v)", bound, err)
|
|
}
|
|
}
|
|
|
|
// ── Half 2: THE RED-PROOF THAT MATTERS — a bound machine must still alarm ───────────────────────
|
|
|
|
// TestCheckBackupDeadlines_BoundButNeverReported_StillAlarms is the case the change could break,
|
|
// and it is a real shape: a machine that was installed and bound and never phoned home. It has a
|
|
// `hosts` row and zero reports — indistinguishable from `david` on every signal EXCEPT the one the
|
|
// guard discriminates on. If the guard is ever "simplified" to key off report presence, customer
|
|
// age or a name pattern, this test goes red.
|
|
func TestCheckBackupDeadlines_BoundButNeverReported_StillAlarms(t *testing.T) {
|
|
st := newUnboundStore(t)
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
// No host-report and no db_dump_completed — the box was bound and never said anything.
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("a BOUND machine that never reported is a real fault and must still alarm; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_BoundThenWentQuiet_StillAlarms: the machine reported once, days ago,
|
|
// and stopped. Nothing about the guard may suppress that.
|
|
//
|
|
// The staleness checker's down-skip is nil here (runDeadline passes nil), which is deliberate: it
|
|
// isolates THIS guard. In production a genuinely down node is skipped by staleness and gets its own
|
|
// node_down event — that path is unchanged and is not what this test is about.
|
|
func TestCheckBackupDeadlines_BoundThenWentQuiet_StillAlarms(t *testing.T) {
|
|
st := newUnboundStore(t)
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
// One host-report whose newest backup evidence is 9 days old, and a db dump that last
|
|
// completed 5 days ago (i.e. not since midnight).
|
|
report := hostReportJSON(t, [][2]string{{rfc(-9 * 24 * time.Hour), "ok"}}, nil)
|
|
if err := st.SaveHostReport("h1", "c1", []byte(report), store.HostReportDenorm{}); err != nil {
|
|
t.Fatalf("SaveHostReport: %v", err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_backup_missed") {
|
|
t.Fatalf("a bound machine that went quiet with stale backups must still raise expected_backup_missed; got %v", got)
|
|
}
|
|
if !has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("a bound machine that went quiet must still raise expected_dbdump_missed; got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestCheckBackupDeadlines_DeletedHost_StillJudged: the host row is gone but a tombstone remains
|
|
// (peti-felhom's live shape). The customer HAD a machine, so this check must not take over the
|
|
// judgement — it hands off to the staleness down-skip exactly as before the change.
|
|
func TestCheckBackupDeadlines_DeletedHost_StillJudged(t *testing.T) {
|
|
st := newUnboundStore(t)
|
|
if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil {
|
|
t.Fatalf("UpsertHost: %v", err)
|
|
}
|
|
if err := st.DeleteHost("h1", false); err != nil {
|
|
t.Fatalf("DeleteHost: %v", err)
|
|
}
|
|
if bound, err := st.HasEverBoundHost("c1"); err != nil || !bound {
|
|
t.Fatalf("a DELETED host is still a machine that was once bound → want (true,nil); got (%v,%v)", bound, err)
|
|
}
|
|
got := runDeadline(t, st)
|
|
if !has(got, "expected_dbdump_missed") {
|
|
t.Fatalf("a customer whose host was deleted was still bound and stays judged here; got %v", got)
|
|
}
|
|
}
|