b080ecf411
oobDegraded tested five things and the sixth never arrived. The agent has emitted `operator_key_configured` on every heartbeat since v0.72.0 — the SAME version that introduced the `oob` stanza carrying it — and store.HostOOBRow mirrored five of the agent's eight OOB fields. With no field for it, encoding/json discarded the fact on arrival, so a box with felhom-sshd active, reachable, a valid config and a configured peer reported `ok` with NO OPERATOR KEY INSTALLED AT ALL. Not a wrong answer: an answer to a question nobody was asking. `operator_peer_configured`, which the hub did read, only says the peer IP is in desired-state — that OOB is MEANT to work, not that entry is possible. Now decoded: operator_key_configured, plus wg_handshake_age_s and healed_at. The last two ride the ALERT TEXT and are deliberately NOT in the predicate — widening a check beyond the fact that is now arriving is how a check stops being read. SCENARIO F, decided on a measurement rather than a preference. operator_key_configured decodes as a POINTER: nil = the agent never said, reported distinctly and never as ok. The version gate was rejected because the field and its stanza shipped in the SAME agent version (v0.72.0), so a stanza without the field cannot come from any released agent; the fleet is 0.113.0/0.127.0 and the vouched floor is 0.127.0. Handled explicitly anyway and pinned, because "cannot happen" is a claim this project has been burned by. THE MESSAGE NAMES THE FAULT. oobDegradedReason is the single source for both predicate and text, so the alert can never name a different fault from the one that fired. The old form derived it separately and had a vocabulary of two — unreachable, or config invalid — with no way to say the key is missing. The operator reads this at 07:00. TESTS DRIVE THE DECODE BOUNDARY. Every hub OOB test before this built a HostOOBRow by hand, and a test written that way CANNOT SEE A FIELD THAT NEVER DECODES — which is how this held a green suite for five weeks. The pre-existing fixture oobReport() also omitted the field, so those scenarios ran against a report shape no released agent produces (same family as R-262). Both fixed. Red-proofs, 8 expected outcomes and 0 wrong, each with the mutation asserted applied: dropping the field returns the false ok; an unconditional check alerts a healthy box; unknown-as-ok restores the silent pass. G-1 CLOSED — scripts/wire_contract_gate.py shipped as ranked, built BEFORE the fixes and seen failing on 40 fields (documentation/tests/wire-contract-gate-2026-08-08/BEFORE.md). Two instrument defects the control caught first: a substring false negative (grep -F healed_at matched privsep_healed_at) and treating dr_recipe as wholly opaque when its top-level sections ARE decoded through an allow-list that already cost offsite_restic (R-122). The prompt for this session said "465 emitted tags, eight unreachable". Checked against the repo: R-260 said "at least eight DECISION-BEARING facts", never eight tags. The real count is 40. R-260 CLOSED (class gated, sharpest instance fixed). R-247 CLOSED (controller v0.209.0). R-264 MINTED and OPEN — the 21 facts with no consumer, allowlisted with reasons so that gating the class could not be mistaken for deciding them. Still open and named: R-246, R-255..R-259, R-261..R-263, and C7's test-comment half. Capability map checked: it claims OOB access is implemented, never monitored, so no row was untrue; what was untrue sat one layer down and the row now records it. repo_gates --fast: all 8 OK. go build/vet/test green in hub, run separately from this commit.
114 lines
4.3 KiB
Go
114 lines
4.3 KiB
Go
package monitor
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// Scenarios D, E and F of the R-260 / G-1 session, asserted on the CONSEQUENCE (does the check
|
|
// report degraded, and does it name the right thing) rather than on the mechanism.
|
|
|
|
func healthyRow() store.HostOOBRow {
|
|
return store.HostOOBRow{
|
|
HostID: "h1", CustomerID: "c1", Present: true,
|
|
FelhomSshdActive: true, Reachable: true, ConfigInvalid: false,
|
|
OperatorPeerConfigured: true,
|
|
OperatorKeyConfigured: true, OperatorKeyReported: true,
|
|
}
|
|
}
|
|
|
|
// Scenario D — the box this session exists for: everything up, no operator key.
|
|
func TestOOBDegraded_NoOperatorKey_IsDegradedAndSaysSo(t *testing.T) {
|
|
r := healthyRow()
|
|
r.OperatorKeyConfigured = false // reported, and false
|
|
|
|
if !oobDegraded(r) {
|
|
t.Fatal("a box with felhom-sshd active, reachable, a valid config, a configured peer and NO OPERATOR KEY reported ok — that is R-260, the exact question this check exists to answer")
|
|
}
|
|
reason := oobDegradedReason(r)
|
|
// §7.4: the operator reads this at 07:00 and must know WHICH of the five things is wrong.
|
|
if !strings.Contains(reason, "authorized_key") {
|
|
t.Errorf("the reason must NAME the missing operator key, not just say access is degraded; got %q", reason)
|
|
}
|
|
for _, wrong := range []string{"unreachable", "config invalid"} {
|
|
if strings.Contains(reason, wrong) {
|
|
t.Errorf("the reason names %q, which is not what is wrong: %q", wrong, reason)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Scenario E — a healthy box must be unchanged. No new alert on a box that is fine.
|
|
func TestOOBDegraded_HealthyBoxUnchanged(t *testing.T) {
|
|
if oobDegraded(healthyRow()) {
|
|
t.Fatalf("a fully healthy box alerted: %q", oobDegradedReason(healthyRow()))
|
|
}
|
|
if got := oobDegradedReason(healthyRow()); got != "" {
|
|
t.Errorf("healthy box produced a reason %q", got)
|
|
}
|
|
}
|
|
|
|
// Scenario F — an agent too old to report the field. Absence must NOT read as "the key is
|
|
// installed". This project has watched an absence read as a fact four times.
|
|
func TestOOBDegraded_StanzaWithoutKeyField_IsNotOK(t *testing.T) {
|
|
r := healthyRow()
|
|
r.OperatorKeyReported = false
|
|
r.OperatorKeyConfigured = false // the zero value an absent field leaves behind
|
|
|
|
if !oobDegraded(r) {
|
|
t.Fatal("an agent that never said whether the operator key is installed was reported ok — that is the defect returning through the version door")
|
|
}
|
|
reason := oobDegradedReason(r)
|
|
if !strings.Contains(reason, "too old") || !strings.Contains(reason, "UNPROVEN") {
|
|
t.Errorf("the unknown case must be reported DISTINCTLY from a known-missing key, and must not claim the key is absent; got %q", reason)
|
|
}
|
|
// and it must NOT be worded as the known-missing-key case
|
|
if strings.Contains(reason, "nobody can log in") {
|
|
t.Errorf("unknown was reported as if it were a known-missing key: %q", reason)
|
|
}
|
|
}
|
|
|
|
// The peer gate is preserved: a box where OOB was never set up is not "broken", and this session
|
|
// must not widen the check beyond the fact that is now arriving.
|
|
func TestOOBDegraded_NoOperatorPeer_StillNotEvaluated(t *testing.T) {
|
|
r := healthyRow()
|
|
r.OperatorPeerConfigured = false
|
|
r.OperatorKeyConfigured = false
|
|
r.OperatorKeyReported = true
|
|
if oobDegraded(r) {
|
|
t.Errorf("a box with no operator peer configured must not alert on a missing key: %q", oobDegradedReason(r))
|
|
}
|
|
}
|
|
|
|
// The pre-existing faults must still fire, and still name themselves correctly.
|
|
func TestOOBDegraded_ExistingFaultsUnchanged(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
mutate func(*store.HostOOBRow)
|
|
expect string
|
|
}{
|
|
{"config invalid", func(r *store.HostOOBRow) { r.ConfigInvalid = true }, "config invalid"},
|
|
{"not active", func(r *store.HostOOBRow) { r.FelhomSshdActive = false }, "not active"},
|
|
{"unreachable", func(r *store.HostOOBRow) { r.Reachable = false }, "unreachable"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
r := healthyRow()
|
|
tc.mutate(&r)
|
|
if !oobDegraded(r) {
|
|
t.Fatalf("%s no longer degrades", tc.name)
|
|
}
|
|
if got := oobDegradedReason(r); !strings.Contains(got, tc.expect) {
|
|
t.Errorf("reason %q does not contain %q", got, tc.expect)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// A host with no oob stanza at all is never evaluated — pre-H1 or feature off.
|
|
func TestOOBDegraded_AbsentStanzaNeverAlerts(t *testing.T) {
|
|
if oobDegraded(store.HostOOBRow{HostID: "h1"}) {
|
|
t.Error("a host with no oob stanza alerted")
|
|
}
|
|
}
|