admin
5121536f10
R-1150: the agent's localapi:backup-cadence-cap is a separate per-host state in HostCapabilityChecker: alone it raises whole_guest_backup_refused (warning, plain text, agent reason in details) and whole_guest_backup_refusal_cleared (info); other critical caps keep agent_capability_degraded/_recovered unchanged. New types operator-only and hub-origin only (not in allowedEventTypes: the suspected controller must not post them). agent_capability_degraded/_recovered join operatorOnlyEvents (the checker's "never the customer" claim was configuration only). R-1154: handleHostEnroll's reuse path re-issues (claim code + off-site login) when no controller wait is open and either deliberate_off_at is held or both the host report and the controller report are older than alerting.stale_threshold (wired from main); once per silence (host_reenroll_reissued record, also written at mint); a live box gets nothing re-issued and the skip is recorded (host_reenroll_not_reissued). 05 section 8 [DESIGN]. Tests red-proved; go build/vet/test green.
301 lines
14 KiB
Go
301 lines
14 KiB
Go
package monitor
|
|
|
|
import (
|
|
"io"
|
|
"log"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
// reportWith builds a host-report body carrying the given capability snapshot (the only field the
|
|
// capability checker reads). One critical-degraded entry makes the host "degraded".
|
|
func reportWith(caps string) []byte {
|
|
return []byte(`{"host_id":"h1","capabilities":` + caps + `}`)
|
|
}
|
|
|
|
const capAllOK = `[{"name":"guest-init-pid","feature":"drive-gate","critical":true,"status":"ok"}]`
|
|
const capCritDegraded = `[{"name":"guest-init-pid","feature":"drive-gate guest-sees","critical":true,"status":"degraded","reason":"sudo policy denied"}]`
|
|
const capNonCritDegraded = `[{"name":"disk-smart","feature":"SMART","critical":false,"status":"degraded","reason":"binary not found"}]`
|
|
|
|
func newCapStore(t *testing.T) *store.Store {
|
|
t.Helper()
|
|
st, err := store.New(filepath.Join(t.TempDir(), "test.db"), log.New(io.Discard, "", 0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { st.Close() })
|
|
st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "ck", RetrievalPassword: "p"})
|
|
st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"})
|
|
return st
|
|
}
|
|
|
|
// TestHostCapabilityChecker covers the full transition contract (§10): seed (no event), ok→degraded
|
|
// (one event), steady degraded (none), degraded→ok (recovered), and that a non-critical degradation
|
|
// never trips the host state.
|
|
func TestHostCapabilityChecker(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
|
|
|
var events []string
|
|
onEvent := func(_, eventType, _, _, _, _ string) { events = append(events, eventType) }
|
|
|
|
// Seed ok → no event on init.
|
|
cc := NewHostCapabilityChecker(st, onEvent, log.New(io.Discard, "", 0))
|
|
if cc.GetState("h1") != "ok" {
|
|
t.Fatalf("seed state = %s, want ok", cc.GetState("h1"))
|
|
}
|
|
if len(events) != 0 {
|
|
t.Fatalf("seed must not emit, got %v", events)
|
|
}
|
|
|
|
// ok → degraded: exactly ONE operator event.
|
|
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if cc.GetState("h1") != "degraded" {
|
|
t.Fatalf("state = %s, want degraded", cc.GetState("h1"))
|
|
}
|
|
if len(events) != 1 || events[0] != "agent_capability_degraded" {
|
|
t.Fatalf("want one agent_capability_degraded, got %v", events)
|
|
}
|
|
|
|
// Steady degraded: NO duplicate event.
|
|
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if len(events) != 1 {
|
|
t.Fatalf("steady degraded must not re-emit, got %v", events)
|
|
}
|
|
|
|
// degraded → ok: recovered event.
|
|
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if cc.GetState("h1") != "ok" {
|
|
t.Fatalf("state = %s, want ok", cc.GetState("h1"))
|
|
}
|
|
if len(events) != 2 || events[1] != "agent_capability_recovered" {
|
|
t.Fatalf("want recovered event, got %v", events)
|
|
}
|
|
}
|
|
|
|
// A NON-critical degradation must NOT flip the host to degraded (no operator noise).
|
|
func TestHostCapabilityChecker_NonCriticalIgnored(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capAllOK), store.HostReportDenorm{})
|
|
var events []string
|
|
cc := NewHostCapabilityChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
|
|
st.SaveHostReport("h1", "c1", reportWith(capNonCritDegraded), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if cc.GetState("h1") != "ok" {
|
|
t.Fatalf("non-critical degradation flipped state to %s, want ok", cc.GetState("h1"))
|
|
}
|
|
if len(events) != 0 {
|
|
t.Fatalf("non-critical degradation must not alert, got %v", events)
|
|
}
|
|
}
|
|
|
|
// A pre-v0.44.0 agent (no capabilities array) is "ok" — an old agent can't trip a false alert.
|
|
func TestHostCapabilityChecker_OldAgentNoCaps(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", []byte(`{"host_id":"h1"}`), store.HostReportDenorm{})
|
|
cc := NewHostCapabilityChecker(st, func(_, _, _, _, _, _ string) {}, log.New(io.Discard, "", 0))
|
|
if cc.GetState("h1") != "ok" {
|
|
t.Fatalf("old agent (no caps) state = %s, want ok", cc.GetState("h1"))
|
|
}
|
|
}
|
|
|
|
// F2 RED-PROOF: a host already DEGRADED at hub (re)start is left UNSEEDED by the constructor and the
|
|
// FIRST Check emits once — so a box that was already broken when the hub restarted alerts (vs the
|
|
// pre-fix path which seeded it silently and never alerted). Companion: pre-fix the constructor seeded
|
|
// "degraded" and Check saw oldState==newState → no event.
|
|
func TestHostCapabilityChecker_F2_BornDegradedAlerts(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capCritDegraded), store.HostReportDenorm{})
|
|
var events []string
|
|
cc := NewHostCapabilityChecker(st, func(_, et, _, _, _, _ string) { events = append(events, et) }, log.New(io.Discard, "", 0))
|
|
if len(events) != 0 {
|
|
t.Fatalf("construction must not emit, got %v", events)
|
|
}
|
|
if cc.GetState("h1") != "unknown" {
|
|
t.Fatalf("born-degraded must be left unseeded, GetState = %s want unknown", cc.GetState("h1"))
|
|
}
|
|
cc.Check()
|
|
if len(events) != 1 || events[0] != "agent_capability_degraded" {
|
|
t.Fatalf("born-degraded first Check → one agent_capability_degraded, got %v", events)
|
|
}
|
|
cc.Check() // steady degraded → no duplicate
|
|
if len(events) != 1 {
|
|
t.Fatalf("steady degraded must not re-emit, got %v", events)
|
|
}
|
|
}
|
|
|
|
// ── R-1150: a refused whole-guest backup has its own alarm ───────────────────────────────────────
|
|
|
|
const capCadenceRefused = `[{"name":"guest-init-pid","feature":"drive-gate","critical":true,"status":"ok"},` +
|
|
`{"name":"localapi:backup-cadence-cap","feature":"whole-guest restore points","critical":true,"status":"degraded",` +
|
|
`"reason":"refused a whole-guest backup of guest 9202 on target \"local\" at 2026-10-19T03:15:00Z — more than the schedule allows; the guest's controller may be misbehaving or compromised: check the guest"}]`
|
|
const capCadenceOK = `[{"name":"guest-init-pid","feature":"drive-gate","critical":true,"status":"ok"},` +
|
|
`{"name":"localapi:backup-cadence-cap","feature":"whole-guest restore points","critical":true,"status":"ok"}]`
|
|
const capGrantAndCadenceDegraded = `[{"name":"guest-init-pid","feature":"drive-gate guest-sees","critical":true,"status":"degraded","reason":"sudo policy denied"},` +
|
|
`{"name":"localapi:backup-cadence-cap","feature":"whole-guest restore points","critical":true,"status":"degraded","reason":"refused a whole-guest backup of guest 9202"}]`
|
|
const capGrantDegradedCadenceOK = `[{"name":"guest-init-pid","feature":"drive-gate guest-sees","critical":true,"status":"degraded","reason":"sudo policy denied"},` +
|
|
`{"name":"localapi:backup-cadence-cap","feature":"whole-guest restore points","critical":true,"status":"ok"}]`
|
|
|
|
type capEvent struct{ eventType, severity, message, details string }
|
|
|
|
func recordCapEvents(evs *[]capEvent) EventNotifyFunc {
|
|
return func(_, eventType, severity, message, details, _ string) {
|
|
*evs = append(*evs, capEvent{eventType, severity, message, details})
|
|
}
|
|
}
|
|
|
|
func capTypes(evs []capEvent) []string {
|
|
out := make([]string, len(evs))
|
|
for i, e := range evs {
|
|
out[i] = e.eventType
|
|
}
|
|
return out
|
|
}
|
|
|
|
// The cadence cap ALONE degraded → exactly one whole_guest_backup_refused (warning, plain text, the agent's
|
|
// reason in the details) and NOT agent_capability_degraded; steady → nothing; the window ending → one
|
|
// whole_guest_backup_refusal_cleared (info) and NOT agent_capability_recovered.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with the pre-fix shape restored (capabilityState counting the
|
|
// cadence cap like any grant, no separate refusal state) this test fails with
|
|
//
|
|
// host_capability_test.go: cadence cap alone → want [whole_guest_backup_refused], got [agent_capability_degraded]
|
|
//
|
|
// Restored.
|
|
func TestR1150_CadenceCapAloneRaisesWholeGuestBackupRefused(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceOK), store.HostReportDenorm{})
|
|
var evs []capEvent
|
|
cc := NewHostCapabilityChecker(st, recordCapEvents(&evs), log.New(io.Discard, "", 0))
|
|
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceRefused), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if got := capTypes(evs); len(got) != 1 || got[0] != "whole_guest_backup_refused" {
|
|
t.Fatalf("cadence cap alone → want [whole_guest_backup_refused], got %v", got)
|
|
}
|
|
e := evs[0]
|
|
if e.severity != "warning" {
|
|
t.Errorf("severity = %q, want warning", e.severity)
|
|
}
|
|
if !strings.Contains(e.message, "a whole-guest backup was refused") || !strings.Contains(e.message, "check the guest") {
|
|
t.Errorf("message does not say what happened in plain words: %q", e.message)
|
|
}
|
|
if strings.Contains(e.message, "privileged capability") {
|
|
t.Errorf("message still reads as a broken grant: %q", e.message)
|
|
}
|
|
if !strings.Contains(e.details, "guest 9202") || !strings.Contains(e.details, "localapi:backup-cadence-cap") {
|
|
t.Errorf("details must carry the agent's reason and the capability name: %s", e.details)
|
|
}
|
|
if cc.GetState("h1") != "ok" {
|
|
t.Errorf("the grants state = %s, want ok (the cadence cap is not a grant)", cc.GetState("h1"))
|
|
}
|
|
if cc.GetBackupRefusalState("h1") != "degraded" {
|
|
t.Errorf("refusal state = %s, want degraded", cc.GetBackupRefusalState("h1"))
|
|
}
|
|
// The event is STORED too (the operator's event list), under the new type.
|
|
if at, err := st.LastEventAt("c1", "whole_guest_backup_refused"); err != nil || at.IsZero() {
|
|
t.Errorf("whole_guest_backup_refused not stored: at=%v err=%v", at, err)
|
|
}
|
|
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceRefused), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if len(evs) != 1 {
|
|
t.Fatalf("steady refusal must not re-emit, got %v", capTypes(evs))
|
|
}
|
|
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceOK), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if got := capTypes(evs); len(got) != 2 || got[1] != "whole_guest_backup_refusal_cleared" {
|
|
t.Fatalf("refusal window ended → want [.. whole_guest_backup_refusal_cleared], got %v", got)
|
|
}
|
|
if evs[1].severity != "info" {
|
|
t.Errorf("cleared severity = %q, want info (recorded, never mailed)", evs[1].severity)
|
|
}
|
|
}
|
|
|
|
// Another critical cap degraded → agent_capability_degraded unchanged (the cadence cap never named in it);
|
|
// both degraded on ONE report → exactly one event per state; the grant restored while the refusal stands →
|
|
// agent_capability_recovered alone; nothing for a steady state.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with the pre-fix shape restored the both-degraded report emits a
|
|
// single event naming both caps and the test fails with
|
|
//
|
|
// host_capability_test.go: grant + cadence on one report → want [agent_capability_degraded whole_guest_backup_refused], got [agent_capability_degraded]
|
|
//
|
|
// Restored.
|
|
func TestR1150_OtherCapStillRaisesCapabilityDegraded(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceOK), store.HostReportDenorm{})
|
|
var evs []capEvent
|
|
cc := NewHostCapabilityChecker(st, recordCapEvents(&evs), log.New(io.Discard, "", 0))
|
|
|
|
// A grant alone: the old event, unchanged text.
|
|
st.SaveHostReport("h1", "c1", reportWith(capGrantDegradedCadenceOK), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if got := capTypes(evs); len(got) != 1 || got[0] != "agent_capability_degraded" {
|
|
t.Fatalf("grant alone → want [agent_capability_degraded], got %v", got)
|
|
}
|
|
if !strings.Contains(evs[0].message, "agent privileged capability degraded — guest-init-pid") {
|
|
t.Errorf("agent_capability_degraded text changed: %q", evs[0].message)
|
|
}
|
|
// Back to clean, then both on one report.
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceOK), store.HostReportDenorm{})
|
|
cc.Check()
|
|
evs = nil
|
|
st.SaveHostReport("h1", "c1", reportWith(capGrantAndCadenceDegraded), store.HostReportDenorm{})
|
|
cc.Check()
|
|
got := capTypes(evs)
|
|
if len(got) != 2 || got[0] != "agent_capability_degraded" || got[1] != "whole_guest_backup_refused" {
|
|
t.Fatalf("grant + cadence on one report → want [agent_capability_degraded whole_guest_backup_refused], got %v", got)
|
|
}
|
|
if strings.Contains(evs[0].message, "backup-cadence-cap") || strings.Contains(evs[0].details, "backup-cadence-cap") {
|
|
t.Errorf("agent_capability_degraded must not name the cadence cap: %q / %s", evs[0].message, evs[0].details)
|
|
}
|
|
// Steady: nothing.
|
|
st.SaveHostReport("h1", "c1", reportWith(capGrantAndCadenceDegraded), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if len(evs) != 2 {
|
|
t.Fatalf("steady both-degraded must not re-emit, got %v", capTypes(evs))
|
|
}
|
|
// The grant restored, the refusal still standing: only the grants recover.
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceRefused), store.HostReportDenorm{})
|
|
cc.Check()
|
|
if got := capTypes(evs); len(got) != 3 || got[2] != "agent_capability_recovered" {
|
|
t.Fatalf("grant restored, refusal standing → want [.. agent_capability_recovered], got %v", got)
|
|
}
|
|
if cc.GetBackupRefusalState("h1") != "degraded" {
|
|
t.Errorf("refusal state = %s, want still degraded", cc.GetBackupRefusalState("h1"))
|
|
}
|
|
}
|
|
|
|
// F2 for the refusal: a host already reporting a refused backup when the hub (re)starts alerts ONCE on the
|
|
// first Check (not at construction), and never again while steady.
|
|
//
|
|
// COMPANION RED-PROOF (observed 2026-10-19): with the pre-fix shape restored it fails with
|
|
//
|
|
// host_capability_test.go: born-refused → want exactly [whole_guest_backup_refused], got [agent_capability_degraded]
|
|
//
|
|
// Restored.
|
|
func TestR1150_BornRefusedAlertsOnce(t *testing.T) {
|
|
st := newCapStore(t)
|
|
st.SaveHostReport("h1", "c1", reportWith(capCadenceRefused), store.HostReportDenorm{})
|
|
var evs []capEvent
|
|
cc := NewHostCapabilityChecker(st, recordCapEvents(&evs), log.New(io.Discard, "", 0))
|
|
if len(evs) != 0 {
|
|
t.Fatalf("construction must not emit, got %v", capTypes(evs))
|
|
}
|
|
cc.Check()
|
|
cc.Check()
|
|
if got := capTypes(evs); len(got) != 1 || got[0] != "whole_guest_backup_refused" {
|
|
t.Fatalf("born-refused → want exactly [whole_guest_backup_refused], got %v", got)
|
|
}
|
|
}
|