Files
felhom.eu/hub/internal/monitor/offsite_delivery_honesty_test.go
T
admin 91cabdde1b
gates / gates (push) Successful in 7s
hub v0.93.0: the retention keeps the key it was built to keep (R-198) + three honesty fixes (R-197, R-192, R-196)
R-198 — host_escrow_superseded shipped with `blob` (the K-escrow / PBS datastore key) and
identity_blob was added to host_escrow LATER, never here. The offsite restic REPOSITORY
password lives in identity_blob. So demoteCurrentEscrowTx -- whose own comment calls it "THE
ONE escrow row-copy routine" -- retained the whole-guest key and silently dropped the off-site
data key, which is the secret the retention was built to preserve. And because the copy happens
as the new blob overwrites the old, the destroying act was the ESCROW CEREMONY: the exact thing
a rebuilt box tells its customer to run, on a card promising in Hungarian that the old backups
stay recoverable. Both demo boxes crossed that line on 2026-08-04.

  - identity_blob added to the table (CREATE + additive ALTER) and carried in the shared copy
    routine, so BOTH callers are fixed at once: re-escrow and host-delete demotion.
  - ListSupersededEscrow reads it back; store.HostEscrow gains IdentityBlob.
  - CountCurrentEscrowWithIdentity is the census of who the fix protects.
  - Nothing is backfillable: pre-v0.93.0 retained rows have no blob and their sources are gone.
  - Tests assert the CONSEQUENCE (a retained row can still yield a repo password), which is why
    the pre-existing retention test stayed green for two months asserting the mechanism.

R-197 — SaveHostEscrow returns the hash it replaced; the escrow PUT raises
offsite_repo_key_changed (warning, operator-only, edge-triggered) when both hashes are known and
differ. No hash value travels. Severity chosen for the world v0.93.0 creates: with the identity
blob retained, a changed key is "this history now depends on an older recovery code", not a loss.

R-192 (half) — the stuck alert now reports the two shapes it actually covers, burned and
regressed, each stating its own measurement; the regressed text withdraws the Re-issue
recommendation. Every self-heal refusal leaves a notification_log row with its reason. The
guard's logic is unchanged; its 500-oldest-reports scoping stays OPEN and the window is named in
the alert text so the limitation travels with the number. offsite_delivery_stuck and
offsite_credential_restaged are added to operatorOnlyEvents -- neither was registered and neither
has a customerMessages entry, which is not a block.

R-196 — five comments (not the three the spec expected) claimed ReissueCredentials rotates the
restic repo password. It resets the PROVIDER password and cannot touch the repo password, which
is generated on the box. All five corrected; the staleness mark documented as precautionary. The
BEHAVIOUR stays open.

Not in this release: R-199, R-200, R-201 remain open -- the chain that hands the key back is
still unassembled. Part 5 hit its gate; the orphan card is untouched (R-202).
2026-08-04 12:56:58 +02:00

224 lines
8.6 KiB
Go

package monitor
import (
"encoding/json"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// R-192 (v0.93.0) — the alert must describe what was MEASURED, and a refusal to self-heal must leave a
// trace.
//
// What was wrong, quoted from the live e-mail demo-hp sent daily from 2026-08-03: "one-time password
// consumed 284h19m0s ago and 500 report(s) since carry no offbox target — the credential is likely
// burned … Re-issue delivers a fresh one." OffsiteReportsSinceConsume for that same call was 500:
// every one of those reports carried a target. The message stated the precise negation of its own
// measurement and prescribed a remedy for a failure mode that had not occurred — while the self-heal
// correctly declined, in complete silence.
// regressedFixture puts c1 into demo-hp's real shape: consumed >1h ago, reports WITH an offbox target
// after the consume, then reports WITHOUT — the latest carrying none.
//
// The report timestamps get jitter deliberately: a perfectly regular series lands exactly on
// boundaries and survives mutations that a realistic one catches.
func (h dtHarness) regressedFixture(t *testing.T, withOffsite, without int) {
t.Helper()
if err := h.st.SaveOneTimeSecret("c1", "x"); err != nil {
t.Fatal(err)
}
consumed := time.Now().UTC().Add(-30 * time.Hour).Format("2006-01-02 15:04:05")
staged := time.Now().UTC().Add(-31 * time.Hour).Format("2006-01-02 15:04:05")
if err := h.st.SetOneTimeSecretTimesForTest("c1", staged, consumed); err != nil {
t.Fatal(err)
}
jitter := []int{0, 3, 1, 7, 2, 5, 11, 4}
for i := 0; i < withOffsite; i++ {
if err := h.st.SaveReport("c1", []byte(dtReportWithOffsite)); err != nil {
t.Fatal(err)
}
time.Sleep(time.Duration(jitter[i%len(jitter)]) * time.Millisecond)
}
for i := 0; i < without; i++ {
if err := h.st.SaveReport("c1", []byte(dtReportNoOffsite)); err != nil {
t.Fatal(err)
}
time.Sleep(time.Duration(jitter[(i+3)%len(jitter)]) * time.Millisecond)
}
}
func (h dtHarness) refusalRows(t *testing.T) []store.NotificationLogEntry {
t.Helper()
all, err := h.st.GetRecentNotifications("c1", 50)
if err != nil {
t.Fatal(err)
}
var out []store.NotificationLogEntry
for _, n := range all {
if n.Status == "refused" {
out = append(out, n)
}
}
return out
}
// Scenario F — the message states the measured state, not a denied cause.
// RED-PROOF: restore the old single hardcoded sentence (interpolate ReportsSinceConsume into
// "report(s) since carry no offbox target") → the regressed assertions FAIL.
func TestDeliveryChecker_RegressedShape_MessageMatchesMeasurement(t *testing.T) {
h := newDTHarness(t, true)
h.regressedFixture(t, 6, 2)
h.checker.Check()
saved := h.savedEvents(t, "offsite_delivery_stuck")
if len(saved) != 1 {
t.Fatalf("stuck events = %d, want 1", len(saved))
}
msg := saved[0].Message
// It must NOT assert the thing the data contradicts.
if strings.Contains(msg, "carry no offbox target") || strings.Contains(msg, "likely burned") {
t.Fatalf("R-192(a): the message still asserts a burned credential on a REGRESSED shape — this is "+
"the sentence that stated the negation of its own measurement:\n%s", msg)
}
// It must name the shape it detected and the measurement behind it.
if !strings.Contains(msg, "REGRESSED") {
t.Errorf("the message must name the detected shape, got:\n%s", msg)
}
if !strings.Contains(msg, "6 DID carry an offbox target") {
t.Errorf("the message must state the measured offbox-bearing count (6), got:\n%s", msg)
}
// It must not prescribe the action for a failure mode that did not occur.
if !strings.Contains(msg, "Re-issue is NOT the indicated action") {
t.Errorf("the message must withdraw the Re-issue recommendation for this shape, got:\n%s", msg)
}
// The counting window is named, because CountReportsOffsiteSince reads the OLDEST 500 — an
// instrument that can silently mis-scope must say so where it reports (R-192's other half, open).
if !strings.Contains(msg, "500") {
t.Errorf("the message must name its counting window, got:\n%s", msg)
}
var details map[string]any
if err := json.Unmarshal([]byte(saved[0].DetailsJSON), &details); err != nil {
t.Fatal(err)
}
if details["shape"] != "regressed" {
t.Errorf("details.shape = %v, want regressed", details["shape"])
}
// The field whose absence was defect (a) is now carried.
if got, ok := details["offsite_reports_since_consume"].(float64); !ok || int(got) != 6 {
t.Errorf("details must carry offsite_reports_since_consume=6, got %v", details["offsite_reports_since_consume"])
}
}
// The burned shape keeps its own (correct) text and its recommendation — the fix must not blunt the
// case that was always right.
func TestDeliveryChecker_BurnedShape_KeepsItsRecommendation(t *testing.T) {
h := newDTHarness(t, false)
h.burnedFixture(t, 5)
h.checker.Check()
saved := h.savedEvents(t, "offsite_delivery_stuck")
if len(saved) != 1 {
t.Fatalf("stuck events = %d, want 1", len(saved))
}
msg := saved[0].Message
if !strings.Contains(msg, "BURNED") || !strings.Contains(msg, "NONE carried an offbox target") {
t.Errorf("the burned shape must state its own measurement, got:\n%s", msg)
}
if !strings.Contains(msg, "Re-issue delivers a fresh one") {
t.Errorf("the burned shape must keep its recommendation, got:\n%s", msg)
}
var details map[string]any
_ = json.Unmarshal([]byte(saved[0].DetailsJSON), &details)
if details["shape"] != "burned" {
t.Errorf("details.shape = %v, want burned", details["shape"])
}
}
// Scenario G — a refusal to self-heal leaves a durable record naming its reason.
// RED-PROOF: replace the recordHealRefusal call in the regressed branch with a bare `return`
// (production behaviour up to v0.92.0) → no row → this FAILS.
func TestDeliveryChecker_HealRefusal_LeavesARecord(t *testing.T) {
h := newDTHarness(t, true)
h.regressedFixture(t, 6, 2)
h.checker.Check()
if h.reissuer.count() != 0 {
t.Fatal("the heal must NOT fire on a regressed shape — the guard is unchanged by this fix")
}
rows := h.refusalRows(t)
if len(rows) != 1 {
t.Fatalf("R-192(b): the heal declined and left %d record(s), want 1 — 'we chose not to act' and "+
"'the heal never ran' must not look identical", len(rows))
}
r := rows[0]
if r.Channel != "operator" {
t.Errorf("refusal channel = %q, want operator (the channel the heal's own event would use)", r.Channel)
}
if !strings.Contains(r.ErrorMessage, "regressed-apply shape") {
t.Errorf("the record must name WHY, got %q", r.ErrorMessage)
}
if !strings.Contains(r.ErrorMessage, "6 of the first 8") {
t.Errorf("the record must carry the measurement it refused on, got %q", r.ErrorMessage)
}
}
// The R-39(a) guard's refusal is recorded too — it was the one refusal that already logged, and it
// still must not be the only one that does.
//
// Reaching it needs the real TOCTOU, not a pre-staged row: staging BEFORE Check() makes the derive
// itself return staged_awaiting_consume and the pass never reaches the heal at all. Same technique as
// TestDeliveryChecker_R39aGuard_NeverRestagesOverUnconsumed — the operator stages mid-tick, on the
// stuck event's dispatch, which runs after the derive and before maybeHeal.
func TestDeliveryChecker_R39aRefusal_LeavesARecord(t *testing.T) {
h := newDTHarness(t, true)
h.burnedFixture(t, 5)
base := h.checker.onEvent
h.checker.onEvent = func(cid, et, sev, msg, det, src string) {
if et == "offsite_delivery_stuck" {
if err := h.st.SaveOneTimeSecret("c1", "operator-fresh"); err != nil {
t.Errorf("mid-tick stage: %v", err)
}
if err := h.st.SetOneTimeSecretTimesForTest("c1", "2026-07-23 12:00:00", ""); err != nil {
t.Errorf("mid-tick stamp: %v", err)
}
}
base(cid, et, sev, msg, det, src)
}
h.checker.Check()
if h.reissuer.count() != 0 {
t.Fatal("R-39(a): the heal must never restage over an unconsumed secret")
}
rows := h.refusalRows(t)
if len(rows) != 1 {
t.Fatalf("the R-39(a) refusal left %d record(s), want 1", len(rows))
}
if !strings.Contains(rows[0].ErrorMessage, "R-39(a)") {
t.Errorf("the record must name the guard, got %q", rows[0].ErrorMessage)
}
}
// The refusal record rides the stuck event's 24h cadence: it explains the e-mail the operator just
// received, and must not accumulate one row per monitor tick.
func TestDeliveryChecker_HealRefusal_RidesTheStuckCadence(t *testing.T) {
h := newDTHarness(t, true)
h.regressedFixture(t, 6, 2)
h.checker.Check()
h.checker.Check() // the stuck event is swallowed by its cooldown; so is the refusal record
h.checker.Check()
if n := len(h.refusalRows(t)); n != 1 {
t.Fatalf("refusal records = %d after three passes, want 1 (one per stuck e-mail, not per tick)", n)
}
}