Files
felhom.eu/hub/internal/store/host_delete_test.go
T
admin 91cabdde1b
gates / gates (push) Successful in 7s
hub v0.93.0: the retention keeps the key it was built to keep (R-198) + three honesty fixes (R-197, R-192, R-196)
R-198 — host_escrow_superseded shipped with `blob` (the K-escrow / PBS datastore key) and
identity_blob was added to host_escrow LATER, never here. The offsite restic REPOSITORY
password lives in identity_blob. So demoteCurrentEscrowTx -- whose own comment calls it "THE
ONE escrow row-copy routine" -- retained the whole-guest key and silently dropped the off-site
data key, which is the secret the retention was built to preserve. And because the copy happens
as the new blob overwrites the old, the destroying act was the ESCROW CEREMONY: the exact thing
a rebuilt box tells its customer to run, on a card promising in Hungarian that the old backups
stay recoverable. Both demo boxes crossed that line on 2026-08-04.

  - identity_blob added to the table (CREATE + additive ALTER) and carried in the shared copy
    routine, so BOTH callers are fixed at once: re-escrow and host-delete demotion.
  - ListSupersededEscrow reads it back; store.HostEscrow gains IdentityBlob.
  - CountCurrentEscrowWithIdentity is the census of who the fix protects.
  - Nothing is backfillable: pre-v0.93.0 retained rows have no blob and their sources are gone.
  - Tests assert the CONSEQUENCE (a retained row can still yield a repo password), which is why
    the pre-existing retention test stayed green for two months asserting the mechanism.

R-197 — SaveHostEscrow returns the hash it replaced; the escrow PUT raises
offsite_repo_key_changed (warning, operator-only, edge-triggered) when both hashes are known and
differ. No hash value travels. Severity chosen for the world v0.93.0 creates: with the identity
blob retained, a changed key is "this history now depends on an older recovery code", not a loss.

R-192 (half) — the stuck alert now reports the two shapes it actually covers, burned and
regressed, each stating its own measurement; the regressed text withdraws the Re-issue
recommendation. Every self-heal refusal leaves a notification_log row with its reason. The
guard's logic is unchanged; its 500-oldest-reports scoping stays OPEN and the window is named in
the alert text so the limitation travels with the number. offsite_delivery_stuck and
offsite_credential_restaged are added to operatorOnlyEvents -- neither was registered and neither
has a customerMessages entry, which is not a block.

R-196 — five comments (not the three the spec expected) claimed ReissueCredentials rotates the
restic repo password. It resets the PROVIDER password and cannot touch the repo password, which
is generated on the box. All five corrected; the staleness mark documented as precautionary. The
BEHAVIOUR stays open.

Not in this release: R-199, R-200, R-201 remain open -- the chain that hands the key back is
still unassembled. Part 5 hit its gate; the orphan card is untouched (R-202).
2026-08-04 12:56:58 +02:00

284 lines
11 KiB
Go

package store
// Group C (hub v0.47.0 stale host removal) — DeleteHost cascade + CountHostArtifacts.
// Effect assertions throughout: rows counted per table before/after; refusals prove the
// NON-effect (zero deletions), never just "an error came back".
import (
"errors"
"testing"
"time"
)
// seedHostWithArtifacts creates a host plus one row in EVERY host-scoped table, and one
// CUSTOMER-scoped log bundle (the controller channel) that must survive a host delete.
func seedHostWithArtifacts(t *testing.T, s *Store, hostID, customerID string) {
t.Helper()
if err := s.UpsertHost(&Host{HostID: hostID, CustomerID: customerID, APIKey: "key-" + hostID}); err != nil {
t.Fatal(err)
}
for _, g := range []int{100, 101} {
if err := s.UpsertGuestFromReport(&Guest{GuestID: GuestID(hostID, g), CustomerID: customerID,
HostID: hostID, VMID: g, Status: "stopped"}); err != nil {
t.Fatal(err)
}
}
// One host_reports row via raw insert — SaveHostReport would set hosts.last_report_at
// (→ online), and the fixture host must stay never-reported for the handler tests.
if _, err := s.db.Exec(`INSERT INTO host_reports (host_id, customer_id, report_json) VALUES (?, ?, '{}')`,
hostID, customerID); err != nil {
t.Fatal(err)
}
if err := s.EnqueueSignedJob(hostID, "job-1", []byte("opaque")); err != nil {
t.Fatal(err)
}
if err := s.SaveHostRecoveryCredential(hostID, "root@pam", "recovery-secret"); err != nil {
t.Fatal(err)
}
if _, err := s.SaveHostPBSSecret(hostID, "pbs-secret"); err != nil {
t.Fatal(err)
}
// Agent-scoped log bundle (scope_id = host_id) + a pending request. Order matters:
// SaveLogBundle CONSUMES the matching pending request, so the request is re-issued
// after the bundle lands to leave one row in each table.
if _, err := s.SaveLogBundle(hostID, LogBundleComponentAgent, time.Now(), []string{"line"}); err != nil {
t.Fatal(err)
}
if err := s.RequestLogBundle(hostID, LogBundleComponentAgent); err != nil {
t.Fatal(err)
}
// Customer-scoped log bundle (controller channel) — must be UNTOUCHED by a host delete.
if _, err := s.SaveLogBundle(customerID, LogBundleComponentController, time.Now(), []string{"ctl line"}); err != nil {
t.Fatal(err)
}
// Bound wg peer (raw insert — no endpoint needed for the cascade test; assigned_ip is
// TEXT with a UNIQUE constraint, so any distinct value serves).
if _, err := s.db.Exec(`INSERT INTO wg_peers (pubkey, assigned_ip, host_id, note) VALUES (?, ?, ?, '')`,
"PK-"+hostID, "ip-"+hostID, hostID); err != nil {
t.Fatal(err)
}
if _, _, err := s.SaveHostEscrow(hostID, []byte("opaque-escrow"), "fp", "posture", "2026-07-01T00:00:00Z", ""); err != nil {
t.Fatal(err)
}
}
func countRows(t *testing.T, s *Store, query string, arg string) int {
t.Helper()
var n int
if err := s.db.QueryRow(query, arg).Scan(&n); err != nil {
t.Fatalf("count %q: %v", query, err)
}
return n
}
// hostRowCounts snapshots every table the cascade touches, keyed for exact comparison.
func hostRowCounts(t *testing.T, s *Store, hostID string) map[string]int {
t.Helper()
return map[string]int{
"hosts": countRows(t, s, `SELECT COUNT(*) FROM hosts WHERE host_id = ?`, hostID),
"guests": countRows(t, s, `SELECT COUNT(*) FROM guests WHERE host_id = ?`, hostID),
"host_reports": countRows(t, s, `SELECT COUNT(*) FROM host_reports WHERE host_id = ?`, hostID),
"signed_jobs": countRows(t, s, `SELECT COUNT(*) FROM signed_jobs WHERE host_id = ?`, hostID),
"host_recovery": countRows(t, s, `SELECT COUNT(*) FROM host_recovery WHERE host_id = ?`, hostID),
"host_pbs_secrets": countRows(t, s, `SELECT COUNT(*) FROM host_pbs_secrets WHERE host_id = ?`, hostID),
"log_bundle_requests": countRows(t, s, `SELECT COUNT(*) FROM log_bundle_requests WHERE scope_id = ?`, hostID),
"log_bundles": countRows(t, s, `SELECT COUNT(*) FROM log_bundles WHERE scope_id = ?`, hostID),
"wg_peers": countRows(t, s, `SELECT COUNT(*) FROM wg_peers WHERE host_id = ?`, hostID),
"host_escrow": countRows(t, s, `SELECT COUNT(*) FROM host_escrow WHERE host_id = ?`, hostID),
}
}
func TestDeleteHost_CascadeAndIsolation(t *testing.T) {
s := newTestStore(t)
seedHostWithArtifacts(t, s, "dr-drill-host", "cust-a")
seedHostWithArtifacts(t, s, "other-host", "cust-b") // must be fully intact afterwards
before := hostRowCounts(t, s, "dr-drill-host")
for table, n := range before {
if n == 0 {
t.Fatalf("fixture gap: %s has no row for dr-drill-host", table)
}
}
otherBefore := hostRowCounts(t, s, "other-host")
if err := s.DeleteHost("dr-drill-host", true); err != nil {
t.Fatalf("DeleteHost: %v", err)
}
// Every host-scoped row is gone.
for table, n := range hostRowCounts(t, s, "dr-drill-host") {
if n != 0 {
t.Errorf("%s: %d row(s) survived the cascade", table, n)
}
}
// The unrelated host's rows are EXACTLY intact.
otherAfter := hostRowCounts(t, s, "other-host")
for table, n := range otherBefore {
if otherAfter[table] != n {
t.Errorf("unrelated host %s: %d → %d rows (must be untouched)", table, n, otherAfter[table])
}
}
// Customer-scoped log bundles (controller channel) are UNTOUCHED — only scope_id ==
// host_id rows die. RED-PROOF 5: widening the delete to all scopes fails this.
if n := countRows(t, s, `SELECT COUNT(*) FROM log_bundles WHERE scope_id = ?`, "cust-a"); n != 1 {
t.Errorf("customer-scoped log bundle count = %d, want 1 (must survive a host delete)", n)
}
}
// Escrow refusal is fail-safe: the typed error comes back and the transaction NEVER ran —
// zero rows deleted anywhere. RED-PROOF 2 (handler-side) models on the same store guard.
func TestDeleteHost_EscrowRefusalIsNonEffect(t *testing.T) {
s := newTestStore(t)
seedHostWithArtifacts(t, s, "escrow-host", "cust-c")
before := hostRowCounts(t, s, "escrow-host")
err := s.DeleteHost("escrow-host", false)
if !errors.Is(err, ErrHostEscrowPresent) {
t.Fatalf("DeleteHost without escrow ack = %v, want ErrHostEscrowPresent", err)
}
after := hostRowCounts(t, s, "escrow-host")
for table, n := range before {
if after[table] != n {
t.Errorf("refusal deleted from %s: %d → %d (must be a non-effect)", table, n, after[table])
}
}
}
// Without an escrow row, deleteEscrow=false succeeds; with one, deleteEscrow=true removes it.
func TestDeleteHost_EscrowFlagSemantics(t *testing.T) {
s := newTestStore(t)
// No escrow → plain delete works without the ack.
if err := s.UpsertHost(&Host{HostID: "plain", CustomerID: "c", APIKey: "k"}); err != nil {
t.Fatal(err)
}
if err := s.DeleteHost("plain", false); err != nil {
t.Fatalf("DeleteHost without escrow: %v", err)
}
if n := countRows(t, s, `SELECT COUNT(*) FROM hosts WHERE host_id = ?`, "plain"); n != 0 {
t.Error("plain host not deleted")
}
// Escrow + ack → escrow row removed with the host.
seedHostWithArtifacts(t, s, "ack-host", "cust-d")
if err := s.DeleteHost("ack-host", true); err != nil {
t.Fatalf("DeleteHost with ack: %v", err)
}
if n := countRows(t, s, `SELECT COUNT(*) FROM host_escrow WHERE host_id = ?`, "ack-host"); n != 0 {
t.Error("escrow row survived deleteEscrow=true")
}
// Empty host id → refused before touching anything.
if err := s.DeleteHost("", true); err == nil {
t.Error("DeleteHost(\"\") must be refused")
}
}
// v0.53.0 F-14 provenance — the deletion record is written IN the delete tx, with
// escrow_acked reflecting an ACTUAL acknowledged destruction (ack over a present escrow).
// RED-PROOF (Part 1): dropping the provenance INSERT from DeleteHost fails the acked case
// (LatestHostDeletion returns nil — the gate finds nothing).
func TestDeleteHost_ProvenanceRecord(t *testing.T) {
s := newTestStore(t)
// Escrow-ack delete → record with escrow_acked = true.
seedHostWithArtifacts(t, s, "prov-acked", "cust-f14")
if err := s.DeleteHost("prov-acked", true); err != nil {
t.Fatalf("DeleteHost: %v", err)
}
rec, err := s.LatestHostDeletion("cust-f14")
if err != nil {
t.Fatalf("LatestHostDeletion: %v", err)
}
if rec == nil {
t.Fatal("no deletion record written by the escrow-ack delete")
}
if rec.HostID != "prov-acked" || rec.CustomerID != "cust-f14" || !rec.EscrowAcked {
t.Errorf("record = %+v, want host=prov-acked customer=cust-f14 escrow_acked=true", rec)
}
if rec.DeletedAt.IsZero() {
t.Error("deleted_at not populated")
}
// Delete WITHOUT escrow (none present) → record exists but escrow_acked = false, even
// though deleteEscrow=true was passed: ticking the box over NOTHING is not an
// acknowledged destruction.
if err := s.UpsertHost(&Host{HostID: "prov-noescrow", CustomerID: "cust-noesc", APIKey: "k"}); err != nil {
t.Fatal(err)
}
if err := s.DeleteHost("prov-noescrow", true); err != nil {
t.Fatalf("DeleteHost: %v", err)
}
rec, err = s.LatestHostDeletion("cust-noesc")
if err != nil || rec == nil {
t.Fatalf("LatestHostDeletion = %+v, %v; want a record", rec, err)
}
if rec.EscrowAcked {
t.Error("escrow_acked = true for a host with NO escrow row — vacuous ack must record false")
}
// The refused delete (escrow present, no ack) writes NOTHING — the tx never ran.
seedHostWithArtifacts(t, s, "prov-refused", "cust-refused")
if err := s.DeleteHost("prov-refused", false); !errors.Is(err, ErrHostEscrowPresent) {
t.Fatalf("expected escrow refusal, got %v", err)
}
if rec, _ := s.LatestHostDeletion("cust-refused"); rec != nil {
t.Errorf("refused delete wrote a provenance record: %+v", rec)
}
// Customer with no deletions ever → nil, nil (the pre-v0.53.0 shape — manual path).
if rec, err := s.LatestHostDeletion("cust-never"); err != nil || rec != nil {
t.Errorf("LatestHostDeletion(no deletions) = %+v, %v; want nil, nil", rec, err)
}
}
// The gate reads the MOST RECENT record: an old acked deletion must not whitelist a newer
// un-acked one (the F-14 law is about the deletion that orphaned the CURRENT tenancy).
func TestLatestHostDeletion_NewestWins(t *testing.T) {
s := newTestStore(t)
seedHostWithArtifacts(t, s, "gen1-host", "cust-seq")
if err := s.DeleteHost("gen1-host", true); err != nil { // acked
t.Fatal(err)
}
if err := s.UpsertHost(&Host{HostID: "gen2-host", CustomerID: "cust-seq", APIKey: "k2"}); err != nil {
t.Fatal(err)
}
if err := s.DeleteHost("gen2-host", false); err != nil { // no escrow → un-acked record
t.Fatal(err)
}
rec, err := s.LatestHostDeletion("cust-seq")
if err != nil || rec == nil {
t.Fatalf("LatestHostDeletion = %+v, %v", rec, err)
}
if rec.HostID != "gen2-host" || rec.EscrowAcked {
t.Errorf("latest record = %+v, want the NEWER un-acked gen2-host row", rec)
}
}
func TestCountHostArtifacts(t *testing.T) {
s := newTestStore(t)
seedHostWithArtifacts(t, s, "impact-host", "cust-e")
a, err := s.CountHostArtifacts("impact-host")
if err != nil {
t.Fatalf("CountHostArtifacts: %v", err)
}
if a.Guests != 2 || a.Reports != 1 || a.LogBundles != 1 {
t.Errorf("counts = %+v, want guests=2 reports=1 log_bundles=1", a)
}
if !a.EscrowPresent || !a.WGPeerBound || !a.PBSSecretPresent || !a.RecoveryPresent {
t.Errorf("presence flags = %+v, want all true", a)
}
// The customer-scoped bundle must NOT be counted (host scope only).
empty, err := s.CountHostArtifacts("no-such-host")
if err != nil {
t.Fatalf("CountHostArtifacts(empty): %v", err)
}
if empty != (HostArtifacts{}) {
t.Errorf("unknown host artifacts = %+v, want zero value", empty)
}
}