package store import ( "database/sql" "encoding/json" "errors" "fmt" "log" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/semver" _ "modernc.org/sqlite" ) // Store handles SQLite persistence for customer reports. type Store struct { db *sql.DB logger *log.Logger // defaultMinControllerVersion is the config/env-supplied global FLOOR fallback (Phase 2 managed // updates). Used only when neither a per-customer override nor a hub_settings row is set. defaultMinControllerVersion string } // SetDefaultMinControllerVersion sets the config/env-supplied global floor fallback. Called once at // startup from main (DEFAULT_MIN_CONTROLLER_VERSION). A hub_settings row, when present, takes // precedence over this value (see GetGlobalMinControllerVersion). func (s *Store) SetDefaultMinControllerVersion(v string) { s.defaultMinControllerVersion = v } // CustomerSummary holds the latest status for a customer (for dashboard). type CustomerSummary struct { CustomerID string CustomerName string ControllerVersion string ReceivedAt time.Time HealthStatus string CPUPercent float64 MemoryPercent float64 ContainerTotal int ContainerRunning int BackupLastSnapshot *time.Time ReportJSON string ControllerURL string // Computed fields (not stored) TimeSinceReport time.Duration DiskSummary string } // sqliteDSNParams are the connection pragmas, and getting the SYNTAX right is the whole point. // // ── R-172: this DSN was WRONG for the hub's entire life, and it failed SILENTLY ────────────────── // // It used to read `?_journal_mode=WAL&_busy_timeout=5000`. That is **mattn/go-sqlite3** syntax. This // hub uses **modernc.org/sqlite**, whose `applyQueryParams` reads only `_pragma`, `_time_format`, // `_time_integer_format`, `_txlock` and `_inttotime` — anything else is **ignored without an error**. // So the hub ran in the default rollback-journal mode with busy_timeout=0 while its own source said // otherwise: a configuration asserting an invariant the code did not provide, the same class as the // comments in `CLAUDE.md`'s false-invariant table. // // The observable that proved it: a 128 MB `/data/hub.db` with **no `-wal`/`-shm` file beside it while // the database was open**. In WAL mode those files must exist. Consequence, measured on 2026-08-02: // 13 `SQLITE_BUSY` collisions in one pod lifetime, each returning HTTP 500 to a host report, and two // consecutive misses crossing the 30-minute staleness threshold — a false `host_stale` alarm plus an // operator e-mail for a host that was up and healthy throughout. // // Each parameter, and why it is not optional: // // - journal_mode(WAL) — in rollback-journal mode a writer excludes readers and vice versa, so // rendering an operator page could block a host report. WAL lets readers and one writer proceed // concurrently. It is a property of the DATABASE FILE, so it persists once set. // - busy_timeout(5000) — writers still serialise against each other. Without a timeout SQLite // returns SQLITE_BUSY *immediately* rather than waiting; 5 s is far longer than any write here. // - txlock=immediate — THE ONE THAT IS EASY TO MISS. `database/sql`'s Begin() is DEFERRED by // default, so a transaction that reads and then writes must upgrade its lock, and a failed // upgrade returns SQLITE_BUSY_SNAPSHOT, which **busy_timeout does not retry**. This store has // 10+ `db.Begin()` sites and they are all write paths (customer delete/reset, wg, appliance, // pbsdr, telemetry, log bundles). Taking the write lock up front converts that un-retryable // failure into an ordinary wait covered by busy_timeout above. WAL + busy_timeout WITHOUT this // would leave a known un-retryable path open and ship half a fix. // // TestStorePragmasAreActuallyApplied asserts what the DATABASE reports, never what string was passed // — asserting the DSN would have passed happily for the entire life of the bug. const sqliteDSNParams = "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_txlock=immediate" // New creates a new store and initializes the schema. func New(dbPath string, logger *log.Logger) (*Store, error) { db, err := sql.Open("sqlite", dbPath+sqliteDSNParams) if err != nil { return nil, fmt.Errorf("opening database: %w", err) } s := &Store{db: db, logger: logger} if err := s.migrate(); err != nil { db.Close() return nil, fmt.Errorf("migrating database: %w", err) } return s, nil } func (s *Store) migrate() error { _, err := s.db.Exec(` CREATE TABLE IF NOT EXISTS reports ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, received_at DATETIME NOT NULL DEFAULT (datetime('now')), report_json TEXT NOT NULL, health_status TEXT, cpu_percent REAL, memory_percent REAL, container_total INTEGER, container_running INTEGER, backup_last_snapshot DATETIME, controller_version TEXT ); CREATE INDEX IF NOT EXISTS idx_reports_customer ON reports(customer_id, received_at DESC); CREATE TABLE IF NOT EXISTS customer_notifications ( customer_id TEXT PRIMARY KEY, email TEXT NOT NULL DEFAULT '', enabled_events TEXT NOT NULL DEFAULT '[]', created_at DATETIME DEFAULT (datetime('now')), updated_at DATETIME DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS notification_log ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, event_type TEXT NOT NULL, severity TEXT NOT NULL, message TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending', error_message TEXT, created_at DATETIME DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_notification_log_customer ON notification_log(customer_id, created_at DESC); CREATE TABLE IF NOT EXISTS customer_configs ( customer_id TEXT PRIMARY KEY, customer_name TEXT NOT NULL DEFAULT '', domain TEXT NOT NULL DEFAULT '', email TEXT NOT NULL DEFAULT '', retrieval_password TEXT NOT NULL, api_key TEXT NOT NULL, config_json TEXT NOT NULL DEFAULT '{}', created_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); `) if err != nil { return err } // v0.1.8: add controller_url column (idempotent — ignore error if already exists) s.db.Exec("ALTER TABLE reports ADD COLUMN controller_url TEXT") // v0.2.1: add status column to customer_configs (idempotent) s.db.Exec("ALTER TABLE customer_configs ADD COLUMN status TEXT NOT NULL DEFAULT 'active'") // v0.15.0: per-customer minimum controller version (the managed-update FLOOR). Empty = no // per-customer override → the effective floor falls back to the global default (hub_settings / // config). Idempotent. s.db.Exec("ALTER TABLE customer_configs ADD COLUMN min_controller_version TEXT NOT NULL DEFAULT ''") // v0.26.0: per-customer config_version — a monotonic counter bumped on every config save. The // report ACK advertises it; the controller compares it against its last-applied version and, on a // change, re-pulls controller.yaml + self-restarts (pull-based config delivery — no inbound). It is // a STORED COUNTER, never a hash of the rendered YAML (configgen emits a fresh session_secret + // timestamp every call, so a content hash would change spuriously). Idempotent. Existing rows seed // at 1, so an already-running box records that as its baseline on its next report without restarting. s.db.Exec("ALTER TABLE customer_configs ADD COLUMN config_version INTEGER NOT NULL DEFAULT 1") // v0.51.0 (DR-tier-by-default): per-customer DR-tier flag. NEW customers default ON (the // create handler sets it); the column default 0 is the LEGACY initialization — an existing // customer is only flipped ON by the one-time backfill (initialize from reality: its host // already carries an ENABLED pbs_dr descriptor; never auto-cascade a legacy box). The // backfill runs exactly once — only when the ALTER actually added the column — and is // DEFERRED to the end of migrate(): the hosts table it reads is created further down (a // fresh DB reaches the ALTER before the CREATE). _, drTierAlterErr := s.db.Exec("ALTER TABLE customer_configs ADD COLUMN dr_tier INTEGER NOT NULL DEFAULT 0") // v0.15.0: hub_settings — a tiny key/value table for operator-set globals that must survive // restarts (currently only the global controller-version floor). The config/env DEFAULT_MIN_ // CONTROLLER_VERSION is the FALLBACK; a row here (set via the operator UI) overrides it. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS hub_settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); `) if err != nil { return err } // v0.3.0: events table for hub-native monitoring _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, event_type TEXT NOT NULL, severity TEXT NOT NULL DEFAULT 'info', message TEXT NOT NULL DEFAULT '', details_json TEXT NOT NULL DEFAULT '{}', source TEXT NOT NULL DEFAULT 'controller', created_at DATETIME NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_events_customer_created ON events(customer_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_events_type ON events(event_type, created_at DESC); `) if err != nil { return err } // v0.3.0: add cooldown_hours to customer_notifications (idempotent) s.db.Exec("ALTER TABLE customer_notifications ADD COLUMN cooldown_hours INTEGER DEFAULT 6") // v0.3.0: add channel column to notification_log (idempotent) s.db.Exec("ALTER TABLE notification_log ADD COLUMN channel TEXT NOT NULL DEFAULT 'customer'") // v0.4.0: app telemetry tables _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS app_telemetry ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, app_name TEXT NOT NULL, display_name TEXT NOT NULL DEFAULT '', reported_at DATETIME NOT NULL, memory_current_mb REAL DEFAULT 0, memory_avg_mb REAL DEFAULT 0, memory_peak_mb REAL DEFAULT 0, cpu_avg_percent REAL DEFAULT 0, catalog_estimate TEXT DEFAULT '', catalog_limit TEXT DEFAULT '', log_errors INTEGER DEFAULT 0, log_warnings INTEGER DEFAULT 0, containers_json TEXT DEFAULT '[]', issues_json TEXT DEFAULT '[]' ); CREATE INDEX IF NOT EXISTS idx_app_telemetry_lookup ON app_telemetry(app_name, reported_at); CREATE INDEX IF NOT EXISTS idx_app_telemetry_customer ON app_telemetry(customer_id, app_name, reported_at); CREATE INDEX IF NOT EXISTS idx_app_telemetry_prune ON app_telemetry(reported_at); CREATE TABLE IF NOT EXISTS app_log_issues ( id INTEGER PRIMARY KEY AUTOINCREMENT, app_name TEXT NOT NULL, fingerprint TEXT NOT NULL, severity TEXT NOT NULL, message TEXT NOT NULL, first_seen DATETIME NOT NULL, last_seen DATETIME NOT NULL, occurrence_count INTEGER DEFAULT 1, affected_customers TEXT DEFAULT '[]', UNIQUE(app_name, fingerprint) ); CREATE INDEX IF NOT EXISTS idx_app_log_issues_app ON app_log_issues(app_name, last_seen DESC); `) if err != nil { return err } // Phase-1 retire (2026-06-16): the Infra Backup mechanism is gone. It pushed // plaintext customer secrets to the hub (the app-secret encryption key, restic // password, Cloudflare tokens — a zero-knowledge violation) and had been dead since // slice 8C. Drop both tables and VACUUM so the freed pages — which still hold the // plaintext — are physically reclaimed from the DB file, not merely delinked. // Gated on existence so ordinary restarts don't pay the VACUUM cost. var infraTables int s.db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('infra_backup_versions','infra_backups')`).Scan(&infraTables) if infraTables > 0 { if _, err = s.db.Exec(`DROP TABLE IF EXISTS infra_backup_versions; DROP TABLE IF EXISTS infra_backups;`); err != nil { return fmt.Errorf("dropping retired infra_backup tables: %w", err) } // VACUUM rewrites the database file, discarding the freed (plaintext) pages. if _, err = s.db.Exec(`VACUUM`); err != nil { return fmt.Errorf("vacuum after infra_backup drop: %w", err) } // WAL checkpoint(TRUNCATE) so no plaintext lingers in the -wal sidecar either. s.db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) if s.logger != nil { s.logger.Printf("[INFO] Retired infra-backup: dropped %d table(s) and VACUUMed to reclaim plaintext pages", infraTables) } } // v0.7.0: host-domain (slice 3). Purely additive — the controller path // (reports/customer_configs) is untouched; the schema cutover is slice 10. // Columns marked INERT exist now so slice 10 needs no ALTER; nothing reads or // writes them this slice. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS hosts ( host_id TEXT PRIMARY KEY, customer_id TEXT NOT NULL, api_key TEXT NOT NULL, agent_version TEXT NOT NULL DEFAULT '', last_report_at DATETIME, desired_json TEXT NOT NULL DEFAULT '{}', desired_generation INTEGER NOT NULL DEFAULT 0, dr_record_json TEXT NOT NULL DEFAULT '{}', created_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_hosts_customer ON hosts(customer_id); CREATE TABLE IF NOT EXISTS guests ( guest_id TEXT PRIMARY KEY, customer_id TEXT NOT NULL, host_id TEXT NOT NULL, vmid INTEGER NOT NULL, display_name TEXT NOT NULL DEFAULT '', status TEXT NOT NULL DEFAULT 'unknown', controller_version TEXT NOT NULL DEFAULT '', last_seen_at DATETIME, api_key TEXT NOT NULL DEFAULT '', desired_spec_json TEXT NOT NULL DEFAULT '{}', created_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_guests_host ON guests(host_id); CREATE INDEX IF NOT EXISTS idx_guests_customer ON guests(customer_id); CREATE TABLE IF NOT EXISTS host_reports ( id INTEGER PRIMARY KEY AUTOINCREMENT, host_id TEXT NOT NULL, customer_id TEXT NOT NULL, received_at DATETIME NOT NULL DEFAULT (datetime('now')), report_json TEXT NOT NULL, agent_version TEXT, cpu_percent REAL, memory_percent REAL, disk_percent REAL, guest_total INTEGER, guest_running INTEGER, cloudflared_status TEXT ); CREATE INDEX IF NOT EXISTS idx_host_reports_host ON host_reports(host_id, received_at DESC); CREATE INDEX IF NOT EXISTS idx_host_reports_customer ON host_reports(customer_id, received_at DESC); -- host_escrow (slice 7, doc 03 §8a): the OPAQUE R-wrapped PBS-key escrow blob. The hub -- stores the ciphertext bytes against the host and NEVER decrypts them (it has no recovery -- code). One row per host; a re-upload (rotation) is last-write-wins. Restore-mode serving -- (handing the blob back to a re-enrolling box) is slice 10. CREATE TABLE IF NOT EXISTS host_escrow ( host_id TEXT PRIMARY KEY, blob BLOB NOT NULL, key_fingerprint TEXT NOT NULL DEFAULT '', posture TEXT NOT NULL DEFAULT '', created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); -- host_escrow_superseded (v0.60.0, offsite-continuity): RETAINED old escrow blobs. Viktor's -- ruling (data protection first): when a NEW escrow blob supersedes an old one whose sealed -- restic-password sha DIFFERS, the old row is COPIED here BEFORE host_escrow is overwritten — -- so the old passphrase stays customer-R-recoverable (turns the reinstall-orphan incident from -- "history destroyed" into "history recoverable with the recovery code"). Append-only; the hub -- never decrypts; NO pruning (the blobs are tiny + R-encrypted; custody unchanged). The ACK and -- restore-serving read host_escrow (the CURRENT row) — never this table. -- -- THE RULING ABOVE WAS NOT MET FOR TWO MONTHS, AND THIS IS THE RECORD OF IT (R-198, fixed -- v0.93.0). This table shipped with the blob column — the K-escrow, i.e. the PBS datastore key -- — and identity_blob was added to host_escrow LATER (the slice-10D ALTER below), never here. -- The offsite restic REPOSITORY password lives in identity_blob, not in blob. So the retention -- preserved the whole-guest key and silently dropped the off-site data key: precisely the -- secret the reinstall-orphan incident was about. Worse, the copy happens as the new blob -- overwrites the old, so the destroying act was the ESCROW CEREMONY — the exact thing a -- rebuilt box asks its customer to run, on a card promising the old backups stay recoverable. -- Both demo boxes crossed that line on 2026-08-04 (07:15:36 and 07:20:08) and their previous -- repository passwords are unrecoverable, recovery code or not. -- identity_blob is now carried (see demoteCurrentEscrowTx, which is still THE ONE row-copy -- routine). Pinned by TestSaveHostEscrow_RetainsIdentityBlob and -- TestDeleteHost_DemotesIdentityBlob — the routine is proven through BOTH of its callers, -- because a shared routine tested through one caller is how a fix gets believed on a path -- nobody exercised. Evidence: audits/RECON-offsite-dr-chain-2026-08-04.md §7. CREATE TABLE IF NOT EXISTS host_escrow_superseded ( id INTEGER PRIMARY KEY AUTOINCREMENT, host_id TEXT NOT NULL, blob BLOB NOT NULL, key_fingerprint TEXT NOT NULL DEFAULT '', posture TEXT NOT NULL DEFAULT '', created_at DATETIME NOT NULL, restic_pw_sha256 TEXT NOT NULL DEFAULT '', superseded_at DATETIME NOT NULL DEFAULT (datetime('now')), identity_blob BLOB ); CREATE INDEX IF NOT EXISTS idx_host_escrow_superseded_host ON host_escrow_superseded(host_id); -- signed_jobs (slice 10A): the per-host queue of OPAQUE operator-signed destructive-op -- blobs. The hub STORES + SERVES them; it never forges one (there is no signing key -- hub-side) and never executes them (execution + signature verification is slice 10B). -- HasSignedOps on the control envelope is "this host has >=1 pending job". A job is opaque -- bytes (the signed-op envelope the agent verifies); the hub treats it as a blob. CREATE TABLE IF NOT EXISTS signed_jobs ( id INTEGER PRIMARY KEY AUTOINCREMENT, host_id TEXT NOT NULL, job_id TEXT NOT NULL, blob BLOB NOT NULL, created_at DATETIME NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_signed_jobs_host ON signed_jobs(host_id, id); `) if err != nil { return err } // Slice 10D (DR capstone) — additive columns on existing tables (fire-and-forget; a duplicate // column on re-run is ignored). `recovery_mode_until` gates restore-directive serving + re-enroll // (NULL/past = off; future = recovery mode active, auto-expires). host_escrow gains the IDENTITY // blob (age-wrapped {tunnel_token, pbs_token}) + the NON-secret DR directive (pbs repo/namespace, // expected key fingerprint, tunnel id) — the hub serves these only in recovery mode; no usable // secret is hub-held (the blobs need R, which the hub never has). s.db.Exec(`ALTER TABLE hosts ADD COLUMN recovery_mode_until DATETIME`) s.db.Exec(`ALTER TABLE host_escrow ADD COLUMN identity_blob BLOB`) s.db.Exec(`ALTER TABLE host_escrow ADD COLUMN directive_json TEXT NOT NULL DEFAULT '{}'`) // R-198 (v0.93.0) — the SAME column on the RETAINED table. It was added above and not here, and // that omission is what made the retention keep the wrong key for two months (see the comment on // host_escrow_superseded). Additive and tolerated on re-run, exactly like the lines above; it // changes no existing row. Rows superseded BEFORE this ships were written without the identity // blob and their source rows are already overwritten — there is nothing to backfill, and the // v0.93.0 report records that as a looked-at fact rather than a deduction. s.db.Exec(`ALTER TABLE host_escrow_superseded ADD COLUMN identity_blob BLOB`) // SLICE 3 (escrow auto-confirm) — sha256 hex of the offsite restic repo password sealed in the // identity blob. The hash of a 256-bit random secret is non-reversible/non-brute-forceable — safe to // store and serve; it lets the controller VERIFY "the escrow covers the CURRENT repo password" // instead of trusting blob-presence. NULL/'' = a legacy or password-less blob (never auto-confirms). s.db.Exec(`ALTER TABLE host_escrow ADD COLUMN restic_pw_sha256 TEXT`) // v0.57.0 (2.3, escrow honesty on offsite re-issue) — stale_at marks an escrow blob that may no // longer cover the box's live repository password. While set, the hub stops advertising "ceremony // done" and withholds restic_pw_sha256 from the auto-confirm ACK. NULL = current; a fresh ceremony // (SaveHostEscrow) clears it. // ⚠ NOTHING SETS IT as of v0.95.0 (R-196 / R-204 item 2). The only writer was the PRECAUTIONARY // mark on offsite re-issue, which guessed rather than measured and so blocked off-site backups on // boxes whose key had not changed. The column and its readers stay; see MarkEscrowStale for what a // legitimate future writer would have to prove first. s.db.Exec(`ALTER TABLE host_escrow ADD COLUMN stale_at DATETIME`) // dr_recipe (SPIKE-dr-recipe-2026-06-16): the secret-free DR reconstruction recipe, stored // PLAINTEXT (it has NO secrets — the clean inverse of the retired infra_backup). Two halves keyed // by customer: the agent's storage/guest/PBS half (host_half_json, from the host-report) and the // controller's customer/apps half (app_half_json, from the controller report); the hub assembles // them on read. DEDICATED table, separate from the opaque host_escrow. One row per customer; // each half is last-write-wins and preserves the other. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS dr_recipe ( customer_id TEXT PRIMARY KEY, recipe_version INTEGER NOT NULL DEFAULT 1, host_id TEXT NOT NULL DEFAULT '', host_half_json TEXT NOT NULL DEFAULT '', app_half_json TEXT NOT NULL DEFAULT '', updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); `) if err != nil { return err } // S1 offsite connectivity (doc 06 §3.2): the WG endpoint record + peer registry. The hub is the // source of truth; the endpoint converges on it (SSH push, internal/wgsync). No `status` column // on wg_peers — presence in the table IS the desired state; the doc-06 `status` field belongs to // the S2 host-join. assigned_ip stores the BARE IP (no /32 — that's presentation, appended by // the API layer and the sync payload). UNIQUE(assigned_ip) is the allocator's race backstop. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS wg_endpoints ( endpoint_id TEXT PRIMARY KEY, dns_name TEXT NOT NULL, wg_port INTEGER NOT NULL, server_pubkey TEXT NOT NULL, tunnel_subnet TEXT NOT NULL, pbs_tunnel_ip TEXT NOT NULL, created_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); CREATE TABLE IF NOT EXISTS wg_peers ( pubkey TEXT PRIMARY KEY, assigned_ip TEXT NOT NULL UNIQUE, host_id TEXT NOT NULL DEFAULT '', note TEXT NOT NULL DEFAULT '', created_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); -- S2: one BOUND peer per host (partial index — S1's unbound admin/test rows unaffected). CREATE UNIQUE INDEX IF NOT EXISTS idx_wg_peers_host ON wg_peers(host_id) WHERE host_id != ''; `) if err != nil { return err } // host_recovery (TASK G1): the break-glass root@pam console credential, vaulted at rest and // operator-retrievable. UNLIKE host_escrow (opaque, hub-can't-open), this IS a hub-held secret the // operator retrieves to reach the PVE web console (pveproxy — a failure domain distinct from sshd) // when both the sshd path AND the agent-independent auto-heal have failed. One row per host, // last-write-wins (day-0 sets it; --rotate re-sets). Never in desired-state, never logged. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS host_recovery ( host_id TEXT PRIMARY KEY, username TEXT NOT NULL, secret TEXT NOT NULL, set_at DATETIME NOT NULL DEFAULT (datetime('now')), updated_at DATETIME NOT NULL DEFAULT (datetime('now')) ); `) if err != nil { return err } // offsite provisioning (SLICE 1): the ONE-TIME transient storage-box/subaccount password. The hub // generates it at provision time, delivers it to the controller EXACTLY ONCE (consume endpoint), then // it is dead — the controller installs its own key + the hub resets the box password. It is transient // custody, NOT the ConfigJSON (which is served every pull). One row per customer; consumed_at marks it // spent. Never logged, never in any served config. (No hub-side at-rest cipher exists; the DB file is // 0600 and the value is short-lived + single-use.) _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS one_time_secrets ( customer_id TEXT PRIMARY KEY, value TEXT NOT NULL, created_at DATETIME NOT NULL DEFAULT (datetime('now')), consumed_at DATETIME ); `) if err != nil { return err } // PBS DR tier (SLICE 1, v0.44.0): the one-time PBS token secret, HOST-scoped — the sibling of // one_time_secrets (customer/controller custody) for host/agent custody. The hub receives the // secret over the tenantsync channel, stores it here, and the AGENT consumes it exactly once // (POST /api/v1/hosts/{id}/pbs/consume-token, per-host key). consumed_at marks it spent; a // re-issue supersedes any unconsumed value. Never logged, never in desired-state/ConfigJSON. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS host_pbs_secrets ( host_id TEXT PRIMARY KEY, value TEXT NOT NULL, created_at DATETIME NOT NULL DEFAULT (datetime('now')), consumed_at DATETIME, generation INTEGER NOT NULL DEFAULT 0 ); `) if err != nil { return err } // R-39 fleet fix (v0.68.0) — the PBS secret GENERATION: a monotonic per-host counter advanced by // every fresh MINT and by nothing else. Must be declared AFTER the CREATE above (an ALTER placed // earlier in this function silently no-ops, because the table does not exist yet). // // Why it has to exist. The agent's re-apply trigger is a change in the DESCRIPTOR CONTENT HASH // (`felhom-agent internal/pbsdr/manager.go` descriptorHash). An ep0 credential re-issue re-keys // the SECRET of an existing token, so token_id, fingerprint, datastore and namespace all come // back byte-identical — the descriptor does not move, the converged agent short-circuits, the // fresh secret is never consumed, and the box serves a revoked credential while reporting // `applied`. That is the 2026-07-18 N100 failure exactly (R-39). Stamping this counter into the // descriptor is what finally makes a re-key LOOK different to the agent. // // It is deliberately NOT created_at (two mints inside one second collide) and there is no row id // to borrow: this table is keyed by host_id and UPSERTed last-write-wins, so a "new row" never // exists. A counter column is the only monotonic source available here. // // RestageHostPBSSecret must NOT touch it: a re-stage re-arms the SAME secret, the descriptor // content genuinely has not changed, and bumping would trigger a pointless agent refetch loop // (that method's own contract says so). s.db.Exec(`ALTER TABLE host_pbs_secrets ADD COLUMN generation INTEGER NOT NULL DEFAULT 0`) // v0.50.0 — customer-claim password arc (DRILL-day0-vm F-4): one row per customer holding the // ACTIVE claim/reset code state. code_hash is bcrypt(code) — the plaintext exists ONLY inside // the email send (same custody rule as the retrieval passphrase). generation is monotonic: a // resend/reset rotates the code (generation+1) and the controller refuses codes of an already- // consumed generation. claimed_at is set once (first successful claim) and NEVER cleared by a // rotation — a reset code on a claimed box must not un-claim it. emailed_at records the last // send; reset_day/reset_count are the hub-side 3/day reset-request limiter. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS customer_claims ( customer_id TEXT PRIMARY KEY, code_hash TEXT NOT NULL, generation INTEGER NOT NULL DEFAULT 1, issued_at DATETIME NOT NULL, emailed_at DATETIME, claimed_at DATETIME, reset_day TEXT NOT NULL DEFAULT '', reset_count INTEGER NOT NULL DEFAULT 0 ); `) if err != nil { return err } // v0.43.0 — remote app-log diagnostics. Additive columns on app_log_issues: // context = JSON array of ±5 redacted lines around the FIRST occurrence (first capture // wins — stable repro context, no churn); context_customer = whose box it came from // (provenance); dismissed_at = dismissal instead of futile deletion (the controller // re-sends recurring issues — a delete is undone minutes later; a dismissal stays until // a genuinely NEW occurrence with last_seen > dismissed_at resurfaces the row). s.db.Exec(`ALTER TABLE app_log_issues ADD COLUMN context TEXT`) s.db.Exec(`ALTER TABLE app_log_issues ADD COLUMN context_customer TEXT`) s.db.Exec(`ALTER TABLE app_log_issues ADD COLUMN dismissed_at DATETIME`) // log_tail_requests: the operator's pending "send me this app's logs" intents — the // pull-based ACK flag (the hub NEVER connects into a box; the controller sees the flag // in its report ACK and ships the tail on its next cycle). One active request per // (customer, app) — a re-click refreshes requested_at. Cleared when the tail arrives // (consume-once). app_log_tails: the received tails, transient — keep the last 2 per // (customer, app), older pruned at insert. _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS log_tail_requests ( customer_id TEXT NOT NULL, app_name TEXT NOT NULL, requested_at DATETIME NOT NULL, PRIMARY KEY (customer_id, app_name) ); CREATE TABLE IF NOT EXISTS app_log_tails ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, app_name TEXT NOT NULL, collected_at DATETIME NOT NULL, received_at DATETIME NOT NULL, lines_json TEXT NOT NULL ); `) if err != nil { return err } // Component log bundles (v0.46.0 observability, see logbundle.go): the operator's // pending pull intents per (scope, component) — scope = customer_id for the // controller (report ACK channel) / host_id for the agent (heartbeat envelope) — // plus the received gzip bundles (72 h TTL, purge on the 60 s sweep; a secret-gate // hit stores a BLOCKED flag row with no payload). _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS log_bundle_requests ( scope_id TEXT NOT NULL, component TEXT NOT NULL, requested_at DATETIME NOT NULL, PRIMARY KEY (scope_id, component) ); CREATE TABLE IF NOT EXISTS log_bundles ( id INTEGER PRIMARY KEY AUTOINCREMENT, scope_id TEXT NOT NULL, component TEXT NOT NULL, collected_at DATETIME NOT NULL, received_at DATETIME NOT NULL, size_bytes INTEGER NOT NULL, gz BLOB, blocked INTEGER NOT NULL DEFAULT 0, blocked_reason TEXT NOT NULL DEFAULT '' ); `) if err != nil { return err } // v0.53.0 — host-deletion provenance (F-14, operator ruling 2026-07-13): one row per DeleteHost, // written INSIDE the delete transaction. escrow_acked records whether the host was removed // through the escrow-ack flow (the operator explicitly acknowledged destroying a PRESENT escrow // row — acknowledged key destruction). The PBS-DR enable path may auto-re-issue a surviving ep0 // tenancy ONLY when the customer's most recent record here has escrow_acked=1; no record (all // pre-v0.53.0 deletions — deliberately NO backfill) or an un-acked record keeps the manual // re-issue path the only one (never-silently-re-key law). _, err = s.db.Exec(` CREATE TABLE IF NOT EXISTS host_deletions ( id INTEGER PRIMARY KEY AUTOINCREMENT, host_id TEXT NOT NULL, customer_id TEXT NOT NULL, deleted_at DATETIME NOT NULL DEFAULT (datetime('now')), escrow_acked INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_host_deletions_customer ON host_deletions(customer_id, id DESC); -- customer_resets (v0.61.0, F-14 style): the RESET provenance + resumable per-leg journal. One -- row per reset attempt; legs_json holds {hetzner,pbs,db_purge → pending|ok|failed|manual}. The -- DB purge runs LAST (publish-last), so a re-run reads the journal to know what still needs -- teardown. escrow_acked records the ruling-1 acknowledgment. NEVER pruned (audit outlives -- every lifecycle tier). CREATE TABLE IF NOT EXISTS customer_resets ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, started_at DATETIME NOT NULL DEFAULT (datetime('now')), completed_at DATETIME, escrow_acked INTEGER NOT NULL DEFAULT 0, legs_json TEXT NOT NULL DEFAULT '{}' ); CREATE INDEX IF NOT EXISTS idx_customer_resets_customer ON customer_resets(customer_id, id DESC); -- appliance_registrations (v0.62.0, R-21 slice C — the universal secret-free ISO): a box -- booted from the GENERIC ISO registers itself here as an UNCLAIMED appliance, the operator -- binds it to a customer, and one poll delivers the customer-id + retrieval passphrase ONCE. -- Keyed by (uuid, mac_set): the N100 DMI verdict says serials are unusable ("Default string"), -- and cheap boards ship DUPLICATE SMBIOS UUIDs — the MAC set is the tiebreaker, so the same -- uuid with a different mac_set is a DISTINCT appliance. token_hash = sha256(appliance token); -- the token itself is never stored. status: registered→bound→delivered (one-shot) | discarded. -- This table's own timestamps ARE the provenance for the pre-bind phase (no customer to scope a -- customer-events row to yet — mirrors host_deletions/customer_resets self-contained provenance). CREATE TABLE IF NOT EXISTS appliance_registrations ( id INTEGER PRIMARY KEY AUTOINCREMENT, uuid TEXT NOT NULL, mac_set TEXT NOT NULL, ssh_host_pubkeys TEXT NOT NULL DEFAULT '', hw_summary TEXT NOT NULL DEFAULT '', token_hash TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'registered', customer_id TEXT, install_mode TEXT, extra_args TEXT, first_seen DATETIME NOT NULL DEFAULT (datetime('now')), last_seen DATETIME NOT NULL DEFAULT (datetime('now')), bound_at DATETIME, delivered_at DATETIME, discarded_at DATETIME, pairing_code TEXT NOT NULL DEFAULT '', UNIQUE(uuid, mac_set) ); CREATE INDEX IF NOT EXISTS idx_appliance_status ON appliance_registrations(status, last_seen DESC); CREATE INDEX IF NOT EXISTS idx_appliance_token ON appliance_registrations(token_hash); -- selfbind_tokens (v0.66.0, R-27 slice 1 — customer self-bind): the 7-day tokenized capability -- link the operator emails. token_hash is sha256 at rest (never the token). Single-active per -- customer (delete-then-insert on re-mint). attempts lock at 5 (Viktor's ruling); consumed_at is -- the one-shot flip. NO appliance data here — the code+passphrase check happens at bind time. CREATE TABLE IF NOT EXISTS selfbind_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, customer_id TEXT NOT NULL, token_hash TEXT NOT NULL UNIQUE, attempts INTEGER NOT NULL DEFAULT 0, locked INTEGER NOT NULL DEFAULT 0, created_at DATETIME NOT NULL DEFAULT (datetime('now')), expires_at DATETIME NOT NULL, emailed_at DATETIME, consumed_at DATETIME ); CREATE INDEX IF NOT EXISTS idx_selfbind_token ON selfbind_tokens(token_hash); `) if err != nil { return err } // v0.66.0 (R-27 slice 1): pairing_code on pre-existing appliance rows (idempotent — errors if the // column already exists, which is fine on a fresh DB where the CREATE above already added it). s.db.Exec("ALTER TABLE appliance_registrations ADD COLUMN pairing_code TEXT NOT NULL DEFAULT ''") // v0.51.0 dr_tier one-time legacy backfill — see the ALTER above; runs last so every table // it touches (hosts, customer_configs) exists on a fresh DB too (where it finds nothing). if drTierAlterErr == nil { if err := s.backfillDRTierFromDescriptors(); err != nil { return fmt.Errorf("dr_tier backfill: %w", err) } } return nil } // NotificationPrefs holds per-customer notification preferences. type NotificationPrefs struct { CustomerID string Email string EnabledEvents []string CooldownHours int } // GetNotificationPrefs returns notification preferences for a customer. func (s *Store) GetNotificationPrefs(customerID string) (*NotificationPrefs, error) { var email, eventsJSON string var cooldownHours int err := s.db.QueryRow( "SELECT email, enabled_events, COALESCE(cooldown_hours, 6) FROM customer_notifications WHERE customer_id = ?", customerID, ).Scan(&email, &eventsJSON, &cooldownHours) if err != nil { if err == sql.ErrNoRows { return nil, nil } return nil, err } var events []string if err := json.Unmarshal([]byte(eventsJSON), &events); err != nil { s.logger.Printf("[WARN] Corrupt enabled_events JSON for %s: %v", customerID, err) } if cooldownHours <= 0 { cooldownHours = 6 } return &NotificationPrefs{ CustomerID: customerID, Email: email, EnabledEvents: events, CooldownHours: cooldownHours, }, nil } // SaveNotificationPrefs creates or updates notification preferences for a customer. func (s *Store) SaveNotificationPrefs(customerID, email string, enabledEvents []string, cooldownHours int) error { eventsJSON, _ := json.Marshal(enabledEvents) if cooldownHours <= 0 { cooldownHours = 6 } _, err := s.db.Exec(` INSERT INTO customer_notifications (customer_id, email, enabled_events, cooldown_hours, updated_at) VALUES (?, ?, ?, ?, datetime('now')) ON CONFLICT(customer_id) DO UPDATE SET email = excluded.email, enabled_events = excluded.enabled_events, cooldown_hours = excluded.cooldown_hours, updated_at = datetime('now')`, customerID, email, string(eventsJSON), cooldownHours, ) return err } // LogNotification records a notification attempt. func (s *Store) LogNotification(customerID, eventType, severity, message, status, errorMsg, channel string) error { if channel == "" { channel = "customer" } _, err := s.db.Exec(` INSERT INTO notification_log (customer_id, event_type, severity, message, status, error_message, channel) VALUES (?, ?, ?, ?, ?, ?, ?)`, customerID, eventType, severity, message, status, errorMsg, channel, ) return err } // NotificationLogEntry represents a single notification log record. type NotificationLogEntry struct { EventType string Severity string Message string Status string // "sent", "skipped", "failed", "suppressed" (R-182: a cooldown drop, recorded rather than silent) ErrorMessage string Channel string // "operator" or "customer" CreatedAt time.Time } // GetRecentNotifications returns the most recent notification log entries for a customer. func (s *Store) GetRecentNotifications(customerID string, limit int) ([]NotificationLogEntry, error) { rows, err := s.db.Query(` SELECT event_type, severity, message, status, COALESCE(error_message, ''), COALESCE(channel, 'customer'), created_at FROM notification_log WHERE customer_id = ? ORDER BY created_at DESC LIMIT ?`, customerID, limit) if err != nil { return nil, err } defer rows.Close() var entries []NotificationLogEntry for rows.Next() { var e NotificationLogEntry var createdAt, errorMsg string if err := rows.Scan(&e.EventType, &e.Severity, &e.Message, &e.Status, &errorMsg, &e.Channel, &createdAt); err != nil { return nil, err } e.CreatedAt = parseSQLiteTime(createdAt) e.ErrorMessage = errorMsg entries = append(entries, e) } return entries, rows.Err() } // LastCustomerSentAt returns the most recent notification_log created_at over the given event // types on the CUSTOMER channel with status='sent', and whether any such row exists. It is the // pairing-evidence query for recovery notifications (v0.71.0, audit F11): "was the customer told // about the down since they were last told about a recovery?" Uses the // (customer_id, created_at DESC) index. An empty eventTypes slice returns (zero, false, nil). func (s *Store) LastCustomerSentAt(customerID string, eventTypes []string) (time.Time, bool, error) { if len(eventTypes) == 0 { return time.Time{}, false, nil } placeholders := make([]string, len(eventTypes)) args := make([]interface{}, 0, len(eventTypes)+1) args = append(args, customerID) for i, et := range eventTypes { placeholders[i] = "?" args = append(args, et) } var createdAt sql.NullString err := s.db.QueryRow(` SELECT MAX(created_at) FROM notification_log WHERE customer_id = ? AND channel = 'customer' AND status = 'sent' AND event_type IN (`+strings.Join(placeholders, ",")+`)`, args..., ).Scan(&createdAt) if err != nil { return time.Time{}, false, err } if !createdAt.Valid || createdAt.String == "" { return time.Time{}, false, nil } return parseSQLiteTime(createdAt.String), true, nil } // SeedNotificationPrefs creates a customer_notifications row IF AND ONLY IF none exists — // insert-if-absent, never an upsert (a customer-edited row must never be overwritten by a seed; // audit F12). An empty email is a no-op: seeding an unnotifiable row would only mask the gap. // Returns whether a row was created. func (s *Store) SeedNotificationPrefs(customerID, email string, enabledEvents []string) (bool, error) { if email == "" { return false, nil } eventsJSON, _ := json.Marshal(enabledEvents) res, err := s.db.Exec(` INSERT OR IGNORE INTO customer_notifications (customer_id, email, enabled_events, cooldown_hours) VALUES (?, ?, ?, 6)`, customerID, email, string(eventsJSON), ) if err != nil { return false, err } n, err := res.RowsAffected() if err != nil { return false, err } return n > 0, nil } // SaveReport stores a new report. The reportJSON should be the raw JSON payload. func (s *Store) SaveReport(customerID string, reportJSON []byte) error { // Parse denormalized fields from the JSON var parsed struct { ControllerVersion string `json:"controller_version"` ControllerURL string `json:"controller_url"` System struct { CPUPercent float64 `json:"cpu_percent"` MemoryPercent float64 `json:"memory_percent"` } `json:"system"` Containers struct { Total int `json:"total"` Running int `json:"running"` } `json:"containers"` Backup struct { LastSnapshot *time.Time `json:"last_snapshot"` } `json:"backup"` Health struct { Status string `json:"status"` } `json:"health"` } if err := json.Unmarshal(reportJSON, &parsed); err != nil { s.logger.Printf("[WARN] Cannot parse report fields for denormalization: %v", err) } var backupSnapshot *string if parsed.Backup.LastSnapshot != nil { t := parsed.Backup.LastSnapshot.Format(time.RFC3339) backupSnapshot = &t } _, err := s.db.Exec(` INSERT INTO reports (customer_id, report_json, health_status, cpu_percent, memory_percent, container_total, container_running, backup_last_snapshot, controller_version, controller_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, customerID, string(reportJSON), parsed.Health.Status, parsed.System.CPUPercent, parsed.System.MemoryPercent, parsed.Containers.Total, parsed.Containers.Running, backupSnapshot, parsed.ControllerVersion, parsed.ControllerURL, ) return err } // reportOffsitePresence is the minimal parse for "does this controller report show an offsite tier // that is actually CONFIGURED on the box" — the R-70 delivery-state signal, i.e. applied-on-the-box. // // ⚠ TIGHTENED 2026-08-05 (hub v0.96.0, R-204 item 4), and the reason is exactly the trap this // project keeps recording. The predicate used to be bare PRESENCE, under a comment asserting *"the // report builder attaches `offsite` only when the box actually has an offbox target configured, so // presence == applied-on-the-box"*. Controller v0.199.0 breaks that premise deliberately: a REBUILT // box now attaches an offsite object carrying `enabled:false` and a declared `state` in order to ASK // for a credential. Left as presence-only, this function would have read that request for help as // proof the tier was applied — turning `DeliveryStateFor` into DeliveryApplied for the exact boxes // that are stranded, and suppressing the stuck event for them. // // The tightening is `enabled == true`, and it is PROVABLY A NO-OP for every report shape that exists // today: `backup.OffboxReportStatus` returned nil unless `t != nil && t.Enabled`, so an attached // object has ALWAYS carried `enabled:true`. Pinned by // TestReportHasOffsite_EnabledOnly — including a case asserting the pre-v0.199.0 shape still reads // true, so the equivalence is measured rather than argued. type reportOffsitePresence struct { Offsite *struct { Enabled bool `json:"enabled"` } `json:"offsite"` } func reportHasOffsite(reportJSON string) bool { var p reportOffsitePresence if err := json.Unmarshal([]byte(reportJSON), &p); err != nil { return false // unparseable report → no offsite evidence } return p.Offsite != nil && p.Offsite.Enabled } // LatestReportOffsitePresence reports whether the customer's most recent controller report exists // and whether it carries an offsite status object (R-70 detector input). func (s *Store) LatestReportOffsitePresence(customerID string) (found bool, receivedAt time.Time, hasOffsite bool, err error) { var recv, reportJSON string err = s.db.QueryRow(`SELECT received_at, report_json FROM reports WHERE customer_id = ? ORDER BY id DESC LIMIT 1`, customerID).Scan(&recv, &reportJSON) if err == sql.ErrNoRows { return false, time.Time{}, false, nil } if err != nil { return false, time.Time{}, false, err } return true, parseSQLiteTime(recv), reportHasOffsite(reportJSON), nil } // CountReportsOffsiteSince counts the customer's controller reports received strictly after `since` // (UTC) and how many of them carry an offsite status object (R-70 detector input: "N consecutive // reports since consume without offbox" == total>0 && withOffsite==0). Capped at 500 rows per call — // far beyond any detector threshold; the cap only bounds memory. func (s *Store) CountReportsOffsiteSince(customerID string, since time.Time) (total, withOffsite int, err error) { rows, err := s.db.Query(`SELECT report_json FROM reports WHERE customer_id = ? AND received_at > ? ORDER BY id LIMIT 500`, customerID, since.UTC().Format("2006-01-02 15:04:05")) if err != nil { return 0, 0, err } defer rows.Close() for rows.Next() { var reportJSON string if err := rows.Scan(&reportJSON); err != nil { return 0, 0, err } total++ if reportHasOffsite(reportJSON) { withOffsite++ } } return total, withOffsite, rows.Err() } // LatestEscrowTimeForCustomer returns the newest escrow-blob timestamp across the customer's // hosts (updated_at, falling back to created_at), or zero when no blob exists. Part-7 v0.73.0: // one leg of the never-ran offsite-staleness anchor — the escrow ceremony is the moment runs // become POSSIBLE, so the 48 h staleness threshold counts from it, not from the dawn of time. // Reduced in Go (not SQL MAX) because created_at formats vary (RFC3339 vs SQLite space form) and // lexicographic MAX would misorder them; parseSQLiteTime handles both. func (s *Store) LatestEscrowTimeForCustomer(customerID string) (time.Time, error) { rows, err := s.db.Query(` SELECT he.created_at, he.updated_at FROM host_escrow he INNER JOIN hosts h ON h.host_id = he.host_id WHERE h.customer_id = ?`, customerID) if err != nil { return time.Time{}, err } defer rows.Close() var newest time.Time for rows.Next() { var createdAt, updatedAt string if err := rows.Scan(&createdAt, &updatedAt); err != nil { return time.Time{}, err } for _, raw := range []string{createdAt, updatedAt} { if t := parseSQLiteTime(raw); t.After(newest) { newest = t } } } return newest, rows.Err() } // LastEventAt returns the created_at of the most recent event of the given type for a customer // (zero time when none). Durable across hub restarts — used as the cooldown/rate-limit source for // hub-emitted detector events (R-70/R-71c: a repeating pattern must surface as repeating events on // a bounded cadence, never as a silent retry loop OR a restart-reset flood). func (s *Store) LastEventAt(customerID, eventType string) (time.Time, error) { var createdAt string err := s.db.QueryRow(`SELECT created_at FROM events WHERE customer_id = ? AND event_type = ? ORDER BY id DESC LIMIT 1`, customerID, eventType).Scan(&createdAt) if err == sql.ErrNoRows { return time.Time{}, nil } if err != nil { return time.Time{}, err } return parseSQLiteTime(createdAt), nil } // GetCustomers returns the latest report summary for each customer. func (s *Store) GetCustomers() ([]CustomerSummary, error) { rows, err := s.db.Query(` SELECT r.customer_id, r.received_at, r.report_json, r.health_status, r.cpu_percent, r.memory_percent, r.container_total, r.container_running, r.backup_last_snapshot, r.controller_version, r.controller_url FROM reports r INNER JOIN ( SELECT customer_id, MAX(received_at) as max_time FROM reports GROUP BY customer_id ) latest ON r.customer_id = latest.customer_id AND r.received_at = latest.max_time ORDER BY r.customer_id`) if err != nil { return nil, err } defer rows.Close() var customers []CustomerSummary for rows.Next() { var c CustomerSummary var receivedAt string var backupSnapshot sql.NullString var controllerURL sql.NullString if err := rows.Scan(&c.CustomerID, &receivedAt, &c.ReportJSON, &c.HealthStatus, &c.CPUPercent, &c.MemoryPercent, &c.ContainerTotal, &c.ContainerRunning, &backupSnapshot, &c.ControllerVersion, &controllerURL); err != nil { return nil, err } c.ReceivedAt = parseSQLiteTime(receivedAt) c.TimeSinceReport = time.Since(c.ReceivedAt) if backupSnapshot.Valid { t, err := time.Parse(time.RFC3339, backupSnapshot.String) if err == nil { c.BackupLastSnapshot = &t } } if controllerURL.Valid { c.ControllerURL = controllerURL.String } // Parse customer_name from JSON var report struct { CustomerName string `json:"customer_name"` } if err := json.Unmarshal([]byte(c.ReportJSON), &report); err != nil { s.logger.Printf("[WARN] Cannot parse customer_name from report JSON for %s: %v", c.CustomerID, err) } c.CustomerName = report.CustomerName // Parse disk summary c.DiskSummary = parseDiskSummary(c.ReportJSON) customers = append(customers, c) } return customers, rows.Err() } // GetCustomer returns the latest report for a specific customer. func (s *Store) GetCustomer(customerID string) (*CustomerSummary, error) { row := s.db.QueryRow(` SELECT customer_id, received_at, report_json, health_status, cpu_percent, memory_percent, container_total, container_running, backup_last_snapshot, controller_version, controller_url FROM reports WHERE customer_id = ? ORDER BY received_at DESC LIMIT 1`, customerID) var c CustomerSummary var receivedAt string var backupSnapshot sql.NullString var controllerURL sql.NullString if err := row.Scan(&c.CustomerID, &receivedAt, &c.ReportJSON, &c.HealthStatus, &c.CPUPercent, &c.MemoryPercent, &c.ContainerTotal, &c.ContainerRunning, &backupSnapshot, &c.ControllerVersion, &controllerURL); err != nil { if err == sql.ErrNoRows { return nil, nil } return nil, err } c.ReceivedAt = parseSQLiteTime(receivedAt) c.TimeSinceReport = time.Since(c.ReceivedAt) if backupSnapshot.Valid { t, err := time.Parse(time.RFC3339, backupSnapshot.String) if err == nil { c.BackupLastSnapshot = &t } } if controllerURL.Valid { c.ControllerURL = controllerURL.String } var report struct { CustomerName string `json:"customer_name"` } json.Unmarshal([]byte(c.ReportJSON), &report) c.CustomerName = report.CustomerName c.DiskSummary = parseDiskSummary(c.ReportJSON) return &c, nil } // GetCustomerHistory returns report history for a customer. func (s *Store) GetCustomerHistory(customerID string, since time.Duration) ([]CustomerSummary, error) { cutoff := time.Now().Add(-since).Format("2006-01-02 15:04:05") rows, err := s.db.Query(` SELECT customer_id, received_at, report_json, health_status, cpu_percent, memory_percent, container_total, container_running, backup_last_snapshot, controller_version, controller_url FROM reports WHERE customer_id = ? AND received_at >= ? ORDER BY received_at DESC`, customerID, cutoff) if err != nil { return nil, err } defer rows.Close() var history []CustomerSummary for rows.Next() { var c CustomerSummary var receivedAt string var backupSnapshot sql.NullString var controllerURL sql.NullString if err := rows.Scan(&c.CustomerID, &receivedAt, &c.ReportJSON, &c.HealthStatus, &c.CPUPercent, &c.MemoryPercent, &c.ContainerTotal, &c.ContainerRunning, &backupSnapshot, &c.ControllerVersion, &controllerURL); err != nil { return nil, err } c.ReceivedAt = parseSQLiteTime(receivedAt) c.TimeSinceReport = time.Since(c.ReceivedAt) if backupSnapshot.Valid { t, err := time.Parse(time.RFC3339, backupSnapshot.String) if err == nil { c.BackupLastSnapshot = &t } } if controllerURL.Valid { c.ControllerURL = controllerURL.String } history = append(history, c) } return history, rows.Err() } // (Infra-backup store methods retired 2026-06-16. The tables are dropped + VACUUMed // in migrate(); the push/get/versions handlers and the operator panel are gone.) // Prune deletes reports older than the given number of days. func (s *Store) Prune(maxDays int) (int64, error) { cutoff := time.Now().AddDate(0, 0, -maxDays).Format("2006-01-02 15:04:05") res, err := s.db.Exec("DELETE FROM reports WHERE received_at < ?", cutoff) if err != nil { return 0, err } n, _ := res.RowsAffected() // v0.7.0: prune the parallel host-domain report stream, same retention. if hres, herr := s.db.Exec("DELETE FROM host_reports WHERE received_at < ?", cutoff); herr == nil { hn, _ := hres.RowsAffected() n += hn } return n, nil } // Close closes the database connection. func (s *Store) Close() error { return s.db.Close() } // backfillDRTierFromDescriptors is the ONE-TIME legacy initialization for the v0.51.0 dr_tier // column (called only when the ALTER just added it): a customer whose host already carries an // ENABLED pbs_dr descriptor in desired_json gets dr_tier=1 — initialize from reality; everyone // else stays 0 (never auto-cascade a legacy box). JSON is inspected in Go, not by LIKE, so field // order/whitespace can't fool it. func (s *Store) backfillDRTierFromDescriptors() error { rows, err := s.db.Query(`SELECT customer_id, desired_json FROM hosts WHERE desired_json LIKE '%pbs_dr%'`) if err != nil { return err } defer rows.Close() enabled := map[string]bool{} for rows.Next() { var customerID, desired string if err := rows.Scan(&customerID, &desired); err != nil { return err } var doc struct { PBSDR *struct { Enabled bool `json:"enabled"` } `json:"pbs_dr"` } if json.Unmarshal([]byte(desired), &doc) == nil && doc.PBSDR != nil && doc.PBSDR.Enabled { enabled[customerID] = true } } if err := rows.Err(); err != nil { return err } for customerID := range enabled { if _, err := s.db.Exec(`UPDATE customer_configs SET dr_tier = 1 WHERE customer_id = ?`, customerID); err != nil { return err } } return nil } // CustomerConfig holds a pre-provisioned customer configuration. type CustomerConfig struct { CustomerID string CustomerName string Domain string Email string RetrievalPassword string APIKey string ConfigJSON string // JSON object with customer-specific override fields Status string // "active" or "blocked" // MinControllerVersion is the per-customer minimum controller version (managed-update FLOOR // override). Empty = use the global default. Set/cleared via the operator UI. MinControllerVersion string // ConfigVersion is the monotonic config counter (bumped on every SaveCustomerConfig). The report // ACK advertises it; the controller re-pulls + self-restarts when it changes. Never a YAML hash. ConfigVersion int // DRTier (v0.51.0, DR-tier-by-default) is the per-customer DR-tier flag — the operator INTENT // the pbsdr cascade converges toward. Default ON for NEW customers (create handler); legacy // rows were initialized from reality by the one-time backfill (enabled descriptor → ON). // OFF = zero Felhom-side cost (no ep0 namespace is provisioned) and offsite provisioning is // refused (the escrow ceremony depends on the PBS key — drill F-6, closed by policy). DRTier bool CreatedAt time.Time UpdatedAt time.Time } // SaveCustomerConfig creates or updates a customer configuration. Every save BUMPS config_version // (new rows start at 1; updates increment) — this is the load-bearing signal that drives the // controller's pull-based config-refresh via the report ACK. Bumping here covers every field that // feeds the generated controller.yaml (identity + the config_json overrides). NOTE: the floor // (min_controller_version), block/unblock status, and retrieval-password regen are deliberately NOT // config.yaml content and intentionally do NOT bump it (they have their own signals or none). func (s *Store) SaveCustomerConfig(cfg *CustomerConfig) error { _, err := s.db.Exec(` INSERT INTO customer_configs (customer_id, customer_name, domain, email, retrieval_password, api_key, config_json, min_controller_version, dr_tier, config_version, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, datetime('now')) ON CONFLICT(customer_id) DO UPDATE SET customer_name = excluded.customer_name, domain = excluded.domain, email = excluded.email, retrieval_password = excluded.retrieval_password, api_key = excluded.api_key, config_json = excluded.config_json, min_controller_version = excluded.min_controller_version, dr_tier = excluded.dr_tier, config_version = customer_configs.config_version + 1, updated_at = datetime('now')`, cfg.CustomerID, cfg.CustomerName, cfg.Domain, cfg.Email, cfg.RetrievalPassword, cfg.APIKey, cfg.ConfigJSON, cfg.MinControllerVersion, cfg.DRTier, ) return err } // GetCustomerConfig returns a customer configuration by ID, or nil if not found. func (s *Store) GetCustomerConfig(customerID string) (*CustomerConfig, error) { var cfg CustomerConfig var createdAt, updatedAt string err := s.db.QueryRow(` SELECT customer_id, customer_name, domain, email, retrieval_password, api_key, config_json, status, min_controller_version, dr_tier, config_version, created_at, updated_at FROM customer_configs WHERE customer_id = ?`, customerID, ).Scan(&cfg.CustomerID, &cfg.CustomerName, &cfg.Domain, &cfg.Email, &cfg.RetrievalPassword, &cfg.APIKey, &cfg.ConfigJSON, &cfg.Status, &cfg.MinControllerVersion, &cfg.DRTier, &cfg.ConfigVersion, &createdAt, &updatedAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } cfg.CreatedAt = parseSQLiteTime(createdAt) cfg.UpdatedAt = parseSQLiteTime(updatedAt) return &cfg, nil } // SaveOneTimeSecret stores (last-write-wins) the one-time transient offsite password for a customer, // resetting the consumed flag (a fresh provision supersedes any prior unconsumed value). Never logged. func (s *Store) SaveOneTimeSecret(customerID, value string) error { _, err := s.db.Exec(` INSERT INTO one_time_secrets (customer_id, value, created_at, consumed_at) VALUES (?, ?, datetime('now'), NULL) ON CONFLICT(customer_id) DO UPDATE SET value = excluded.value, created_at = datetime('now'), consumed_at = NULL`, customerID, value) return err } // ConsumeOneTimeSecret returns the customer's one-time offsite password and marks it consumed in the SAME // transaction (single use). A second call — or a call when none is stored — returns ("", sql.ErrNoRows). // The value is never logged. func (s *Store) ConsumeOneTimeSecret(customerID string) (string, error) { tx, err := s.db.Begin() if err != nil { return "", err } defer tx.Rollback() var value string err = tx.QueryRow(`SELECT value FROM one_time_secrets WHERE customer_id = ? AND consumed_at IS NULL`, customerID).Scan(&value) if err != nil { return "", err // sql.ErrNoRows when absent OR already consumed } if _, err := tx.Exec(`UPDATE one_time_secrets SET consumed_at = datetime('now') WHERE customer_id = ?`, customerID); err != nil { return "", err } if err := tx.Commit(); err != nil { return "", err } return value, nil } // RestageOneTimeSecret re-arms an ALREADY-STORED one-time offsite secret for re-consumption by // clearing its consumed flag — WITHOUT changing the secret value, WITHOUT inserting a row, and // WITHOUT any storage-provider call. It is the off-site sibling of RestageHostPBSSecret (pbsdr.go), // and it exists because a REBUILT box has no credential of its own: its predecessor spent the // one-time password (R-193 / R-204 item 4). // // IT IS POSSIBLE AT ALL ONLY BECAUSE THE VALUE SURVIVES A CONSUME, and that was established from // this file rather than assumed from the PBS analogy (the two secrets are different objects with // different lifecycles, and assuming a shared shape is how two sessions confused the credentials): // ConsumeOneTimeSecret sets `consumed_at` and NOTHING ELSE — the `value` column is never cleared or // overwritten, and the schema declares it `TEXT NOT NULL`. So the row still holds a serviceable // password after consumption, and re-arming it costs no external call and converges in one report // tick. Pinned by TestRestageOneTimeSecret_ReArmsTheSameValue. // // Returns restaged=true when a row existed (its consumed flag is now cleared, so // ConsumeOneTimeSecret will serve it once more); restaged=false when NO secret is stored for the // customer — the caller must then escalate to a fresh mint (ReissueCredentials). The value is never // read or logged here. func (s *Store) RestageOneTimeSecret(customerID string) (bool, error) { res, err := s.db.Exec(`UPDATE one_time_secrets SET consumed_at = NULL WHERE customer_id = ?`, customerID) if err != nil { return false, err } n, err := res.RowsAffected() if err != nil { return false, err } return n > 0, nil } // LatestReportOffsiteDeclaration returns the id of the customer's newest controller report and the // state DECLARED on its `offsite` object (controller >= v0.199.0; "" on anything older, on a report // with no offsite object, and on an unparseable one). // // The report id is the debounce currency: the reconciler counts DISTINCT reports, not its own ticks, // so a stuck box is confirmed by fresh evidence rather than by the passage of time. // // found=false when the customer has no reports at all — distinguished from "a report that declares // nothing" so the caller never treats an absent report as a healthy one. func (s *Store) LatestReportOffsiteDeclaration(customerID string) (found bool, reportID int64, state string, err error) { var id int64 var reportJSON string err = s.db.QueryRow(`SELECT id, report_json FROM reports WHERE customer_id = ? ORDER BY id DESC LIMIT 1`, customerID).Scan(&id, &reportJSON) if err == sql.ErrNoRows { return false, 0, "", nil } if err != nil { return false, 0, "", err } var p struct { Offsite *struct { State string `json:"state"` } `json:"offsite"` } if json.Unmarshal([]byte(reportJSON), &p) == nil && p.Offsite != nil { state = p.Offsite.State } return true, id, state, nil } // OneTimeSecretInfo is the delivery-state metadata of a customer's one-time offsite secret — // timestamps ONLY, the value column is deliberately never selected (R-70 detector input; the // customer-scoped sibling of PBSDRHealStates' SecretUnconsumedFor). type OneTimeSecretInfo struct { CustomerID string CreatedAt time.Time ConsumedAt time.Time // zero = staged, not yet consumed } // GetOneTimeSecretInfo returns the timestamps of a customer's one-time offsite secret row, or nil // when none exists. Never reads the value. func (s *Store) GetOneTimeSecretInfo(customerID string) (*OneTimeSecretInfo, error) { var createdAt string var consumedAt sql.NullString err := s.db.QueryRow(`SELECT created_at, consumed_at FROM one_time_secrets WHERE customer_id = ?`, customerID).Scan(&createdAt, &consumedAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } info := &OneTimeSecretInfo{CustomerID: customerID, CreatedAt: parseSQLiteTime(createdAt)} if consumedAt.Valid { info.ConsumedAt = parseSQLiteTime(consumedAt.String) } return info, nil } // SetOneTimeSecretTimesForTest back-dates a one-time secret's timestamps (SQLite datetime strings; // consumedAt "" leaves it NULL) so grace/age behavior is testable without sleeping. TEST-ONLY — // mirrors SetHostPBSSecretCreatedAtForTest. func (s *Store) SetOneTimeSecretTimesForTest(customerID, createdAt, consumedAt string) error { if consumedAt == "" { _, err := s.db.Exec(`UPDATE one_time_secrets SET created_at = ?, consumed_at = NULL WHERE customer_id = ?`, createdAt, customerID) return err } _, err := s.db.Exec(`UPDATE one_time_secrets SET created_at = ?, consumed_at = ? WHERE customer_id = ?`, createdAt, consumedAt, customerID) return err } // ListCustomerConfigs returns all customer configurations ordered by ID. func (s *Store) ListCustomerConfigs() ([]CustomerConfig, error) { rows, err := s.db.Query(` SELECT customer_id, customer_name, domain, email, retrieval_password, api_key, config_json, status, min_controller_version, dr_tier, config_version, created_at, updated_at FROM customer_configs ORDER BY customer_id`) if err != nil { return nil, err } defer rows.Close() var configs []CustomerConfig for rows.Next() { var cfg CustomerConfig var createdAt, updatedAt string if err := rows.Scan(&cfg.CustomerID, &cfg.CustomerName, &cfg.Domain, &cfg.Email, &cfg.RetrievalPassword, &cfg.APIKey, &cfg.ConfigJSON, &cfg.Status, &cfg.MinControllerVersion, &cfg.DRTier, &cfg.ConfigVersion, &createdAt, &updatedAt); err != nil { return nil, err } cfg.CreatedAt = parseSQLiteTime(createdAt) cfg.UpdatedAt = parseSQLiteTime(updatedAt) configs = append(configs, cfg) } return configs, rows.Err() } // DeleteCustomerConfig deletes a customer configuration AND purges the customer's escrow custody // (v0.60.1). The customer Danger-zone Delete is the ONE true purge point for recovery-key custody: // host deletion only DEMOTES a blob to retained custody (never destroys), so removing the customer is // the deliberate, acknowledged point where that retained custody is permanently removed. In one tx it // deletes host_escrow AND host_escrow_superseded for ALL the customer's hosts — INCLUDING hosts // already deleted (whose demoted blobs survive in host_escrow_superseded), resolved via the F-14 // host_deletions provenance so a host-delete-then-customer-delete ordering leaves nothing orphaned. // The broader offboarding lifecycle (Hetzner sub-account, WG peer, Storage-Box data, the host rows // themselves) is NOT this method — see the delete/re-create rehearsal (ROADMAP R-3). func (s *Store) DeleteCustomerConfig(customerID string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() // Current hosts' escrow. if _, err := tx.Exec(`DELETE FROM host_escrow WHERE host_id IN (SELECT host_id FROM hosts WHERE customer_id = ?)`, customerID); err != nil { return fmt.Errorf("DeleteCustomerConfig %s: purge host_escrow: %w", customerID, err) } // Retained (superseded) blobs for BOTH current and already-deleted hosts of this customer. if _, err := tx.Exec(` DELETE FROM host_escrow_superseded WHERE host_id IN ( SELECT host_id FROM hosts WHERE customer_id = ? UNION SELECT host_id FROM host_deletions WHERE customer_id = ? )`, customerID, customerID); err != nil { return fmt.Errorf("DeleteCustomerConfig %s: purge host_escrow_superseded: %w", customerID, err) } if _, err := tx.Exec(`DELETE FROM customer_configs WHERE customer_id = ?`, customerID); err != nil { return fmt.Errorf("DeleteCustomerConfig %s: delete config: %w", customerID, err) } return tx.Commit() } // GetCustomerConfigByAPIKey looks up a customer config by its unique API key. // Returns nil if no matching key is found. func (s *Store) GetCustomerConfigByAPIKey(apiKey string) (*CustomerConfig, error) { var cfg CustomerConfig var createdAt, updatedAt string err := s.db.QueryRow(` SELECT customer_id, customer_name, domain, email, retrieval_password, api_key, config_json, status, min_controller_version, dr_tier, config_version, created_at, updated_at FROM customer_configs WHERE api_key = ?`, apiKey, ).Scan(&cfg.CustomerID, &cfg.CustomerName, &cfg.Domain, &cfg.Email, &cfg.RetrievalPassword, &cfg.APIKey, &cfg.ConfigJSON, &cfg.Status, &cfg.MinControllerVersion, &cfg.DRTier, &cfg.ConfigVersion, &createdAt, &updatedAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } cfg.CreatedAt = parseSQLiteTime(createdAt) cfg.UpdatedAt = parseSQLiteTime(updatedAt) return &cfg, nil } // ── Customer-claim password arc (v0.50.0, DRILL-day0-vm F-4) ────────────────────────────── // ClaimState is one customer's active claim/reset-code state. CodeHash is bcrypt(code) — the // store NEVER holds a plaintext code. Claimed means the customer has completed the claim (set // their own password) at least once; rotations never clear it. type ClaimState struct { CustomerID string CodeHash string Generation int IssuedAt time.Time EmailedAt *time.Time ClaimedAt *time.Time ResetDay string ResetCount int } // Claimed reports whether the claim has been completed at least once. func (c *ClaimState) Claimed() bool { return c != nil && c.ClaimedAt != nil } // GetClaim returns the customer's claim state, or nil when none exists. func (s *Store) GetClaim(customerID string) (*ClaimState, error) { var cs ClaimState var issuedAt string var emailedAt, claimedAt sql.NullString err := s.db.QueryRow(` SELECT customer_id, code_hash, generation, issued_at, emailed_at, claimed_at, reset_day, reset_count FROM customer_claims WHERE customer_id = ?`, customerID, ).Scan(&cs.CustomerID, &cs.CodeHash, &cs.Generation, &issuedAt, &emailedAt, &claimedAt, &cs.ResetDay, &cs.ResetCount) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } cs.IssuedAt = parseSQLiteTime(issuedAt) if emailedAt.Valid { t := parseSQLiteTime(emailedAt.String) cs.EmailedAt = &t } if claimedAt.Valid { t := parseSQLiteTime(claimedAt.String) cs.ClaimedAt = &t } return &cs, nil } // RotateClaimCode installs a fresh code hash: creates the row (generation 1) or rotates it // (generation+1, new issued_at, emailed_at cleared until the send is confirmed). claimed_at is // deliberately PRESERVED — rotating a claimed customer's code (a password reset) never un-claims // the box. Returns the new generation. func (s *Store) RotateClaimCode(customerID, codeHash string) (int, error) { _, err := s.db.Exec(` INSERT INTO customer_claims (customer_id, code_hash, generation, issued_at) VALUES (?, ?, 1, datetime('now')) ON CONFLICT(customer_id) DO UPDATE SET code_hash = excluded.code_hash, generation = customer_claims.generation + 1, issued_at = datetime('now'), emailed_at = NULL`, customerID, codeHash) if err != nil { return 0, err } var gen int if err := s.db.QueryRow(`SELECT generation FROM customer_claims WHERE customer_id = ?`, customerID).Scan(&gen); err != nil { return 0, err } return gen, nil } // MarkClaimEmailed records that the active code was delivered to the registered address. func (s *Store) MarkClaimEmailed(customerID string) error { _, err := s.db.Exec(`UPDATE customer_claims SET emailed_at = datetime('now') WHERE customer_id = ?`, customerID) return err } // MarkClaimed records the first successful claim (idempotent — an already-set claimed_at stays). // Returns true when this call performed the unclaimed→claimed transition. func (s *Store) MarkClaimed(customerID string) (bool, error) { res, err := s.db.Exec(`UPDATE customer_claims SET claimed_at = datetime('now') WHERE customer_id = ? AND claimed_at IS NULL`, customerID) if err != nil { return false, err } n, _ := res.RowsAffected() return n > 0, nil } // BumpResetCount enforces the hub-side reset-request limiter: increments today's counter and // returns the post-increment count (the caller compares against the daily cap). The day key // rolls over automatically (UTC date). func (s *Store) BumpResetCount(customerID string) (int, error) { day := time.Now().UTC().Format("2006-01-02") _, err := s.db.Exec(` UPDATE customer_claims SET reset_count = CASE WHEN reset_day = ? THEN reset_count + 1 ELSE 1 END, reset_day = ? WHERE customer_id = ?`, day, day, customerID) if err != nil { return 0, err } var count int if err := s.db.QueryRow(`SELECT reset_count FROM customer_claims WHERE customer_id = ?`, customerID).Scan(&count); err != nil { return 0, err } return count, nil } // SetCustomerConfigStatus sets the status (active/blocked) for a customer config. func (s *Store) SetCustomerConfigStatus(customerID, status string) error { _, err := s.db.Exec(` UPDATE customer_configs SET status = ?, updated_at = datetime('now') WHERE customer_id = ?`, status, customerID, ) return err } // SetMinControllerVersion sets (or clears, with "") the per-customer controller-version floor // override. The customer config must already exist. func (s *Store) SetMinControllerVersion(customerID, version string) error { _, err := s.db.Exec(` UPDATE customer_configs SET min_controller_version = ?, updated_at = datetime('now') WHERE customer_id = ?`, version, customerID, ) return err } // GetGlobalMinControllerVersion returns the operator-set global floor from hub_settings if present, // else the config/env-supplied default. Empty string = no global floor. func (s *Store) GetGlobalMinControllerVersion() string { var v string err := s.db.QueryRow(`SELECT value FROM hub_settings WHERE key = 'min_controller_version'`).Scan(&v) if err == nil && v != "" { return v } // No DB override (missing row or empty value) → fall back to the config/env default. return s.defaultMinControllerVersion } // NewestReportedControllerVersion returns the highest controller version ANY customer has reported, or // "" when none has. Semver-ordered in Go, not in SQL: `MAX(controller_version)` would compare lexically // and rank 0.99.0 above 0.186.0 — which is the exact pair this gate has to get right. // // It reads `reports.controller_version`, the column SaveReport denormalises out of every report // (store.go:903). It deliberately does NOT read `guests.controller_version`: that column exists in the // schema (:294) and **nothing writes it**, so a gate keyed on it would always see "" and fail open — // an inert gate, which is the exact class R-29 is about. Verified by grep before writing this. // // R-120's gate signal. The hub cannot ask "what is the newest controller that exists" — it has no // registry credential and makes no outbound call at vouch time — but it does know what the FLEET is // running, and that is the signal that matters: the failure this exists to catch is a golden left // behind a controller **already deployed**. It has happened three times (R-111, R-115, R-120) and on // the R-120 occurrence felhom-pve was reporting 0.186.0 while the manifest vouched a 0.185.1 golden — // exactly the comparison below. // // KNOWN BLIND SPOT, stated rather than papered over: a controller no box has ever run is invisible // here, so a golden baked behind an unreleased controller still passes. That is a real limit and it is // not the failure mode that has bitten — the three instances were all "deployed newer than baked". func (s *Store) NewestReportedControllerVersion() string { rows, err := s.db.Query(`SELECT DISTINCT controller_version FROM reports WHERE controller_version IS NOT NULL AND controller_version != ''`) if err != nil { return "" // unreadable → the gate degrades to "cannot compare", never to a false refusal } defer rows.Close() newest := "" for rows.Next() { var v string if rows.Scan(&v) != nil { continue } v = strings.TrimPrefix(strings.TrimSpace(v), "v") if v == "" { continue } if newest == "" || semver.Compare(v, newest) > 0 { newest = v } } return newest } // GlobalFloorResolution is the full picture of the effective global floor for the operator UI: the // resolved value + WHICH source won + both raw inputs. It makes the "a manifest save silently armed // a live floor" incident (publish-train 0.81/0.113) permanently visible — the operator can see the // DB override vs the env fallback at a glance. type GlobalFloorResolution struct { Effective string // the value GetGlobalMinControllerVersion returns ("" = no floor) Source string // "db" | "env" | "none" DBValue string // the hub_settings row value ("" = no row / cleared) EnvValue string // the DEFAULT_MIN_CONTROLLER_VERSION fallback } // ResolveGlobalFloor reports the effective floor AND its source (DB hub_settings row vs the env // default). Mirrors GetGlobalMinControllerVersion's precedence exactly — do not fork the rule. func (s *Store) ResolveGlobalFloor() GlobalFloorResolution { res := GlobalFloorResolution{EnvValue: s.defaultMinControllerVersion} var db string if err := s.db.QueryRow(`SELECT value FROM hub_settings WHERE key = 'min_controller_version'`).Scan(&db); err == nil { res.DBValue = db } switch { case res.DBValue != "": res.Effective, res.Source = res.DBValue, "db" case res.EnvValue != "": res.Effective, res.Source = res.EnvValue, "env" default: res.Source = "none" } return res } // SetGlobalMinControllerVersion persists the operator-set global floor (overriding the config/env // default). Pass "" to clear the override and fall back to the default. func (s *Store) SetGlobalMinControllerVersion(version string) error { _, err := s.db.Exec(` INSERT INTO hub_settings (key, value, updated_at) VALUES ('min_controller_version', ?, datetime('now')) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`, version, ) return err } // ArtifactManifest is the operator-vouched current artifact set (agent binary + golden archive) // served to the host-bootstrap script so it can verify-before-install. The hub is the TRUST ROOT // for these checksums (a different root than Gitea, which only STORES the bytes): the script fetches // each artifact from Gitea with the config-retrieve git token, then checks its sha256 against the // value recorded here before installing/using it. Empty fields = nothing published yet (the script // then falls back to the local golden / fails clearly on a missing binary). type ArtifactManifest struct { AgentVersion string `json:"agent_version"` AgentSHA256 string `json:"agent_sha256"` GoldenVersion string `json:"golden_version"` GoldenSHA256 string `json:"golden_sha256"` // MinAgent is the MINIMUM host-agent version this golden's controller requires (the controller // CHANGELOG `MinAgent:` value the operator vouches at manifest time). Empty = an UNCOUPLED // release: no per-box agent gating. When set, the hub HOLDS the controller-version floor for any // box whose agent is below it (Part D) — mechanising the publish-train "agent BEFORE controller // floor" rule instead of leaving it to operator discipline. MinAgent string `json:"min_agent"` // WrapperSHA256 is the sha256 of the PBS-DR apply wrapper (configs/felhom-pbs-apply) the operator // has vouched (R-50b(a), v0.68.0). // // Unlike the agent binary and the golden, this artifact is installed from // `raw/branch/main` by felhom-host-install.sh — UNVERSIONED, with no tag, no pin and no checksum. // It is a root-owned 0755 file and the pinned sudoers vector for the PBS storage verbs, so "which // wrapper is on this host?" was previously unanswerable from any manifest: two hosts installed a // week apart could carry different privileged code while reporting the same agent version. // Recording the hash here does not fix the delivery channel (that is R-50b(b)/(c)) — it makes // DRIFT VISIBLE, which is the cheap honest first step. WrapperSHA256 string `json:"wrapper_sha256"` } // hub_settings keys for the artifact manifest (BUNDLE slice). Stored as discrete key/value rows in // the existing hub_settings table — same mechanism as the controller-version floor, so it survives // restarts and needs no schema change. const ( settingArtifactAgentVersion = "artifact_agent_version" settingArtifactAgentSHA256 = "artifact_agent_sha256" settingArtifactGoldenVersion = "artifact_golden_version" settingArtifactGoldenSHA256 = "artifact_golden_sha256" settingArtifactMinAgent = "artifact_min_agent" settingArtifactWrapperSHA256 = "artifact_wrapper_sha256" // R-50b(a): the vouched felhom-pbs-apply hash ) // settingOperatorPasswordHash is the hub_settings key for the operator login password bcrypt hash, // set via the Configuration UI (v0.54.0). When present it OVERRIDES the config/env seed // (auth.password_hash in hub.yaml) — the same DB-override-wins precedence as the controller-version // floor. The ConfigMap value stays the break-glass fallback: clear this row (or edit the manifest + // redeploy) to reset a lost password. const settingOperatorPasswordHash = "operator_password_hash" // GetOperatorPasswordHash returns the UI-set operator password bcrypt hash, or "" when none has been // set (the config/env seed is then authoritative). func (s *Store) GetOperatorPasswordHash() string { return s.getSetting(settingOperatorPasswordHash) } // SetOperatorPasswordHash persists a new operator password bcrypt hash set via the Configuration UI. func (s *Store) SetOperatorPasswordHash(hash string) error { return s.setSetting(settingOperatorPasswordHash, hash) } // getSetting reads a single hub_settings value ("" if the row is absent). func (s *Store) getSetting(key string) string { var v string if err := s.db.QueryRow(`SELECT value FROM hub_settings WHERE key = ?`, key).Scan(&v); err != nil { return "" } return v } // setSetting upserts a single hub_settings value. func (s *Store) setSetting(key, value string) error { _, err := s.db.Exec(` INSERT INTO hub_settings (key, value, updated_at) VALUES (?, ?, datetime('now')) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = datetime('now')`, key, value, ) return err } // GetArtifactManifest returns the operator-recorded current artifact set. All-empty when nothing // has been published yet. func (s *Store) GetArtifactManifest() ArtifactManifest { return ArtifactManifest{ AgentVersion: s.getSetting(settingArtifactAgentVersion), AgentSHA256: s.getSetting(settingArtifactAgentSHA256), GoldenVersion: s.getSetting(settingArtifactGoldenVersion), GoldenSHA256: s.getSetting(settingArtifactGoldenSHA256), MinAgent: s.getSetting(settingArtifactMinAgent), WrapperSHA256: s.getSetting(settingArtifactWrapperSHA256), } } // SetArtifactManifest persists the operator-recorded current artifact set (all four fields). Each // field is stored independently so a partial form submission (e.g. agent only) still round-trips. func (s *Store) SetArtifactManifest(m ArtifactManifest) error { if err := s.setSetting(settingArtifactAgentVersion, m.AgentVersion); err != nil { return err } if err := s.setSetting(settingArtifactAgentSHA256, m.AgentSHA256); err != nil { return err } if err := s.setSetting(settingArtifactGoldenVersion, m.GoldenVersion); err != nil { return err } if err := s.setSetting(settingArtifactGoldenSHA256, m.GoldenSHA256); err != nil { return err } if err := s.setSetting(settingArtifactMinAgent, m.MinAgent); err != nil { return err } return s.setSetting(settingArtifactWrapperSHA256, m.WrapperSHA256) } // EffectiveMinControllerVersion resolves the floor that actually applies to a customer: the // per-customer override when set (non-empty), otherwise the global floor (hub_settings → config/env // default). Returns "" when no floor applies at all (Phase 2 inert for that customer). func (s *Store) EffectiveMinControllerVersion(customerID string) string { cfg, err := s.GetCustomerConfig(customerID) if err == nil && cfg != nil && cfg.MinControllerVersion != "" { return cfg.MinControllerVersion } return s.GetGlobalMinControllerVersion() } // ManagedFloorDecision is the per-box outcome of the MinAgent conditional floor (Part D): the floor // to actually serve this customer's controller, whether it is being HELD (and why), and the inputs. type ManagedFloorDecision struct { Floor string // the controller-version floor to SERVE ("" = serve none) Held bool // true = the floor is withheld (see HeldBeyondGolden for which of the two reasons) AgentVersion string // the box's reported agent version ("" = unknown → held when MinAgent is set) // RequestedFloor is the floor that was asked for, kept for the message: Floor is cleared to "" on // a hold, and a reason naming an empty floor tells the operator nothing. RequestedFloor string // MinAgent is the manifest's vouched MinAgent — the agent requirement of the GOLDEN's controller // (`ArtifactManifest.MinAgent`). ⚠ It describes the golden, NOT necessarily the version this // decision is about; that gap is what HeldBeyondGolden closes. Empty = uncoupled, no gating. MinAgent string // GoldenVersion is the vouched golden's controller version — the version MinAgent describes. GoldenVersion string // HeldBeyondGolden distinguishes the two hold reasons, because they need different operator text: // false = the box's agent is below MinAgent (the original Part D hold); true = the FLOOR points // ABOVE the vouched golden, so the manifest's MinAgent does not describe the version being served // and the hub does not know its agent requirement (R-216). HeldBeyondGolden bool } // HoldReason is the ONE operator-facing sentence for a held floor, so the two hold reasons can never // drift apart across the log line and the dashboard again. // // ⚠ It exists because v0.97.0 introduced a second reason and, for one deploy, both surfaces still // printed the first: the live hub logged `managed floor HELD for c11: agent "0.125.0" < MinAgent // 0.113.0` — a comparison that is FALSE (0.125.0 is above 0.113.0) for a box held because its floor // sat above the golden. That is `CLAUDE.md`'s corollary exactly: when a verdict changes which field it // counts from, the alarm text has to change with it, or a true alarm reads as one to dismiss. // Returns "" when not held. func (d ManagedFloorDecision) HoldReason() string { switch { case !d.Held: return "" case d.HeldBeyondGolden: return fmt.Sprintf("held: floor %s is ABOVE the vouched golden %s, so its agent requirement is unknown — vouch a golden carrying the floor's controller (publish-train rule 1)", d.EffectiveFloorForMessage(), goldenOrUnknown(d.GoldenVersion)) default: return fmt.Sprintf("held: agent %s < MinAgent %s", agentOrUnknownVersion(d.AgentVersion), d.MinAgent) } } // EffectiveFloorForMessage reports the floor that was REQUESTED, for the message — Floor is cleared to // "" when held, and a message naming an empty floor tells the operator nothing. func (d ManagedFloorDecision) EffectiveFloorForMessage() string { if d.RequestedFloor != "" { return d.RequestedFloor } return d.Floor } func goldenOrUnknown(v string) string { if v == "" { return "(none vouched)" } return v } func agentOrUnknownVersion(v string) string { if v == "" { return "(unknown)" } return v } // ResolveManagedFloor decides the controller-version floor to serve a customer, HOLDING it rather // than pushing a controller past the agent it depends on (Part D — the hub-enforced "agent BEFORE // controller floor" rule). // // ⚠ R-216, and the reason this function changed. `publish-train-rules.md` rule 3 states the rule // about the FLOOR's controller: *"The floor may not effectively push a box past a controller whose // MinAgent that box's agent does not yet meet."* The implementation compared against // `ArtifactManifest.MinAgent`, which by its own doc comment is *"the MINIMUM host-agent version this // GOLDEN's controller requires"*. Those are the same number only while the floor sits at or below the // golden — the arrangement rule 5's ISO gate assumes. Raise a floor above the vouched golden (which // the day-0 runbook explicitly recommends after a golden rebuild, and which a per-customer override // makes trivial) and the guard compares against a version it is not serving. // // Measured live on 2026-08-05 (CAMPAIGN-11 Phase 1): golden 0.192.0 with MinAgent 0.113.0, a // per-customer floor of 0.200.0, and a box on agent 0.120.0. 0.120.0 ≥ 0.113.0, so the floor was // served — and the box was pushed onto a controller needing agent 0.125.0. Its customer was then // told their correct recovery code was wrong. The guard whose comment says it exists to *"never push // a controller past its agent"* had just done exactly that. // // THE FIX, and why it needs no new operator input: the hub cannot know the agent requirement of a // controller version it was never told about, so it must not pretend to. A floor ABOVE the vouched // golden is served blind today; from v0.97.0 it is HELD, with its own reason. Nobody types a number // twice — the existing vouched MinAgent keeps its exact meaning, and the guard simply stops applying // it to versions it does not describe. The operator's remedy is the one the publish train already // prescribes: vouch a golden that carries the floor's controller (rule 1, "manifest before floor"). // // Logic: // - effective floor "" → nothing to serve (no floor configured); // - manifest MinAgent "" → UNCOUPLED release: serve the floor as-is (no agent gating); // - floor ABOVE the vouched golden's controller → HOLD (requirement unknown) + flag; // - agent_version known AND ≥ MinAgent → serve the floor; // - agent_version below MinAgent, OR unknown/unparseable → HOLD (serve no directive) + flag. // // A held box is VISIBLE (the dashboard renders the reason), never silently stale. func (s *Store) ResolveManagedFloor(customerID string) ManagedFloorDecision { d := ManagedFloorDecision{Floor: s.EffectiveMinControllerVersion(customerID)} d.RequestedFloor = d.Floor if d.Floor == "" { return d } man := s.GetArtifactManifest() d.MinAgent, d.GoldenVersion = man.MinAgent, man.GoldenVersion if d.MinAgent == "" { return d // uncoupled release — no agent gate (Scenario C: unchanged from before) } if h, err := s.GetHostByCustomer(customerID); err == nil && h != nil { d.AgentVersion = h.AgentVersion } // R-216: the floor names a controller the manifest does not describe → the agent requirement is // UNKNOWN, so hold. Only when both versions parse — an unreadable golden must not gate the fleet. if semver.Valid(d.Floor) && semver.Valid(d.GoldenVersion) && semver.Compare(d.Floor, d.GoldenVersion) > 0 { d.Held, d.HeldBeyondGolden = true, true d.Floor = "" return d } if d.AgentVersion != "" && semver.Valid(d.AgentVersion) && semver.Valid(d.MinAgent) && semver.Compare(d.AgentVersion, d.MinAgent) >= 0 { return d // agent is new enough — serve the floor } // Agent too old, or unknown/unparseable → hold the floor (never push a controller past its agent). d.Held = true d.Floor = "" return d } // IsCustomerBlocked returns true if the customer config has status "blocked". func (s *Store) IsCustomerBlocked(customerID string) bool { var status string err := s.db.QueryRow( "SELECT status FROM customer_configs WHERE customer_id = ?", customerID, ).Scan(&status) return err == nil && status == "blocked" } // UpdateRetrievalPassword updates the retrieval password for a customer config. func (s *Store) UpdateRetrievalPassword(customerID, newPassword string) error { _, err := s.db.Exec(` UPDATE customer_configs SET retrieval_password = ?, updated_at = datetime('now') WHERE customer_id = ?`, newPassword, customerID, ) return err } // --- Event system --- // Event represents a single event record. type Event struct { ID int64 CustomerID string EventType string Severity string // "info", "warning", "error" Message string DetailsJSON string // raw JSON Source string // "controller" or "hub" CreatedAt time.Time } // SaveEvent inserts a new event and returns its ID. func (s *Store) SaveEvent(customerID, eventType, severity, message, detailsJSON, source string) (int64, error) { if detailsJSON == "" { detailsJSON = "{}" } if source == "" { source = "controller" } res, err := s.db.Exec(` INSERT INTO events (customer_id, event_type, severity, message, details_json, source) VALUES (?, ?, ?, ?, ?, ?)`, customerID, eventType, severity, message, detailsJSON, source, ) if err != nil { return 0, err } return res.LastInsertId() } // GetRecentEvents returns the most recent events for a customer, newest first. func (s *Store) GetRecentEvents(customerID string, limit int) ([]Event, error) { rows, err := s.db.Query(` SELECT id, customer_id, event_type, severity, message, details_json, source, created_at FROM events WHERE customer_id = ? ORDER BY created_at DESC LIMIT ?`, customerID, limit) if err != nil { return nil, err } defer rows.Close() return scanEvents(rows) } // GetEventsByType returns events of a specific type for a customer since a given time. func (s *Store) GetEventsByType(customerID, eventType string, since time.Time) ([]Event, error) { rows, err := s.db.Query(` SELECT id, customer_id, event_type, severity, message, details_json, source, created_at FROM events WHERE customer_id = ? AND event_type = ? AND created_at >= ? ORDER BY created_at DESC`, customerID, eventType, since.UTC().Format("2006-01-02 15:04:05")) if err != nil { return nil, err } defer rows.Close() return scanEvents(rows) } // GetLatestEventByType returns the most recent event of a given type for a customer. func (s *Store) GetLatestEventByType(customerID, eventType string) (*Event, error) { var e Event var createdAt string err := s.db.QueryRow(` SELECT id, customer_id, event_type, severity, message, details_json, source, created_at FROM events WHERE customer_id = ? AND event_type = ? ORDER BY created_at DESC LIMIT 1`, customerID, eventType, ).Scan(&e.ID, &e.CustomerID, &e.EventType, &e.Severity, &e.Message, &e.DetailsJSON, &e.Source, &createdAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } e.CreatedAt = parseSQLiteTime(createdAt) return &e, nil } // GetAllRecentEvents returns the most recent events across all customers. func (s *Store) GetAllRecentEvents(limit int) ([]Event, error) { rows, err := s.db.Query(` SELECT id, customer_id, event_type, severity, message, details_json, source, created_at FROM events ORDER BY created_at DESC LIMIT ?`, limit) if err != nil { return nil, err } defer rows.Close() return scanEvents(rows) } // CountEventsBySeverity returns a count of events per severity for a customer since a given time. func (s *Store) CountEventsBySeverity(customerID string, since time.Time) (map[string]int, error) { rows, err := s.db.Query(` SELECT severity, COUNT(*) FROM events WHERE customer_id = ? AND created_at >= ? GROUP BY severity`, customerID, since.UTC().Format("2006-01-02 15:04:05")) if err != nil { return nil, err } defer rows.Close() counts := make(map[string]int) for rows.Next() { var sev string var count int if err := rows.Scan(&sev, &count); err != nil { return nil, err } counts[sev] = count } return counts, rows.Err() } // PruneEvents deletes events older than the given number of days. func (s *Store) PruneEvents(maxDays int) (int64, error) { cutoff := time.Now().AddDate(0, 0, -maxDays).UTC().Format("2006-01-02 15:04:05") res, err := s.db.Exec("DELETE FROM events WHERE created_at < ?", cutoff) if err != nil { return 0, err } return res.RowsAffected() } // GetActiveCustomerIDs returns customer IDs from customer_configs where status is 'active'. func (s *Store) GetActiveCustomerIDs() ([]string, error) { rows, err := s.db.Query("SELECT customer_id FROM customer_configs WHERE status = 'active'") if err != nil { return nil, err } defer rows.Close() var ids []string for rows.Next() { var id string if err := rows.Scan(&id); err != nil { return nil, err } ids = append(ids, id) } return ids, rows.Err() } // Ping verifies the database is accessible. func (s *Store) Ping() error { var n int return s.db.QueryRow("SELECT 1").Scan(&n) } func scanEvents(rows *sql.Rows) ([]Event, error) { var events []Event for rows.Next() { var e Event var createdAt string if err := rows.Scan(&e.ID, &e.CustomerID, &e.EventType, &e.Severity, &e.Message, &e.DetailsJSON, &e.Source, &createdAt); err != nil { return nil, err } e.CreatedAt = parseSQLiteTime(createdAt) events = append(events, e) } return events, rows.Err() } // parseSQLiteTime tries multiple formats that modernc.org/sqlite may return. func parseSQLiteTime(s string) time.Time { formats := []string{ "2006-01-02 15:04:05", // SQLite datetime('now') "2006-01-02T15:04:05Z", // RFC3339 without fractional time.RFC3339, // 2006-01-02T15:04:05Z07:00 time.RFC3339Nano, // with fractional seconds "2006-01-02 15:04:05+00:00", // with explicit UTC offset "2006-01-02 15:04:05.999999999", // with fractional, no TZ } for _, f := range formats { if t, err := time.Parse(f, s); err == nil { return t } } // Last resort: if string is non-empty, log it for debugging if s != "" { log.Printf("[WARN] Could not parse timestamp: %q", s) } return time.Time{} // zero time } func parseDiskSummary(reportJSON string) string { var report struct { Storage []struct { Mount string `json:"mount"` Percent float64 `json:"percent"` } `json:"storage"` } // ignore parse errors — show "–" on failure json.Unmarshal([]byte(reportJSON), &report) //nolint:errcheck var parts []string for _, s := range report.Storage { parts = append(parts, fmt.Sprintf("%.0f%%", s.Percent)) } if len(parts) == 0 { return "–" } result := parts[0] for _, p := range parts[1:] { result += "/" + p } return result } // ---- v0.7.0: host-domain (slice 3) ---- // Additive store surface for the agent's host-report stream. The controller-path // methods above are untouched. // Host is one customer agent. Mixes operator-intent columns (Desired*, DRRecord — // INERT until slice 10) with box-reported reality (AgentVersion, LastReportAt). type Host struct { HostID string CustomerID string APIKey string AgentVersion string LastReportAt *time.Time DesiredJSON string DesiredGeneration int64 DRRecordJSON string RecoveryModeUntil *time.Time // slice 10D: recovery mode active until this time (nil/past = off) CreatedAt time.Time UpdatedAt time.Time } // InRecoveryMode reports whether the host is currently in recovery mode (set + not expired). func (h *Host) InRecoveryMode(now time.Time) bool { return h.RecoveryModeUntil != nil && now.Before(*h.RecoveryModeUntil) } // Guest is one controller LXC. Reality columns are report-driven; APIKey and // DesiredSpecJSON are INERT until slice 10 and must survive report upserts. type Guest struct { GuestID string CustomerID string HostID string VMID int DisplayName string Status string ControllerVersion string LastSeenAt *time.Time APIKey string DesiredSpecJSON string CreatedAt time.Time UpdatedAt time.Time } // HostReportDenorm are the denormalized fields pulled from a host-report for the // dashboard / staleness, mirroring the reports table's denorm pattern. type HostReportDenorm struct { AgentVersion string CPUPercent float64 MemoryPercent float64 DiskPercent float64 GuestTotal int GuestRunning int CloudflaredStatus string } // HostStaleRow is the minimal per-host recency row the dead-man's-switch reads. type HostStaleRow struct { HostID string CustomerID string LastReportAt time.Time } // GuestID derives the interim guest primary key from host + vmid. The hub owns the // id scheme (locked decision 3) so the slice-10 swap to durable ids is hub-only. func GuestID(hostID string, vmid int) string { return hostID + "/" + strconv.Itoa(vmid) } func scanHost(scan func(dest ...any) error) (*Host, error) { var h Host var lastReport, recoveryUntil sql.NullString var createdAt, updatedAt string err := scan(&h.HostID, &h.CustomerID, &h.APIKey, &h.AgentVersion, &lastReport, &h.DesiredJSON, &h.DesiredGeneration, &h.DRRecordJSON, &recoveryUntil, &createdAt, &updatedAt) if err != nil { return nil, err } if lastReport.Valid { t := parseSQLiteTime(lastReport.String) h.LastReportAt = &t } if recoveryUntil.Valid && recoveryUntil.String != "" { t := parseSQLiteTime(recoveryUntil.String) h.RecoveryModeUntil = &t } h.CreatedAt = parseSQLiteTime(createdAt) h.UpdatedAt = parseSQLiteTime(updatedAt) return &h, nil } const hostSelectCols = `host_id, customer_id, api_key, agent_version, last_report_at, desired_json, desired_generation, dr_record_json, recovery_mode_until, created_at, updated_at` // GetHostByAPIKey looks up a host by its per-host hub key. Returns nil (no error) // if no match — parallels GetCustomerConfigByAPIKey. func (s *Store) GetHostByAPIKey(apiKey string) (*Host, error) { h, err := scanHost(s.db.QueryRow(`SELECT `+hostSelectCols+` FROM hosts WHERE api_key = ?`, apiKey).Scan) if err == sql.ErrNoRows { return nil, nil } return h, err } // GetHost looks up a host by id. Returns nil (no error) if not found. func (s *Store) GetHost(hostID string) (*Host, error) { h, err := scanHost(s.db.QueryRow(`SELECT `+hostSelectCols+` FROM hosts WHERE host_id = ?`, hostID).Scan) if err == sql.ErrNoRows { return nil, nil } return h, err } // GetHostByCustomer returns the customer's host, or nil (no error) if none exists. // Backs the passphrase-authed host-enroll mint-once-reuse path (Day-0 option C): on // the second enroll the existing credential is reused, not re-minted. A customer is // expected to have at most one host in the Day-0 model; if more than one ever exists, // the most-recently-updated wins (we never mint a duplicate on a reuse). Uses the // idx_hosts_customer index. Mirrors GetHostByAPIKey's nil-on-not-found contract. func (s *Store) GetHostByCustomer(customerID string) (*Host, error) { h, err := scanHost(s.db.QueryRow(`SELECT `+hostSelectCols+ ` FROM hosts WHERE customer_id = ? ORDER BY updated_at DESC LIMIT 1`, customerID).Scan) if err == sql.ErrNoRows { return nil, nil } return h, err } // HasEverBoundHost reports whether a machine was EVER bound to this customer — a live row in // `hosts` OR a tombstone in `host_deletions`. It answers "was anything ever expected of this // customer", which is the question the deadline verdicts actually need (R-195). // // It is deliberately NOT "has a report arrived", and the distinction is the whole point: a box // that was installed, bound, and then went silent IS bound, and its silence is a real fault that // must keep alarming. Only a customer that never had a machine at all is UNKNOWN. // // `host_deletions` is included because a customer whose host was removed HAD one — the deadline // caller reaches its down-skip for that shape, and this predicate must not quietly take over a // judgement the staleness checker owns. func (s *Store) HasEverBoundHost(customerID string) (bool, error) { var n int if err := s.db.QueryRow( `SELECT EXISTS(SELECT 1 FROM hosts WHERE customer_id = ?) OR EXISTS(SELECT 1 FROM host_deletions WHERE customer_id = ?)`, customerID, customerID).Scan(&n); err != nil { return false, err } return n != 0, nil } // ListHostsByCustomer returns the customer's hosts ordered by host_id (v0.47.0 — the // customer page's Host tab is a LIST by design: 1 host today, N for a later HA cluster). // Uses the idx_hosts_customer index. func (s *Store) ListHostsByCustomer(customerID string) ([]Host, error) { rows, err := s.db.Query(`SELECT `+hostSelectCols+ ` FROM hosts WHERE customer_id = ? ORDER BY host_id`, customerID) if err != nil { return nil, err } defer rows.Close() var hosts []Host for rows.Next() { h, err := scanHost(rows.Scan) if err != nil { return nil, err } hosts = append(hosts, *h) } return hosts, rows.Err() } // ListHosts returns all hosts (debug / host-domain views). func (s *Store) ListHosts() ([]Host, error) { rows, err := s.db.Query(`SELECT ` + hostSelectCols + ` FROM hosts ORDER BY host_id`) if err != nil { return nil, err } defer rows.Close() var hosts []Host for rows.Next() { h, err := scanHost(rows.Scan) if err != nil { return nil, err } hosts = append(hosts, *h) } return hosts, rows.Err() } // ErrHostEscrowPresent is returned by DeleteHost when the host still has a key-escrow row // and the caller did not explicitly acknowledge deleting it (fail-safe-to-refuse — an escrow // blob may be the ONLY remaining path to a customer's backup keys). var ErrHostEscrowPresent = errors.New("host has key escrow; deletion requires the explicit escrow acknowledgement") // HostArtifacts summarizes what a host deletion would remove — counts/booleans ONLY (the // impact preview must never carry a secret or blob). type HostArtifacts struct { Guests int Reports int LogBundles int // log_bundles rows with scope_id == host_id (the agent channel ONLY) EscrowPresent bool WGPeerBound bool PBSSecretPresent bool RecoveryPresent bool } // CountHostArtifacts reports the per-table blast radius of deleting a host (v0.47.0 stale // host removal). LogBundles counts ONLY host-scoped rows — customer-scoped bundles (the // controller channel, scope_id == customer_id) belong to the customer and are never touched. func (s *Store) CountHostArtifacts(hostID string) (HostArtifacts, error) { var a HostArtifacts counts := []struct { dst *int query string }{ {&a.Guests, `SELECT COUNT(*) FROM guests WHERE host_id = ?`}, {&a.Reports, `SELECT COUNT(*) FROM host_reports WHERE host_id = ?`}, {&a.LogBundles, `SELECT COUNT(*) FROM log_bundles WHERE scope_id = ?`}, } for _, c := range counts { if err := s.db.QueryRow(c.query, hostID).Scan(c.dst); err != nil { return a, err } } flags := []struct { dst *bool query string }{ {&a.EscrowPresent, `SELECT EXISTS(SELECT 1 FROM host_escrow WHERE host_id = ?)`}, {&a.WGPeerBound, `SELECT EXISTS(SELECT 1 FROM wg_peers WHERE host_id = ?)`}, {&a.PBSSecretPresent, `SELECT EXISTS(SELECT 1 FROM host_pbs_secrets WHERE host_id = ?)`}, {&a.RecoveryPresent, `SELECT EXISTS(SELECT 1 FROM host_recovery WHERE host_id = ?)`}, } for _, f := range flags { var n int if err := s.db.QueryRow(f.query, hostID).Scan(&n); err != nil { return a, err } *f.dst = n != 0 } return a, nil } // DeleteHost removes a host and every host-scoped artifact in ONE transaction (v0.47.0 // stale host removal). The online-gate lives in the web handler — the store deletes what // it is told to. Guards: // - empty hostID → refused (would DELETE the ” scope rows); // - escrow present without deleteEscrow → ErrHostEscrowPresent, the tx never starts. // // The wg_peers delete is INSIDE the tx on purpose — a crash between a host delete and a // separate peer delete would strand a bound peer the reconciler keeps pushing. The wgsync // reconciler's 5-minute declarative full-list push converges the endpoint after the row // disappears — no bump, no reconciler change. log_bundle rows die by scope_id == host_id // (agent channel); customer-scoped bundles (scope_id == customer_id) are NOT touched. // // v0.53.0 (F-14 provenance): every delete also writes a host_deletions row IN THE SAME tx. // escrow_acked = deleteEscrow AND an escrow row was actually present — "removed through the // escrow-ack flow" means the operator acknowledged the host removal and the current escrow blob was // DEMOTED to retained custody (v0.60.1: moved into host_escrow_superseded, not destroyed), not // merely that the checkbox was ticked over nothing. The flag's F-14 gate semantics are unchanged. func (s *Store) DeleteHost(hostID string, deleteEscrow bool) error { if hostID == "" { return fmt.Errorf("DeleteHost: empty host_id") } var escrowPresent int if err := s.db.QueryRow(`SELECT EXISTS(SELECT 1 FROM host_escrow WHERE host_id = ?)`, hostID).Scan(&escrowPresent); err != nil { return fmt.Errorf("DeleteHost %s: escrow check: %w", hostID, err) } if !deleteEscrow && escrowPresent != 0 { return ErrHostEscrowPresent } tx, err := s.db.Begin() if err != nil { return fmt.Errorf("DeleteHost %s: begin: %w", hostID, err) } defer tx.Rollback() // Provenance first (reads the host row this tx is about to delete). A host_id that has no // row deletes nothing anyway — skip the record rather than inventing an empty customer_id. var customerID string switch err := tx.QueryRow(`SELECT customer_id FROM hosts WHERE host_id = ?`, hostID).Scan(&customerID); err { case nil: acked := 0 if deleteEscrow && escrowPresent != 0 { acked = 1 } if _, err := tx.Exec(`INSERT INTO host_deletions (host_id, customer_id, escrow_acked) VALUES (?, ?, ?)`, hostID, customerID, acked); err != nil { return fmt.Errorf("DeleteHost %s: provenance record: %w", hostID, err) } case sql.ErrNoRows: // no host row — fall through, the deletes below are no-ops default: return fmt.Errorf("DeleteHost %s: customer lookup: %w", hostID, err) } // v0.60.1: host deletion is a LIFECYCLE event — the current escrow blob is DEMOTED to retained // custody (copied into host_escrow_superseded, copy-BEFORE-delete in this same tx), NEVER // destroyed; existing superseded rows are spared. No operator path through host lifecycle can // lose a blob. The customer Danger-zone Delete is the one true purge point (deleteCustomer). if deleteEscrow { if _, derr := demoteCurrentEscrowTx(tx, hostID); derr != nil { return fmt.Errorf("DeleteHost %s: demote escrow to retained custody: %w", hostID, derr) } if _, derr := tx.Exec(`DELETE FROM host_escrow WHERE host_id = ?`, hostID); derr != nil { return fmt.Errorf("DeleteHost %s: remove current escrow row: %w", hostID, derr) } } stmts := []string{ `DELETE FROM guests WHERE host_id = ?`, `DELETE FROM host_reports WHERE host_id = ?`, `DELETE FROM signed_jobs WHERE host_id = ?`, `DELETE FROM host_recovery WHERE host_id = ?`, `DELETE FROM host_pbs_secrets WHERE host_id = ?`, `DELETE FROM log_bundle_requests WHERE scope_id = ?`, `DELETE FROM log_bundles WHERE scope_id = ?`, `DELETE FROM wg_peers WHERE host_id = ?`, `DELETE FROM hosts WHERE host_id = ?`, } for _, q := range stmts { if _, err := tx.Exec(q, hostID); err != nil { return fmt.Errorf("DeleteHost %s: %q: %w", hostID, q, err) } } return tx.Commit() } // HostDeletion is one host-removal provenance record (v0.53.0, F-14). EscrowAcked means the // operator removed the host through the escrow-ack flow — the current key custody was DEMOTED to // retained custody (v0.60.1), not destroyed — the ONLY state that permits the PBS-DR auto-re-issue. type HostDeletion struct { HostID string CustomerID string DeletedAt time.Time EscrowAcked bool } // LatestHostDeletion returns the customer's MOST RECENT host-deletion record (nil when the // customer has none — every pre-v0.53.0 deletion, by design: no backfill invents provenance). // The latest record is the one that orphaned a surviving ep0 tenancy, so the F-14 gate reads // exactly this row — an older acked record must not whitelist a newer un-acked deletion. func (s *Store) LatestHostDeletion(customerID string) (*HostDeletion, error) { var d HostDeletion var deletedAt string var acked int err := s.db.QueryRow(` SELECT host_id, customer_id, deleted_at, escrow_acked FROM host_deletions WHERE customer_id = ? ORDER BY id DESC LIMIT 1`, customerID, ).Scan(&d.HostID, &d.CustomerID, &deletedAt, &acked) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } d.DeletedAt = parseSQLiteTime(deletedAt) d.EscrowAcked = acked != 0 return &d, nil } // UpsertHost creates or updates a host identity (used by the admin mint). On // conflict it updates only operator-settable identity fields + updated_at; it does // NOT touch the reality columns (agent_version/last_report_at) or the inert intent // columns (desired_*/dr_record_json) — those are owned elsewhere. func (s *Store) UpsertHost(h *Host) error { _, err := s.db.Exec(` INSERT INTO hosts (host_id, customer_id, api_key, updated_at) VALUES (?, ?, ?, datetime('now')) ON CONFLICT(host_id) DO UPDATE SET customer_id = excluded.customer_id, api_key = excluded.api_key, updated_at = datetime('now')`, h.HostID, h.CustomerID, h.APIKey, ) return err } // HostEscrow is the opaque R-wrapped escrow blob stored for a host (doc 03 §8a). Blob is // ciphertext the hub cannot open. type HostEscrow struct { HostID string Blob []byte KeyFingerprint string Posture string CreatedAt string UpdatedAt string // ResticPwSHA256 (SLICE 3) — the non-reversible hash of the offsite repo password the identity blob // covers ("" = legacy/password-less blob). Safe to store/serve; the password itself never reaches the hub. ResticPwSHA256 string // StaleAt (v0.57.0, 2.3) — non-empty when the offsite password was re-issued after this blob was // sealed: the blob is stale (seals a password that no longer opens the repo). Cleared by a fresh ceremony. StaleAt string // IdentityBlob (R-198, v0.93.0) — the age-wrapped identity bundle, which is where the offsite // restic REPOSITORY password lives. Populated by ListSupersededEscrow so a retained blob is // reachable from Go at all; nil for pre-v0.93.0 retained rows and for hosts that never uploaded // one. Opaque: useless without the customer's recovery code, which the hub never holds. IdentityBlob []byte } // SaveHostEscrow stores (last-write-wins) the OPAQUE escrow blob for a host. The hub keeps the // bytes and NEVER decrypts them — there is no decrypt path. createdAt is the agent's timestamp. // resticPwSHA256 is "" when the ceremony sealed no staged password (stored as-is; never auto-confirms). // SaveHostEscrow returns superseded=true when it RETAINED a different-passphrase old blob into // host_escrow_superseded before overwriting the current row (Part B, v0.60.0). A same-sha re-upload // (idempotent re-ceremony of the same password) refreshes the current row and does NOT create a // superseded row. // // R-197 (v0.93.0): it also returns prevResticPwSHA256 — the hash the row being replaced sealed ("" when // no row existed, or when it was a legacy hash-less blob). Both halves of "did the box's offsite DATA // key change?" have been in this database since SLICE 3 and NOTHING compared them; demo-felhom's key // changed on 2026-08-03 and no signal of any kind fired for thirteen hours. The comparison is the // caller's (api.handleHostEscrowPut) because the event needs the customer id; returning the value is // this function's part. The VALUE is a non-reversible hash of a 256-bit random secret and is never // logged, mailed or written to a report — see the caller. // demoteCurrentEscrowTx copies the host's CURRENT host_escrow row (if any) into // host_escrow_superseded as a retained blob, inside the given tx. This is THE ONE escrow row-copy // routine (v0.60.0): SaveHostEscrow uses it to retain a superseded different-passphrase blob before // overwriting, and DeleteHost (v0.60.1) uses it to DEMOTE the current blob to retained custody // instead of destroying it. Returns the number of rows copied (0 when the host has no current row). // The hub never decrypts; custody is unchanged. // // R-198 (v0.93.0): `identity_blob` is copied too. It was omitted from this SELECT for two months, so // every supersession retained the PBS datastore key and destroyed the offsite restic repository // password — the one secret the retention exists to preserve. Copying more opaque bytes gains the hub // NO knowledge: it still has no recovery code and no decrypt path. // // ORDERING THIS DEPENDS ON, stated because it is load-bearing and invisible from here: the identity // blob is written by SaveHostDRBundle AFTER SaveHostEscrow returns (api/handler.go, the escrow PUT), // so at demote time host_escrow still holds the OLD identity blob. If that order ever changes, this // routine silently retains the NEW blob under the OLD blob's hash — pinned by // TestSaveHostEscrow_RetainsIdentityBlob, which asserts the retained bytes are the previous ones. func demoteCurrentEscrowTx(tx *sql.Tx, hostID string) (int64, error) { res, err := tx.Exec(` INSERT INTO host_escrow_superseded (host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, superseded_at, identity_blob) SELECT host_id, blob, key_fingerprint, posture, created_at, COALESCE(restic_pw_sha256, ''), datetime('now'), identity_blob FROM host_escrow WHERE host_id = ?`, hostID) if err != nil { return 0, err } return res.RowsAffected() } func (s *Store) SaveHostEscrow(hostID string, blob []byte, keyFingerprint, posture, createdAt, resticPwSHA256 string) (superseded bool, prevResticPwSHA256 string, err error) { tx, err := s.db.Begin() if err != nil { return false, "", err } defer func() { if err != nil { tx.Rollback() } }() // Retain the current row iff it exists AND seals a DIFFERENT restic password (the incident: a // recreated volume mints a new passphrase; the old must stay recoverable with its recovery code). var curSHA string var exists bool switch scanErr := tx.QueryRow(`SELECT COALESCE(restic_pw_sha256,'') FROM host_escrow WHERE host_id = ?`, hostID).Scan(&curSHA); scanErr { case nil: exists = true case sql.ErrNoRows: exists = false default: err = scanErr return false, "", err } if exists { prevResticPwSHA256 = curSHA // R-197: the caller compares; "" = no row or a legacy hash-less blob } if exists && curSHA != resticPwSHA256 { if _, err = demoteCurrentEscrowTx(tx, hostID); err != nil { return false, prevResticPwSHA256, err } superseded = true } if _, err = tx.Exec(` INSERT INTO host_escrow (host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, updated_at) VALUES (?, ?, ?, ?, ?, ?, datetime('now')) ON CONFLICT(host_id) DO UPDATE SET blob = excluded.blob, key_fingerprint = excluded.key_fingerprint, posture = excluded.posture, created_at = excluded.created_at, restic_pw_sha256 = excluded.restic_pw_sha256, stale_at = NULL, updated_at = datetime('now')`, hostID, blob, keyFingerprint, posture, createdAt, resticPwSHA256); err != nil { return false, prevResticPwSHA256, err } err = tx.Commit() return superseded, prevResticPwSHA256, err } // CountSupersededEscrow returns how many retained (superseded) escrow blobs the hub holds for a host // (the operator "N superseded escrow blobs retained" surface). 0 on no rows. func (s *Store) CountSupersededEscrow(hostID string) (int, error) { var n int err := s.db.QueryRow(`SELECT COUNT(*) FROM host_escrow_superseded WHERE host_id = ?`, hostID).Scan(&n) return n, err } // ListSupersededEscrow returns the retained (superseded) escrow blobs for a host, newest-superseded // first. Opaque bytes — the hub never decrypts. Seeds the future guided-recovery flow (R-26). func (s *Store) ListSupersededEscrow(hostID string) ([]HostEscrow, error) { rows, err := s.db.Query(` SELECT host_id, blob, key_fingerprint, posture, created_at, restic_pw_sha256, superseded_at, identity_blob FROM host_escrow_superseded WHERE host_id = ? ORDER BY id DESC`, hostID) if err != nil { return nil, err } defer rows.Close() var out []HostEscrow for rows.Next() { var e HostEscrow if err := rows.Scan(&e.HostID, &e.Blob, &e.KeyFingerprint, &e.Posture, &e.CreatedAt, &e.ResticPwSHA256, &e.UpdatedAt, &e.IdentityBlob); err != nil { return nil, err } out = append(out, e) } return out, rows.Err() } // CountCurrentEscrowWithIdentity returns how many hosts hold a CURRENT escrow row, and how many of // those carry an identity blob — i.e. the population whose offsite repository password R-198's fix // now protects from the next ceremony, and the remainder for whom there is nothing to protect // because no identity blob was ever uploaded. Read-only; no blob or hash leaves this call. func (s *Store) CountCurrentEscrowWithIdentity() (total, withIdentity int, err error) { err = s.db.QueryRow(` SELECT COUNT(*), COALESCE(SUM(identity_blob IS NOT NULL), 0) FROM host_escrow`).Scan(&total, &withIdentity) return total, withIdentity, err } // MarkEscrowStale flags a host's escrow blob as stale (v0.57.0, 2.3). // // ⚠ IT HAS NO CALLER as of hub v0.95.0 (R-196 / R-204 item 2), and that is deliberate, not an // oversight. Its ONE caller was `offsite.ReissueCredentials`, which called it on every re-issue on // the PRECAUTIONARY grounds that the box's re-apply might mint a fresh repository password. It // usually does not, so the call marked healthy escrows stale — and because a stale flag WITHHOLDS // restic_pw_sha256 from the ACK (GetEscrowStatusForCustomer, below), it blocked every off-site backup // on those boxes and asked the customer for a ceremony that would supersede a perfectly good key. The // full reasoning, and the two measured signals that cover the real case, are at // offsite.ReissueCredentials. // // KEPT, not deleted, because the FLAG is still live and correct — `stale_at` is read by the ACK, the // operator card and the PBS-DR view, and a future EVIDENTIAL caller (one that has measured a key // change rather than guessed at one) is the right way to set it. Pinned by // TestReissue_DoesNotMarkAHealthyEscrowStale: if a caller reappears without that evidence, it fails. // // No-op when no escrow row exists; idempotent (only stamps the first mark since the last ceremony; a // fresh ceremony clears stale_at via SaveHostEscrow's ON CONFLICT). func (s *Store) MarkEscrowStale(hostID string) error { _, err := s.db.Exec(`UPDATE host_escrow SET stale_at = datetime('now') WHERE host_id = ? AND stale_at IS NULL`, hostID) return err } // GetHostEscrow returns the stored opaque escrow for a host (nil if none). Used by tests and // (future, slice 10) restore-mode serving. The hub returns bytes verbatim; it never decrypts. func (s *Store) GetHostEscrow(hostID string) (*HostEscrow, error) { var e HostEscrow err := s.db.QueryRow(` SELECT host_id, blob, key_fingerprint, posture, created_at, updated_at, COALESCE(restic_pw_sha256, ''), COALESCE(stale_at, '') FROM host_escrow WHERE host_id = ?`, hostID). Scan(&e.HostID, &e.Blob, &e.KeyFingerprint, &e.Posture, &e.CreatedAt, &e.UpdatedAt, &e.ResticPwSHA256, &e.StaleAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } return &e, nil } // EscrowStatus (SLICE 3) is the non-secret escrow summary served in the report ACK so the controller can // VERIFY-and-auto-confirm: not "a blob exists" but "the blob covers the CURRENT repo password" (hash match). type EscrowStatus struct { IdentityBlobPresent bool `json:"identity_blob_present"` ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"` CreatedAt string `json:"created_at,omitempty"` // Stale (v0.57.0, 2.3) — true when `stale_at` is stamped on the host's escrow row. ⚠ SINCE hub // v0.95.0 (R-196 / R-204 item 2) NOTHING STAMPS IT: the one caller was the precautionary // re-issue mark, and it was removed for marking healthy escrows stale. See MarkEscrowStale. // When stale the ResticPwSHA256 is WITHHELD (emptied) so the controller cannot auto-confirm against // a hash that may no longer match the live repo password — the ceremony must run again. That // withholding is exactly why the precautionary caller had to go: it BLINDED the controller's own // hash comparison, which is the measurement that actually detects a changed repository password. Stale bool `json:"escrow_stale,omitempty"` // SupersededPresent (hub v0.97.0, R-222) — this host has at least one RETAINED superseded escrow // row carrying an identity blob. It exists so the recovery screen can distinguish two situations // that are otherwise identical from the box's side: a genuinely wrong recovery code, and a code // that is right about an EARLIER sealed package the hub is deliberately keeping. // // Measured on 2026-08-05 (CAMPAIGN-11 Phase 3, step 7): the customer entered the correct code for // the orphaned history, the unseal failed closed against the CURRENT package — correctly — and the // screen told them to check their typing. Nothing on the box could have known better. // // It is a BOOLEAN, deliberately. It says "an earlier package is kept"; it does not serve one, does // not say whose code opens it, and grants no read path — that link is still unbuilt (R-199's // inventory) and this field must not be mistaken for it. SupersededPresent bool `json:"superseded_present,omitempty"` // SupersededAt (hub v0.97.0, R-222) — when the most recent supersession happened (RFC3339-ish, as // stored). Empty when none. Non-secret: a timestamp. SupersededAt string `json:"superseded_at,omitempty"` } // GetEscrowStatusForCustomer returns the escrow status of the customer's host (nil if the customer has no // escrow row). With multiple hosts (not the current model), the most recently updated escrow wins. func (s *Store) GetEscrowStatusForCustomer(customerID string) (*EscrowStatus, error) { var st EscrowStatus var identityPresent int var staleAt string err := s.db.QueryRow(` SELECT (e.identity_blob IS NOT NULL), COALESCE(e.restic_pw_sha256, ''), e.created_at, COALESCE(e.stale_at, '') FROM host_escrow e JOIN hosts h ON h.host_id = e.host_id WHERE h.customer_id = ? ORDER BY e.updated_at DESC LIMIT 1`, customerID). Scan(&identityPresent, &st.ResticPwSHA256, &st.CreatedAt, &staleAt) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } st.IdentityBlobPresent = identityPresent == 1 // v0.57.0 (2.3): a stale blob must NOT auto-confirm — withhold the hash and flag it so the // controller stays pending and the escrow wizard is offered again. if staleAt != "" { st.Stale = true st.ResticPwSHA256 = "" } // R-222 (v0.97.0): does a RETAINED earlier package exist for this host? Only rows that actually // carry an identity blob count — the pre-v0.93.0 rows have `identity_blob` NULL and retain nothing // the recovery screen could ever be talking about, so counting them would make the screen offer an // explanation that is false for exactly the boxes hurt by the original defect. var supersededAt sql.NullString if err := s.db.QueryRow(` SELECT MAX(sup.superseded_at) FROM host_escrow_superseded sup JOIN hosts h ON h.host_id = sup.host_id WHERE h.customer_id = ? AND sup.identity_blob IS NOT NULL`, customerID).Scan(&supersededAt); err == nil && supersededAt.Valid { st.SupersededPresent = true st.SupersededAt = supersededAt.String } return &st, nil } // SetHostDesired sets a host's desired-state JSON and ATOMICALLY bumps its desired_generation // (slice 10A — the operator "admin-set" write). Returns the NEW generation. The generation is // the cheap change-signal carried on every heartbeat envelope; the agent re-fetches the full // desired-state only when it advances. Errors with sql.ErrNoRows if the host does not exist. func (s *Store) SetHostDesired(hostID string, desiredJSON []byte) (int64, error) { res, err := s.db.Exec(` UPDATE hosts SET desired_json = ?, desired_generation = desired_generation + 1, updated_at = datetime('now') WHERE host_id = ?`, string(desiredJSON), hostID) if err != nil { return 0, err } if n, _ := res.RowsAffected(); n == 0 { return 0, sql.ErrNoRows // unknown host } var gen int64 if err := s.db.QueryRow(`SELECT desired_generation FROM hosts WHERE host_id = ?`, hostID).Scan(&gen); err != nil { return 0, err } return gen, nil } // BumpHostDesired advances a host's desired_generation WITHOUT touching desired_json (S2). Used // when hub-OWNED served state changes (the merge-at-read wireguard block) — the stored operator // blob is not the thing that moved, so SetHostDesired (which replaces it) must not be used. // Returns the new generation; sql.ErrNoRows for an unknown host. func (s *Store) BumpHostDesired(hostID string) (int64, error) { res, err := s.db.Exec(` UPDATE hosts SET desired_generation = desired_generation + 1, updated_at = datetime('now') WHERE host_id = ?`, hostID) if err != nil { return 0, err } if n, _ := res.RowsAffected(); n == 0 { return 0, sql.ErrNoRows // unknown host } var gen int64 if err := s.db.QueryRow(`SELECT desired_generation FROM hosts WHERE host_id = ?`, hostID).Scan(&gen); err != nil { return 0, err } return gen, nil } // BumpAllHostGenerations advances EVERY host's desired_generation (TASK H1). Used when a FLEET-WIDE // hub-owned served-state value changes — the operator OOB peer's /32 flows into every host's // merge-at-read wireguard block (oob_peer_ip), so every agent must re-fetch + re-render. Returns the // number of hosts bumped. func (s *Store) BumpAllHostGenerations() (int64, error) { res, err := s.db.Exec(`UPDATE hosts SET desired_generation = desired_generation + 1, updated_at = datetime('now')`) if err != nil { return 0, err } n, _ := res.RowsAffected() return n, nil } // SignedJob is one OPAQUE operator-signed destructive-op blob queued for a host (slice 10A). The // hub stores + serves the bytes; it never forges, opens, or executes them (10B owns verify+run). type SignedJob struct { JobID string Blob []byte CreatedAt string } // EnqueueSignedJob appends an opaque signed-op blob to a host's queue (slice 10A). Operator-side; // the hub holds no signing key — the blob arrives pre-signed. func (s *Store) EnqueueSignedJob(hostID, jobID string, blob []byte) error { _, err := s.db.Exec(`INSERT INTO signed_jobs (host_id, job_id, blob) VALUES (?, ?, ?)`, hostID, jobID, blob) return err } // GetSignedJobs returns a host's pending signed-op blobs, oldest first (slice 10A serving). func (s *Store) GetSignedJobs(hostID string) ([]SignedJob, error) { rows, err := s.db.Query(`SELECT job_id, blob, created_at FROM signed_jobs WHERE host_id = ? ORDER BY id ASC`, hostID) if err != nil { return nil, err } defer rows.Close() var jobs []SignedJob for rows.Next() { var j SignedJob if err := rows.Scan(&j.JobID, &j.Blob, &j.CreatedAt); err != nil { return nil, err } jobs = append(jobs, j) } return jobs, rows.Err() } // CountSignedJobs returns the number of pending signed-op blobs for a host (drives the envelope's // has_signed_ops flag — the cheap "fetch your jobs" notification). func (s *Store) CountSignedJobs(hostID string) (int, error) { var n int err := s.db.QueryRow(`SELECT COUNT(*) FROM signed_jobs WHERE host_id = ?`, hostID).Scan(&n) return n, err } // DeleteSignedJob removes a processed job from a host's queue (slice 10B completion). The agent // calls it after executing OR terminally rejecting a job. Idempotent (deleting an absent job is a // no-op, returns nil) — a retried completion must not error. func (s *Store) DeleteSignedJob(hostID, jobID string) error { _, err := s.db.Exec(`DELETE FROM signed_jobs WHERE host_id = ? AND job_id = ?`, hostID, jobID) return err } // ---- slice 10D: DR capstone (recovery mode, DR bundle, re-enroll) ---------------------------- // SetRecoveryMode arms recovery mode for a host until `until` (the operator toggle; bounded // auto-expiry). While active, the hub serves the restore directive + allows re-enroll. Errors // ErrNoRows for an unknown host. func (s *Store) SetRecoveryMode(hostID string, until time.Time) error { res, err := s.db.Exec(`UPDATE hosts SET recovery_mode_until = ?, updated_at = datetime('now') WHERE host_id = ?`, until.UTC().Format("2006-01-02 15:04:05"), hostID) if err != nil { return err } if n, _ := res.RowsAffected(); n == 0 { return sql.ErrNoRows } return nil } // ClearRecoveryMode disables recovery mode (operator confirm, or after re-enroll completes). func (s *Store) ClearRecoveryMode(hostID string) error { _, err := s.db.Exec(`UPDATE hosts SET recovery_mode_until = NULL, updated_at = datetime('now') WHERE host_id = ?`, hostID) return err } // RotateHostAPIKey replaces a host's API key (the re-enroll credential rotation — the old box's hub // access is revoked the instant this commits; purely hub-internal, no Cloudflare/PBS write needed). func (s *Store) RotateHostAPIKey(hostID, newAPIKey string) error { res, err := s.db.Exec(`UPDATE hosts SET api_key = ?, updated_at = datetime('now') WHERE host_id = ?`, newAPIKey, hostID) if err != nil { return err } if n, _ := res.RowsAffected(); n == 0 { return sql.ErrNoRows } return nil } // SaveHostDRBundle stores the IDENTITY escrow blob + the NON-secret DR directive alongside the // existing K-escrow blob (slice 10D.1). The K-escrow row must already exist (slice-7 escrow upload); // this updates the additive 10D columns. The hub holds only ciphertext + non-secret directive. func (s *Store) SaveHostDRBundle(hostID string, identityBlob []byte, directiveJSON string) error { if directiveJSON == "" { directiveJSON = "{}" } res, err := s.db.Exec(`UPDATE host_escrow SET identity_blob = ?, directive_json = ?, updated_at = datetime('now') WHERE host_id = ?`, identityBlob, directiveJSON, hostID) if err != nil { return err } if n, _ := res.RowsAffected(); n == 0 { return sql.ErrNoRows // no K-escrow row yet — upload the escrow first } return nil } // HostDRBundle is the full DR directive served to a re-enrolling box (slice 10D): the two OPAQUE // escrow blobs (K + identity — useless without R) + the non-secret directive fields. type HostDRBundle struct { KEscrowBlob []byte IdentityBlob []byte DirectiveJSON string } // GetHostDRBundle returns a host's DR bundle (nil if no escrow row). The blobs are opaque — the hub // cannot open them (it has no R). func (s *Store) GetHostDRBundle(hostID string) (*HostDRBundle, error) { var b HostDRBundle var directive sql.NullString err := s.db.QueryRow(`SELECT blob, identity_blob, directive_json FROM host_escrow WHERE host_id = ?`, hostID). Scan(&b.KEscrowBlob, &b.IdentityBlob, &directive) if err == sql.ErrNoRows { return nil, nil } if err != nil { return nil, err } if directive.Valid { b.DirectiveJSON = directive.String } return &b, nil } // SaveHostReport inserts a host_reports row and bumps the host's reality columns // (agent_version/last_report_at/updated_at) — never the inert intent columns. func (s *Store) SaveHostReport(hostID, customerID string, reportJSON []byte, d HostReportDenorm) error { _, err := s.db.Exec(` INSERT INTO host_reports (host_id, customer_id, report_json, agent_version, cpu_percent, memory_percent, disk_percent, guest_total, guest_running, cloudflared_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, hostID, customerID, string(reportJSON), d.AgentVersion, d.CPUPercent, d.MemoryPercent, d.DiskPercent, d.GuestTotal, d.GuestRunning, d.CloudflaredStatus, ) if err != nil { return err } _, err = s.db.Exec(` UPDATE hosts SET agent_version = ?, last_report_at = datetime('now'), updated_at = datetime('now') WHERE host_id = ?`, d.AgentVersion, hostID) return err } // GetLatestHostReportJSON returns the most recent host-report payload (report_json) // for a customer, by received_at, or ("", nil) if the customer has no host-report. // The backup deadline check uses it to read the agent's own backup reality // (pbs_snapshots + vzdump backups) — the authoritative offsite-backup signal // post-slice-8C, replacing the no-longer-emitted backup_completed event. func (s *Store) GetLatestHostReportJSON(customerID string) (string, error) { var j string err := s.db.QueryRow( `SELECT report_json FROM host_reports WHERE customer_id = ? ORDER BY received_at DESC LIMIT 1`, customerID, ).Scan(&j) if err == sql.ErrNoRows { return "", nil } if err != nil { return "", err } return j, nil } // HostReportRow is one retained host-report: when the hub received it, and its payload. type HostReportRow struct { ReceivedAt time.Time ReportJSON string } // GetHostReportsSince returns the customer's retained host-reports received at or after // `since`, NEWEST FIRST. The hub keeps ~retention.max_days of history (90 by default), which // is what makes it possible to ask "when did I last SEE evidence of a backup?" rather than // only "what does the latest report say?". // // R-81: this is the anchor source for the backup-deadline check. The agent's backup record // store is IN-MEMORY (felhom-agent/internal/backup/store.go — "lost on restart; the cadence // re-populates"), so a restart empties `backups` in every report until the next backup runs. // The hub has memory the agent does not; reading across the window is what turns that blind // window from a false alarm into a correctly-silent verdict. Newest-first so the caller can // stop as soon as it has seen enough (see monitor.newestBackupEvidence). func (s *Store) GetHostReportsSince(customerID string, since time.Time) ([]HostReportRow, error) { rows, err := s.db.Query( `SELECT received_at, report_json FROM host_reports WHERE customer_id = ? AND received_at >= ? ORDER BY received_at DESC`, customerID, since.UTC().Format("2006-01-02 15:04:05"), ) if err != nil { return nil, err } defer rows.Close() var out []HostReportRow for rows.Next() { var at, j string if err := rows.Scan(&at, &j); err != nil { return nil, err } out = append(out, HostReportRow{ReceivedAt: parseSQLiteTime(at).UTC(), ReportJSON: j}) } return out, rows.Err() } // GetFirstHostReportAt returns when the hub received its FIRST retained host-report for the // customer, or (zero, nil) when it holds none. // // R-81: this is the observation anchor. "No backup evidence anywhere" is only meaningful // relative to how long the hub has been watching — a box registered an hour ago has no // evidence yet and is NOT failing. Same shape as the v0.73.0 offsite never-ran anchor: // absence becomes a fault only once it has outlived the existing threshold, measured from a // point where evidence first became POSSIBLE. // // Caveat, deliberately accepted: retention prunes at max_days, so for a host older than the // window this returns the prune horizon rather than true first-contact. That only makes the // anchor MORE conservative for long-lived hosts (the window has long since elapsed either // way), and it never shortens a newborn's grace. func (s *Store) GetFirstHostReportAt(customerID string) (time.Time, error) { var at string err := s.db.QueryRow( `SELECT received_at FROM host_reports WHERE customer_id = ? ORDER BY received_at ASC LIMIT 1`, customerID, ).Scan(&at) if err == sql.ErrNoRows { return time.Time{}, nil } if err != nil { return time.Time{}, err } return parseSQLiteTime(at).UTC(), nil } // SetHostReportsReceivedAtForTest back-dates ALL of a customer's existing host_reports rows // to the given SQLite datetime string, so anchor/window behaviour is testable without // sleeping. TEST-ONLY — mirrors SetOneTimeSecretTimesForTest. Rows saved AFTER the call keep // datetime('now'), which is how a test stages "old reports, then a fresh one". func (s *Store) SetHostReportsReceivedAtForTest(customerID, sqliteTime string) error { _, err := s.db.Exec(`UPDATE host_reports SET received_at = ? WHERE customer_id = ?`, sqliteTime, customerID) return err } // UpsertGuestFromReport upserts the REALITY columns of a guest. On conflict it // must NOT clobber the inert columns (api_key / desired_spec_json). func (s *Store) UpsertGuestFromReport(g *Guest) error { _, err := s.db.Exec(` INSERT INTO guests (guest_id, customer_id, host_id, vmid, display_name, status, controller_version, last_seen_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'), datetime('now')) ON CONFLICT(guest_id) DO UPDATE SET vmid = excluded.vmid, display_name = excluded.display_name, status = excluded.status, controller_version = excluded.controller_version, last_seen_at = datetime('now'), updated_at = datetime('now')`, g.GuestID, g.CustomerID, g.HostID, g.VMID, g.DisplayName, g.Status, g.ControllerVersion, ) return err } // guestRealitySelectCols are the report-driven reality columns (plus identity/timestamps) // of a guest. It deliberately OMITS the secret/inert columns (api_key, desired_spec_json): // the read-only Hosts view never renders them, so they are not selected. const guestRealitySelectCols = `guest_id, customer_id, host_id, vmid, display_name, status, controller_version, last_seen_at, created_at, updated_at` func scanGuest(scan func(dest ...any) error) (*Guest, error) { var g Guest var lastSeen sql.NullString var createdAt, updatedAt string err := scan(&g.GuestID, &g.CustomerID, &g.HostID, &g.VMID, &g.DisplayName, &g.Status, &g.ControllerVersion, &lastSeen, &createdAt, &updatedAt) if err != nil { return nil, err } if lastSeen.Valid && lastSeen.String != "" { t := parseSQLiteTime(lastSeen.String) g.LastSeenAt = &t } g.CreatedAt = parseSQLiteTime(createdAt) g.UpdatedAt = parseSQLiteTime(updatedAt) return &g, nil } // ListGuestsForHost returns the guests (controller LXCs) enrolled on a host, ordered by // vmid. Reads only reality columns (no api_key / desired_spec_json). Returns an empty // slice (never nil-error) when the host has no guests — the read-only Hosts detail view. func (s *Store) ListGuestsForHost(hostID string) ([]Guest, error) { rows, err := s.db.Query(`SELECT `+guestRealitySelectCols+` FROM guests WHERE host_id = ? ORDER BY vmid`, hostID) if err != nil { return nil, err } defer rows.Close() guests := []Guest{} for rows.Next() { g, err := scanGuest(rows.Scan) if err != nil { return nil, err } guests = append(guests, *g) } return guests, rows.Err() } // GetHostStaleness returns per-host recency for the dead-man's-switch. Hosts that // have never reported (NULL last_report_at) are skipped — a freshly-minted host is // not "down" until it has checked in at least once. func (s *Store) GetHostStaleness() ([]HostStaleRow, error) { rows, err := s.db.Query(`SELECT host_id, customer_id, last_report_at FROM hosts WHERE last_report_at IS NOT NULL`) if err != nil { return nil, err } defer rows.Close() var out []HostStaleRow for rows.Next() { var r HostStaleRow var last string if err := rows.Scan(&r.HostID, &r.CustomerID, &last); err != nil { return nil, err } r.LastReportAt = parseSQLiteTime(last) out = append(out, r) } return out, rows.Err() } // CapabilityStatus mirrors the agent's capability.Status wire shape (felhom-agent v0.44.0): one // privileged `sudo -n` grant the non-root agent depends on, and whether it is currently usable. type CapabilityStatus struct { Name string `json:"name"` Feature string `json:"feature"` Critical bool `json:"critical"` Status string `json:"status"` // "ok" | "degraded" Reason string `json:"reason,omitempty"` } // HostCapabilityRow is the per-host capability snapshot the HostCapabilityChecker reads — extracted // from the latest host-report's report_json (no dedicated column; the array rides the report body). type HostCapabilityRow struct { HostID string CustomerID string Capabilities []CapabilityStatus } // GetHostCapabilities returns the latest capability snapshot per host (from the most recent // host_reports row). Hosts whose latest report carries no capabilities array (a pre-v0.44.0 agent) // yield an empty slice — the checker treats that as "ok/unknown" and never alerts, so an old agent // can't trip a false degraded. func (s *Store) GetHostCapabilities() ([]HostCapabilityRow, error) { rows, err := s.db.Query(` SELECT hr.host_id, hr.customer_id, hr.report_json FROM host_reports hr JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest ON hr.id = latest.mx`) if err != nil { return nil, err } defer rows.Close() var out []HostCapabilityRow for rows.Next() { var r HostCapabilityRow var reportJSON string if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { return nil, err } var body struct { Capabilities []CapabilityStatus `json:"capabilities"` } _ = json.Unmarshal([]byte(reportJSON), &body) // a malformed/old body → nil caps → no alert r.Capabilities = body.Capabilities out = append(out, r) } return out, rows.Err() } // HostLeafRow is the per-host served-leaf-fingerprint the HostLeafChecker reads — extracted from the // latest host-report's report_json (no dedicated column; the fp rides the report body, same as the // capabilities snapshot). LeafFP is "" for a pre-v0.48.0 agent / local-API-disabled host. type HostLeafRow struct { HostID string CustomerID string LeafFP string } // HostDiskRow is the per-host root-filesystem usage the HostDiskChecker reads: the denormalized // disk_percent column (the threshold signal) plus total/used bytes parsed from report_json (event detail // only). DiskPercent is the HOST root fs (agent HostMetrics) — distinct from the controller's GUEST cgroup // disk. A NULL/absent disk_percent (a pre-disk-reporting agent) yields 0 → the checker treats it as ok. type HostDiskRow struct { HostID string CustomerID string DiskPercent float64 DiskTotalBytes int64 DiskUsedBytes int64 } // GetHostDiskUsage returns the latest root-fs usage per host (MAX(id) per host, mirroring // GetHostCapabilities / GetHostLeafFingerprints). disk_percent comes from the denorm column; total/used // bytes are parsed from the report body's host block for the event detail (no schema migration needed). func (s *Store) GetHostDiskUsage() ([]HostDiskRow, error) { rows, err := s.db.Query(` SELECT hr.host_id, hr.customer_id, hr.disk_percent, hr.report_json FROM host_reports hr JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest ON hr.id = latest.mx`) if err != nil { return nil, err } defer rows.Close() var out []HostDiskRow for rows.Next() { var r HostDiskRow var dp sql.NullFloat64 var reportJSON string if err := rows.Scan(&r.HostID, &r.CustomerID, &dp, &reportJSON); err != nil { return nil, err } r.DiskPercent = dp.Float64 var body struct { Host struct { DiskTotalBytes int64 `json:"disk_total_bytes"` DiskUsedBytes int64 `json:"disk_used_bytes"` } `json:"host"` } _ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old body → zero bytes (detail only) r.DiskTotalBytes = body.Host.DiskTotalBytes r.DiskUsedBytes = body.Host.DiskUsedBytes out = append(out, r) } return out, rows.Err() } // HostStorageTargetRow is one (host, storage target) fill observation, parsed from the latest report's // storage_targets[] (no denorm column — modeled on GetHostDiskUsage's report_json parse). Percent is // used_fraction×100 (the agent reports a 0..1 fraction). A target with no usable fraction yields 0 (ok). type HostStorageTargetRow struct { HostID string CustomerID string Name string // Proxmox storage id Type string // local | local-dir | lvmthin | usb | nfs | cifs | pbs MountPath string // host mountpoint ("" for network/lvm/root-backed dir) Percent float64 // 0..100 (used_fraction × 100) TotalBytes int64 UsedBytes int64 } // GetHostStorageTargets returns the per-storage fill of every host's LATEST report (MAX(id) per host), // parsed from report_json.storage_targets[] (mirrors GetHostDiskUsage). The StorageFillChecker keys on // (host, name) and excludes the root-backed builtin — see monitor/storage_fill.go. func (s *Store) GetHostStorageTargets() ([]HostStorageTargetRow, error) { rows, err := s.db.Query(` SELECT hr.host_id, hr.customer_id, hr.report_json FROM host_reports hr JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest ON hr.id = latest.mx`) if err != nil { return nil, err } defer rows.Close() var out []HostStorageTargetRow for rows.Next() { var hostID, customerID, reportJSON string if err := rows.Scan(&hostID, &customerID, &reportJSON); err != nil { return nil, err } var body struct { StorageTargets []struct { Name string `json:"name"` Type string `json:"type"` MountPath string `json:"mount_path"` UsedFraction float64 `json:"used_fraction"` TotalBytes int64 `json:"total_bytes"` UsedBytes int64 `json:"used_bytes"` } `json:"storage_targets"` } _ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old body → no targets (never a false alert) for _, t := range body.StorageTargets { out = append(out, HostStorageTargetRow{ HostID: hostID, CustomerID: customerID, Name: t.Name, Type: t.Type, MountPath: t.MountPath, Percent: t.UsedFraction * 100, TotalBytes: t.TotalBytes, UsedBytes: t.UsedBytes, }) } } return out, rows.Err() } // GetHostLeafFingerprints returns the latest reported local-API leaf fp per host (mirrors // GetHostCapabilities — MAX(id) per host, parsed from report_json so there is no schema migration). func (s *Store) GetHostLeafFingerprints() ([]HostLeafRow, error) { rows, err := s.db.Query(` SELECT hr.host_id, hr.customer_id, hr.report_json FROM host_reports hr JOIN (SELECT host_id, MAX(id) AS mx FROM host_reports GROUP BY host_id) latest ON hr.id = latest.mx`) if err != nil { return nil, err } defer rows.Close() var out []HostLeafRow for rows.Next() { var r HostLeafRow var reportJSON string if err := rows.Scan(&r.HostID, &r.CustomerID, &reportJSON); err != nil { return nil, err } var body struct { LeafFingerprint string `json:"leaf_fingerprint"` } _ = json.Unmarshal([]byte(reportJSON), &body) // malformed/old body → "" → no alert r.LeafFP = body.LeafFingerprint out = append(out, r) } return out, rows.Err() }