The hub's 30-day check rests on the box's own reports: a box could name an app that never existed, or stop listing a living one, wait 30 days, and get every snapshot agreed beside the cap, with no operator signal. - One window agrees to at most the size of its own cap, from the hub's OWN count of the store; the oldest files first; the rest waits for the next window. A narrowing of D84, decided by CC unattended — the operator may reverse it (07 §8.2f holds the question, both options and their cost). - offsite_end_claim_unseen (warning, mailed, operator-only): a box first names an app the hub never saw installed — once per household and app, at most 8 per household, 30 days before anything can be agreed. - offsite_ended_large_agreement (warning, mailed, operator-only): a request for more than half of the store waits one week; that window agrees to none. - 07 §8.2f: the limit rewritten (no honest ended app is needed, only 30 days), what bounds it, what is still open. 08 §6.5: the two events. Also carries the red-proof outputs of the controller's review fixes 2 and 3 (felhom-controller 3928c3d and the next commit). Tests seen red on bb226f23; 15 mutations seen red (redproof-review-1.txt). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
3729 lines
181 KiB
Go
3729 lines
181 KiB
Go
package api
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/subtle"
|
||
"database/sql"
|
||
"encoding/base64"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/accesslog"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/assets"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/claim"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/configgen"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/intent"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/mailhold"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/mailrelay"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/notify"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys"
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
)
|
||
|
||
// ConfigTemplateProvider returns the controller.yaml template for config generation.
|
||
type ConfigTemplateProvider interface {
|
||
Template() string
|
||
}
|
||
|
||
// LatestVersionProvider returns the latest controller image version known to the hub's registry
|
||
// checker (e.g. "0.86.0"), or "" if unknown. Satisfied by *web.VersionChecker; nil when the
|
||
// registry checker is disabled. Used to advertise latest on the controller report ACK.
|
||
type LatestVersionProvider interface {
|
||
LatestVersion() string
|
||
}
|
||
|
||
// Poker is the agent-plane immediate-sync seam (v0.63.0): satisfied by *poke.Notifier. A system-
|
||
// initiated desired-state write here (admin-set, operator-peer bump) fires a contentless, fire-and-
|
||
// forget nudge so the box ticks in seconds. nil = poke disabled — mutations still persist; the box
|
||
// picks them up on its next ≤15-min cycle. Both methods are safe to call on a nil *poke.Notifier,
|
||
// but every call site still guards with `if h.poker != nil` (the field itself may be nil).
|
||
type Poker interface {
|
||
PokeHost(hostID string)
|
||
PokeAllHosts()
|
||
}
|
||
|
||
// Handler handles API endpoints for report ingest and customer queries.
|
||
type Handler struct {
|
||
store *store.Store
|
||
offsiteKeys OffsiteKeyService // decision 69 key registrar (nil → 503)
|
||
osUpdates OSUpdateService // `11` §8 step 2, the guest fast lane (nil → 503, no block merged)
|
||
apiKey string
|
||
resendAPIKey string
|
||
fromEmail string
|
||
logger *log.Logger
|
||
httpClient *http.Client
|
||
templateProvider ConfigTemplateProvider
|
||
dispatcher *notify.Dispatcher
|
||
assetsMgr *assets.Manager
|
||
latestVersion LatestVersionProvider
|
||
|
||
// floorNotes (R-472): customer → the last served-floor decision logged, so the SERVED line is written
|
||
// once per change. Zero value is ready to use.
|
||
floorNotes sync.Map
|
||
|
||
// App-email passthrough (POST /api/v1/mail). nil sender = endpoint returns 503.
|
||
mailSender mailrelay.Sender
|
||
// mailHold is the hub-wide mail gate (MAIL-HOLD marker; internal/mailhold). nil never holds.
|
||
mailHold *mailhold.Hold
|
||
mailLimiter *mailRateLimiter
|
||
mailFromAllow map[string]bool
|
||
|
||
// applianceLimiter (v0.62.0, R-21 slice C) throttles the ONE unauthenticated endpoint,
|
||
// POST /api/v1/appliance/register, per client IP — the unclaimed population is tiny and the
|
||
// ingress already geo-restricts to HU, so this is a cheap anti-abuse bound, not a fleet lever.
|
||
applianceLimiter *ipRateLimiter
|
||
|
||
// S1 offsite connectivity: the wgsync reconciler seam (internal/api/wg.go). nil = peer-sync
|
||
// disabled — mutations still persist, responses carry sync:"disabled".
|
||
wgSyncer WGSyncer
|
||
|
||
// claimEngine is the customer-claim code engine (v0.50.0). nil = claim arc disabled: no codes
|
||
// issued, no claim field in ACKs/configs — pre-arc behavior exactly.
|
||
claimEngine *claim.Engine
|
||
|
||
// wgRegisteredHook (v0.51.0, DR-tier-by-default scenario A) fires after a host's FIRST WG
|
||
// peer registration — main.go wires it to the web server's PBSDRAutoProvision so a DR-ON
|
||
// customer's pbs_dr descriptor lands hands-free (WG registers → provision → the agent's next
|
||
// desired-state tick). nil = no cascade hook (pre-v0.51.0 behavior). Runs in a detached
|
||
// goroutine; must never delay or fail the registration response.
|
||
wgRegisteredHook func(ctx context.Context, customerID string)
|
||
|
||
// offsiteReissuer (F3, v0.57.0) re-issues the customer's offsite credentials on clean-slate
|
||
// re-enrollment — main.go wires it to the web server's ReissueOffsiteForCustomer (same machinery
|
||
// as the manual "Re-issue offsite credentials" button). nil = no auto re-issue; a no-op when
|
||
// offsite isn't provisioned/enabled for the customer. WHAT IT STILL DOES (R-1158, 2026-10-19): it
|
||
// resets the Storage Box sub-account password that only the HUB uses (offsitekeys.TargetFor — the
|
||
// registrar's login); no box reads it since hub v0.127.0 (consume-password answers 410, every box
|
||
// reaches the store with its own registered key). Kept because it hands the hub a fresh, working
|
||
// login at every clean reinstall — which repairs a broken stored one — not because the box needs it.
|
||
offsiteReissuer func(ctx context.Context, customerID string) error
|
||
|
||
// intentHub (v0.58.0, Direction-2 immediate-sync) is the in-memory per-customer generation
|
||
// notifier that GET /api/v1/wait long-polls against. nil = wait endpoint returns 503 (the box
|
||
// falls back to the 15-min cycle). Shared with the web server, whose intent handlers Bump it.
|
||
intentHub *intent.Hub
|
||
|
||
// poker (v0.63.0, Direction-2a agent-plane immediate-sync) fires a fire-and-forget nudge after a
|
||
// system-initiated HOST desired-state write (admin-set desired-state, operator-peer bump) so the
|
||
// box ticks in seconds instead of ≤15 min. Shared with the web server (same *poke.Notifier). nil
|
||
// = poke disabled (a no-op; the report cycle still reconciles).
|
||
poker Poker
|
||
|
||
// staleThreshold (R-1154, v0.159.0) is the hub's report-liveness threshold (alerting.stale_threshold — the
|
||
// same value hostStatus and both staleness checkers read). The host-enroll REUSE path judges by it whether
|
||
// the old box is gone (a reinstall over the old record → the operator's warning mail) or live (reenroll.go).
|
||
// 0 = not wired: no verdict, a log line only. main wires it.
|
||
staleThreshold time.Duration
|
||
}
|
||
|
||
// SetStaleThreshold wires the report-liveness threshold the host-enroll reuse path judges by (R-1154).
|
||
func (h *Handler) SetStaleThreshold(d time.Duration) {
|
||
h.staleThreshold = d
|
||
}
|
||
|
||
// SetClaimEngine wires the customer-claim code engine (nil-safe everywhere it is used).
|
||
func (h *Handler) SetClaimEngine(e *claim.Engine) {
|
||
h.claimEngine = e
|
||
}
|
||
|
||
// SetOffsiteReissuer wires the clean-slate re-enroll offsite re-issue seam (nil-safe).
|
||
func (h *Handler) SetOffsiteReissuer(f func(ctx context.Context, customerID string) error) {
|
||
h.offsiteReissuer = f
|
||
}
|
||
|
||
// SetWGRegisteredHook wires the post-WG-registration cascade hook (v0.51.0; nil-safe).
|
||
func (h *Handler) SetWGRegisteredHook(f func(ctx context.Context, customerID string)) {
|
||
h.wgRegisteredHook = f
|
||
}
|
||
|
||
// SetLatestVersionProvider wires the registry version checker so the controller report ACK can
|
||
// advertise the latest available version (Phase 2). nil-safe (no latest_version field emitted).
|
||
func (h *Handler) SetLatestVersionProvider(p LatestVersionProvider) {
|
||
h.latestVersion = p
|
||
}
|
||
|
||
// SetIntentHub wires the operator-intent notifier for GET /api/v1/wait (v0.58.0; nil-safe — an
|
||
// unset hub makes the wait endpoint return 503).
|
||
func (h *Handler) SetIntentHub(hub *intent.Hub) {
|
||
h.intentHub = hub
|
||
}
|
||
|
||
// SetPoker wires the agent-plane immediate-sync notifier (v0.63.0; nil-safe — an unset poker makes
|
||
// the admin desired-state writes fire no nudge, and the box picks the change up on its next cycle).
|
||
func (h *Handler) SetPoker(p Poker) {
|
||
h.poker = p
|
||
}
|
||
|
||
// New creates a new API handler.
|
||
func New(store *store.Store, apiKey, resendAPIKey, fromEmail string, templateProvider ConfigTemplateProvider, logger *log.Logger) *Handler {
|
||
return &Handler{
|
||
store: store,
|
||
apiKey: apiKey,
|
||
resendAPIKey: resendAPIKey,
|
||
fromEmail: fromEmail,
|
||
logger: logger,
|
||
httpClient: &http.Client{Timeout: 10 * time.Second},
|
||
templateProvider: templateProvider,
|
||
applianceLimiter: newIPRateLimiter(20), // 20 registrations/min/IP burst — booting boxes retry ~30s
|
||
}
|
||
}
|
||
|
||
// SetMailHold wires the hub-wide mail gate: while the MAIL-HOLD marker exists, /notify and the app-mail relay send
|
||
// nothing (internal/mailhold).
|
||
func (h *Handler) SetMailHold(m *mailhold.Hold) {
|
||
h.mailHold = m
|
||
}
|
||
|
||
// SetDispatcher sets the notification dispatcher for event-triggered emails.
|
||
func (h *Handler) SetDispatcher(d *notify.Dispatcher) {
|
||
h.dispatcher = d
|
||
}
|
||
|
||
// SetAssetManager sets the asset manager for serving app assets to controllers.
|
||
func (h *Handler) SetAssetManager(am *assets.Manager) {
|
||
h.assetsMgr = am
|
||
}
|
||
|
||
// checkAuth verifies the Bearer token against the global API key or a per-customer API key.
|
||
// Returns true if authorized.
|
||
func (h *Handler) checkAuth(r *http.Request) bool {
|
||
_, _, ok := h.checkAuthCustomer(r)
|
||
return ok
|
||
}
|
||
|
||
// checkAuthCustomer verifies the Bearer token and returns the authenticated customer identity.
|
||
// For per-customer keys: returns (customerID, false, true).
|
||
// For global key: returns ("", true, true) — caller must allow any customer_id.
|
||
// On failure: returns ("", false, false).
|
||
func (h *Handler) checkAuthCustomer(r *http.Request) (customerID string, isGlobal bool, ok bool) {
|
||
auth := r.Header.Get("Authorization")
|
||
if !strings.HasPrefix(auth, "Bearer ") {
|
||
return "", false, false
|
||
}
|
||
token := strings.TrimPrefix(auth, "Bearer ")
|
||
|
||
// Check global key first
|
||
if h.apiKey != "" && subtle.ConstantTimeCompare([]byte(token), []byte(h.apiKey)) == 1 {
|
||
accesslog.Mark(r.Context(), "global") // R-1140: the KIND only, for the hub's access log
|
||
return "", true, true
|
||
}
|
||
|
||
// Check per-customer key
|
||
cfg, err := h.store.GetCustomerConfigByAPIKey(token)
|
||
if err != nil || cfg == nil {
|
||
return "", false, false
|
||
}
|
||
accesslog.Mark(r.Context(), "customer")
|
||
return cfg.CustomerID, false, true
|
||
}
|
||
|
||
// checkAuthHost resolves a Bearer token to a HOST identity (the agent's auth
|
||
// path). It is a sibling of checkAuthCustomer — the controller path is unchanged.
|
||
// - global key -> ("", "", true, true) caller trusts body.host_id
|
||
// - per-host key -> (hostID, customerID, false, true)
|
||
// - failure -> ("", "", false, false)
|
||
func (h *Handler) checkAuthHost(r *http.Request) (hostID, customerID string, isGlobal, ok bool) {
|
||
auth := r.Header.Get("Authorization")
|
||
if !strings.HasPrefix(auth, "Bearer ") {
|
||
return "", "", false, false
|
||
}
|
||
token := strings.TrimPrefix(auth, "Bearer ")
|
||
|
||
// Global key first (same constant-time compare as checkAuthCustomer).
|
||
if h.apiKey != "" && subtle.ConstantTimeCompare([]byte(token), []byte(h.apiKey)) == 1 {
|
||
accesslog.Mark(r.Context(), "global")
|
||
return "", "", true, true
|
||
}
|
||
|
||
host, err := h.store.GetHostByAPIKey(token)
|
||
if err != nil || host == nil {
|
||
return "", "", false, false
|
||
}
|
||
accesslog.Mark(r.Context(), "host")
|
||
return host.HostID, host.CustomerID, false, true
|
||
}
|
||
|
||
// requireGlobalKey answers 403 and returns false unless the request carries the operator/global key.
|
||
// The same check as the /admin/wg/* handlers (wg.go), in one place for routes that read every customer.
|
||
func (h *Handler) requireGlobalKey(w http.ResponseWriter, r *http.Request) bool {
|
||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok || !isGlobal {
|
||
http.Error(w, "Forbidden: global key required", http.StatusForbidden)
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
// ServeHTTP routes API requests.
|
||
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||
path := strings.TrimPrefix(r.URL.Path, "/api/v1")
|
||
|
||
// An operator access token (`05` §17, prefix fhp_) never reaches the box API: not a box route, not a global-key
|
||
// route. Refused here, before any handler, so no route can accept one by accident. Pinned by
|
||
// TestAPI_AccessTokenIsRefusedEverywhere.
|
||
if auth := r.Header.Get("Authorization"); strings.HasPrefix(strings.TrimSpace(strings.TrimPrefix(auth, "Bearer ")), store.APITokenPrefix) {
|
||
h.logger.Printf("[WARN] access token refused on the box API: %s %s", r.Method, r.URL.Path)
|
||
http.Error(w, "Forbidden: access tokens are not accepted on /api/v1", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
switch {
|
||
case r.Method == http.MethodPost && path == "/report":
|
||
h.handleReport(w, r)
|
||
// Direction-2 immediate-sync (v0.58.0): the box long-polls here; the hub completes it on any
|
||
// operator-intent bump for the box's customer, then the box fires its ordinary report.
|
||
case r.Method == http.MethodGet && path == "/wait":
|
||
h.handleWait(w, r)
|
||
// R-21 slice C — the universal ISO. register is the ONE unauthenticated endpoint (per-IP
|
||
// rate-limited); poll is Bearer appliance-token. Both minimal, no enumeration oracle.
|
||
case r.Method == http.MethodPost && path == "/appliance/register":
|
||
h.handleApplianceRegister(w, r)
|
||
case r.Method == http.MethodGet && path == "/appliance/poll":
|
||
h.handleAppliancePoll(w, r)
|
||
case r.Method == http.MethodPost && path == "/host-report":
|
||
h.handleHostReport(w, r)
|
||
case r.Method == http.MethodPost && path == "/host-enroll":
|
||
h.handleHostEnroll(w, r)
|
||
case r.Method == http.MethodPost && path == "/admin/hosts":
|
||
h.handleAdminCreateHost(w, r)
|
||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
|
||
h.handleHostEscrowPut(w, r, hostID)
|
||
// R-199 (v0.94.0): the box-authenticated MIRROR of the PUT above — a host reads back its own
|
||
// opaque identity blob so it can be unsealed with the customer's recovery code. Distinct from the
|
||
// operator-driven DR path in dr.go, which stays exactly as it is (see handleHostEscrowGet).
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow")
|
||
h.handleHostEscrowGet(w, r, hostID)
|
||
// R-311 (v0.103.0): the RETAINED siblings of the row above. The two cases cannot collide — one
|
||
// ends `/escrow`, the other `/escrow/retained` — but do NOT "tidy" them into a single prefix
|
||
// match: a prefix match would route retained reads to the CURRENT row, which is silently the
|
||
// wrong package and is exactly the confusion this endpoint exists to end.
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/escrow/retained"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/escrow/retained")
|
||
h.handleHostEscrowRetainedGet(w, r, hostID)
|
||
// G1 break-glass: day-0 vaults the root@pam console credential (self-scoped host key); the
|
||
// operator retrieves it via the /admin/ path (global key only).
|
||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/recovery-credential")
|
||
h.handleHostRecoveryCredentialPut(w, r, hostID)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-credential"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-credential")
|
||
h.handleAdminGetRecoveryCredential(w, r, hostID)
|
||
// DR capstone (slice 10D). Recovery-mode toggle (global key); re-enroll + restore-directive
|
||
// (gated on recovery mode — no old key needed, the box is lost).
|
||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-mode")
|
||
h.handleSetRecoveryMode(w, r, hostID)
|
||
case r.Method == http.MethodDelete && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/recovery-mode"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/recovery-mode")
|
||
h.handleClearRecoveryMode(w, r, hostID)
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/re-enroll"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/re-enroll")
|
||
h.handleReEnroll(w, r, hostID)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/restore-directive"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/restore-directive")
|
||
h.handleGetRestoreDirective(w, r, hostID)
|
||
// S2 offsite connectivity: box-facing WG pubkey registration (per-host key, self-scoped).
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/wg"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/wg")
|
||
h.handleRegisterHostWG(w, r, hostID)
|
||
// PBS DR tier (SLICE 1): the agent's consume-once fetch of its PBS token secret (api/pbsdr.go).
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/pbs/consume-token"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/pbs/consume-token")
|
||
h.handleConsumePBSToken(w, r, hostID)
|
||
// OS updates (hub v0.130.0): the agent's report after every OS-leg run — per-host key, self-scoped.
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/os-report"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/os-report")
|
||
h.handleOSReport(w, r, hostID)
|
||
// Desired-state serving (slice 10A) — per-host-key, self-scoped (a host reads only its own).
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/desired-state"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/desired-state")
|
||
h.handleGetDesiredState(w, r, hostID)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/jobs"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/jobs")
|
||
h.handleGetJobs(w, r, hostID)
|
||
// Job completion (slice 10B) — per-host-key, self-scoped: DELETE /hosts/{id}/jobs/{job_id}.
|
||
case r.Method == http.MethodDelete && strings.HasPrefix(path, "/hosts/") && strings.Contains(path, "/jobs/"):
|
||
rest := strings.TrimPrefix(path, "/hosts/")
|
||
if i := strings.Index(rest, "/jobs/"); i > 0 {
|
||
h.handleDeleteJob(w, r, rest[:i], rest[i+len("/jobs/"):])
|
||
} else {
|
||
http.NotFound(w, r)
|
||
}
|
||
// Admin-set (slice 10A) — global/operator key only; bumps the generation.
|
||
case r.Method == http.MethodPut && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/desired-state"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/desired-state")
|
||
h.handleAdminSetDesiredState(w, r, hostID)
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/admin/hosts/") && strings.HasSuffix(path, "/jobs"):
|
||
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/admin/hosts/"), "/jobs")
|
||
h.handleAdminEnqueueJob(w, r, hostID)
|
||
// S1 offsite connectivity — WG endpoint record + peer registry (global key only, api/wg.go).
|
||
// DELETE carries the pubkey in the body: base64 '/'+'+' keep pubkeys out of URL paths.
|
||
case r.Method == http.MethodPut && path == "/admin/wg/endpoint":
|
||
h.handleAdminSetWGEndpoint(w, r)
|
||
case r.Method == http.MethodGet && path == "/admin/wg/endpoint":
|
||
h.handleAdminGetWGEndpoint(w, r)
|
||
case r.Method == http.MethodPost && path == "/admin/wg/peers":
|
||
h.handleAdminAddWGPeer(w, r)
|
||
case r.Method == http.MethodDelete && path == "/admin/wg/peers":
|
||
h.handleAdminDeleteWGPeer(w, r)
|
||
case r.Method == http.MethodGet && path == "/admin/wg/peers":
|
||
h.handleAdminListWGPeers(w, r)
|
||
// H1: the fleet operator OOB peer (register/rotate at an explicit /32) + read-back.
|
||
case r.Method == http.MethodPut && path == "/admin/wg/operator-peer":
|
||
h.handleAdminSetOperatorPeer(w, r)
|
||
case r.Method == http.MethodGet && path == "/admin/wg/operator-peer":
|
||
h.handleAdminGetOperatorPeer(w, r)
|
||
case r.Method == http.MethodPost && path == "/claim/reset-request":
|
||
h.handleClaimResetRequest(w, r)
|
||
case r.Method == http.MethodPost && path == "/event":
|
||
h.handleEvent(w, r)
|
||
case r.Method == http.MethodPost && path == "/mail":
|
||
h.handleMail(w, r)
|
||
case r.Method == http.MethodPost && path == "/notify":
|
||
h.handleNotify(w, r)
|
||
case r.Method == http.MethodPost && path == "/preferences":
|
||
h.handleSavePreferences(w, r)
|
||
case r.Method == http.MethodGet && path == "/preferences":
|
||
h.handleGetPreferences(w, r) // R-1199 reopened: a box with no local list reads the household's
|
||
case r.Method == http.MethodGet && path == "/customers":
|
||
h.handleCustomers(w, r)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/customers/"):
|
||
parts := strings.Split(strings.TrimPrefix(path, "/customers/"), "/")
|
||
customerID := parts[0]
|
||
if len(parts) > 1 && parts[1] == "history" {
|
||
h.handleCustomerHistory(w, r, customerID)
|
||
} else {
|
||
h.handleCustomer(w, r, customerID)
|
||
}
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/recovery/"):
|
||
customerID := strings.TrimPrefix(path, "/recovery/")
|
||
h.handleRecovery(w, r, customerID)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/config/"):
|
||
customerID := strings.TrimPrefix(path, "/config/")
|
||
h.handleConfigRetrieve(w, r, customerID)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/box-config/"):
|
||
h.handleBoxConfig(w, r, strings.TrimPrefix(path, "/box-config/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/consume-password/"):
|
||
customerID := strings.TrimPrefix(path, "/offsite/consume-password/")
|
||
h.handleOffsiteConsumePassword(w, r, customerID)
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/register-key/"):
|
||
h.handleOffsiteRegisterKey(w, r, strings.TrimPrefix(path, "/offsite/register-key/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/confirm-key/"):
|
||
h.handleOffsiteConfirmKey(w, r, strings.TrimPrefix(path, "/offsite/confirm-key/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/move-aside/"):
|
||
h.handleOffsiteMoveAside(w, r, strings.TrimPrefix(path, "/offsite/move-aside/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/window-open/"):
|
||
h.handleOffsiteWindowOpen(w, r, strings.TrimPrefix(path, "/offsite/window-open/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/window-close/"):
|
||
h.handleOffsiteWindowClose(w, r, strings.TrimPrefix(path, "/offsite/window-close/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/abandon-request/"):
|
||
h.handleOffsiteAbandon(w, r, "request", strings.TrimPrefix(path, "/offsite/abandon-request/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/abandon-cancel/"):
|
||
h.handleOffsiteAbandon(w, r, "cancel", strings.TrimPrefix(path, "/offsite/abandon-cancel/"))
|
||
case r.Method == http.MethodPost && strings.HasPrefix(path, "/offsite/abandon-status/"):
|
||
h.handleOffsiteAbandon(w, r, "status", strings.TrimPrefix(path, "/offsite/abandon-status/"))
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/artifacts/"):
|
||
customerID := strings.TrimPrefix(path, "/artifacts/")
|
||
h.handleArtifactManifest(w, r, customerID)
|
||
case r.Method == http.MethodGet && path == "/assets/manifest":
|
||
h.handleAssetsManifest(w, r)
|
||
case r.Method == http.MethodGet && strings.HasPrefix(path, "/assets/file/"):
|
||
filename := strings.TrimPrefix(path, "/assets/file/")
|
||
h.handleAssetFile(w, r, filename)
|
||
default:
|
||
http.NotFound(w, r)
|
||
}
|
||
}
|
||
|
||
func (h *Handler) handleReport(w http.ResponseWriter, r *http.Request) {
|
||
authCustomerID, isGlobal, ok := h.checkAuthCustomer(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) // 1MB limit
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// Extract customer_id from JSON
|
||
var payload struct {
|
||
CustomerID string `json:"customer_id"`
|
||
}
|
||
if err := json.Unmarshal(body, &payload); err != nil || payload.CustomerID == "" {
|
||
http.Error(w, "Invalid payload: customer_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// Validate customer_id matches authenticated customer (unless global key)
|
||
if !isGlobal && authCustomerID != payload.CustomerID {
|
||
http.Error(w, "Forbidden: customer_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
if err := h.store.SaveReport(payload.CustomerID, body); err != nil {
|
||
h.logger.Printf("[ERROR] Failed to save report from %s: %v", payload.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
// D72 (`07` §8.2f part 4): the hub's own record of which apps this box runs, by the hub's clock.
|
||
h.noteAppList(payload.CustomerID, body)
|
||
|
||
// Parse and save app telemetry (backward-compatible — old controllers won't have this field)
|
||
var telemetryPayload struct {
|
||
AppTelemetry []store.AppTelemetryRecord `json:"app_telemetry"`
|
||
}
|
||
if err := json.Unmarshal(body, &telemetryPayload); err == nil && len(telemetryPayload.AppTelemetry) > 0 {
|
||
if err := h.store.SaveAppTelemetry(payload.CustomerID, time.Now(), telemetryPayload.AppTelemetry); err != nil {
|
||
h.logger.Printf("[WARN] Failed to save app telemetry for %s: %v", payload.CustomerID, err)
|
||
}
|
||
}
|
||
|
||
// On-demand log tails (v0.43.0) — the controller ships these on the cycle after the ACK
|
||
// requested them. Storing one clears its pending request (consume-once, in SaveAppLogTail)
|
||
// so the next ACK stops advertising it. Backward-compatible: old controllers never send this.
|
||
var tailPayload struct {
|
||
LogTails []struct {
|
||
App string `json:"app"`
|
||
CollectedAt time.Time `json:"collected_at"`
|
||
Lines []string `json:"lines"`
|
||
} `json:"log_tails"`
|
||
}
|
||
if err := json.Unmarshal(body, &tailPayload); err == nil {
|
||
for _, lt := range tailPayload.LogTails {
|
||
if lt.App == "" {
|
||
continue
|
||
}
|
||
if err := h.store.SaveAppLogTail(payload.CustomerID, lt.App, lt.CollectedAt, lt.Lines); err != nil {
|
||
h.logger.Printf("[WARN] Failed to save log tail %s/%s: %v", payload.CustomerID, lt.App, err)
|
||
} else {
|
||
h.logger.Printf("[INFO] Log tail received for %s/%s (%d lines)", payload.CustomerID, lt.App, len(lt.Lines))
|
||
}
|
||
}
|
||
}
|
||
|
||
// Controller self-log tail (v0.46.0) — the controller's OWN debug ring, shipped on the
|
||
// cycle after the ACK's controller_log_requested. SaveLogBundle runs the secret gate
|
||
// (a hit stores a BLOCKED flag row, nothing else) and clears the pending request
|
||
// (consume-once). Backward-compatible: old controllers never send this.
|
||
var selfTailPayload struct {
|
||
ControllerLogTail *struct {
|
||
CollectedAt time.Time `json:"collected_at"`
|
||
Lines []string `json:"lines"`
|
||
} `json:"controller_log_tail"`
|
||
}
|
||
if err := json.Unmarshal(body, &selfTailPayload); err == nil && selfTailPayload.ControllerLogTail != nil {
|
||
lt := selfTailPayload.ControllerLogTail
|
||
blocked, berr := h.store.SaveLogBundle(payload.CustomerID, store.LogBundleComponentController, lt.CollectedAt, lt.Lines)
|
||
switch {
|
||
case berr != nil:
|
||
h.logger.Printf("[WARN] Failed to save controller log bundle for %s: %v", payload.CustomerID, berr)
|
||
case blocked:
|
||
h.logger.Printf("[WARN] controller log bundle for %s BLOCKED: possible secret in log content — nothing stored", payload.CustomerID)
|
||
default:
|
||
h.logger.Printf("[INFO] controller log bundle received for %s (%d lines)", payload.CustomerID, len(lt.Lines))
|
||
}
|
||
}
|
||
|
||
// Operator actions (R-314/R-279/R-177, `09` §3 decision 185): the box's results for the actions
|
||
// the ACK listed. Each closes its row — matched on the id AND this report's customer, so a result
|
||
// naming another customer's action changes nothing — and becomes a hub-minted event (stored only,
|
||
// never dispatched: an operator record, not a customer mail). Old controllers never send this.
|
||
h.ingestOperatorActionResults(payload.CustomerID, body)
|
||
|
||
// DR recipe — persist the controller's secret-free customer/apps half (preserving any host half).
|
||
// Backward-compatible (old controllers won't have this field); a failure must not drop the report.
|
||
var drPayload struct {
|
||
DRRecipe json.RawMessage `json:"dr_recipe"`
|
||
}
|
||
if err := json.Unmarshal(body, &drPayload); err == nil && len(drPayload.DRRecipe) > 0 {
|
||
var ver drRecipeVersionOnly
|
||
_ = json.Unmarshal(drPayload.DRRecipe, &ver)
|
||
if err := h.store.SaveDRRecipeAppHalf(payload.CustomerID, ver.RecipeVersion, drPayload.DRRecipe); err != nil {
|
||
h.logger.Printf("[WARN] Failed to save DR-recipe app-half for %s: %v", payload.CustomerID, err)
|
||
} else {
|
||
h.logger.Printf("[INFO] DR-recipe app-half stored for customer %s (v%d)", payload.CustomerID, ver.RecipeVersion)
|
||
}
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Received report from %s (%d bytes)", payload.CustomerID, len(body))
|
||
|
||
// Build response with optional customer_blocked flag
|
||
resp := map[string]interface{}{"status": "ok"}
|
||
if custCfg, err := h.store.GetCustomerConfig(payload.CustomerID); err == nil && custCfg != nil {
|
||
if custCfg.Status == "blocked" {
|
||
resp["customer_blocked"] = true
|
||
}
|
||
// Config-refresh (v0.26.0): advertise the per-customer config_version. The controller compares
|
||
// it against its last-applied version and, on a change, re-pulls controller.yaml + self-restarts
|
||
// (pull-based config delivery — the hub never connects into the box). Only emitted for
|
||
// config-managed customers (a report-only box without a config row gets no field and is unaffected).
|
||
resp["config_version"] = custCfg.ConfigVersion
|
||
|
||
// Customer-claim arc (v0.50.0, F-4): ensure a claim code exists for every reporting managed
|
||
// customer (idempotent — the live-box entry point; Day-0 boxes get theirs at config retrieve),
|
||
// ingest the controller's reported claimed flag (set-only — a wiped settings.json can never
|
||
// un-claim), and serve the ACTIVE code hash + generation in the ACK. The hash is bcrypt (non-
|
||
// reversible) — safe to serve on the authenticated report channel; the plaintext code exists
|
||
// only in the customer's mailbox.
|
||
if h.claimEngine != nil {
|
||
cs, cerr := h.claimEngine.EnsureIssued(custCfg)
|
||
if cerr != nil {
|
||
h.logger.Printf("[WARN] claim issue for %s on report: %v", payload.CustomerID, cerr)
|
||
}
|
||
var claimedPayload struct {
|
||
Claimed *bool `json:"claimed"`
|
||
}
|
||
if err := json.Unmarshal(body, &claimedPayload); err == nil &&
|
||
claimedPayload.Claimed != nil && *claimedPayload.Claimed {
|
||
if err := h.claimEngine.MarkClaimed(custCfg); err != nil {
|
||
h.logger.Printf("[WARN] claim mark-claimed for %s: %v", payload.CustomerID, err)
|
||
} else if cs != nil && cs.ClaimedAt == nil {
|
||
cs, _ = h.store.GetClaim(payload.CustomerID) // refresh for the ACK below
|
||
}
|
||
}
|
||
if cs != nil {
|
||
resp["claim"] = map[string]interface{}{
|
||
"code_hash": cs.CodeHash,
|
||
"generation": cs.Generation,
|
||
"issued_at": cs.IssuedAt.UTC().Format(time.RFC3339),
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── R-241 (v0.98.0) — THE ABANDONMENT PURGE, ACTED ON BEFORE THE ACK IS BUILT ───────────────
|
||
//
|
||
// The box has deleted its set-aside off-site history at the end of a 14-day countdown the customer
|
||
// chose and could see, and now asks the hub to drop the sealed package that protected it, so the
|
||
// two halves go together. Removing only one leaves a state that asks a question nobody can answer
|
||
// — a package that opens nothing, or ciphertext nobody can decrypt.
|
||
//
|
||
// IT ACTS ON A DECLARATION, NEVER AN INFERENCE, on the same principle as `offsite.state`: the hub
|
||
// cannot see that a remote store was deleted, and the box can. The declaration repeats on every
|
||
// report until the ACK below stops carrying a superseded package, so a lost request retries by
|
||
// itself instead of leaving the pair half-removed.
|
||
//
|
||
// PLACED IMMEDIATELY BEFORE THE ACK deliberately: `GetEscrowStatusForCustomer` is read after this
|
||
// runs, so the SAME response that carries the request's effect is what closes the box's
|
||
// two-phase commit. No second round-trip, and no window in which the box thinks it is still owed.
|
||
//
|
||
// Only RETAINED rows go — never `host_escrow`, which covers the key the box is using now and is
|
||
// what makes its live backups recoverable.
|
||
var abandonPayload struct {
|
||
Offsite *struct {
|
||
AbandonPurgeRequested bool `json:"abandon_purge_requested"`
|
||
} `json:"offsite"`
|
||
}
|
||
abandonDeclared := false
|
||
if err := json.Unmarshal(body, &abandonPayload); err == nil &&
|
||
abandonPayload.Offsite != nil && abandonPayload.Offsite.AbandonPurgeRequested {
|
||
abandonDeclared = true
|
||
// R-1182: only the packages that open a set-aside copy the hub DELETED — never every retained package.
|
||
n, perr := h.store.PurgeAbandonedEscrowForCustomer(payload.CustomerID)
|
||
switch {
|
||
case perr != nil:
|
||
// Never swallowed: the box keeps declaring, so this retries next cycle — but an operator
|
||
// must be able to see that the pair is half-removed right now.
|
||
h.logger.Printf("[ERROR] offsite-abandon: %s DECLARED the set-aside history deleted, but purging the retained recovery package FAILED — the two halves are apart until this succeeds: %v", payload.CustomerID, perr)
|
||
case n > 0:
|
||
h.logger.Printf("[INFO] offsite-abandon: %s deleted its set-aside off-site history and the retained recovery package was purged (%d row(s)) — both halves are gone", payload.CustomerID, n)
|
||
// Hub-internal, like the pbsdr_* / offsite_selfheal_* events — not gated by
|
||
// allowedEventTypes, which governs the box-pushed POST /event surface.
|
||
if _, eerr := h.store.SaveEvent(payload.CustomerID, "offsite_abandon_purged", "info",
|
||
fmt.Sprintf("Az ügyfél korábbi távoli mentései és a hozzájuk tartozó megőrzött helyreállítási csomag is törölve (%d csomag). Az ügyfél döntése alapján, a 14 napos türelmi idő lejárta után.", n),
|
||
"", "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] offsite-abandon: %s purge succeeded but the audit event could not be saved: %v", payload.CustomerID, eerr)
|
||
}
|
||
default:
|
||
// Idempotent: already purged, and the box simply has not seen the confirming ACK yet.
|
||
h.logger.Printf("[DEBUG] offsite-abandon: %s declared a purge; no retained package remained (already done)", payload.CustomerID)
|
||
}
|
||
}
|
||
|
||
// SLICE 3 — escrow status for the hub-verified auto-confirm: the controller flips its offbox
|
||
// EscrowState pending→escrowed ONLY when sha256(its local repo password) matches restic_pw_sha256
|
||
// (blob-presence alone must never confirm — a stale blob may not cover the current key). The hash is
|
||
// non-reversible (256-bit random secret) — safe to serve; omitted entirely when no escrow row exists.
|
||
// R-1176: a rebuilt box's FIRST ACK must already carry the household's package, or the box mints a fresh key.
|
||
h.carryEscrow(payload.CustomerID)
|
||
if es, err := h.store.GetEscrowStatusForCustomer(payload.CustomerID); err == nil && es != nil {
|
||
// R-1182: other retained packages are now rightly KEPT after an abandonment, so `superseded_present` can stay
|
||
// true for good; this field tells the box its own declaration is finished. Only on a declaring report.
|
||
if abandonDeclared {
|
||
if left, lerr := h.store.AbandonedEscrowLeft(payload.CustomerID); lerr == nil && left == 0 {
|
||
es.AbandonPurgeDone = true
|
||
}
|
||
}
|
||
resp["escrow"] = es
|
||
}
|
||
|
||
// v0.43.0 — pending log-tail requests (same additive ACK-flag pattern as escrow): the
|
||
// controller collects the named apps' tails and ships them on its NEXT report; the field
|
||
// is omitted when nothing is pending. The hub never connects into the box.
|
||
if apps, err := h.store.GetPendingLogTailRequests(payload.CustomerID); err == nil && len(apps) > 0 {
|
||
resp["log_tail_requests"] = apps
|
||
}
|
||
|
||
// v0.46.0 — pending CONTROLLER self-log pull (same additive ACK-flag pattern): the
|
||
// controller ships controller_log_tail on its NEXT report; omitted when nothing pending.
|
||
if pending, err := h.store.PendingLogBundleRequest(payload.CustomerID, store.LogBundleComponentController); err == nil && pending {
|
||
resp["controller_log_requested"] = true
|
||
}
|
||
|
||
// Decision 185: the operator's pending actions for this box, listed until each result arrives.
|
||
// Read AFTER the results above were recorded, so an action answered in this report is not
|
||
// listed back in the same reply. Omitted when none (an old controller ignores the field).
|
||
if acts, err := h.store.PendingOperatorActions(payload.CustomerID); err != nil {
|
||
h.logger.Printf("[WARN] operator actions for %s could not be read (not listed this cycle): %v", payload.CustomerID, err)
|
||
} else if len(acts) > 0 {
|
||
resp["operator_actions"] = acts
|
||
h.logger.Printf("[DEBUG] operator actions listed for %s: %d", payload.CustomerID, len(acts))
|
||
}
|
||
|
||
// Phase 2 managed updates: advertise the effective controller-version FLOOR (per-customer override
|
||
// else global default) and the latest available version. The controller compares its current
|
||
// version against the floor and auto-updates when below it (latest stays the customer's opt-in
|
||
// "update to latest" button — NOT the auto-target). Both fields are omitted when empty, so an old
|
||
// controller that ignores them, or a hub with no floor configured, behaves exactly as before.
|
||
// Part D: the per-box MinAgent conditional floor — HOLD the controller-version floor for a box
|
||
// whose host agent is below the current golden's required MinAgent (never push a controller past
|
||
// the agent it depends on). A held box gets NO directive (behaves as if no floor) but is flagged
|
||
// on the dashboard, never silently stale.
|
||
if fd := h.store.ResolveManagedFloor(payload.CustomerID); fd.Floor != "" {
|
||
resp["min_controller_version"] = fd.Floor
|
||
// R-472: say where the served floor's agent requirement came from — once per change, not per
|
||
// report (a box reports every few minutes, and a line repeated that often is a line nobody reads).
|
||
note := fd.Floor + "|" + fd.MinAgentSource + "|" + fd.MinAgent
|
||
if prev, ok := h.floorNotes.Load(payload.CustomerID); !ok || prev.(string) != note {
|
||
h.floorNotes.Store(payload.CustomerID, note)
|
||
src := fd.MinAgentSource
|
||
if src == "" {
|
||
src = "none (uncoupled release)"
|
||
}
|
||
h.logger.Printf("[INFO] managed floor SERVED for %s: floor %s, agent requirement %q from %s (golden %s)",
|
||
payload.CustomerID, fd.Floor, fd.MinAgent, src, fd.GoldenVersion)
|
||
}
|
||
} else if fd.HeldSpoiled {
|
||
// R-1085: a withdrawn build — ONE [WARN] per change naming the tag and why, not one per report.
|
||
note := "spoiled|" + fd.RequestedFloor
|
||
if prev, ok := h.floorNotes.Load(payload.CustomerID); !ok || prev.(string) != note {
|
||
h.floorNotes.Store(payload.CustomerID, note)
|
||
h.logger.Printf("[WARN] managed floor HELD for %s: %s (controller floor withheld)",
|
||
payload.CustomerID, fd.HoldReason())
|
||
}
|
||
} else if fd.Held {
|
||
h.floorNotes.Delete(payload.CustomerID)
|
||
// ONE sentence, from the decision itself — the two hold reasons must never drift apart
|
||
// across this line and the dashboard (see ManagedFloorDecision.HoldReason).
|
||
h.logger.Printf("[INFO] managed floor HELD for %s: %s (controller floor withheld)",
|
||
payload.CustomerID, fd.HoldReason())
|
||
}
|
||
if h.latestVersion != nil {
|
||
if latest := h.latestVersion.LatestVersion(); latest != "" {
|
||
resp["latest_version"] = latest
|
||
}
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// defaultHostPollSeconds is the cadence the hub hands every agent this slice (no
|
||
// per-host override UI yet — that is a later slice).
|
||
const defaultHostPollSeconds = 900
|
||
|
||
// maxHostReportBytes bounds a host-report body. Larger than the controller path's
|
||
// 1 MiB because host reports carry the full guest list + (later) storage/backup
|
||
// arrays. We read one byte past it and reject explicitly (413) rather than letting
|
||
// LimitReader silently truncate — a truncated-but-valid JSON would otherwise be
|
||
// accepted as a partial report, dropping guests from the mirror.
|
||
const maxHostReportBytes = 4 << 20 // 4 MiB
|
||
|
||
// hostReportPayload is the subset of the agent host-report (slice-3 contract,
|
||
// §3 / agent spec §4) the hub needs for denorm + guest reality. The remaining fields
|
||
// (backups/restore_tests/pbs_snapshots/audit_tail) are ignored, so an empty or absent
|
||
// collection is accepted without error.
|
||
//
|
||
// storage_targets (slice 5) is now parsed: the agent populates it, and the hub accepts
|
||
// + persists it. Persistence is the full report_json row (which carries the targets
|
||
// verbatim) plus the denorm counts below — the RICH manifest schema (desired class/role/
|
||
// policy/creds) is hub-owned and lands in slice 10; this slice only mirrors what the agent
|
||
// observes.
|
||
type hostReportPayload struct {
|
||
HostID string `json:"host_id"`
|
||
AgentVersion string `json:"agent_version"`
|
||
Host struct {
|
||
CPUPercent float64 `json:"cpu_percent"`
|
||
MemoryPercent float64 `json:"memory_percent"`
|
||
DiskPercent float64 `json:"disk_percent"`
|
||
// R-1192 follow-up (agent ≥ the swap release): the host's swap and its guests' swap allowance, MB. nil = an
|
||
// older agent → not evaluated.
|
||
HostSwapMB *int64 `json:"host_swap_mb"`
|
||
GuestSwapMB *int64 `json:"guest_swap_mb"`
|
||
} `json:"host"`
|
||
// Wireguard (R-1204): the box's own view of its tunnel to ep0. nil = an agent with no tunnel stanza.
|
||
Wireguard *hostWireguard `json:"wireguard"`
|
||
// ProxmoxTrust (R-1380, R-1396; `03` §12a): how the agent trusts Proxmox's certificate and, when something about it
|
||
// is wrong, the CAUSE. nil = an agent without the stanza (older than release 2b).
|
||
ProxmoxTrust *hostProxmoxTrust `json:"proxmox_trust"`
|
||
// GuestPowerGaveUp (R-1383, D78): the guests the agent's guest-power watchdog stopped trying to start (three
|
||
// failed starts). nil/empty = none, or an agent that does not report it.
|
||
GuestPowerGaveUp []hostGuestPowerGaveUp `json:"guest_power_gave_up"`
|
||
// SentAt (R-1384, D79): the box's clock at the moment the agent SENDS the report (stamped in the agent's
|
||
// Client.Report, after the collect). "" = an agent that does not stamp it → the clock is not judged.
|
||
// reported_at is NOT used for this: it is stamped when the collect starts, and a collect can take long.
|
||
SentAt string `json:"sent_at"`
|
||
Guests []struct {
|
||
VMID int `json:"vmid"`
|
||
Name string `json:"name"`
|
||
Status string `json:"status"`
|
||
ControllerVersion string `json:"controller_version"`
|
||
} `json:"guests"`
|
||
StorageTargets []hostStorageTarget `json:"storage_targets"`
|
||
Backups []hostBackup `json:"backups"` // slice 6
|
||
RestoreTests []hostRestoreTest `json:"restore_tests"` // slice 6
|
||
PBSSnapshots []hostPBSSnapshot `json:"pbs_snapshots"` // slice 6 Phase B
|
||
// ForeignKeyArchives (R-366 slice 2, agent >= the 2026-10-07 release): per tier, the whole-guest archives the
|
||
// restore-test skipped as written with another key. Absent = not evaluated / an older agent → the hub keeps its
|
||
// state; `tiers: []` = evaluated, none.
|
||
ForeignKeyArchives *struct {
|
||
Tiers []hostForeignKeyArchives `json:"tiers"`
|
||
} `json:"foreign_key_archives"`
|
||
Cloudflared struct {
|
||
Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown; absent since R-1070 (never alarms) (older: active | inactive | …)
|
||
Detail string `json:"detail,omitempty"`
|
||
} `json:"cloudflared"`
|
||
// DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim,
|
||
// ignore-unknown (forward-compat). Persisted to dr_recipe, assembled with the controller half.
|
||
DRRecipe json.RawMessage `json:"dr_recipe"`
|
||
// LogTail (v0.46.0) — the agent's on-demand debug-ring tail, present only on the
|
||
// heartbeat right after the envelope's log_tail_requested (agent ≥ 0.83.0).
|
||
LogTail *struct {
|
||
CollectedAt time.Time `json:"collected_at"`
|
||
Lines []string `json:"lines"`
|
||
} `json:"log_tail"`
|
||
// DeliberateOff (R-998, agent >= 0.155.0): the household asked this box to power off, in its current boot.
|
||
// Absent = not (or an older agent). deliberate_off.go turns it into the host row's hold.
|
||
DeliberateOff *struct {
|
||
RequestedAt string `json:"requested_at"`
|
||
} `json:"deliberate_off"`
|
||
// OperatorActionResults (2026-10-29, the agent's operator actions): the answers to the previous replies'
|
||
// operator_actions — the controller's shape. Absent on an older agent.
|
||
OperatorActionResults []hostOperatorActionResult `json:"operator_action_results"`
|
||
}
|
||
|
||
// hostOperatorActionResult mirrors the agent's hub.OperatorActionResult.
|
||
type hostOperatorActionResult struct {
|
||
ID int64 `json:"id"`
|
||
Outcome string `json:"outcome"`
|
||
Message string `json:"message"`
|
||
}
|
||
|
||
// hostForeignKeyArchives mirrors the agent's hub.ForeignKeyArchives (R-366 slice 2).
|
||
type hostForeignKeyArchives struct {
|
||
Target string `json:"target"`
|
||
Count int `json:"count"`
|
||
Oldest string `json:"oldest"`
|
||
Newest string `json:"newest"`
|
||
}
|
||
|
||
// drRecipeVersionOnly extracts just recipe_version from a half's JSON (ignore-unknown). 0 if absent.
|
||
type drRecipeVersionOnly struct {
|
||
RecipeVersion int `json:"recipe_version"`
|
||
}
|
||
|
||
// hostPBSSnapshot mirrors the agent's hub.PBSSnapshot wire contract (slice 6 Phase B). The
|
||
// hub persists it via report_json and surfaces a FAILED verify prominently (the loudest
|
||
// offsite-DR signal — same treatment as a failed restore-test).
|
||
type hostPBSSnapshot struct {
|
||
Namespace string `json:"namespace"`
|
||
BackupType string `json:"backup_type"`
|
||
BackupID string `json:"backup_id"`
|
||
BackupTime string `json:"backup_time"`
|
||
SizeBytes int64 `json:"size_bytes"`
|
||
Owner string `json:"owner"`
|
||
Protected bool `json:"protected"`
|
||
Encrypted bool `json:"encrypted"`
|
||
VerifyState string `json:"verify_state"`
|
||
VerifyUPID string `json:"verify_upid,omitempty"`
|
||
}
|
||
|
||
// hostBackup mirrors the agent's hub.Backup wire contract field-for-field; hostRestoreTest is a deliberate
|
||
// SUBSET of hub.RestoreTest — the agent's mount_parity, mount_inventory and skipped are NOT modelled here (R-262,
|
||
// pinned by TestR262_RestoreTestFieldsAreAKnownSubset) (slice 6, doc 03 §8). DUPLICATED contract — the golden stays
|
||
// byte-identical with felhom-agent's copy and the key-set tests guard drift. The hub
|
||
// persists these via report_json (no new columns this slice) and surfaces a FAILED
|
||
// restore-test prominently (the loudest DR signal). The rich backup policy is slice 10.
|
||
type hostBackup struct {
|
||
TargetID string `json:"target_id"`
|
||
VMID int `json:"vmid"`
|
||
Archive string `json:"archive"`
|
||
Mode string `json:"mode"`
|
||
CrashConsistent bool `json:"crash_consistent"`
|
||
SizeBytes int64 `json:"size_bytes"`
|
||
Success bool `json:"success"`
|
||
Error string `json:"error,omitempty"`
|
||
StartedAt string `json:"started_at"`
|
||
DurationSeconds float64 `json:"duration_seconds"`
|
||
UncoveredVolumes []string `json:"uncovered_volumes"`
|
||
}
|
||
|
||
type hostRestoreTest struct {
|
||
SourceArchive string `json:"source_archive"`
|
||
SourceTier string `json:"source_tier"`
|
||
ScratchVMID int `json:"scratch_vmid"`
|
||
Pass bool `json:"pass"`
|
||
Verified string `json:"verified"`
|
||
Error string `json:"error,omitempty"`
|
||
TestedAt string `json:"tested_at"`
|
||
DurationSeconds float64 `json:"duration_seconds"`
|
||
// Warnings are the guest-start task's warning line(s) on a PASS (e.g. the systemd-nesting
|
||
// advisory). The verdict is liveness-only, so a passed restore-test can carry warnings.
|
||
Warnings []string `json:"warnings,omitempty"`
|
||
// WarningsRecognized is true iff every warning is the known-benign anchor. Absent ⇒ false,
|
||
// which is the SAFE default: the hub then treats it as an unrecognized warning (the louder
|
||
// path), so a missing flag can only over-notice, never hide a real warning.
|
||
WarningsRecognized bool `json:"warnings_recognized,omitempty"`
|
||
}
|
||
|
||
// hostStorageTarget mirrors the agent's hub.StorageTarget wire contract field-for-field.
|
||
// It is a DUPLICATED contract (no shared types module yet); testdata/host-report.golden.json
|
||
// must stay byte-identical with felhom-agent's copy and the key-set test guards drift.
|
||
// The hub does not act on these yet beyond persisting + counting them (slice 10 adds the
|
||
// authoritative manifest), but mirroring the full shape keeps the cross-repo contract honest.
|
||
type hostStorageTarget struct {
|
||
Name string `json:"name"`
|
||
Type string `json:"type"`
|
||
DurableID string `json:"durable_id"`
|
||
State string `json:"state"`
|
||
Reachable bool `json:"reachable"`
|
||
TotalBytes int64 `json:"total_bytes"`
|
||
UsedBytes int64 `json:"used_bytes"`
|
||
AvailBytes int64 `json:"avail_bytes"`
|
||
UsedFraction float64 `json:"used_fraction"`
|
||
Content string `json:"content"`
|
||
MountPath string `json:"mount_path"`
|
||
BackingDevice string `json:"backing_device"`
|
||
ClassHint string `json:"class_hint"`
|
||
Role string `json:"role"`
|
||
// Physical-disk identity (agent v0.157.0, felhom.eu architecture/16-storage-physical-identity.md §3):
|
||
// agent-computed; the hub stores it with the report and shows it on the host page. No alarm (build A).
|
||
Kind string `json:"kind,omitempty"`
|
||
PhysicalDisks []string `json:"physical_disks,omitempty"`
|
||
Identity string `json:"identity,omitempty"`
|
||
IdentityReason string `json:"identity_reason,omitempty"`
|
||
ThinPool *struct {
|
||
DataUsedFraction float64 `json:"data_used_fraction"`
|
||
MetadataUsedFraction *float64 `json:"metadata_used_fraction"`
|
||
} `json:"thin_pool,omitempty"`
|
||
Smart struct {
|
||
Health string `json:"health"`
|
||
TemperatureC *int `json:"temperature_c"`
|
||
PowerOnHours *int `json:"power_on_hours"`
|
||
ReallocatedSectors *int `json:"reallocated_sectors"`
|
||
PendingSectors *int `json:"pending_sectors"`
|
||
OfflineUncorrectable *int `json:"offline_uncorrectable"`
|
||
// R-330 (agent v0.150.0+): SATA 187/188/199 raw counters; absent (older agent, NVMe/USB, not
|
||
// reported) = nil = unknown. Accepted and stored with the report; nothing alarms on them yet.
|
||
ReportedUncorrect *int64 `json:"reported_uncorrect,omitempty"`
|
||
CommandTimeout *int64 `json:"command_timeout,omitempty"`
|
||
UDMACRCErrors *int64 `json:"udma_crc_errors,omitempty"`
|
||
CriticalWarning *int `json:"critical_warning"`
|
||
MediaErrors *int `json:"media_errors"`
|
||
PercentageUsed *int `json:"percentage_used"`
|
||
} `json:"smart"`
|
||
}
|
||
|
||
// handleHostReport ingests the agent's host-report (the heartbeat) and returns the
|
||
// control envelope (agent spec §5).
|
||
func (h *Handler) handleHostReport(w http.ResponseWriter, r *http.Request) {
|
||
received := time.Now() // before the body is read: the clock check (R-1384) compares the box's send time with THIS
|
||
hostID, custID, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, maxHostReportBytes+1))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if len(body) > maxHostReportBytes {
|
||
http.Error(w, "Payload too large", http.StatusRequestEntityTooLarge)
|
||
return
|
||
}
|
||
var rep hostReportPayload
|
||
if err := json.Unmarshal(body, &rep); err != nil || rep.HostID == "" {
|
||
http.Error(w, "Invalid payload: host_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
if isGlobal {
|
||
// Global-key bootstrap: trust body.host_id but require the host to exist
|
||
// (it must be minted first) and resolve its customer from the row.
|
||
host, err := h.store.GetHost(rep.HostID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] host lookup failed for %s: %v", rep.HostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if host == nil {
|
||
http.Error(w, "Unknown host_id (mint via /admin/hosts first)", http.StatusBadRequest)
|
||
return
|
||
}
|
||
hostID, custID = rep.HostID, host.CustomerID
|
||
} else if rep.HostID != hostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
running := 0
|
||
for _, g := range rep.Guests {
|
||
if g.Status == "running" {
|
||
running++
|
||
}
|
||
}
|
||
denorm := store.HostReportDenorm{
|
||
AgentVersion: rep.AgentVersion,
|
||
CPUPercent: rep.Host.CPUPercent,
|
||
MemoryPercent: rep.Host.MemoryPercent,
|
||
DiskPercent: rep.Host.DiskPercent,
|
||
GuestTotal: len(rep.Guests),
|
||
GuestRunning: running,
|
||
CloudflaredStatus: rep.Cloudflared.Status,
|
||
}
|
||
if err := h.store.SaveHostReport(hostID, custID, body, denorm); err != nil {
|
||
h.logger.Printf("[ERROR] Failed to save host-report from %s: %v", hostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
// R-1176: the agent's report is the first a new box sends — carry the household's package before its
|
||
// controller exists, so the controller's first ACK finds it.
|
||
if custID != "" {
|
||
h.carryEscrow(custID)
|
||
}
|
||
h.checkTunnel(hostID, custID, rep.Cloudflared.Status, rep.Cloudflared.Detail)
|
||
h.checkHostSwap(hostID, custID, rep.Host.HostSwapMB, rep.Host.GuestSwapMB)
|
||
h.checkWGTunnel(hostID, custID, rep.Wireguard)
|
||
h.checkProxmoxTrust(hostID, custID, rep.ProxmoxTrust)
|
||
h.checkBoxClock(hostID, custID, rep.SentAt, received)
|
||
runningGuests := map[int]bool{}
|
||
for _, g := range rep.Guests {
|
||
if g.Status == "running" {
|
||
runningGuests[g.VMID] = true
|
||
}
|
||
}
|
||
knownGuests := map[int]bool{}
|
||
for _, g := range rep.Guests {
|
||
knownGuests[g.VMID] = true
|
||
}
|
||
h.checkGuestPowerGaveUp(hostID, custID, rep.GuestPowerGaveUp, runningGuests, knownGuests)
|
||
h.checkCrash(hostID, custID, body)
|
||
h.checkDeliberateOff(hostID, custID, &rep)
|
||
|
||
for _, g := range rep.Guests {
|
||
status := g.Status
|
||
if status == "" {
|
||
status = "unknown"
|
||
}
|
||
guest := &store.Guest{
|
||
GuestID: store.GuestID(hostID, g.VMID),
|
||
CustomerID: custID,
|
||
HostID: hostID,
|
||
VMID: g.VMID,
|
||
DisplayName: g.Name,
|
||
Status: status,
|
||
ControllerVersion: g.ControllerVersion,
|
||
}
|
||
if err := h.store.UpsertGuestFromReport(guest); err != nil {
|
||
// A guest upsert failure must not drop the whole report (liveness).
|
||
h.logger.Printf("[WARN] Failed to upsert guest %s: %v", guest.GuestID, err)
|
||
}
|
||
}
|
||
|
||
// storage_targets (slice 5): persisted as part of report_json above. Count + surface
|
||
// disconnected ones in the log (the slice-10 manifest will reconcile them; for now the
|
||
// signal is the visibility — a disconnected target is the storage analog of host-down).
|
||
disconnected := 0
|
||
for _, st := range rep.StorageTargets {
|
||
if st.State == "disconnected" {
|
||
disconnected++
|
||
}
|
||
}
|
||
if disconnected > 0 {
|
||
h.logger.Printf("[WARN] host %s reports %d disconnected storage target(s) of %d",
|
||
hostID, disconnected, len(rep.StorageTargets))
|
||
}
|
||
|
||
// restore_tests (slice 6): a FAILED self-restore-test is the loudest DR signal there is
|
||
// — surface it prominently. A PASS that carried start warnings (e.g. the systemd-nesting
|
||
// advisory) is surfaced too: INFO when every warning is recognized-benign, escalated to
|
||
// WARN when an UNRECOGNIZED warning stood out (as loud as a failed PBS verify is for
|
||
// backups), so a real restore warning can't hide behind a green pass. A backup whose
|
||
// vzdump failed is also worth a warning.
|
||
for _, rt := range rep.RestoreTests {
|
||
switch {
|
||
case !rt.Pass:
|
||
h.logger.Printf("[WARN] host %s restore-test FAILED: archive=%s tier=%s scratch=%d err=%q",
|
||
hostID, rt.SourceArchive, rt.SourceTier, rt.ScratchVMID, rt.Error)
|
||
case len(rt.Warnings) == 0:
|
||
// clean pass — nothing to surface here (counted in the summary line below).
|
||
case rt.WarningsRecognized:
|
||
h.logger.Printf("[INFO] host %s restore-test passed WITH WARNINGS (recognized): archive=%s tier=%s warnings=%v",
|
||
hostID, rt.SourceArchive, rt.SourceTier, rt.Warnings)
|
||
default:
|
||
h.logger.Printf("[WARN] host %s restore-test passed WITH UNRECOGNIZED WARNINGS: archive=%s tier=%s warnings=%v",
|
||
hostID, rt.SourceArchive, rt.SourceTier, rt.Warnings)
|
||
}
|
||
}
|
||
if rep.ForeignKeyArchives != nil {
|
||
h.noteForeignKeyArchives(custID, hostID, rep.ForeignKeyArchives.Tiers)
|
||
}
|
||
for _, bk := range rep.Backups {
|
||
if !bk.Success {
|
||
h.logger.Printf("[WARN] host %s backup FAILED: target=%s vmid=%d err=%q",
|
||
hostID, bk.TargetID, bk.VMID, bk.Error)
|
||
}
|
||
}
|
||
// pbs_snapshots (slice 6 Phase B): a FAILED PBS verify is the loudest offsite-DR signal.
|
||
for _, ps := range rep.PBSSnapshots {
|
||
if ps.VerifyState == "failed" {
|
||
h.logger.Printf("[WARN] host %s PBS verify FAILED: %s/%s ns=%s owner=%s",
|
||
hostID, ps.BackupType, ps.BackupID, ps.Namespace, ps.Owner)
|
||
}
|
||
}
|
||
|
||
h.logger.Printf("[INFO] host-report from %s (%d guests, %d storage targets, %d backups, %d restore-tests, %d pbs-snapshots, %d bytes)",
|
||
hostID, len(rep.Guests), len(rep.StorageTargets), len(rep.Backups), len(rep.RestoreTests), len(rep.PBSSnapshots), len(body))
|
||
|
||
// Agent log tail (v0.46.0) — the debug-ring bundle a prior envelope requested.
|
||
// SaveLogBundle runs the secret gate + clears the pending request (consume-once).
|
||
// A failure must NOT drop the heartbeat; just warn.
|
||
if rep.LogTail != nil {
|
||
blocked, berr := h.store.SaveLogBundle(hostID, store.LogBundleComponentAgent, rep.LogTail.CollectedAt, rep.LogTail.Lines)
|
||
switch {
|
||
case berr != nil:
|
||
h.logger.Printf("[WARN] Failed to save agent log bundle for %s: %v", hostID, berr)
|
||
case blocked:
|
||
h.logger.Printf("[WARN] agent log bundle for %s BLOCKED: possible secret in log content — nothing stored", hostID)
|
||
default:
|
||
h.logger.Printf("[INFO] agent log bundle received for %s (%d lines)", hostID, len(rep.LogTail.Lines))
|
||
}
|
||
}
|
||
|
||
// DR recipe — persist the agent's secret-free storage/guest/PBS half (preserving any app half).
|
||
// A failure here must NOT drop the heartbeat (the report already saved); just warn.
|
||
if len(rep.DRRecipe) > 0 && custID != "" {
|
||
var ver drRecipeVersionOnly
|
||
_ = json.Unmarshal(rep.DRRecipe, &ver)
|
||
if err := h.store.SaveDRRecipeHostHalf(custID, hostID, ver.RecipeVersion, rep.DRRecipe); err != nil {
|
||
h.logger.Printf("[WARN] Failed to save DR-recipe host-half for customer %s (host %s): %v", custID, hostID, err)
|
||
} else {
|
||
h.logger.Printf("[INFO] DR-recipe host-half stored for customer %s (host %s, v%d)", custID, hostID, ver.RecipeVersion)
|
||
}
|
||
}
|
||
|
||
blocked := false
|
||
if cc, err := h.store.GetCustomerConfig(custID); err == nil && cc != nil && cc.Status == "blocked" {
|
||
blocked = true
|
||
}
|
||
|
||
// Control envelope (slice 10A): the cheap change-notification. desired_generation is the
|
||
// host's current generation (the agent re-fetches the full desired-state only when it
|
||
// advances past its cached one); has_signed_ops flags a non-empty signed-jobs queue (the
|
||
// agent fetches/executes them in 10B). Both degrade safely to their slice-4 defaults on a
|
||
// store error — a heartbeat must never fail on the control channel.
|
||
var desiredGen int64
|
||
if host, err := h.store.GetHost(hostID); err == nil && host != nil {
|
||
desiredGen = host.DesiredGeneration
|
||
}
|
||
hasSignedOps := false
|
||
if n, err := h.store.CountSignedJobs(hostID); err == nil && n > 0 {
|
||
hasSignedOps = true
|
||
}
|
||
resp := map[string]interface{}{
|
||
"status": "ok",
|
||
"poll_interval_seconds": defaultHostPollSeconds,
|
||
"blocked": blocked,
|
||
"desired_generation": desiredGen,
|
||
"has_signed_ops": hasSignedOps,
|
||
}
|
||
// v0.46.0 — pending agent log pull: the NEXT heartbeat carries log_tail (agent ≥
|
||
// 0.83.0; older agents ignore the flag and the request stays visibly pending).
|
||
if pending, err := h.store.PendingLogBundleRequest(hostID, store.LogBundleComponentAgent); err == nil && pending {
|
||
resp["log_tail_requested"] = true
|
||
}
|
||
// The agent's operator actions (2026-10-29): the results this report carries close their rows (one event each);
|
||
// what is still pending rides this reply. A store error drops nothing from the heartbeat.
|
||
h.ingestHostOperatorActionResults(hostID, custID, rep.OperatorActionResults)
|
||
if acts, err := h.store.PendingHostOperatorActions(hostID); err != nil {
|
||
h.logger.Printf("[WARN] host operator actions for %s: %v", hostID, err)
|
||
} else if len(acts) > 0 {
|
||
resp["operator_actions"] = acts
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// handleAdminCreateHost mints a host identity (host_id + per-host api_key).
|
||
//
|
||
// PROVISIONAL (slice-3 bootstrap): global-key only, so the demo agent can
|
||
// authenticate before enrollment (slices 7–8) exists. Enrollment will mint host
|
||
// identity + pin signing keys; this endpoint should be removed/locked down then
|
||
// (tracked under doc 05 §11 auth-tightening at cutover).
|
||
func (h *Handler) handleAdminCreateHost(w http.ResponseWriter, r *http.Request) {
|
||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok || !isGlobal {
|
||
http.Error(w, "Forbidden: global key required", http.StatusForbidden)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var req struct {
|
||
CustomerID string `json:"customer_id"`
|
||
HostID string `json:"host_id"`
|
||
DisplayName string `json:"display_name"`
|
||
}
|
||
if err := json.Unmarshal(body, &req); err != nil || req.CustomerID == "" {
|
||
http.Error(w, "Invalid payload: customer_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
cc, err := h.store.GetCustomerConfig(req.CustomerID)
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cc == nil {
|
||
http.Error(w, "Unknown customer_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
hostID := req.HostID
|
||
if hostID == "" {
|
||
sfx, err := configgen.RandomHex(3) // 6 hex chars — human-legible for the demo
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
hostID = req.CustomerID + "-" + sfx
|
||
}
|
||
apiKey, err := configgen.RandomHex(32)
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if err := h.store.UpsertHost(&store.Host{HostID: hostID, CustomerID: req.CustomerID, APIKey: apiKey}); err != nil {
|
||
h.logger.Printf("[ERROR] Failed to mint host for %s: %v", req.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] provisional host mint: %s (customer %s)", hostID, req.CustomerID)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusCreated)
|
||
json.NewEncoder(w).Encode(map[string]string{"host_id": hostID, "api_key": apiKey})
|
||
}
|
||
|
||
// handleHostEnroll is the passphrase-authed, mint-once-reuse host enrollment for Day-0
|
||
// (option C, SPIKE-day0-firstboot-handshake-2026-06-26). It is the sibling of the
|
||
// global-key handleAdminCreateHost: the operator/host-bootstrap script carries ONLY the
|
||
// customer's retrieval passphrase (no global key in the field deploy path), POSTs the
|
||
// customer_id, and gets back the host credential — minted on first call, REUSED byte-for-
|
||
// byte on every subsequent call (so re-running the bootstrap never orphans a live agent's
|
||
// key). Auth (passphrase) is checked BEFORE any mint — a bad-auth call never writes a row.
|
||
// The proven GET /config/{id} controller pull and POST /admin/hosts escape hatch are
|
||
// untouched.
|
||
func (h *Handler) handleHostEnroll(w http.ResponseWriter, r *http.Request) {
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var req struct {
|
||
CustomerID string `json:"customer_id"`
|
||
}
|
||
if err := json.Unmarshal(body, &req); err != nil || req.CustomerID == "" {
|
||
http.Error(w, "Invalid payload: customer_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// Passphrase auth — mirrors handleConfigRetrieve exactly (header, 404-then-401 order,
|
||
// constant-time compare). Happens BEFORE any mint.
|
||
password := r.Header.Get("X-Retrieval-Password")
|
||
if password == "" {
|
||
http.Error(w, "Unauthorized: X-Retrieval-Password header required", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
cc, err := h.store.GetCustomerConfig(req.CustomerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] host-enroll: customer lookup failed for %s: %v", req.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cc == nil {
|
||
http.Error(w, "Not found", http.StatusNotFound)
|
||
return
|
||
}
|
||
// R-879: a sealed passphrase that did not open is a hub fault (500), never a wrong password.
|
||
if cc.SecretsUnreadable {
|
||
h.logger.Printf("[ERROR] customer %s: sealed secrets do not open (OFFSITE_SECRET_KEY)", cc.CustomerID)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if subtle.ConstantTimeCompare([]byte(password), []byte(cc.RetrievalPassword)) != 1 {
|
||
http.Error(w, "Unauthorized: invalid password", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
// Mint-once-reuse: an existing host for this customer is returned as-is (idempotent) — the SAME host id and
|
||
// key, so re-running the installer never orphans a live agent's key and a reinstalled box takes over the
|
||
// record. The clean-slate re-issues do NOT run here: they are operator session-only powers and the
|
||
// passphrase must not reach them (R-1154, reenroll.go). A reinstall over the old record is made LOUD — an
|
||
// operator-only warning mail saying to delete the old host and re-run, or re-issue by hand.
|
||
existing, err := h.store.GetHostByCustomer(req.CustomerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] host-enroll: host lookup failed for %s: %v", req.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if existing != nil {
|
||
// R-879: never re-serve an empty key because the seal did not open.
|
||
if existing.SecretsUnreadable || existing.APIKey == "" {
|
||
h.logger.Printf("[ERROR] host-enroll: host %s key unreadable (sealed key does not open)", existing.HostID)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
h.noteReenrollOverExistingHost(cc, existing, time.Now())
|
||
// R-1159: once the box uses its own key, the passphrase no longer re-serves the host key.
|
||
if closed, why := h.passphraseKeysClosed(req.CustomerID); closed {
|
||
h.refusePassphraseKeys(w, r, req.CustomerID, "POST /host-enroll", why)
|
||
return
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(map[string]string{"host_id": existing.HostID, "api_key": existing.APIKey})
|
||
return
|
||
}
|
||
|
||
// First enroll: mint (mirrors handleAdminCreateHost's mint block).
|
||
sfx, err := configgen.RandomHex(3) // 6 hex chars — host_id suffix
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
hostID := req.CustomerID + "-" + sfx
|
||
apiKey, err := configgen.RandomHex(32)
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if err := h.store.UpsertHost(&store.Host{HostID: hostID, CustomerID: req.CustomerID, APIKey: apiKey}); err != nil {
|
||
h.logger.Printf("[ERROR] host-enroll: failed to mint host for %s: %v", req.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] host enrolled: %s (customer %s)", hostID, req.CustomerID)
|
||
|
||
// F2 (v0.57.0) — clean-slate reinstall of an existing customer. This mint path fires exactly
|
||
// once per fresh host record (the operator deleted the stale host, so re-enroll mints), so it is
|
||
// the single-shot re-enroll hook — and the operator's delete is its gate (R-1154: the reuse path
|
||
// above never re-issues; it tells the operator to delete). For a CLAIMED customer the fresh box has no
|
||
// password; auto-issue a reset code so the customer isn't stranded at the claim page hunting for
|
||
// the manual "request a new code" button (delivery rides the report ACK). No-op for an unclaimed
|
||
// customer (first provision). Also re-issue the off-site login (F3) — since hub v0.127.0 that is the
|
||
// HUB's own Storage Box login, not something the fresh box receives (R-1158; see offsiteReissuer).
|
||
h.reissueOnReenroll(cc)
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusCreated)
|
||
json.NewEncoder(w).Encode(map[string]string{"host_id": hostID, "api_key": apiKey})
|
||
}
|
||
|
||
// reissueOnReenroll runs the F2+F3 clean-slate re-enrollment side effects for a customer whose box
|
||
// was wiped and re-minted a host record: re-issue the claim code (if claimed) and re-issue offsite
|
||
// credentials (if provisioned). Every action that fires emits a visible customer event. Best-effort
|
||
// and non-fatal — a failure here never fails the enrollment (the box is already minted).
|
||
func (h *Handler) reissueOnReenroll(cc *store.CustomerConfig) {
|
||
// F2 — claim continuity.
|
||
if h.claimEngine != nil {
|
||
if gen, reissued, err := h.claimEngine.ReissueForReenroll(cc); err != nil {
|
||
h.logger.Printf("[WARN] claim re-issue on re-enroll for %s failed: %v", cc.CustomerID, err)
|
||
} else if reissued {
|
||
if _, serr := h.store.SaveEvent(cc.CustomerID, "claim_reissued_reenroll", "info",
|
||
fmt.Sprintf("Új beállító kódot küldtünk a szerver újratelepítése után (%d. generáció) az ügyfél címére.", gen),
|
||
"", "hub"); serr != nil {
|
||
h.logger.Printf("[WARN] save claim_reissued_reenroll for %s: %v", cc.CustomerID, serr)
|
||
}
|
||
}
|
||
}
|
||
// F3 — the off-site login: a fresh Storage Box sub-account password, stored for the HUB's registrar
|
||
// (R-1158: no box receives it since hub v0.127.0; the fresh box registers its own key with the
|
||
// controller API key). ⚠ CORRECTED 2026-08-04 (R-196): this
|
||
// used to claim "the re-issuer resets the restic repo password, which makes the OLD escrow blob
|
||
// stale". It does not and cannot — the re-issuer resets the PROVIDER account password; the
|
||
// repository password is generated on the box and never leaves it except sealed under R. The
|
||
// provisioner does mark the escrow stale, but PRECAUTIONARILY (see the reasoning at
|
||
// offsite.ReissueCredentials), not because this call rotated anything.
|
||
// Skips silently when offsite isn't provisioned/enabled.
|
||
if h.offsiteReissuer != nil {
|
||
if err := h.offsiteReissuer(context.Background(), cc.CustomerID); err != nil {
|
||
h.logger.Printf("[WARN] offsite re-issue on re-enroll for %s failed: %v", cc.CustomerID, err)
|
||
}
|
||
}
|
||
// Direction-2 (v0.63.0): wake a long-polling controller so the re-staged claim code / offsite
|
||
// password ride the next ACK in seconds, not on the 15-min cycle. Both legs above are
|
||
// best-effort; an over-bump costs one cheap wake. (On the clean-slate path the controller
|
||
// usually does not exist yet — its startup fetch covers that shape; a bump landing during a
|
||
// fresh controller's FIRST hold is recorded as baseline without firing — the known open
|
||
// observation, fixed later by carrying intent_gen in the report ACK. Out of scope here.)
|
||
if h.intentHub != nil {
|
||
h.intentHub.Bump(cc.CustomerID)
|
||
}
|
||
}
|
||
|
||
// escrowUploadRequest is the agent→hub wire shape for the OPAQUE PBS recovery-code escrow blob
|
||
// (slice 7, doc 03 §8a). It MUST stay in lockstep with the agent's emit struct
|
||
// (felhom-agent cmd/felhom-agent escrowUploadRequest). The hub stores the bytes and NEVER decrypts
|
||
// them (it has no recovery code).
|
||
type escrowUploadRequest struct {
|
||
BlobB64 string `json:"blob_b64"` // base64 of the opaque R-wrapped blob (ciphertext)
|
||
KeyFingerprint string `json:"key_fingerprint"` // for operator display only
|
||
Posture string `json:"posture"` // e.g. "zero_knowledge"
|
||
CreatedAt string `json:"created_at"` // RFC3339
|
||
// Slice 10D.1 — optional identity escrow, stored alongside the K-escrow (opaque). An old agent may still send a
|
||
// `directive`; it is ignored (R-105, decision 169 — retired, never read).
|
||
IdentityBlobB64 string `json:"identity_blob_b64,omitempty"` // age-wrapped {tunnel_token, pbs_token}
|
||
// SLICE 3 — sha256 hex of the restic repo password sealed in the identity blob (non-reversible hash
|
||
// of a 256-bit random secret — safe to store/serve; present only when a staged password was folded in).
|
||
ResticPwSHA256 string `json:"restic_pw_sha256,omitempty"`
|
||
}
|
||
|
||
// handleHostEscrowPut stores a host's opaque escrow blob (doc 03 §8a). Authed with the PER-HOST key
|
||
// (a host may only write its own escrow; the global operator key is also accepted). The hub keeps
|
||
// the ciphertext and never opens it. Last-write-wins (rotation). No serving this slice (slice 10).
|
||
func (h *Handler) handleHostEscrowPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// A per-host key may only write ITS OWN escrow; the global key may write any.
|
||
if !isGlobal && authHostID != pathHostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) // 1 MB cap; the blob is ~hundreds of bytes
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var req escrowUploadRequest
|
||
if err := json.Unmarshal(body, &req); err != nil || req.BlobB64 == "" {
|
||
http.Error(w, "Invalid payload: blob_b64 required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
blob, err := base64.StdEncoding.DecodeString(req.BlobB64)
|
||
if err != nil || len(blob) == 0 {
|
||
http.Error(w, "Invalid payload: blob_b64 not valid base64", http.StatusBadRequest)
|
||
return
|
||
}
|
||
createdAt := req.CreatedAt
|
||
if createdAt == "" {
|
||
createdAt = time.Now().UTC().Format(time.RFC3339)
|
||
}
|
||
// Slice 10D.1: optionally the IDENTITY escrow blob alongside the K-escrow (opaque — no usable secret hub-side).
|
||
// R-1254: decoded BEFORE any write (a bad one used to 400 after the new key was already stored) and saved in the
|
||
// SAME transaction as the key and its hash — all of the new package or all of the old. The directive is retired
|
||
// (R-105).
|
||
var idBlob []byte
|
||
if req.IdentityBlobB64 != "" {
|
||
b, derr := base64.StdEncoding.DecodeString(req.IdentityBlobB64)
|
||
if derr != nil || len(b) == 0 {
|
||
http.Error(w, "Invalid payload: identity_blob_b64 not valid base64", http.StatusBadRequest)
|
||
return
|
||
}
|
||
idBlob = b
|
||
}
|
||
// Store the OPAQUE bytes. No decrypt path exists — the hub cannot open this. Part B (v0.60.0):
|
||
// when this upload supersedes a DIFFERENT-passphrase old blob, the old one is RETAINED (not
|
||
// overwritten) so its recovery-code-recoverable history survives (Viktor's data-first ruling).
|
||
superseded, prevPwSHA, serr := h.store.SaveHostEscrowWithIdentity(pathHostID, blob, idBlob, req.KeyFingerprint, req.Posture, createdAt, req.ResticPwSHA256)
|
||
if errors.Is(serr, store.ErrEscrowWouldDropRepoKey) {
|
||
// R-1218: the current package seals the off-site repository password and this one does not — keeping the good
|
||
// one current is the whole point; the box shows the ceremony as failed instead of silently weakening the code.
|
||
h.logger.Printf("[WARN] escrow for host %s REFUSED: the upload seals no off-site repository password while the current package does — the current package stays (R-1218)", pathHostID)
|
||
http.Error(w, "Refused: the new recovery package covers no off-site key while the stored one does; the stored package stays (R-1218)", http.StatusConflict)
|
||
return
|
||
}
|
||
if serr != nil {
|
||
h.logger.Printf("[ERROR] Failed to store escrow for host %s: %v", pathHostID, serr)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if superseded {
|
||
n, _ := h.store.CountSupersededEscrow(pathHostID)
|
||
h.logger.Printf("[INFO] escrow for host %s superseded an escrow with a different passphrase or backup key (R-366) — RETAINED (now %d superseded blob(s) held)", pathHostID, n)
|
||
// Hub-internal audit event (not gated by allowedEventTypes) — tied to the owning customer.
|
||
if host, herr := h.store.GetHost(pathHostID); herr == nil && host != nil && host.CustomerID != "" {
|
||
details, _ := json.Marshal(map[string]any{"host_id": pathHostID, "retained_count": n})
|
||
if _, eerr := h.store.SaveEvent(host.CustomerID, "escrow_superseded", "info",
|
||
"A korábbi helyreállítási csomag megőrizve (új kulcs érkezett).", string(details), "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] escrow_superseded event save failed for %s: %v", pathHostID, eerr)
|
||
}
|
||
// R-197: the box's offsite DATA key demonstrably changed. Both halves of that comparison
|
||
// have been stored since SLICE 3 and nothing read them — demo-felhom's key changed on
|
||
// 2026-08-03 and nothing said so for thirteen hours.
|
||
h.maybeEmitRepoKeyChanged(host.CustomerID, pathHostID, prevPwSHA, req.ResticPwSHA256, n)
|
||
}
|
||
}
|
||
if idBlob != nil {
|
||
h.logger.Printf("[INFO] stored DR bundle for host %s (identity %d bytes, same transaction as the key — R-1254)", pathHostID, len(idBlob))
|
||
}
|
||
h.logger.Printf("[INFO] stored opaque escrow blob for host %s (%d bytes, posture=%s, fp=%s)",
|
||
pathHostID, len(blob), req.Posture, req.KeyFingerprint)
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok"}`))
|
||
}
|
||
|
||
// eventRepoKeyChanged (R-197) — the box's offsite restic REPOSITORY password changed, proven by the
|
||
// hub's own stored hashes. Hub-internal (not in allowedEventTypes, like escrow_superseded) and
|
||
// registered operator-only in notify.operatorOnlyEvents.
|
||
const eventRepoKeyChanged = "offsite_repo_key_changed"
|
||
|
||
// eventForeignKeyArchives (R-366 slice 2, `09` §3 decision 168) — the box's whole-guest copies include archives an
|
||
// earlier install wrote with another key, which this box cannot open. Hub-internal, severity info: ONE line on the
|
||
// operator's timeline per change of the set (not per archive, not per report); never mailed, never the household's.
|
||
const eventForeignKeyArchives = "restore_test_foreign_key_archives"
|
||
|
||
// noteForeignKeyArchives turns a CHANGE of the reported set into one operator event (the caller skips an absent
|
||
// stanza, which keeps the state); an empty set clears it without an event. Pinned by TestR366_ForeignKeyArchives*.
|
||
func (h *Handler) noteForeignKeyArchives(customerID, hostID string, set []hostForeignKeyArchives) {
|
||
parts := make([]string, 0, len(set))
|
||
for _, f := range set {
|
||
if f.Count > 0 {
|
||
parts = append(parts, fmt.Sprintf("%s:%d:%s:%s", f.Target, f.Count, f.Oldest, f.Newest))
|
||
}
|
||
}
|
||
sort.Strings(parts)
|
||
sig := strings.Join(parts, ";")
|
||
if sig == h.store.ForeignKeyArchiveSignature(hostID) {
|
||
return
|
||
}
|
||
if err := h.store.SetForeignKeyArchiveSignature(hostID, sig); err != nil {
|
||
h.logger.Printf("[WARN] host %s: storing the foreign-key archive state failed: %v", hostID, err)
|
||
return
|
||
}
|
||
if sig == "" {
|
||
h.logger.Printf("[INFO] host %s: no whole-guest archive written with another key any more (R-366)", hostID)
|
||
return
|
||
}
|
||
var lines []string
|
||
total := 0
|
||
for _, f := range set {
|
||
if f.Count > 0 {
|
||
total += f.Count
|
||
lines = append(lines, fmt.Sprintf("%d on %s (%s … %s)", f.Count, f.Target, dayOf(f.Oldest), dayOf(f.Newest)))
|
||
}
|
||
}
|
||
retained, _ := h.store.CountSupersededEscrow(hostID)
|
||
msg := fmt.Sprintf("Host %s: %d whole-guest backup archive(s) were written with ANOTHER key (an earlier install of this box) — %s. "+
|
||
"This box cannot open them, so its restore test skips them. The hub holds %d retained escrow blob(s) for this host; "+
|
||
"the server-side prune removes the old archives as new copies land (R-366).", hostID, total, strings.Join(lines, "; "), retained)
|
||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "archives": set, "retained_escrow": retained})
|
||
if _, err := h.store.SaveEvent(customerID, eventForeignKeyArchives, "info", msg, string(details), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save failed for %s: %v", eventForeignKeyArchives, hostID, err)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] host %s: %d whole-guest archive(s) written with another key — operator event recorded (R-366)", hostID, total)
|
||
}
|
||
|
||
// dayOf returns the date part of an RFC3339 time ("" stays "").
|
||
func dayOf(ts string) string {
|
||
if len(ts) >= 10 {
|
||
return ts[:10]
|
||
}
|
||
return ts
|
||
}
|
||
|
||
// eventEscrowBlobServed (R-199) — a host retrieved its own sealed identity blob. Hub-internal,
|
||
// operator-only. See handleHostEscrowGet for why every retrieval is loud.
|
||
const eventEscrowBlobServed = "escrow_blob_served"
|
||
|
||
// escrowSelfServiceRetrieval is THE SINGLE DECISION POINT for the §8.2/§8.3 trade (R-199).
|
||
//
|
||
// true (§8.2, shipped v0.94.0) — a host may read its own blob whenever it authenticates as itself.
|
||
// false (§8.3, the fallback) — the same read additionally requires operator-armed recovery mode.
|
||
//
|
||
// It is one condition on purpose: the operator may overrule the trade below, and switching must cost a
|
||
// boolean rather than a redesign. Everything else in the recovery chain is identical either way.
|
||
const escrowSelfServiceRetrieval = true
|
||
|
||
// handleHostEscrowGet serves a host its OWN opaque identity-escrow blob (R-199, v0.94.0).
|
||
//
|
||
// WHAT THIS GIVES OUT, WHY IT IS SAFE, AND WHAT IT CHANGES ABOUT WHO IS REQUIRED — recorded here so the
|
||
// next reader finds the trade rather than inferring it (the dr.go header convention).
|
||
//
|
||
// WHAT: the age-wrapped `IdentityBundle` — opaque ciphertext. It carries the offsite restic repository
|
||
// password, the tunnel token, the PBS token and the WG key. The hub stores these bytes and has no
|
||
// decrypt path; the recovery code R that opens them exists only in the customer's hands.
|
||
//
|
||
// WHY IT IS SAFE TO GIVE OUT: the blob is useless without R (age scrypt + ChaCha20-Poly1305; a wrong R
|
||
// fails closed at the KDF, never to a plausible-but-wrong bundle), and a 10-word EFF code carries ~129
|
||
// bits. The caller already authenticates as this host for its report, its desired state, its WG
|
||
// registration and its PBS token — this adds no new identity, only a new object, and it is the exact
|
||
// MIRROR of the PUT above, which is how the blob got here in the first place.
|
||
//
|
||
// WHAT IT CHANGES, STATED PLAINLY BECAUSE IT IS THE WHOLE OF THE TRADE: before this, obtaining the blob
|
||
// required the OPERATOR to arm recovery mode with the global key (dr.go). Now whoever controls a
|
||
// rebuilt box can obtain it with that box's own credential. That is a real reduction in the number of
|
||
// parties required. They still cannot open it. The mitigation is that the capability is AUDITED rather
|
||
// than silent: every successful retrieval raises an operator event (below), because a silent capability
|
||
// is the shape this project has spent two weeks removing.
|
||
//
|
||
// THE OPERATOR-DRIVEN DR PATH IS UNTOUCHED. `handleReEnroll` / `handleGetRestoreDirective` keep their
|
||
// recovery-mode gate and their global-key arming, and they serve the K-escrow and the directive as
|
||
// well. This endpoint serves ONE object to ONE authenticated owner. Do not merge them.
|
||
func (h *Handler) handleHostEscrowGet(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// SELF-SCOPED: a per-host key reads only its OWN escrow. The global operator key may read any —
|
||
// the same asymmetry the PUT has. Without this line any host key is a fleet-wide blob reader.
|
||
if !isGlobal && authHostID != pathHostID {
|
||
h.logger.Printf("[WARN] escrow GET REFUSED: host %s asked for %s's blob (self-scope)", authHostID, pathHostID)
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
host, err := h.store.GetHost(pathHostID)
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if host == nil {
|
||
http.Error(w, "Unknown host_id", http.StatusNotFound)
|
||
return
|
||
}
|
||
// §8.3 fallback lives here and nowhere else.
|
||
if !escrowSelfServiceRetrieval && !host.InRecoveryMode(time.Now().UTC()) {
|
||
h.logger.Printf("[WARN] escrow GET REFUSED for %s — self-service retrieval is disabled and recovery mode is not armed", pathHostID)
|
||
http.Error(w, "Forbidden: host not in recovery mode (operator must arm it)", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
bundle, berr := h.store.GetHostDRBundle(pathHostID)
|
||
if berr != nil {
|
||
h.logger.Printf("[ERROR] escrow GET for %s: %v", pathHostID, berr)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
// A host with no sealed bundle gets a CLEAN ANSWER, not a fault: 200 with present=false. A 404
|
||
// here would be indistinguishable from an unknown host, and an empty 200 without the flag would be
|
||
// indistinguishable from a zero-length blob — both read as "something is broken" to a caller whose
|
||
// situation is simply "no ceremony has run yet".
|
||
if bundle == nil || len(bundle.IdentityBlob) == 0 {
|
||
h.logger.Printf("[INFO] escrow GET for %s: no identity blob stored (no ceremony has run)", pathHostID)
|
||
writeJSON(w, http.StatusOK, map[string]any{"host_id": pathHostID, "present": false, "identity_escrow_b64": ""})
|
||
return
|
||
}
|
||
|
||
// THE MITIGATION (§8.2). Recorded BEFORE the bytes leave, so a retrieval cannot be served without
|
||
// its audit row; a save failure is logged and does NOT block the response (the blob is opaque and
|
||
// refusing it would break a recovery over an audit hiccup — but the log line always exists).
|
||
//
|
||
// SEVERITY = warning, i.e. it reaches the operator by e-mail. Retrieval is not routine today: it
|
||
// happens during a recovery and nowhere else. IF a customer-facing self-service flow ever makes it
|
||
// routine, revisit this — but revisit it deliberately, do not let it decay to info because the
|
||
// mail became annoying.
|
||
if host.CustomerID != "" {
|
||
msg := fmt.Sprintf("Recovery blob served: host %s retrieved its own sealed identity escrow (%d opaque bytes). "+
|
||
"This is the recovery path in use — the blob cannot be opened without the customer's recovery code, which the hub never holds. "+
|
||
"If no recovery is in progress on that box, investigate.", pathHostID, len(bundle.IdentityBlob))
|
||
details, _ := json.Marshal(map[string]any{
|
||
"host_id": pathHostID,
|
||
"blob_bytes": len(bundle.IdentityBlob),
|
||
"self_scope": !isGlobal,
|
||
})
|
||
if _, eerr := h.store.SaveEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s (serving anyway): %v", eventEscrowBlobServed, pathHostID, eerr)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub")
|
||
}
|
||
}
|
||
h.logger.Printf("[WARN] escrow blob SERVED to host %s (%d opaque bytes, self_scope=%v) — recovery path in use",
|
||
pathHostID, len(bundle.IdentityBlob), !isGlobal)
|
||
writeJSON(w, http.StatusOK, map[string]any{
|
||
"host_id": pathHostID,
|
||
"present": true,
|
||
"identity_escrow_b64": base64.StdEncoding.EncodeToString(bundle.IdentityBlob),
|
||
})
|
||
}
|
||
|
||
// handleHostEscrowRetainedGet serves a host its own RETAINED (superseded) sealed identity packages.
|
||
//
|
||
// ── WHY THIS EXISTS (R-311) ─────────────────────────────────────────────────────────────────────
|
||
//
|
||
// `ListSupersededEscrow` has been the only reader of a retained identity blob since v0.93.0 and had
|
||
// ZERO production callers — every call site was a test. The consequence, measured on the fixture
|
||
// 2026-08-12: a recovery code that demonstrably opens a retained package (proven by hand: unsealed,
|
||
// and it restored planted files byte-identical from a store the box could no longer open) was
|
||
// reported to the customer as a code that opened nothing. The screen already hedged that with two
|
||
// possible causes (R-222/R-226) and said it could not tell them apart. **It could not tell them
|
||
// apart because nothing ever looked.** This endpoint is what makes looking possible.
|
||
//
|
||
// ── WHAT THIS DOES NOT CHANGE ───────────────────────────────────────────────────────────────────
|
||
//
|
||
// The hub still cannot read any of it. Sealed bytes in, sealed bytes out; there is no decrypt path
|
||
// here and the hub has never held a recovery code. What DOES widen is volume: a host key that could
|
||
// previously fetch one opaque package can now fetch N. The trade is stated rather than waved through
|
||
// — see the audit — and it is bounded three ways: the same self-scope as the current row, the same
|
||
// recovery-mode gate, and an explicit cap so a host with a long supersession history cannot turn one
|
||
// request into an unbounded read.
|
||
//
|
||
// Rows whose `identity_blob` is NULL are NOT served and are counted separately. They are the
|
||
// pre-v0.93.0 rows; they retain the PBS key and not the repository password, so they can never open
|
||
// anything the caller is asking about. Serving them would make the agent try packages that cannot
|
||
// succeed and would let the screen claim an earlier package is openable when it is not — the same
|
||
// false-explanation trap `SupersededPresent` already avoids at store.go. They are counted because
|
||
// their EXISTENCE is a true and load-bearing fact: on those boxes a correct old code opens nothing,
|
||
// and the honest reason is a defect of ours, not the customer's typing.
|
||
const retainedEscrowServeCap = 16
|
||
|
||
// RetainedEscrowPackage / RetainedEscrowResponse are the WIRE for GET /hosts/<id>/escrow/retained.
|
||
//
|
||
// They are named types rather than a `map[string]any` on purpose: the wire-contract gate resolves a
|
||
// declared ROOT by TYPE, so an untyped map is a cross-repo contract the gate cannot see. This wire is
|
||
// declared in `scripts/wire_contract_gate.py` ROOTS — hub → agent — and the agent's mirror is
|
||
// `felhom-agent/internal/hub.RetainedEscrowResponse`. Change a tag here and the gate fails there,
|
||
// which is the entire point.
|
||
type RetainedEscrowPackage struct {
|
||
// Index is a label WITHIN ONE RESPONSE. Not durable, never persisted, never a lookup key.
|
||
Index int `json:"index"`
|
||
// SupersededAt is when this package stopped being the current one. It is the only field here a
|
||
// customer ever sees — it is how they recognise which recovery code they are holding.
|
||
SupersededAt string `json:"superseded_at"`
|
||
KeyFingerprint string `json:"key_fingerprint"`
|
||
// IdentityEscrowB64 is OPAQUE ciphertext. The hub cannot open it and never could.
|
||
IdentityEscrowB64 string `json:"identity_escrow_b64"`
|
||
}
|
||
|
||
type RetainedEscrowResponse struct {
|
||
HostID string `json:"host_id"`
|
||
Count int `json:"count"`
|
||
// UnopenableCount is the number of retained rows withheld because they carry no key material
|
||
// (every pre-v0.93.0 row). Underivable by the caller and load-bearing: on a box with only those,
|
||
// a perfectly correct old code opens nothing and the reason is a defect of ours.
|
||
UnopenableCount int `json:"unopenable_count"`
|
||
TruncatedCount int `json:"truncated_count"`
|
||
Packages []RetainedEscrowPackage `json:"packages"`
|
||
}
|
||
|
||
func (h *Handler) handleHostEscrowRetainedGet(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// SELF-SCOPED, identically to the current-row GET. Without this line any host key is a fleet-wide
|
||
// reader of every retained package, which is strictly worse than the same hole on one row.
|
||
if !isGlobal && authHostID != pathHostID {
|
||
h.logger.Printf("[WARN] retained escrow GET REFUSED: host %s asked for %s's packages (self-scope)", authHostID, pathHostID)
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
host, err := h.store.GetHost(pathHostID)
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if host == nil {
|
||
http.Error(w, "Unknown host_id", http.StatusNotFound)
|
||
return
|
||
}
|
||
// The SAME §8.3 gate as the current row. A retained package is not less sensitive than the
|
||
// current one; if self-service retrieval is ever switched off, it must go dark with it.
|
||
if !escrowSelfServiceRetrieval && !host.InRecoveryMode(time.Now().UTC()) {
|
||
h.logger.Printf("[WARN] retained escrow GET REFUSED for %s — self-service retrieval is disabled and recovery mode is not armed", pathHostID)
|
||
http.Error(w, "Forbidden: host not in recovery mode (operator must arm it)", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
retained, rerr := h.store.ListSupersededEscrow(pathHostID)
|
||
if rerr != nil {
|
||
h.logger.Printf("[ERROR] retained escrow GET for %s: %v", pathHostID, rerr)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
packages := make([]RetainedEscrowPackage, 0, len(retained))
|
||
unopenable := 0
|
||
for _, e := range retained {
|
||
if len(e.IdentityBlob) == 0 {
|
||
unopenable++
|
||
continue
|
||
}
|
||
if len(packages) >= retainedEscrowServeCap {
|
||
continue
|
||
}
|
||
packages = append(packages, RetainedEscrowPackage{
|
||
Index: len(packages),
|
||
SupersededAt: e.UpdatedAt,
|
||
KeyFingerprint: e.KeyFingerprint,
|
||
IdentityEscrowB64: base64.StdEncoding.EncodeToString(e.IdentityBlob),
|
||
})
|
||
}
|
||
truncated := 0
|
||
if n := len(retained) - unopenable; n > len(packages) {
|
||
truncated = n - len(packages)
|
||
}
|
||
|
||
// The audit row is written BEFORE the bytes leave, exactly as the current-row GET does, so a
|
||
// retrieval cannot be served without its record. Severity matches that path deliberately: this is
|
||
// a recovery in progress and nothing else, and it must reach the operator by e-mail.
|
||
if host.CustomerID != "" && len(packages) > 0 {
|
||
msg := fmt.Sprintf("Retained recovery packages served: host %s retrieved %d retained sealed package(s) (%d unopenable pre-v0.93.0 row(s) withheld). "+
|
||
"This is the recovery path in use — the packages cannot be opened without the customer's recovery code, which the hub never holds. "+
|
||
"If no recovery is in progress on that box, investigate.", pathHostID, len(packages), unopenable)
|
||
details, _ := json.Marshal(map[string]any{
|
||
"host_id": pathHostID,
|
||
"served": len(packages),
|
||
"unopenable": unopenable,
|
||
"truncated": truncated,
|
||
"self_scope": !isGlobal,
|
||
})
|
||
if _, eerr := h.store.SaveEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s (serving anyway): %v", eventEscrowBlobServed, pathHostID, eerr)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(host.CustomerID, eventEscrowBlobServed, "warning", msg, string(details), "hub")
|
||
}
|
||
}
|
||
h.logger.Printf("[WARN] retained escrow SERVED to host %s (%d package(s), %d unopenable withheld, %d over cap, self_scope=%v)",
|
||
pathHostID, len(packages), unopenable, truncated, !isGlobal)
|
||
writeJSON(w, http.StatusOK, RetainedEscrowResponse{
|
||
HostID: pathHostID,
|
||
Count: len(packages),
|
||
UnopenableCount: unopenable,
|
||
TruncatedCount: truncated,
|
||
Packages: packages,
|
||
})
|
||
}
|
||
|
||
// maybeEmitRepoKeyChanged raises ONE operator signal per supersession when the sealed offsite repo
|
||
// password demonstrably changed. Both hashes have been stored since SLICE 3 (host_escrow and, since
|
||
// v0.60.0, host_escrow_superseded) and NOTHING compared them: demo-felhom's repository password
|
||
// changed on 2026-08-03, orphaning 36 snapshots / 1.14 GB, and no event, e-mail, card or log line
|
||
// said so for thirteen hours — the comparison that eventually found it is this one
|
||
// (audits/RECON-offsite-dr-chain-2026-08-04.md, R-197).
|
||
//
|
||
// THE PREDICATE IS DELIBERATELY NARROW: both hashes known AND different. A first-ever hash (prev "")
|
||
// is onboarding, not a change; a hash-less supersession (now "") cannot show a change happened; an
|
||
// identical hash is a re-ceremony of the SAME password, which is a normal healthy act and must stay
|
||
// silent or a customer is punished for re-running a ceremony. The in-between shapes are LOGGED rather
|
||
// than dropped, so "we chose not to alarm" and "the check did not run" never look identical.
|
||
//
|
||
// SEVERITY = warning, chosen for the world v0.93.0 creates rather than the one it inherits. Before
|
||
// R-198 a changed key meant the previous history was unopenable by anyone, ever — that would have
|
||
// argued for error. From v0.93.0 the superseding ceremony RETAINS the old identity blob, so the
|
||
// previous history stays recoverable with the recovery code that sealed it: the situation is "this
|
||
// customer's off-site history now depends on an older recovery code", which is operator-actionable
|
||
// (check the orphan card, expect a fresh repository) and is not a loss. warning also routes: the
|
||
// dispatcher notifies on warning/error/critical and treats info as an intentional non-notify, and
|
||
// the whole point of this row is that the operator learns on the day.
|
||
//
|
||
// EDGE-TRIGGERED: called only from the superseded branch of the escrow PUT, i.e. once per
|
||
// supersession, never per report. No timer lives here — the dispatcher owns cooldown.
|
||
//
|
||
// NO HASH VALUE TRAVELS. The message and the details name the host, the customer and the retained
|
||
// count only. The hashes are non-reversible, but a hash is still a fingerprint of a live secret and
|
||
// this project's rule is that values do not leave the store.
|
||
func (h *Handler) maybeEmitRepoKeyChanged(customerID, hostID, prevSHA, newSHA string, retained int) {
|
||
switch {
|
||
case prevSHA == "":
|
||
h.logger.Printf("[INFO] escrow for host %s: no previous repo-password hash recorded (first hash, or a legacy hash-less blob) — repo-key-change check not applicable", hostID)
|
||
return
|
||
case newSHA == "":
|
||
h.logger.Printf("[WARN] escrow for host %s: the NEW blob carries no repo-password hash (hash-less supersession) — whether the repository key changed CANNOT be determined from the hub's data", hostID)
|
||
return
|
||
case prevSHA == newSHA:
|
||
return // same password re-sealed: a healthy re-ceremony (Scenario E — silence is correct)
|
||
}
|
||
msg := fmt.Sprintf("Offsite repository key CHANGED for host %s: the new escrow seals a different repository password than the one it replaced. "+
|
||
"The previous off-site history is no longer opened by this box's current key. The superseding blob was retained (%d held), so that history stays recoverable with the recovery code that sealed it — verify the box's off-site tier reports a repository rather than an orphan card, and expect the next backup to start a fresh history.",
|
||
hostID, retained)
|
||
details, _ := json.Marshal(map[string]any{
|
||
"host_id": hostID,
|
||
"retained_count": retained,
|
||
"repo_key": "changed", // never the hash values
|
||
})
|
||
if _, err := h.store.SaveEvent(customerID, eventRepoKeyChanged, "warning", msg, string(details), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save failed for %s: %v", eventRepoKeyChanged, hostID, err)
|
||
return // audit row first: an e-mail without its event row lies (the OffsiteChecker convention)
|
||
}
|
||
h.logger.Printf("[WARN] offsite repository key CHANGED for host %s (customer %s) — previous history now depends on the superseded recovery code; %d retained blob(s)",
|
||
hostID, customerID, retained)
|
||
if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(customerID, eventRepoKeyChanged, "warning", msg, string(details), "hub")
|
||
}
|
||
}
|
||
|
||
// handleHostRecoveryCredentialPut vaults a host's break-glass root@pam console credential (TASK G1).
|
||
// SELF-SCOPED (a host key writes only its own; global may write any) — day-0 posts it with the
|
||
// host api_key. The secret is stored at rest and NEVER logged (only the username + a length are
|
||
// logged). This is the human fallback for when both the sshd path AND the agent-independent
|
||
// auto-heal have failed: the operator retrieves it to reach the PVE web console (pveproxy — a
|
||
// failure domain distinct from sshd).
|
||
func (h *Handler) handleHostRecoveryCredentialPut(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authHostID != pathHostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16)) // 64 KiB cap; a username+password is tiny
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var req struct {
|
||
Username string `json:"username"`
|
||
Password string `json:"password"`
|
||
}
|
||
if err := json.Unmarshal(body, &req); err != nil || req.Username == "" || req.Password == "" {
|
||
http.Error(w, "Invalid payload: username + password required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// The host must exist (mint-first) — a per-host key already proves it; the global path re-checks.
|
||
if isGlobal {
|
||
host, herr := h.store.GetHost(pathHostID)
|
||
if herr != nil || host == nil {
|
||
http.Error(w, "Unknown host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
}
|
||
if err := h.store.SaveHostRecoveryCredential(pathHostID, req.Username, req.Password); err != nil {
|
||
h.logger.Printf("[ERROR] Failed to vault recovery credential for host %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
// SECRET DISCIPLINE: log the username + a length only — NEVER the password.
|
||
h.logger.Printf("[INFO] vaulted break-glass recovery credential for host %s (user=%s, secret %d chars)",
|
||
pathHostID, req.Username, len(req.Password))
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok"}`))
|
||
}
|
||
|
||
// handleAdminGetRecoveryCredential returns a host's vaulted break-glass credential to the OPERATOR
|
||
// (global key only — a per-host key must NOT read its own console password back out). This is the
|
||
// authenticated retrieval path the break-glass runbook uses. The response body carries the secret by
|
||
// necessity; it is never written to the hub log.
|
||
func (h *Handler) handleAdminGetRecoveryCredential(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok || !isGlobal {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized) // operator/global key ONLY
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
cred, err := h.store.GetHostRecoveryCredential(pathHostID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to read recovery credential for host %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cred == nil {
|
||
http.Error(w, "No recovery credential vaulted for this host", http.StatusNotFound)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] operator retrieved break-glass recovery credential for host %s (user=%s)", pathHostID, cred.Username)
|
||
resp, _ := json.Marshal(map[string]string{
|
||
"host_id": cred.HostID,
|
||
"username": cred.Username,
|
||
"password": cred.Secret,
|
||
"set_at": cred.SetAt.UTC().Format(time.RFC3339),
|
||
})
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write(resp)
|
||
}
|
||
|
||
// handleGetDesiredState serves a host its authoritative desired-state (slice 10A). Per-host key,
|
||
// SELF-SCOPED: a host reads ONLY its own (the global operator key may read any). The agent fetches
|
||
// this when the heartbeat envelope's desired_generation has advanced past its cached one. The
|
||
// response carries the generation the state corresponds to, so the agent caches it atomically.
|
||
func (h *Handler) handleGetDesiredState(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authHostID != pathHostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
host, err := h.store.GetHost(pathHostID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] desired-state lookup for %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if host == nil {
|
||
http.Error(w, "Unknown host_id", http.StatusNotFound)
|
||
return
|
||
}
|
||
desired := host.DesiredJSON
|
||
if strings.TrimSpace(desired) == "" {
|
||
desired = "{}"
|
||
}
|
||
// S2: merge the hub-OWNED wireguard block at read time (no peer → pass-through unchanged;
|
||
// the stored operator blob is never modified). See api/wg.go mergeWireguard.
|
||
desired = h.mergeWireguard(pathHostID, desired)
|
||
desired = h.mergeOSUpdate(pathHostID, desired)
|
||
resp := map[string]interface{}{
|
||
"generation": host.DesiredGeneration,
|
||
"desired_state": json.RawMessage(desired), // opaque to the hub — agent owns the schema
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// handleGetJobs serves a host its pending signed-op blobs (slice 10A). Per-host key, SELF-SCOPED.
|
||
// The blobs are OPAQUE (the hub never forged or opened them); the agent verifies + executes them
|
||
// in 10B. 10A only serves the queue.
|
||
func (h *Handler) handleGetJobs(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authHostID != pathHostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
jobs, err := h.store.GetSignedJobs(pathHostID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] jobs lookup for %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
out := make([]map[string]string, 0, len(jobs))
|
||
for _, j := range jobs {
|
||
out = append(out, map[string]string{
|
||
"job_id": j.JobID,
|
||
"blob_b64": base64.StdEncoding.EncodeToString(j.Blob),
|
||
"created_at": j.CreatedAt,
|
||
})
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(map[string]interface{}{"jobs": out})
|
||
}
|
||
|
||
// handleDeleteJob clears a processed job from a host's queue (slice 10B). Per-host key,
|
||
// SELF-SCOPED (a host clears only its own jobs; the global key may clear any). Idempotent.
|
||
func (h *Handler) handleDeleteJob(w http.ResponseWriter, r *http.Request, pathHostID, jobID string) {
|
||
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if pathHostID == "" || jobID == "" {
|
||
http.Error(w, "Missing host_id or job_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authHostID != pathHostID {
|
||
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
removed, err := h.store.DeleteSignedJobN(pathHostID, jobID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] delete job %s for %s: %v", jobID, pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if removed > 0 {
|
||
// only a job that WAS in this box's queue can be told as refused: the id comes from the box, and a box must
|
||
// not mint operator events by clearing ids it invents. It also makes a repeated DELETE silent.
|
||
h.noteSignedJobOutcome(pathHostID, jobID, r.URL.Query().Get("outcome"), r.URL.Query().Get("reason"), r.URL.Query().Get("op"))
|
||
}
|
||
h.logger.Printf("[INFO] host %s cleared signed-op job %s (executed or rejected)", pathHostID, jobID)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok"}`))
|
||
}
|
||
|
||
// handleAdminSetDesiredState sets a host's desired-state (slice 10A). GLOBAL/operator key ONLY —
|
||
// a per-host key cannot author its own intent. The body is the desired-state JSON (opaque to the
|
||
// hub: it stores + serves bytes, never validates/interprets the schema — the agent/CLI owns it).
|
||
// Writing BUMPS desired_generation so the next heartbeat signals the agent to re-fetch.
|
||
func (h *Handler) handleAdminSetDesiredState(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok || !isGlobal {
|
||
http.Error(w, "Forbidden: global key required", http.StatusForbidden)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// Validate it is well-formed JSON (the hub does not interpret the schema, but a malformed
|
||
// blob would break the agent's parse — reject it at the door).
|
||
if !json.Valid(body) {
|
||
http.Error(w, "Invalid payload: body must be JSON", http.StatusBadRequest)
|
||
return
|
||
}
|
||
// S2: the wireguard block is HUB-owned, merged at read time — an operator copy-paste of a
|
||
// served desired-state must never write it back into the stored blob (it would go stale and
|
||
// shadow the live assignment). Reject at the door.
|
||
var topKeys map[string]json.RawMessage
|
||
if err := json.Unmarshal(body, &topKeys); err == nil {
|
||
if _, has := topKeys["wireguard"]; has {
|
||
http.Error(w, "wireguard is hub-owned; register via POST /hosts/{id}/wg", http.StatusBadRequest)
|
||
return
|
||
}
|
||
}
|
||
gen, err := h.store.SetHostDesired(pathHostID, body)
|
||
if err == sql.ErrNoRows {
|
||
http.Error(w, "Unknown host_id", http.StatusNotFound)
|
||
return
|
||
}
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] set desired-state for %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] admin-set desired-state for host %s (generation now %d, %d bytes)", pathHostID, gen, len(body))
|
||
if h.poker != nil {
|
||
h.poker.PokeHost(pathHostID) // agent-plane immediate-sync (Direction-2a): generation bumped → nudge the box now
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok", "generation": gen})
|
||
}
|
||
|
||
// handleAdminEnqueueJob appends an opaque signed-op blob to a host's queue (slice 10A). GLOBAL key
|
||
// ONLY. The blob is pre-signed off-hub (the hub holds no signing key); the hub stores it verbatim.
|
||
// This is the minimal operator path to seed the queue so HasSignedOps/serving are exercisable; the
|
||
// rich operator/signing UX is later. Execution is 10B.
|
||
func (h *Handler) handleAdminEnqueueJob(w http.ResponseWriter, r *http.Request, pathHostID string) {
|
||
_, _, isGlobal, ok := h.checkAuthHost(r)
|
||
if !ok || !isGlobal {
|
||
http.Error(w, "Forbidden: global key required", http.StatusForbidden)
|
||
return
|
||
}
|
||
if pathHostID == "" {
|
||
http.Error(w, "Missing host_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
host, err := h.store.GetHost(pathHostID)
|
||
if err != nil || host == nil {
|
||
http.Error(w, "Unknown host_id", http.StatusNotFound)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var req struct {
|
||
JobID string `json:"job_id"`
|
||
BlobB64 string `json:"blob_b64"`
|
||
}
|
||
if err := json.Unmarshal(body, &req); err != nil || req.BlobB64 == "" {
|
||
http.Error(w, "Invalid payload: blob_b64 required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
blob, err := base64.StdEncoding.DecodeString(req.BlobB64)
|
||
if err != nil || len(blob) == 0 {
|
||
http.Error(w, "Invalid payload: blob_b64 not valid base64", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if req.JobID == "" {
|
||
req.JobID, _ = configgen.RandomHex(8)
|
||
}
|
||
if err := h.store.EnqueueSignedJob(pathHostID, req.JobID, blob); err != nil {
|
||
h.logger.Printf("[ERROR] enqueue job for %s: %v", pathHostID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] enqueued signed-op job %s for host %s (%d bytes)", req.JobID, pathHostID, len(blob))
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusCreated)
|
||
json.NewEncoder(w).Encode(map[string]interface{}{"status": "ok", "job_id": req.JobID})
|
||
}
|
||
|
||
// handleClaimResetRequest is the controller-forwarded "Elfelejtett jelszó" (v0.50.0): the box
|
||
// asks the hub to email a fresh reset code to the REGISTERED customer address — the requester
|
||
// never chooses the destination. Auth: the customer's own report Bearer key (self-scoped).
|
||
// The response is deliberately neutral 200 on every authorized outcome (cap reached, email
|
||
// failure) — the customer-facing message is always "ha az e-mail cím regisztrálva van…"; the
|
||
// real outcome goes to the operator log + notification_log.
|
||
func (h *Handler) handleClaimResetRequest(w http.ResponseWriter, r *http.Request) {
|
||
authCustomerID, isGlobal, ok := h.checkAuthCustomer(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 4096))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
var payload struct {
|
||
CustomerID string `json:"customer_id"`
|
||
}
|
||
if err := json.Unmarshal(body, &payload); err != nil || payload.CustomerID == "" {
|
||
http.Error(w, "Invalid payload: customer_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authCustomerID != payload.CustomerID {
|
||
http.Error(w, "Forbidden: customer_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
if h.claimEngine == nil {
|
||
http.Error(w, "Claim engine not available", http.StatusServiceUnavailable)
|
||
return
|
||
}
|
||
cfg, err := h.store.GetCustomerConfig(payload.CustomerID)
|
||
if err != nil || cfg == nil {
|
||
http.Error(w, "Not found", http.StatusNotFound)
|
||
return
|
||
}
|
||
if err := h.claimEngine.RequestReset(cfg); err != nil {
|
||
// Neutral to the box; loud to the operator (cap reached / send failure / no email).
|
||
h.logger.Printf("[WARN] claim reset-request for %s: %v", payload.CustomerID, err)
|
||
} else {
|
||
h.logger.Printf("[INFO] claim reset-request for %s: reset code emailed to the registered address", payload.CustomerID)
|
||
}
|
||
// v0.52.0 (take-two F-15): serve the ACTIVE code state in the response — same shape and same
|
||
// bcrypt-only guarantee as the report ACK — so the box accepts the emailed code the moment it
|
||
// lands instead of waiting for the next ACK (~15 min). Served on every authorized outcome: on
|
||
// a cap-reached refusal it is the unrotated row (a controller-side no-op by generation).
|
||
resp := map[string]interface{}{"status": "ok"}
|
||
if cs, err := h.store.GetClaim(payload.CustomerID); err == nil && cs != nil {
|
||
resp["claim"] = map[string]interface{}{
|
||
"code_hash": cs.CodeHash,
|
||
"generation": cs.Generation,
|
||
"issued_at": cs.IssuedAt.UTC().Format(time.RFC3339),
|
||
}
|
||
}
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// allowedEventTypes lists all valid event_type values the Hub accepts.
|
||
var allowedEventTypes = map[string]bool{
|
||
// R-85: restore-test signals. Hub-GENERATED (source "hub"), but listed here on purpose —
|
||
// allowedEventTypes is the project's single register of legitimate event types, and R-77's
|
||
// lesson was that an event type missing from it ships as an inert seam. Neither has a
|
||
// customerMessages entry, so both stay operator-tier.
|
||
"restore_test_failed": true,
|
||
"restore_test_stale": true,
|
||
|
||
// R-97a: the WHOLE-GUEST (vzdump) backup tier's outcome. Controller-pushed from
|
||
// `internal/quiesce`, which until now had no route to the hub at all — on 2026-07-27 three failed
|
||
// whole-guest backups and twelve app-stack stop/starts produced ZERO events.
|
||
//
|
||
// DELIBERATELY NOT `backup_failed`/`backup_completed`. Those two carry customerMessages entries
|
||
// AND sit in demo-felhom's live enabled_events, so reusing them would email the CUSTOMER, in
|
||
// Hungarian, that their backup failed — while it is still retrying behind the R-88 breaker. A
|
||
// customer can take no action on a failed whole-guest backup.
|
||
//
|
||
// OPERATOR-ONLY IS ENFORCED BY `notify.operatorOnlyEvents`, NOT by the absence of a
|
||
// customerMessages entry. v0.78.0 claimed the latter and was WRONG (corrected in v0.79.0/R-97c):
|
||
// `FormatCustomerEmail` treats a missing entry as a fallback to the raw message, and the only
|
||
// customer gate is `prefs.EnabledEvents` — configuration, which a customer or a future code path
|
||
// can change. The register is checked before customer dispatch and logs a `skipped/operator_only`
|
||
// row. Adding a type here does NOT make it operator-only; add it to that register too.
|
||
"whole_guest_backup_failed": true,
|
||
"whole_guest_backup_recovered": true,
|
||
|
||
// R-158 / R-167 (controller v0.191.0, decision D-c): a per-app Tier-1 recovery-unit capture
|
||
// failed. Until then a `[WARN]` line in the controller reached no hub channel at all — the fifth
|
||
// instance in this project of a mechanism built and left disconnected.
|
||
//
|
||
// DELIBERATELY NOT `backup_failed`, for exactly the reason recorded above for the whole-guest
|
||
// pair: that type carries a customerMessages entry AND sits in the controller's
|
||
// DefaultEnabledEvents, so reusing it emails the CUSTOMER, in Hungarian, about a failure they
|
||
// cannot act on. R-158's original proposal named `backup_failed`; D-c routes this to the
|
||
// operator, and where the two disagree D-c wins.
|
||
//
|
||
// OPERATOR-ONLY IS ENFORCED BY `notify.operatorOnlyEvents` — see the paragraph above. This entry
|
||
// alone does NOT make it operator-only.
|
||
"recovery_unit_capture_failed": true,
|
||
// R-182. The per-run backup digest: one event at the end of a run, listing every app whose
|
||
// backup failed or was refused. Allowlisting it is NOT what keeps it away from customers —
|
||
// `notify.operatorOnlyEvents` is (see the comment there); both entries ship together and
|
||
// `backup_run_digest_event_test.go` pins the pair.
|
||
"backup_run_failures": true,
|
||
|
||
// v0.120.0 (`09` §3 decision 15): an app update the box UNDID by itself (warning) and one that ended
|
||
// HELD — the undo failed too, or the update held for any reason (error). BOTH reach the household
|
||
// (one mail per app per outcome — per-app cooldown, notify.perAppCustomerCooldownEvents) and the
|
||
// operator; both carry a Hungarian AND English `mail.event.*` entry, never the raw-English fallback.
|
||
"app_update_undone": true,
|
||
"app_update_held": true,
|
||
// v0.123.0 (decision 28, controller v0.269.0): the box stopped an app that kept crashing / ran out
|
||
// of memory. Household + operator; hu AND en `mail.event.*`; per-app cooldown on both legs.
|
||
"app_stopped_unhealthy": true,
|
||
// D53 (`08` §10, controller with internal/holdreport): the box's held apps — app_held for each hold in place (at once,
|
||
// then daily), app_hold_cleared when one is gone. Severity info: never mailed on receipt; the hub mirrors them in
|
||
// held_apps (handleEvent → recordHeldApp) and the reminder checker mails the OPERATOR after a grace. Operator-only
|
||
// (notify.operatorOnlyEvents); hu AND en `mail.event.*` lines exist only because the coupling rule needs them.
|
||
"app_held": true,
|
||
"app_hold_cleared": true,
|
||
|
||
// R-871 (v0.134.0, controller v0.295.0, `09` decision 109): the box was off at its backup time and made the
|
||
// missed backups when it came back. Info — recorded on the box's timeline, never mailed.
|
||
"backup_catchup_done": true,
|
||
|
||
// Controller-pushed events
|
||
"controller_started": true,
|
||
"claim_lockout": true, // v0.50.0 — claim/reset code brute-force lockout tripped
|
||
"controller_updated": true,
|
||
"backup_completed": true,
|
||
"backup_failed": true,
|
||
"db_dump_completed": true,
|
||
"db_dump_failed": true,
|
||
"backup_integrity_ok": true,
|
||
"backup_integrity_failed": true,
|
||
// controller v0.231.0 (R-87) — the nightly off-site PROOF found a backup that is READABLE and
|
||
// contains none of the app's data. DELIBERATELY NOT `backup_integrity_failed`, which is the
|
||
// nearest existing type and would be the wrong sentence: that one means the STORE IS DAMAGED and
|
||
// carries a Hungarian template saying the integrity check found an error. Here the store is sound
|
||
// and the CONTENT is missing — a different fact, a different cause and a different action.
|
||
// Telling a customer their backups are damaged when they are not is the more expensive mistake,
|
||
// and it is the one R-359's own `looksLikeRepositoryDamage` is shaped to avoid.
|
||
//
|
||
// NO `customerMessages` entry, deliberately: the controller sends a dynamic Hungarian message
|
||
// naming the app and what is missing, and a template would DISCARD those specifics
|
||
// (templates.go priority) — the `offbox_enlarge_blocked` / `disk_health_degraded` precedent.
|
||
// Operator routing is enforced by `notify.operatorOnlyEvents`, in this same commit; this entry
|
||
// alone does NOT make it operator-only (the v0.78.0 defect that register records).
|
||
"offsite_proof_empty": true,
|
||
// R-431 — the hub raises this itself; allowlisted so a hub-origin event is never 400'd.
|
||
"offsite_snapshots_dropped": true,
|
||
// R-243 — the hub raises these itself (monitor/offsite_escrow_pending.go); allowlisted like the line above.
|
||
"offsite_escrow_pending": true,
|
||
"offsite_escrow_pending_cleared": true,
|
||
// R-304 option C — controller (same day) sends it on an unlock that opens / may open an older package. Operator-only
|
||
// (notify.operatorOnlyEvents); NO customerMessages entry.
|
||
"recovery_older_package": true,
|
||
// controller v0.289.0 (decision 69): the customer-chosen deletion of set-aside history is deferred
|
||
// to the operator — the box's append-only key cannot delete. Operator-only (notify.operatorOnlyEvents).
|
||
"offbox_abandon_deferred": true,
|
||
"crossdrive_completed": true,
|
||
"crossdrive_failed": true,
|
||
// controller v0.134.1 — enlarged offsite push refused by the quota gate (warning; the controller's
|
||
// dynamic Hungarian message is customer-grade — deliberately NO customerMessages entry, which would
|
||
// discard the numbers (templates.go:129 priority)).
|
||
"offbox_enlarge_blocked": true,
|
||
// controller v0.142.0 — offsite-repo continuity: the remote repo is orphaned (reinstall shape) /
|
||
// was reset (move-aside + re-init). Customer-grade messages below.
|
||
"offbox_repo_orphaned": true,
|
||
"offbox_repo_reset": true,
|
||
"storage_disconnected": true,
|
||
"storage_reconnected": true,
|
||
// controller v0.184.0 (E-2) — the assigned whole-guest backup TARGET drive is absent. Distinct
|
||
// from storage_disconnected on purpose: that one says "a drive went away and some apps may have
|
||
// stopped"; this one says "the thing that makes a backup survive a disk failure is gone", which
|
||
// is a different action for the customer and a different urgency for the operator. Before this
|
||
// the only signal was the tier's own failure at its next due cycle — up to ~24 h on the daily
|
||
// local tier — i.e. the R-100 shape: a real fault visible only after a deadline elapsed.
|
||
"backup_target_absent": true,
|
||
"backup_target_restored": true,
|
||
"disk_warning": true,
|
||
"disk_critical": true,
|
||
// controller v0.169.0 — per-disk SMART degradation (Rendben→Figyelmeztetés/Hiba). The controller
|
||
// sends a dynamic Hungarian message (disk label + the triggering attribute names), so — like
|
||
// offbox_enlarge_blocked — there is deliberately NO customerMessages entry (which would discard the
|
||
// specifics via the templates.go fallback priority).
|
||
"disk_health_degraded": true,
|
||
"health_degraded": true,
|
||
"health_critical": true,
|
||
"health_recovered": true,
|
||
"app_deployed": true,
|
||
// R-536 (controller v0.244.0). `app_deployed` used to fire beside the 202 that merely ACCEPTED a
|
||
// deploy, so an install interrupted five seconds later stood on the timeline as a completed one
|
||
// — measured 2026-09-16 on the drill box with mealie, which ended `not_deployed`. The accept-time
|
||
// fact is worth keeping, so it becomes its own type, and the failure gets one too rather than
|
||
// being silence. `app_deployed` now means the compose up succeeded and the durable state was
|
||
// written.
|
||
//
|
||
// Both are customer-tier like `app_deployed` itself (NOT in operatorOnlyEvents): a household that
|
||
// pressed „Telepítés" is the party who wants to know it did not finish.
|
||
"app_deploy_started": true,
|
||
"app_deploy_failed": true,
|
||
"app_removed": true,
|
||
"app_start_failed": true, // controller fix-3 (CAMPAIGN-3): a deployed app is not running
|
||
"disaster_recovery_started": true,
|
||
"disaster_recovery_completed": true,
|
||
// Controller→agent channel health (controller v0.90.0) — operator-only (not customer toggles)
|
||
"agent_channel_pin_mismatch": true,
|
||
"agent_channel_unauthorized": true,
|
||
"agent_channel_unreachable": true,
|
||
"agent_channel_timeout": true,
|
||
"agent_channel_misconfigured": true,
|
||
"agent_channel_construction_error": true,
|
||
"agent_channel_unknown": true,
|
||
"agent_channel_recovered": true,
|
||
// controller v0.173.0 (R-77): controller.yaml and bootstrap.json disagree on local_api.endpoint.
|
||
// Operator-only, deliberately NOT an agent_channel_* type — during the 2026-07-25 island-migration
|
||
// outage the generic "unreachable" alert was the only signal and it hid a specific config fault.
|
||
"local_api_endpoint_drift": true,
|
||
// Hub-generated events
|
||
"node_stale": true,
|
||
"node_down": true,
|
||
"node_recovered": true,
|
||
// v0.57.0 reinstall arc (F2/F3/2.3) — hub-emitted on clean-slate re-enrollment / offsite re-issue
|
||
"claim_reissued_reenroll": true, // reset code auto-issued to a reinstalled claimed customer
|
||
"offsite_reissued": true, // offsite one-time password re-staged (manual button or re-enroll)
|
||
"escrow_stale": true, // key-escrow blob invalidated by an offsite password re-issue
|
||
// Hub-generated host-domain events (v0.7.0, slice 3)
|
||
"host_stale": true,
|
||
"host_down": true,
|
||
"host_recovered": true,
|
||
// Hub-generated host root-fs disk-pressure (v0.23.0) — distinct from the controller's GUEST disk_*
|
||
"host_disk_warning": true,
|
||
"host_disk_critical": true,
|
||
"storage_fill_warning": true,
|
||
"storage_fill_critical": true,
|
||
"expected_backup_missed": true,
|
||
"expected_dbdump_missed": true,
|
||
// R-523 (hub v0.114.0, agent v0.131.0) — hub-generated from the agent's controller_supervisor
|
||
// report stanza (monitor/controller_supervisor.go). Both operator-only (notify.operatorOnlyEvents).
|
||
"controller_restarted_by_agent": true,
|
||
"controller_crashloop": true,
|
||
// R-539 (hub v0.117.0, agent v0.132.0) — hub-generated from slow_crashloop_since; operator-only.
|
||
"controller_slow_crashloop": true,
|
||
// R-1344 (2026-10-29): hub-generated from the stanza's unresponsive_since; operator-only (notify.operatorOnlyEvents).
|
||
// 2026-10-30: controller_unresponsive_restart from unresponsive_restart_at (agent ≥ 0.171.0); operator-only.
|
||
"controller_unresponsive": true,
|
||
"controller_unresponsive_restart": true,
|
||
// R-550 (hub v0.117.0, controller v0.246.0) — pushed by the controller at startup when it finds a
|
||
// restore that was in flight when the box stopped. For the household: they can run it again.
|
||
"restore_interrupted": true,
|
||
// R-509 / R-511 / R-518 / R-514 (hub v0.114.0). selfbind_link_sent and pbsdr_adopted are
|
||
// hub-internal audit rows (stored, not dispatched). backup_tier_skipped and app_oom are pushed by
|
||
// controller v0.243.0; both operator-only (notify.operatorOnlyEvents).
|
||
"selfbind_link_sent": true,
|
||
"pbsdr_adopted": true,
|
||
"backup_tier_skipped": true,
|
||
"app_oom": true,
|
||
// R-636 (hub v0.121.0, controller v0.265.0): the same container run OOM-killed 20+ times in 30 min.
|
||
// Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||
"app_oom_storm": true,
|
||
// R-659 (hub v0.122.0, controller v0.268.0): an app held after a failed update and a failed undo
|
||
// with NO copy on its box that brings it back whole — support must act. Operator-only
|
||
// (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||
"app_hold_no_whole_copy": true,
|
||
// R-1007 (hub v0.150.0, controller v0.311.0, `09` §3 decision 236): what an app's after_unclean_boot run found
|
||
// and repaired after an unclean host boot — COUNTS ONLY. Operator-only (notify.operatorOnlyEvents) — registered in
|
||
// BOTH in the same commit; the household's side is the app page's lost-files note on its own box.
|
||
"app_repaired_after_crash": true,
|
||
// R-971 (hub v0.157.0, controller v0.328.0, operator decision D2 = A): the box's catalog allow-list REFUSED an app's
|
||
// definition (a correction, an update, a restore, a settings change) — nothing was written, the app runs on its last
|
||
// good compose. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||
"app_definition_refused": true,
|
||
// The upload and CPU watch (hub v0.169.0, controller v0.342.0; `08` §9, operator ruling 2026-10-23): an app sent or burnt
|
||
// far more than its own numbers explain — the sign of a hacked app (Plex2Shell) — and the clear when it stopped.
|
||
// Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same commit.
|
||
"app_unexplained_upload": true,
|
||
"app_unexplained_upload_cleared": true,
|
||
"app_unexplained_cpu": true,
|
||
"app_unexplained_cpu_cleared": true,
|
||
// D62 (operator 2026-10-29; controller „Törlés most" on a restore copy): the household deleted a restore copy by hand
|
||
// — who, app, the restore's date, size. Operator-only (notify.operatorOnlyEvents) — registered in BOTH in the same
|
||
// commit.
|
||
"restore_copy_deleted": true,
|
||
// D84 (`09` §3 decision 309; `07` §8.2f): the household pressed „delete backup data” for an app — its off-site
|
||
// copies end 30 days later. A household RECEIPT (notify.householdReceiptEvents): mailed to the household's own
|
||
// address once per press, never to the operator; `mail.event.offsite_end_scheduled` in both bundles.
|
||
"offsite_end_scheduled": true,
|
||
// Special
|
||
"test": true,
|
||
}
|
||
|
||
// handleEvent processes structured events from controllers (new endpoint, replaces /notify for updated controllers).
|
||
func (h *Handler) handleEvent(w http.ResponseWriter, r *http.Request) {
|
||
authCustomerID, isGlobal, ok := h.checkAuthCustomer(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
var payload struct {
|
||
CustomerID string `json:"customer_id"`
|
||
EventType string `json:"event_type"`
|
||
Severity string `json:"severity"`
|
||
Message string `json:"message"`
|
||
// MessageCustomer (controller v0.256.0, R-558) — the SAME sentence as Message, written in
|
||
// the household's language. Optional forever: an older controller sends none, and a box
|
||
// that never will (a parked one) must keep working exactly as it does today.
|
||
//
|
||
// It is used ONLY to render the household's e-mail. Message stays the Hungarian one that is
|
||
// stored, logged and mailed to the operator, so nothing an operator reads moves.
|
||
MessageCustomer string `json:"message_customer"`
|
||
Details json.RawMessage `json:"details"`
|
||
}
|
||
if err := json.Unmarshal(body, &payload); err != nil {
|
||
http.Error(w, "Invalid JSON", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if payload.CustomerID == "" || payload.EventType == "" {
|
||
http.Error(w, "customer_id and event_type are required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// Validate customer_id matches authenticated customer (unless global key)
|
||
if !isGlobal && authCustomerID != payload.CustomerID {
|
||
http.Error(w, "Forbidden: customer_id mismatch", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
// Validate event_type
|
||
if !allowedEventTypes[payload.EventType] {
|
||
http.Error(w, fmt.Sprintf("Invalid event_type: %s", payload.EventType), http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
// Validate/default severity (exact-match lowercase; unknown values coerce to info).
|
||
//
|
||
// R-387 — THE COERCION STAYS. THE SILENCE DOES NOT.
|
||
//
|
||
// Note the asymmetry two blocks up: an unknown event TYPE is rejected with a loud 400, while an
|
||
// unknown SEVERITY was rewritten without a word. The silent one is the one that hid a real defect
|
||
// for the whole life of two features — `DiskAlertKind.Severity` emitted "warn" until controller
|
||
// v0.215.0, and `app_start_failed` emitted it until v0.223.0. Both were stored, both were
|
||
// coerced here to "info", and "info" is dropped by severityNotifies — so both were mailed to
|
||
// NOBODY, on either leg, while every POST returned 200.
|
||
//
|
||
// The dispatcher has a line whose job is exactly this (`unrecognized severity %q`), and it can
|
||
// never execute, because this block guarantees the value it looks for cannot reach it. **The
|
||
// mechanism built to detect this class was structurally blind to it.**
|
||
//
|
||
// WHY NOT A 400. A rejected event is a LOST event, and losing an alarm is worse than mis-routing
|
||
// one — the same reasoning that made this a coercion in the first place. The producer is our own
|
||
// controller, so the fix belongs at the emitter; this line is how the emitter's mistake becomes
|
||
// VISIBLE the first time it happens instead of never.
|
||
switch payload.Severity {
|
||
case "info", "warning", "error", "critical":
|
||
default:
|
||
h.logger.Printf("[WARN] [api] Event from %s: severity %q is not in {info,warning,error,critical} "+
|
||
"— coercing to \"info\", which severityNotifies DROPS, so this %s alert will reach NOBODY. "+
|
||
"Fix the emitting controller; this event is stored but not routed.",
|
||
payload.CustomerID, payload.Severity, payload.EventType)
|
||
payload.Severity = "info"
|
||
}
|
||
|
||
// NO SEPARATE LENGTH CAP, and that is deliberate rather than an omission. `message` has never
|
||
// had one either: both are bounded by the 1 MB LimitReader on this request body, which is the
|
||
// only cap that has ever existed here. Adding one to the new field alone would make the two
|
||
// sentences behave differently for no measured reason — and a byte-count truncation would cut a
|
||
// UTF-8 sequence in half, turning a long Hungarian sentence into mojibake in a customer's inbox.
|
||
// Store details as JSON string
|
||
detailsStr := "{}"
|
||
if len(payload.Details) > 0 && string(payload.Details) != "null" {
|
||
detailsStr = string(payload.Details)
|
||
}
|
||
|
||
_, err = h.store.SaveEvent(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, detailsStr, "controller")
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to save event from %s: %v", payload.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Event from %s: %s (%s) — %s", payload.CustomerID, payload.EventType, payload.Severity, payload.Message)
|
||
|
||
// D53: mirror the box's held apps. A failure is logged and the event still dispatched — the box repeats
|
||
// app_held daily, so the next one heals the mirror.
|
||
if payload.EventType == "app_held" || payload.EventType == "app_hold_cleared" {
|
||
h.recordHeldApp(payload.CustomerID, payload.EventType, payload.Details)
|
||
}
|
||
|
||
// Dispatch notifications (non-blocking)
|
||
if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessBoxEvent(payload.CustomerID, payload.EventType, payload.Severity,
|
||
payload.Message, payload.MessageCustomer, detailsStr, "controller")
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"ok":true}`))
|
||
}
|
||
|
||
func (h *Handler) handleCustomers(w http.ResponseWriter, r *http.Request) {
|
||
// R-951: operator/global key only. These routes return every customer's name, dashboard address and
|
||
// stored box report; until v0.148.0 they answered with no credential at all. A box's own key must not
|
||
// list customers either (TestCustomersRoutes_*).
|
||
if !h.requireGlobalKey(w, r) {
|
||
return
|
||
}
|
||
customers, err := h.store.GetCustomers()
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to get customers: %v", err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
type customerJSON struct {
|
||
ID string `json:"id"`
|
||
Name string `json:"name"`
|
||
ControllerVersion string `json:"controller_version"`
|
||
ControllerURL string `json:"controller_url,omitempty"`
|
||
HealthStatus string `json:"health_status"`
|
||
LastSeen time.Time `json:"last_seen"`
|
||
CPUPercent float64 `json:"cpu_percent"`
|
||
MemoryPercent float64 `json:"memory_percent"`
|
||
ContainerTotal int `json:"container_total"`
|
||
ContainerRunning int `json:"container_running"`
|
||
BackupLastSnapshot *time.Time `json:"backup_last_snapshot"`
|
||
}
|
||
|
||
result := make([]customerJSON, 0, len(customers))
|
||
for _, c := range customers {
|
||
result = append(result, customerJSON{
|
||
ID: c.CustomerID,
|
||
Name: c.CustomerName,
|
||
ControllerVersion: c.ControllerVersion,
|
||
ControllerURL: c.ControllerURL,
|
||
HealthStatus: c.HealthStatus,
|
||
LastSeen: c.ReceivedAt,
|
||
CPUPercent: c.CPUPercent,
|
||
MemoryPercent: c.MemoryPercent,
|
||
ContainerTotal: c.ContainerTotal,
|
||
ContainerRunning: c.ContainerRunning,
|
||
BackupLastSnapshot: c.BackupLastSnapshot,
|
||
})
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
json.NewEncoder(w).Encode(result)
|
||
}
|
||
|
||
func (h *Handler) handleCustomer(w http.ResponseWriter, r *http.Request, customerID string) {
|
||
// R-951: operator/global key only. These routes return every customer's name, dashboard address and
|
||
// stored box report; until v0.148.0 they answered with no credential at all. A box's own key must not
|
||
// list customers either (TestCustomersRoutes_*).
|
||
if !h.requireGlobalKey(w, r) {
|
||
return
|
||
}
|
||
customer, err := h.store.GetCustomer(customerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to get customer %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if customer == nil {
|
||
http.NotFound(w, r)
|
||
return
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
// Return the full report JSON directly
|
||
w.Write([]byte(customer.ReportJSON))
|
||
}
|
||
|
||
func (h *Handler) handleCustomerHistory(w http.ResponseWriter, r *http.Request, customerID string) {
|
||
// R-951: operator/global key only. These routes return every customer's name, dashboard address and
|
||
// stored box report; until v0.148.0 they answered with no credential at all. A box's own key must not
|
||
// list customers either (TestCustomersRoutes_*).
|
||
if !h.requireGlobalKey(w, r) {
|
||
return
|
||
}
|
||
period := r.URL.Query().Get("period")
|
||
var since time.Duration
|
||
switch period {
|
||
case "7d":
|
||
since = 7 * 24 * time.Hour
|
||
case "30d":
|
||
since = 30 * 24 * time.Hour
|
||
default:
|
||
since = 24 * time.Hour
|
||
}
|
||
|
||
history, err := h.store.GetCustomerHistory(customerID, since)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to get history for %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
type historyEntry struct {
|
||
ReceivedAt time.Time `json:"received_at"`
|
||
HealthStatus string `json:"health_status"`
|
||
CPUPercent float64 `json:"cpu_percent"`
|
||
MemoryPercent float64 `json:"memory_percent"`
|
||
}
|
||
|
||
result := make([]historyEntry, 0, len(history))
|
||
for _, h := range history {
|
||
result = append(result, historyEntry{
|
||
ReceivedAt: h.ReceivedAt,
|
||
HealthStatus: h.HealthStatus,
|
||
CPUPercent: h.CPUPercent,
|
||
MemoryPercent: h.MemoryPercent,
|
||
})
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
json.NewEncoder(w).Encode(result)
|
||
}
|
||
|
||
// handleNotify processes notification events from customer controllers.
|
||
func (h *Handler) handleNotify(w http.ResponseWriter, r *http.Request) {
|
||
// A per-customer key acts only for its own customer (the body's customer_id is checked below); the global key for
|
||
// any. Found 2026-10-09: this route checked only that the key was valid (crosscustomer_prefs_test.go).
|
||
authCustomerID, isGlobal, ok := h.checkAuthCustomer(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
var payload struct {
|
||
CustomerID string `json:"customer_id"`
|
||
EventType string `json:"event_type"`
|
||
Severity string `json:"severity"`
|
||
Message string `json:"message"`
|
||
Details string `json:"details"`
|
||
}
|
||
if err := json.Unmarshal(body, &payload); err != nil || payload.CustomerID == "" || payload.EventType == "" {
|
||
http.Error(w, "Invalid payload: customer_id and event_type required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authCustomerID != payload.CustomerID {
|
||
http.Error(w, "Forbidden: customer_id does not match the key", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Notification from %s: %s (%s) — %s", payload.CustomerID, payload.EventType, payload.Severity, payload.Message)
|
||
|
||
// Check if customer is blocked
|
||
if h.store.IsCustomerBlocked(payload.CustomerID) {
|
||
h.logger.Printf("[INFO] Notification suppressed for blocked customer %s", payload.CustomerID)
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "skipped", "customer blocked", "customer")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":false,"reason":"blocked"}`))
|
||
return
|
||
}
|
||
|
||
// Look up customer notification preferences
|
||
prefs, err := h.store.GetNotificationPrefs(payload.CustomerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to get notification prefs for %s: %v", payload.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
// Check if customer has email configured and event type is enabled
|
||
if prefs == nil || prefs.Email == "" {
|
||
h.logger.Printf("[INFO] No email configured for %s, skipping notification", payload.CustomerID)
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "skipped", "no email configured", "customer")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":false,"reason":"no_email"}`))
|
||
return
|
||
}
|
||
|
||
// Check if event type is in the enabled list (test events always pass)
|
||
eventEnabled := payload.EventType == "test"
|
||
for _, e := range prefs.EnabledEvents {
|
||
if e == payload.EventType {
|
||
eventEnabled = true
|
||
break
|
||
}
|
||
}
|
||
if !eventEnabled {
|
||
h.logger.Printf("[INFO] Event %s not enabled for %s, skipping", payload.EventType, payload.CustomerID)
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "skipped", "event not enabled", "customer")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":false,"reason":"event_disabled"}`))
|
||
return
|
||
}
|
||
|
||
// Send email via Resend API
|
||
if h.resendAPIKey == "" {
|
||
h.logger.Printf("[WARN] Resend API key not configured, cannot send notification email")
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "skipped", "resend api key not configured", "customer")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":false,"reason":"no_api_key"}`))
|
||
return
|
||
}
|
||
|
||
// MAIL-HOLD (internal/mailhold): a restored or quarantined hub sends nothing. The mail is dropped, not queued.
|
||
if err := h.mailHold.Check("customer event "+payload.EventType, payload.CustomerID); err != nil {
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "held", "MAIL-HOLD marker present — dropped", "customer")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":false,"reason":"mail_hold"}`))
|
||
return
|
||
}
|
||
|
||
subject, emailBody := formatNotificationEmail(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, payload.Details)
|
||
sendErr := h.sendResendEmail(prefs.Email, subject, emailBody)
|
||
if sendErr != nil {
|
||
h.logger.Printf("[ERROR] Failed to send notification email to %s: %v", prefs.Email, sendErr)
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "failed", sendErr.Error(), "customer")
|
||
http.Error(w, "Failed to send email", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Notification email sent to %s for %s/%s", prefs.Email, payload.CustomerID, payload.EventType)
|
||
h.store.LogNotification(payload.CustomerID, payload.EventType, payload.Severity, payload.Message, "sent", "", "customer")
|
||
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok","sent":true}`))
|
||
}
|
||
|
||
// handleSavePreferences stores notification preferences pushed from a customer controller.
|
||
func (h *Handler) handleSavePreferences(w http.ResponseWriter, r *http.Request) {
|
||
// A per-customer key acts only for its own customer (the body's customer_id is checked below); the global key for
|
||
// any. Found 2026-10-09: this route checked only that the key was valid (crosscustomer_prefs_test.go).
|
||
authCustomerID, isGlobal, ok := h.checkAuthCustomer(r)
|
||
if !ok {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
|
||
if err != nil {
|
||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
var payload struct {
|
||
CustomerID string `json:"customer_id"`
|
||
Email string `json:"email"`
|
||
EnabledEvents []string `json:"enabled_events"`
|
||
CooldownHours int `json:"cooldown_hours"`
|
||
// EmailCleared (R-922, operator ruling 2026-10-09 option A) is sent true by the controller ONLY when the
|
||
// household deliberately cleared its address on the dashboard. Omitted (false) on every other push —
|
||
// including every push from a controller older than the one that emits it.
|
||
EmailCleared bool `json:"email_cleared"`
|
||
// Source (R-1199 reopened, controller ≥ 0.339.0): "household" — the household saved this list on the box
|
||
// (the settings page, or the re-push of a list saved there); "default" — the box's own default, sent because
|
||
// nothing was saved and the hub holds no household list. Absent = an older controller (prefsPushIsHousehold).
|
||
Source string `json:"source"`
|
||
}
|
||
if err := json.Unmarshal(body, &payload); err != nil || payload.CustomerID == "" {
|
||
http.Error(w, "Invalid payload: customer_id required", http.StatusBadRequest)
|
||
return
|
||
}
|
||
if !isGlobal && authCustomerID != payload.CustomerID {
|
||
http.Error(w, "Forbidden: customer_id does not match the key", http.StatusForbidden)
|
||
return
|
||
}
|
||
|
||
// Empty-email no-clobber guard (v0.71.0, audit F12): a controller push with an empty email
|
||
// (e.g. an unconfigured box) must never wipe a stored non-empty address — the seeded/edited
|
||
// email is the customer's alert lifeline. Events + cooldown from the push still apply; a push
|
||
// with a non-empty email updates everything (customer edits keep working).
|
||
//
|
||
// R-922: the ONE exception is a deliberate clear — `email_cleared: true` with an empty address. The household
|
||
// removed its address, so the hub deletes it (personal data the household withdrew does not stay on our side).
|
||
// The guard cannot tell a clear from an unconfigured box by the address alone, which is why the clear travels
|
||
// as its own flag. A flag beside a NON-empty address is an ordinary update (the address wins). Pinned by
|
||
// TestSavePreferences_DeliberateClearDeletesAddress, TestSavePreferences_EmptyEmailCannotClobber and
|
||
// TestSavePreferences_ClearFlagWithAddressIsAnUpdate.
|
||
// R-1199 reopened: a push that is not the household's own save never overwrites a list the household saved.
|
||
household := prefsPushIsHousehold(payload.Source, payload.Email, payload.EmailCleared)
|
||
if !household {
|
||
if existing, err := h.store.GetNotificationPrefs(payload.CustomerID); err == nil && existing != nil && existing.HouseholdSaved {
|
||
h.logger.Printf("[INFO] Notification prefs push for %s (source=%q) not applied — the household saved its own list (R-1199)", payload.CustomerID, payload.Source)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"kept","household_saved":true}`))
|
||
return
|
||
}
|
||
}
|
||
|
||
saveEmail := payload.Email
|
||
if saveEmail == "" && payload.EmailCleared {
|
||
h.logger.Printf("[INFO] Notification prefs push for %s: household cleared its mail address — deleted", payload.CustomerID)
|
||
} else if saveEmail == "" {
|
||
if existing, err := h.store.GetNotificationPrefs(payload.CustomerID); err == nil && existing != nil && existing.Email != "" {
|
||
saveEmail = existing.Email
|
||
h.logger.Printf("[INFO] Notification prefs push for %s had empty email — preserving stored address", payload.CustomerID)
|
||
}
|
||
}
|
||
|
||
if err := h.store.SaveNotificationPrefsAs(payload.CustomerID, saveEmail, payload.EnabledEvents, payload.CooldownHours, household); err != nil {
|
||
h.logger.Printf("[ERROR] Failed to save notification prefs for %s: %v", payload.CustomerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Notification preferences updated for %s: address set=%t, household=%t, events=%v", payload.CustomerID, saveEmail != "", household, payload.EnabledEvents) // never the address (R-922)
|
||
w.WriteHeader(http.StatusOK)
|
||
w.Write([]byte(`{"status":"ok"}`))
|
||
}
|
||
|
||
// handleRecovery returns the generated controller.yaml for disaster recovery.
|
||
// Auth: X-Retrieval-Password header (same as config retrieval).
|
||
//
|
||
// The infra-backup payload was retired (Phase-1, 2026-06-16): it pushed plaintext
|
||
// customer secrets to the hub (a zero-knowledge violation) and had been dead since
|
||
// slice 8C. DR config now comes from the generated controller.yaml here; the data
|
||
// bytes come from the agent's PBS whole-CT snapshot. A secret-free DR recipe is the
|
||
// later DR slice's job.
|
||
func (h *Handler) handleRecovery(w http.ResponseWriter, r *http.Request, customerID string) {
|
||
if customerID == "" {
|
||
http.Error(w, "Missing customer_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
password := r.Header.Get("X-Retrieval-Password")
|
||
if password == "" {
|
||
http.Error(w, "Unauthorized: X-Retrieval-Password header required", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
cfg, err := h.store.GetCustomerConfig(customerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Recovery: failed to get customer config for %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cfg == nil {
|
||
http.Error(w, "Not found", http.StatusNotFound)
|
||
return
|
||
}
|
||
|
||
// R-879: a sealed passphrase that did not open is a hub fault (500), never a wrong password.
|
||
if cfg.SecretsUnreadable {
|
||
h.logger.Printf("[ERROR] customer %s: sealed secrets do not open (OFFSITE_SECRET_KEY)", cfg.CustomerID)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if subtle.ConstantTimeCompare([]byte(password), []byte(cfg.RetrievalPassword)) != 1 {
|
||
http.Error(w, "Unauthorized: invalid password", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
if closed, why := h.passphraseKeysClosed(customerID); closed {
|
||
h.refusePassphraseKeys(w, r, customerID, "GET /recovery", why)
|
||
return
|
||
}
|
||
|
||
// Generate controller.yaml. The claim state is baked read-only (no issue on the DR path — a
|
||
// recovered box whose settings.json is gone re-gates on the EXISTING hash; the reset flow
|
||
// covers a customer who lost the password with the box).
|
||
var configYAML string
|
||
if h.templateProvider != nil {
|
||
var claimState *store.ClaimState
|
||
if h.claimEngine != nil {
|
||
claimState, _ = h.store.GetClaim(customerID)
|
||
}
|
||
yamlOutput, err := configgen.Generate(h.templateProvider.Template(), cfg, claimState)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Recovery: failed to generate config for %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
configYAML = yamlOutput
|
||
}
|
||
|
||
// infra_backup retired: the response keeps has_infra_backup=false so any old client
|
||
// degrades gracefully to the config_yaml-only path.
|
||
resp := struct {
|
||
CustomerID string `json:"customer_id"`
|
||
ConfigYAML string `json:"config_yaml"`
|
||
HasInfraBackup bool `json:"has_infra_backup"`
|
||
}{
|
||
CustomerID: customerID,
|
||
ConfigYAML: configYAML,
|
||
HasInfraBackup: false,
|
||
}
|
||
|
||
h.logger.Printf("[INFO] Recovery data downloaded for customer %s (config only; infra-backup retired)", customerID)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// handleConfigRetrieve returns a generated controller.yaml for a customer.
|
||
// Auth: X-Retrieval-Password header (not Bearer token).
|
||
func (h *Handler) handleConfigRetrieve(w http.ResponseWriter, r *http.Request, customerID string) {
|
||
if customerID == "" {
|
||
http.Error(w, "Missing customer_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
password := r.Header.Get("X-Retrieval-Password")
|
||
if password == "" {
|
||
http.Error(w, "Unauthorized: X-Retrieval-Password header required", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
cfg, err := h.store.GetCustomerConfig(customerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] Failed to get customer config for %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cfg == nil {
|
||
http.Error(w, "Not found", http.StatusNotFound)
|
||
return
|
||
}
|
||
|
||
// R-879: a sealed passphrase that did not open is a hub fault (500), never a wrong password.
|
||
if cfg.SecretsUnreadable {
|
||
h.logger.Printf("[ERROR] customer %s: sealed secrets do not open (OFFSITE_SECRET_KEY)", cfg.CustomerID)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
// Constant-time comparison to prevent timing attacks
|
||
if subtle.ConstantTimeCompare([]byte(password), []byte(cfg.RetrievalPassword)) != 1 {
|
||
http.Error(w, "Unauthorized: invalid password", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
// R-1159: the passphrase gives keys only until the box uses its own (passphrase_scope.go).
|
||
if closed, why := h.passphraseKeysClosed(customerID); closed {
|
||
h.refusePassphraseKeys(w, r, customerID, "GET /config", why)
|
||
return
|
||
}
|
||
// Customer-claim arc (v0.50.0): the REAL config pull (Day-0 installer / controller refresh) is the Day-0 claim
|
||
// entry point — serveGeneratedConfig issues + emails the first code (idempotent) and bakes the active hash into
|
||
// the generated web.claim_code_hash so the box is gated from FIRST boot. (The operator-UI preview deliberately
|
||
// does NOT issue — it only bakes an existing hash.)
|
||
h.serveGeneratedConfig(w, cfg, "retrieval passphrase")
|
||
}
|
||
|
||
// artifactManifestResponse is the wire shape the host-bootstrap script consumes: the operator-vouched
|
||
// current agent binary + golden archive (version + sha256 each). The script fetches each artifact from
|
||
// Gitea with the config-retrieve git token and verifies its sha256 against THESE values before install
|
||
// — so the hub is the checksum trust root, a different root than Gitea (which only stores the bytes).
|
||
type artifactManifestResponse struct {
|
||
Agent artifactEntry `json:"agent"`
|
||
Golden artifactEntry `json:"golden"`
|
||
// Bundle is the vouched agent's config bundle (R-840); absent when the vouched agent carries none.
|
||
Bundle *artifactEntry `json:"bundle,omitempty"`
|
||
}
|
||
|
||
type artifactEntry struct {
|
||
Version string `json:"version"`
|
||
SHA256 string `json:"sha256"`
|
||
}
|
||
|
||
// handleArtifactManifest serves the current artifact set for a customer. Auth mirrors
|
||
// handleConfigRetrieve EXACTLY (X-Retrieval-Password header, 404-then-401 order, constant-time
|
||
// compare) so a script that can pull the controller.yaml can pull the manifest with the same secret.
|
||
// v0.16.0 returns the GLOBAL current set for every customer (per-customer pinning is a future hook).
|
||
// An unset manifest returns empty fields (not an error) — the script falls back to the local golden
|
||
// and fails clearly on a missing binary.
|
||
func (h *Handler) handleArtifactManifest(w http.ResponseWriter, r *http.Request, customerID string) {
|
||
if customerID == "" {
|
||
http.Error(w, "Missing customer_id", http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
password := r.Header.Get("X-Retrieval-Password")
|
||
if password == "" {
|
||
http.Error(w, "Unauthorized: X-Retrieval-Password header required", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
cfg, err := h.store.GetCustomerConfig(customerID)
|
||
if err != nil {
|
||
h.logger.Printf("[ERROR] artifacts: customer lookup failed for %s: %v", customerID, err)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if cfg == nil {
|
||
http.Error(w, "Not found", http.StatusNotFound)
|
||
return
|
||
}
|
||
// R-879: a sealed passphrase that did not open is a hub fault (500), never a wrong password.
|
||
if cfg.SecretsUnreadable {
|
||
h.logger.Printf("[ERROR] customer %s: sealed secrets do not open (OFFSITE_SECRET_KEY)", cfg.CustomerID)
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
if subtle.ConstantTimeCompare([]byte(password), []byte(cfg.RetrievalPassword)) != 1 {
|
||
http.Error(w, "Unauthorized: invalid password", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
m := h.store.GetArtifactManifest()
|
||
resp := artifactManifestResponse{
|
||
Agent: artifactEntry{Version: m.AgentVersion, SHA256: m.AgentSHA256},
|
||
Golden: artifactEntry{Version: m.GoldenVersion, SHA256: m.GoldenSHA256},
|
||
}
|
||
if m.BundleSHA256 != "" {
|
||
resp.Bundle = &artifactEntry{Version: m.AgentVersion, SHA256: m.BundleSHA256}
|
||
}
|
||
h.logger.Printf("[INFO] Artifact manifest served for customer %s (agent=%s golden=%s)", customerID, m.AgentVersion, m.GoldenVersion)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// sendResendEmail sends an email via the Resend HTTP API.
|
||
func (h *Handler) sendResendEmail(to, subject, textBody string) error {
|
||
payload := map[string]interface{}{
|
||
"from": h.fromEmail,
|
||
"to": []string{to},
|
||
"subject": subject,
|
||
"text": textBody,
|
||
}
|
||
|
||
jsonData, err := json.Marshal(payload)
|
||
if err != nil {
|
||
return fmt.Errorf("marshaling email payload: %w", err)
|
||
}
|
||
|
||
req, err := http.NewRequest("POST", "https://api.resend.com/emails", bytes.NewReader(jsonData))
|
||
if err != nil {
|
||
return fmt.Errorf("creating request: %w", err)
|
||
}
|
||
req.Header.Set("Authorization", "Bearer "+h.resendAPIKey)
|
||
req.Header.Set("Content-Type", "application/json")
|
||
|
||
resp, err := h.httpClient.Do(req)
|
||
if err != nil {
|
||
return fmt.Errorf("sending request: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode >= 400 {
|
||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
|
||
return fmt.Errorf("resend API returned %d: %s", resp.StatusCode, string(respBody))
|
||
}
|
||
|
||
return nil
|
||
}
|
||
|
||
// formatNotificationEmail creates a Hungarian email subject and body.
|
||
func formatNotificationEmail(customerID, eventType, severity, message, details string) (string, string) {
|
||
severityLabel := map[string]string{
|
||
"info": "Információ",
|
||
"warning": "Figyelmeztetés",
|
||
"error": "Hiba",
|
||
"critical": "Kritikus",
|
||
}
|
||
label := severityLabel[severity]
|
||
if label == "" {
|
||
label = severity
|
||
}
|
||
|
||
subject := fmt.Sprintf("[Felhom] %s: %s", label, message)
|
||
|
||
now := time.Now().Format("2006-01-02 15:04")
|
||
emailText := fmt.Sprintf(`Kedves Ügyfél!
|
||
|
||
A Felhom rendszered a következő figyelmeztetést jelezte:
|
||
|
||
%s
|
||
|
||
Részletek:
|
||
- Szerver: %s
|
||
- Időpont: %s
|
||
- Szint: %s
|
||
- Típus: %s`, message, customerID, now, label, eventType)
|
||
|
||
if details != "" {
|
||
emailText += fmt.Sprintf("\n- Megjegyzés: %s", details)
|
||
}
|
||
|
||
emailText += `
|
||
|
||
Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel.
|
||
|
||
Üdvözlettel,
|
||
Felhom.eu monitoring`
|
||
|
||
return subject, emailText
|
||
}
|
||
|
||
// --- Asset endpoints ---
|
||
|
||
func (h *Handler) handleAssetsManifest(w http.ResponseWriter, r *http.Request) {
|
||
if !h.checkAuth(r) {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
if h.assetsMgr == nil {
|
||
http.Error(w, "Assets not configured", http.StatusServiceUnavailable)
|
||
return
|
||
}
|
||
|
||
data, err := h.assetsMgr.MarshalManifestJSON()
|
||
if err != nil {
|
||
http.Error(w, "Internal error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.Write(data)
|
||
}
|
||
|
||
func (h *Handler) handleAssetFile(w http.ResponseWriter, r *http.Request, filename string) {
|
||
if !h.checkAuth(r) {
|
||
http.Error(w, "Unauthorized", http.StatusUnauthorized)
|
||
return
|
||
}
|
||
|
||
if h.assetsMgr == nil {
|
||
http.Error(w, "Assets not configured", http.StatusServiceUnavailable)
|
||
return
|
||
}
|
||
|
||
h.assetsMgr.ServeFile(w, r, filename)
|
||
}
|
||
|
||
// Tunnel alarm (R-841, hub v0.131.0). The agent (>= 0.141.0) reports the box's tunnel as running | not_running |
|
||
// unknown, read from the cloudflared container's own readiness check. `not_running` in TWO reports in a row — more
|
||
// than one report cycle — is an operator alarm (`tunnel_down`, error); the first `running` after that is
|
||
// `tunnel_recovered`. `unknown` never alarms and breaks a run of not_running (R-96 rule 3: a probe that could not
|
||
// ask is not evidence of down). Pinned by TestTunnelAlarm_*.
|
||
const (
|
||
eventTunnelDown = "tunnel_down"
|
||
eventTunnelRecovered = "tunnel_recovered"
|
||
)
|
||
|
||
func (h *Handler) checkTunnel(hostID, custID, status, detail string) {
|
||
st, err := h.store.RecentCloudflaredStatuses(hostID, 3)
|
||
if err != nil || len(st) < 2 {
|
||
return
|
||
}
|
||
down := func(i int) bool { return i < len(st) && st[i] == "not_running" }
|
||
var typ, sev, msg string
|
||
switch {
|
||
case down(0) && down(1) && !down(2):
|
||
typ, sev = eventTunnelDown, "error"
|
||
msg = fmt.Sprintf("The tunnel of %s is NOT running in two reports in a row (%s). The box is not reachable from outside its home.", hostID, detail)
|
||
case st[0] == "running" && down(1) && down(2):
|
||
typ, sev = eventTunnelRecovered, "info"
|
||
msg = fmt.Sprintf("The tunnel of %s is running again.", hostID)
|
||
default:
|
||
return
|
||
}
|
||
h.logger.Printf("[WARN] host %s tunnel: %s (%s)", hostID, typ, detail)
|
||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "status": status, "detail": detail})
|
||
if _, eerr := h.store.SaveEvent(custID, typ, sev, msg, string(details), "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, eerr)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(details), "hub")
|
||
}
|
||
}
|
||
|
||
// The four silent failures, the two a BOX tells (R-1383, D78, `09` §3 decision 299; `08-alarm-ladder.md`).
|
||
//
|
||
// (a) signed_job_refused — the agent clears a signed job from its queue when it ran it AND when it refused it; until
|
||
// now the hub could not tell the two apart („executed or rejected”), so an agent update that was refused left no
|
||
// trace outside the box's journal (clock list CA09-4, CA09-7). An agent of release 3 says how the job ended in the
|
||
// DELETE's query: outcome = done | rejected | failed | malformed, a reason from a CLOSED list, and the op's name.
|
||
// Anything but `done` → one operator-only event, and ONLY for a job that was in that box's queue. No outcome (an older agent) → nothing. The box's words never reach
|
||
// a mail: an unknown reason is told as „other”, an op name is cut to [a-z0-9_].
|
||
// (b) guest_power_gave_up — see checkGuestPowerGaveUp.
|
||
// Pinned by TestSignedJobOutcome_* and TestGuestPowerGaveUp_*.
|
||
const (
|
||
eventSignedJobRefused = "signed_job_refused"
|
||
eventGuestPowerGaveUp = "guest_power_gave_up"
|
||
eventGuestPowerRunning = "guest_power_ok"
|
||
)
|
||
|
||
// signedJobReasons is the closed list of the agent's reasons (signedjobs.reasonCode), each with the operator's
|
||
// sentence. The two clock reasons name the clock, because a box with a wrong clock refuses EVERY job.
|
||
var signedJobReasons = map[string]string{
|
||
"expired": "its validity had run out when the box read it. If the job was fresh, the box's clock is AHEAD",
|
||
"not_yet_valid": "it was not valid yet by the box's clock. The box's clock is BEHIND",
|
||
"window_too_long": "its validity window is longer than the box allows",
|
||
"replay": "the box saw this job's number before (a job runs once)",
|
||
"bad_signature": "the signature did not verify",
|
||
"unknown_signer": "the signing key is not one the box trusts",
|
||
"target": "it names another box",
|
||
"namespace": "it was signed for another use",
|
||
"malformed": "the box could not read it",
|
||
"role_denied": "the signing key may not sign this kind of job",
|
||
"binding_mismatch": "it is signed for another action or device",
|
||
"execution_failed": "it was accepted and its run FAILED on the box",
|
||
}
|
||
|
||
var signedJobOpRe = regexp.MustCompile(`^[a-z0-9_]{1,40}$`)
|
||
|
||
// signedJobIDRe is the alphabet of a job id the hub mints (hex) or the operator's tool names; anything else is not
|
||
// quoted in a mail.
|
||
var signedJobIDRe = regexp.MustCompile(`^[A-Za-z0-9._-]{1,64}$`)
|
||
|
||
// guestPowerMaxEntries bounds the give-up list read from ONE report (a box has a handful of guests).
|
||
const guestPowerMaxEntries = 16
|
||
|
||
// guestPowerMaxMarks bounds how many give-up marks ONE box can hold in the hub's settings at a time.
|
||
const guestPowerMaxMarks = 8
|
||
|
||
// guestPowerMarkKeep: how long the mark of a guest the box stopped listing keeps its slot.
|
||
const guestPowerMarkKeep = 7 * 24 * time.Hour
|
||
|
||
func (h *Handler) noteSignedJobOutcome(hostID, jobID, outcome, reason, op string) {
|
||
switch outcome {
|
||
case "rejected", "failed", "malformed":
|
||
default:
|
||
return // "done", or an agent that does not say
|
||
}
|
||
if !signedJobIDRe.MatchString(jobID) {
|
||
// the id was a row of the hub's own queue (the caller checked), but it is put there by the operator's upload
|
||
// as free text — an id outside the hub's own alphabet is named, not quoted.
|
||
jobID = "(id not shown)"
|
||
}
|
||
text := signedJobReasons[reason]
|
||
if text == "" {
|
||
reason, text = "other", "the box gave a reason this hub does not know"
|
||
}
|
||
if !signedJobOpRe.MatchString(op) {
|
||
op = "unknown"
|
||
}
|
||
custID := ""
|
||
if host, err := h.store.GetHost(hostID); err == nil && host != nil {
|
||
custID = host.CustomerID
|
||
}
|
||
what := "REFUSED"
|
||
if outcome == "failed" {
|
||
what = "ran and FAILED"
|
||
}
|
||
h.logger.Printf("[WARN] host %s signed job %s (%s): %s — %s", hostID, jobID, op, outcome, reason)
|
||
msg := fmt.Sprintf("The signed job %s (%s) for %s was %s: %s. The job is gone from the queue and will not run again. Do: read the cause, then sign a new job.", jobID, op, hostID, what, text)
|
||
details, _ := json.Marshal(map[string]any{"host_id": hostID, "job_id": jobID, "op": op, "outcome": outcome, "reason": reason, "subject": hostID + ":" + jobID})
|
||
if _, err := h.store.SaveEvent(custID, eventSignedJobRefused, "error", msg, string(details), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", eventSignedJobRefused, hostID, err)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(custID, eventSignedJobRefused, "error", msg, string(details), "hub")
|
||
}
|
||
}
|
||
|
||
type hostGuestPowerGaveUp struct {
|
||
VMID int `json:"vmid"`
|
||
Since string `json:"since"`
|
||
Attempts int `json:"attempts"`
|
||
}
|
||
|
||
// checkGuestPowerGaveUp: the agent's guest-power watchdog starts a guest that must run and is stopped; after three
|
||
// failed starts it stops trying, and until now it wrote one line in the host's journal and told nobody (clock list
|
||
// CA07-7) — the household's apps are off and the only outside sign was „the box does not report”, which is not even
|
||
// true here (the HOST reports). One operator-only `guest_power_gave_up` (error) per guest, told once for one down
|
||
// period; one `guest_power_ok` when a later report shows that guest RUNNING. The agent's record is memory: after an
|
||
// agent restart the entry is gone while the guest is still down — so „gone from the list” alone never clears.
|
||
//
|
||
// The list comes from the box, so it is bounded: only a guest that the SAME report lists (and not as running) is
|
||
// read, and at most guestPowerMaxEntries entries — a box cannot mint an alarm per invented vmid.
|
||
func (h *Handler) checkGuestPowerGaveUp(hostID, custID string, gave []hostGuestPowerGaveUp, running, known map[int]bool) {
|
||
if len(gave) > guestPowerMaxEntries {
|
||
h.logger.Printf("[WARN] host %s: guest_power_gave_up lists %d guests — only the first %d are read", hostID, len(gave), guestPowerMaxEntries)
|
||
gave = gave[:guestPowerMaxEntries]
|
||
}
|
||
emit := func(typ, sev, msg string, details map[string]any) {
|
||
dj, _ := json.Marshal(details)
|
||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||
}
|
||
}
|
||
prefix := fmt.Sprintf("guest_power:%s:", hostID)
|
||
key := func(vmid int) string { return prefix + strconv.Itoa(vmid) }
|
||
// The marks this box has now. The report's own guest list is the BOX's word too, so „the report lists the guest”
|
||
// bounds nothing by itself (review, 2026-11-14): one box holds at most guestPowerMaxMarks marks, and a mark for a
|
||
// guest the box no longer lists keeps its slot for guestPowerMarkKeep before it is dropped. So a box that invents
|
||
// new guests at every report gets guestPowerMaxMarks alarms in that time, not that many per report — each alarm
|
||
// is mailed (the mail cooldown is per guest). A destroyed guest's mark still goes, a week later.
|
||
raised := map[int]bool{}
|
||
for _, k := range h.store.OSAlarmKeys(prefix) {
|
||
vmid, err := strconv.Atoi(strings.TrimPrefix(k, prefix))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if !known[vmid] && time.Since(h.store.OSAlarmRaised(k)) > guestPowerMarkKeep {
|
||
_ = h.store.SetOSAlarmRaised(k, time.Time{})
|
||
h.logger.Printf("[INFO] host %s: guest %d has not been reported for %s — its give-up mark is dropped", hostID, vmid, guestPowerMarkKeep)
|
||
continue
|
||
}
|
||
raised[vmid] = true // also a guest the box stopped listing, for guestPowerMarkKeep: its mark keeps its slot
|
||
}
|
||
for _, g := range gave {
|
||
if g.VMID <= 0 || !known[g.VMID] || running[g.VMID] || raised[g.VMID] {
|
||
continue
|
||
}
|
||
if len(raised) >= guestPowerMaxMarks {
|
||
h.logger.Printf("[WARN] host %s: %d give-up marks already — guest %d is not told (a box has a handful of guests)", hostID, len(raised), g.VMID)
|
||
break
|
||
}
|
||
raised[g.VMID] = true
|
||
_ = h.store.SetOSAlarmRaised(key(g.VMID), time.Now())
|
||
h.logger.Printf("[WARN] host %s: the guest-power watchdog gave up on guest %d (%d attempts)", hostID, g.VMID, g.Attempts)
|
||
emit(eventGuestPowerGaveUp, "error",
|
||
fmt.Sprintf("Guest %d on %s is STOPPED and the agent gave up starting it after %d failed attempts. The household's apps are off. The agent does not try again by itself. Do: read `journalctl -u felhom-agent` on the box for „guest-power”, start the guest, and read why it did not start.", g.VMID, hostID, g.Attempts),
|
||
map[string]any{"host_id": hostID, "vmid": g.VMID, "attempts": g.Attempts, "since": trustDate(g.Since), "subject": key(g.VMID)})
|
||
}
|
||
for vmid := range running {
|
||
if !raised[vmid] {
|
||
continue
|
||
}
|
||
_ = h.store.SetOSAlarmRaised(key(vmid), time.Time{})
|
||
h.logger.Printf("[INFO] host %s: guest %d runs again (the guest-power watchdog had given up)", hostID, vmid)
|
||
emit(eventGuestPowerRunning, "info", fmt.Sprintf("Guest %d on %s runs again.", vmid, hostID),
|
||
map[string]any{"host_id": hostID, "vmid": vmid, "subject": key(vmid)})
|
||
}
|
||
}
|
||
|
||
// Box clock alarm (R-1384, D79, `09` §3 decision 300; `08-alarm-ladder.md`). The hub compares the box's clock at
|
||
// the moment of SENDING (`sent_at`) with its own clock at the moment the request arrives. More than one minute apart,
|
||
// in either direction → one operator-only `box_clock_wrong`; back within 30 seconds → one `box_clock_ok` (the gap
|
||
// between 30 and 60 s keeps a clock at the edge from mailing at every report).
|
||
//
|
||
// What this can NOT tell: whose clock is wrong. A slow line can only make a box look BEHIND, and by less than the
|
||
// agent's request timeout (30 s), so it cannot raise the alarm alone. When the HUB's clock is wrong, every healthy box
|
||
// looks wrong in the SAME direction — so the message says so when a second box is off in the same direction.
|
||
// A report with no `sent_at` (an older agent) is never judged. Pinned by TestBoxClock_*.
|
||
const (
|
||
eventBoxClockWrong = "box_clock_wrong"
|
||
eventBoxClockOK = "box_clock_ok"
|
||
boxClockRaiseAbove = time.Minute
|
||
boxClockClearBelow = 30 * time.Second
|
||
)
|
||
|
||
func (h *Handler) checkBoxClock(hostID, custID, sentAt string, received time.Time) {
|
||
if sentAt == "" {
|
||
return
|
||
}
|
||
sent, err := time.Parse(time.RFC3339Nano, sentAt)
|
||
if err != nil {
|
||
h.logger.Printf("[WARN] host %s: sent_at is not a time (%d bytes); the clock is not judged", hostID, len(sentAt))
|
||
return
|
||
}
|
||
skew := sent.Sub(received) // > 0: the box is AHEAD of the hub
|
||
abs := skew
|
||
if abs < 0 {
|
||
abs = -abs
|
||
}
|
||
dir := "ahead"
|
||
if skew < 0 {
|
||
dir = "behind"
|
||
}
|
||
key := func(d string) string { return "box_clock:" + hostID + ":" + d }
|
||
emit := func(typ, sev, msg string, details map[string]any) {
|
||
dj, _ := json.Marshal(details)
|
||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||
}
|
||
}
|
||
raised := ""
|
||
for _, d := range []string{"ahead", "behind"} {
|
||
if !h.store.OSAlarmRaised(key(d)).IsZero() {
|
||
raised = d
|
||
}
|
||
}
|
||
secs := int64(abs / time.Second)
|
||
switch {
|
||
case abs > boxClockRaiseAbove:
|
||
if raised == dir {
|
||
return // said once
|
||
}
|
||
if raised != "" {
|
||
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
|
||
}
|
||
_ = h.store.SetOSAlarmRaised(key(dir), received)
|
||
// the hub's own clock: another box off in the same direction points at the hub, not at the boxes
|
||
var same []string
|
||
if hosts, lerr := h.store.ListHosts(); lerr == nil {
|
||
for _, o := range hosts {
|
||
if o.HostID != hostID && !h.store.OSAlarmRaised("box_clock:"+o.HostID+":"+dir).IsZero() {
|
||
same = append(same, o.HostID)
|
||
}
|
||
}
|
||
}
|
||
sort.Strings(same)
|
||
hint := ""
|
||
if len(same) > 0 {
|
||
hint = fmt.Sprintf(" %d other box(es) are %s too (%s): check the HUB's own clock (DooPlex) first.", len(same), dir, strings.Join(same, ", "))
|
||
}
|
||
h.logger.Printf("[WARN] host %s clock: %d s %s of the hub (same direction: %d other)", hostID, secs, dir, len(same))
|
||
emit(eventBoxClockWrong, "warning",
|
||
fmt.Sprintf("The clock of %s is %d seconds %s the hub's clock. A box with a wrong clock refuses signed jobs and stamps its backups with a wrong time. Read its time service (`chronyc tracking`).%s", hostID, secs, map[string]string{"ahead": "ahead of", "behind": "behind"}[dir], hint),
|
||
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "direction": dir, "same_direction": same})
|
||
case abs <= boxClockClearBelow && raised != "":
|
||
_ = h.store.SetOSAlarmRaised(key(raised), time.Time{})
|
||
h.logger.Printf("[INFO] host %s clock: in step again (%d s; was %s)", hostID, secs, raised)
|
||
emit(eventBoxClockOK, "info", fmt.Sprintf("The clock of %s is in step with the hub's clock again (was %s).", hostID, raised),
|
||
map[string]any{"host_id": hostID, "skew_seconds": int64(skew / time.Second), "was": raised})
|
||
}
|
||
}
|
||
|
||
// Proxmox trust alarm (R-1396, R-1380; `03-host-agent.md` §12a). The agent (release 2b and later) reports how it trusts
|
||
// Proxmox's certificate and names the cause when that is wrong: the proof with the CA copy failed, the copy ends
|
||
// within 30 days, the root step gave no copy, the config could not be written. One operator-only event per (box,
|
||
// cause) — raised once, not at every report — and one `proxmox_trust_ok` when the box reports no problem again.
|
||
// A box with NO stanza (an older agent) never alarms and never clears. Pinned by TestProxmoxTrust_*.
|
||
const (
|
||
eventProxmoxTrustProblem = "proxmox_trust_problem"
|
||
eventProxmoxTrustOK = "proxmox_trust_ok"
|
||
)
|
||
|
||
type hostProxmoxTrust struct {
|
||
Mode string `json:"mode"`
|
||
Problem string `json:"problem"`
|
||
Detail string `json:"detail"`
|
||
CANotAfter string `json:"ca_not_after"`
|
||
CADaysLeft *int `json:"ca_days_left"`
|
||
MovedAt string `json:"moved_at"`
|
||
}
|
||
|
||
// proxmoxTrustProblems is the CLOSED list of causes the hub alarms on (the agent's internal/pvetrust and
|
||
// proxmox.Trust* names). A cause outside it is logged and stored under "other" — never dropped, never trusted as text.
|
||
var proxmoxTrustProblems = map[string]string{
|
||
"pinned_cert_changed": "the pinned certificate is not the one Proxmox serves any more",
|
||
"ca_mismatch": "the root CA copy did not sign the certificate Proxmox serves",
|
||
"cert_expired": "the certificate Proxmox serves, or its root CA, has ended",
|
||
"ca_file_unreadable": "the root CA copy is missing or holds no certificate",
|
||
"name_mismatch": "the certificate Proxmox serves does not name the address the agent calls",
|
||
"ca_copy_unavailable": "the box's root step gave no copy of Proxmox's root CA",
|
||
"ca_copy_does_not_verify": "a call to Proxmox with the root CA copy alone failed; the box keeps its pinned certificate",
|
||
"ca_copy_ends_soon": "the root CA copy ends within 30 days",
|
||
"config_not_written": "the agent could not save its new trust settings; the box keeps its pinned certificate",
|
||
}
|
||
|
||
func (h *Handler) checkProxmoxTrust(hostID, custID string, t *hostProxmoxTrust) {
|
||
if t == nil {
|
||
return
|
||
}
|
||
now := time.Now()
|
||
emit := func(typ, sev, msg string, details map[string]any) {
|
||
dj, _ := json.Marshal(details)
|
||
if _, err := h.store.SaveEvent(custID, typ, sev, msg, string(dj), "hub"); err != nil {
|
||
h.logger.Printf("[WARN] %s event save FAILED for %s: %v", typ, hostID, err)
|
||
} else if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(custID, typ, sev, msg, string(dj), "hub")
|
||
}
|
||
}
|
||
if t.Problem == "" {
|
||
// the box reports no problem: clear every cause that was raised for it, with ONE recovery event
|
||
var cleared []string
|
||
for cause := range proxmoxTrustProblems {
|
||
key := "pve_trust:" + hostID + ":" + cause
|
||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||
cleared = append(cleared, cause)
|
||
}
|
||
}
|
||
if key := "pve_trust:" + hostID + ":other"; !h.store.OSAlarmRaised(key).IsZero() {
|
||
_ = h.store.SetOSAlarmRaised(key, time.Time{})
|
||
cleared = append(cleared, "other")
|
||
}
|
||
if len(cleared) > 0 {
|
||
sort.Strings(cleared)
|
||
h.logger.Printf("[INFO] host %s proxmox trust: no problem any more (was: %s; mode %s)", hostID, strings.Join(cleared, ", "), trustModeWord(t.Mode))
|
||
emit(eventProxmoxTrustOK, "info", fmt.Sprintf("The agent of %s trusts Proxmox again (mode: %s). Was: %s.", hostID, trustModeWord(t.Mode), strings.Join(cleared, ", ")),
|
||
map[string]any{"host_id": hostID, "mode": trustModeWord(t.Mode), "was": cleared})
|
||
}
|
||
return
|
||
}
|
||
mode, caEnds := trustModeWord(t.Mode), trustDate(t.CANotAfter)
|
||
cause, text := t.Problem, proxmoxTrustProblems[t.Problem]
|
||
if text == "" {
|
||
cause, text = "other", "a cause this hub does not know"
|
||
}
|
||
key := "pve_trust:" + hostID + ":" + cause
|
||
if !h.store.OSAlarmRaised(key).IsZero() {
|
||
return // said once; the box repeats it at every report
|
||
}
|
||
_ = h.store.SetOSAlarmRaised(key, now)
|
||
sev := "error"
|
||
if cause == "ca_copy_ends_soon" {
|
||
sev = "warning"
|
||
}
|
||
days := ""
|
||
if t.CADaysLeft != nil {
|
||
days = fmt.Sprintf(" The root CA copy ends in %d day(s) (%s).", *t.CADaysLeft, caEnds)
|
||
}
|
||
h.logger.Printf("[WARN] host %s proxmox trust: %s (mode %s)", hostID, cause, mode)
|
||
emit(eventProxmoxTrustProblem, sev, fmt.Sprintf("Trust in Proxmox on %s: %s (mode: %s).%s Design: `03` §12a.", hostID, text, mode, days),
|
||
map[string]any{"host_id": hostID, "cause": cause, "mode": mode, "ca_not_after": caEnds, "subject": hostID + ":" + cause})
|
||
}
|
||
|
||
// trustModeWord / trustDate: the stanza is the BOX's. Only a word of the closed list and a real date reach an event
|
||
// (and so a mail); anything else is „other” / "" — the comment „never trusted as text” must hold for every field.
|
||
func trustModeWord(m string) string {
|
||
switch m {
|
||
case "fingerprint", "ca", "ca_file_own", "insecure", "system":
|
||
return m
|
||
}
|
||
return "other"
|
||
}
|
||
|
||
func trustDate(s string) string {
|
||
if t, err := time.Parse(time.RFC3339, s); err == nil {
|
||
return t.UTC().Format(time.RFC3339)
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func truncateRunes(s string, n int) string {
|
||
r := []rune(s)
|
||
if len(r) <= n {
|
||
return s
|
||
}
|
||
return string(r[:n])
|
||
}
|
||
|
||
// recordHeldApp applies one app_held / app_hold_cleared to the held_apps mirror (D53, `08` §10). The hub never
|
||
// clears a hold on a box; it only forgets one the box says is gone.
|
||
func (h *Handler) recordHeldApp(customerID, eventType string, raw json.RawMessage) {
|
||
var d struct {
|
||
StackName string `json:"stack_name"`
|
||
Reason string `json:"reason"`
|
||
Since string `json:"since"`
|
||
Why string `json:"why"`
|
||
}
|
||
if err := json.Unmarshal(raw, &d); err != nil || d.StackName == "" {
|
||
h.logger.Printf("[WARN] [held-apps] %s from %s has no readable stack_name — not recorded", eventType, customerID)
|
||
return
|
||
}
|
||
if eventType == "app_hold_cleared" {
|
||
had, err := h.store.ClearHeldApp(customerID, d.StackName)
|
||
if err != nil {
|
||
h.logger.Printf("[WARN] [held-apps] clear %s/%s: %v", customerID, d.StackName, err)
|
||
return
|
||
}
|
||
h.logger.Printf("[INFO] [held-apps] %s/%s no longer held (a row existed: %v)", customerID, d.StackName, had)
|
||
return
|
||
}
|
||
var since time.Time
|
||
if d.Since != "" {
|
||
if t, err := time.Parse(time.RFC3339, d.Since); err == nil {
|
||
since = t
|
||
} else {
|
||
h.logger.Printf("[WARN] [held-apps] %s/%s: since %q is not RFC3339 — the hub's first sighting is the clock", customerID, d.StackName, d.Since)
|
||
}
|
||
}
|
||
if err := h.store.UpsertHeldApp(customerID, d.StackName, d.Reason, since, d.Why, time.Now()); err != nil {
|
||
h.logger.Printf("[WARN] [held-apps] record %s/%s: %v", customerID, d.StackName, err)
|
||
return
|
||
}
|
||
h.logger.Printf("[DEBUG] [held-apps] %s/%s held (%s since %s)", customerID, d.StackName, d.Reason, d.Since)
|
||
}
|
||
|
||
// noteAppList keeps the hub's own record of which apps a box runs (store.NoteAppList) from one controller report:
|
||
// `stacks.deployed` is the app list, `offsite.ended_apps` the apps the box holds a pending off-site end record for.
|
||
// A report with no `stacks` object changes nothing — an absent list is not an empty list. The box's dates in
|
||
// `ended_apps` are not read: the clock is the hub's (D72, `09` §3 decision 292).
|
||
func (h *Handler) noteAppList(customerID string, body []byte) {
|
||
var p struct {
|
||
Stacks *struct {
|
||
Deployed []string `json:"deployed"`
|
||
} `json:"stacks"`
|
||
Offsite *struct {
|
||
EndedApps []struct {
|
||
App string `json:"app"`
|
||
// The box's own dates. Carried on the wire for the operator's reading of a stored report; the
|
||
// hub decides by its own clock and never by these.
|
||
RemovedAt string `json:"removed_at"`
|
||
EndsAt string `json:"ends_at"`
|
||
} `json:"ended_apps"`
|
||
} `json:"offsite"`
|
||
}
|
||
if err := json.Unmarshal(body, &p); err != nil || p.Stacks == nil {
|
||
return
|
||
}
|
||
var claimed []string
|
||
if p.Offsite != nil {
|
||
for _, e := range p.Offsite.EndedApps {
|
||
claimed = append(claimed, e.App)
|
||
}
|
||
}
|
||
unseen, err := h.store.NoteAppListClaims(customerID, p.Stacks.Deployed, claimed, time.Now())
|
||
if err != nil {
|
||
h.logger.Printf("[WARN] [api] report from %s: the hub's own app record could not be updated: %v", customerID, err)
|
||
return
|
||
}
|
||
// Review of 2026-11-15: an app the hub NEVER saw in this box's app list, named in an end record for the first
|
||
// time. It is the honest shape of „removed before the hub kept this record” — and the shape of a box that invents
|
||
// an app to have its copies agreed. The operator hears it now, 30 days before the hub could agree to anything.
|
||
// The store returns each app once and at most a few per household; the name has the shape of an app name.
|
||
for _, u := range unseen {
|
||
day := u.AbsentSince.Add(offsitekeys.EndedAppAfter).UTC().Format("2006-01-02")
|
||
msg := fmt.Sprintf("Off-site: the box of %s holds an end record for the app %q (its off-site copies are to be deleted 30 days after „delete backup data”), "+
|
||
"and the hub never saw that app in this box's app list. The hub's own 30 days start now: from %s it would agree, in a clean-up window, "+
|
||
"to remove that app's off-site snapshots — at most the window's cap per week. If the household did not remove an app of that name, "+
|
||
"look at the box before that day and switch the weekly windows off for it.", customerID, u.App, day)
|
||
details, _ := json.Marshal(map[string]any{"subject": u.App, "app": u.App, "agrees_from": day})
|
||
h.logger.Printf("[WARN] [api] report from %s: an end record names %q, an app the hub never saw installed — operator told; the hub would agree from %s", customerID, u.App, day)
|
||
if _, eerr := h.store.SaveEvent(customerID, offsitekeys.EventEndedClaimUnseen, "warning", msg, string(details), "hub"); eerr != nil {
|
||
h.logger.Printf("[WARN] [api] %s event save FAILED for %s: %v", offsitekeys.EventEndedClaimUnseen, customerID, eerr)
|
||
continue // the audit row first: a mail without its event row lies
|
||
}
|
||
if h.dispatcher != nil {
|
||
go h.dispatcher.ProcessEvent(customerID, offsitekeys.EventEndedClaimUnseen, "warning", msg, string(details), "hub")
|
||
}
|
||
}
|
||
}
|