Files
felhom-agent/internal/hub/client.go
T
admin ede49b610d
gates / gates (push) Successful in 7s
R-344: restore the idle-connection timeout our hand-rolled transports lost
Every client here pins TLS, so none can use http.DefaultTransport and each
hand-rolls its own. A composite literal takes IdleConnTimeout ZERO, which
means retain idle connections forever -- not "use a sane default".
pbsTargetsFromPVE builds a fresh pbs.Client every cycle and drops the
previous one, and an abandoned http.Transport does not close its
connections. One stranded socket per cycle, on both sides, forever.

Measured: 388 established connections on ep0 over 46 h, 194 per box, zero
closed in a 31-minute window. pvestatd and proxmox-backup-client made
162,404 requests in the same window and leaked none.

New leaf package internal/httpx owns the default (90s, http.DefaultTransport's
own value) and NewTransport, which returns a FRESH transport per call and
treats <=0 as "use the default", never "no timeout". pbs.Config gains
IdleConnTimeout for tests only.

hub and proxmox carried the same missing default and are corrected here for
consistency. Neither contributed to the ep0 leak -- both are built once per
process and neither talks to ep0:8007.

Tests count connections SERVER-side and model the abandonment, so they pin
the consequence rather than the field. Two red-proofs, both seen failing:
removing the timeout -> "still holds 5 open connection(s), want 0";
DisableKeepAlives -> "3 sequential requests over 3 connection(s), want 1"
(the leak test PASSES under that one -- it is the worse-fix guard that
catches it).

Not released: hand-installed on demo-hp only so demo-felhom stays the
control. CHANGELOG heading stays UNRELEASED until the publish is authorised.
2026-08-20 11:09:39 +02:00

428 lines
17 KiB
Go

package hub
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/config"
"gitea.dooplex.hu/admin/felhom-agent/internal/httpx"
)
const reportPath = "/api/v1/host-report"
// Client posts host-reports to the hub. Auth is a per-host Bearer key. Transport is
// standard TLS (system roots, or a CAFile pool); verification is always on — the hub
// has a real cert (unlike the Proxmox self-signed path), so there is no insecure mode.
type Client struct {
baseURL string
apiKey string
hostID string // for the slice-10A desired-state/jobs paths (/hosts/{hostID}/…)
hc *http.Client
logger *slog.Logger
}
// NewClient builds a hub client from config (defaults applied). It never logs the key.
func NewClient(cfg config.HubConfig, logger *slog.Logger) (*Client, error) {
cfg = cfg.WithDefaults()
if logger == nil {
logger = slog.Default()
}
tlsCfg := &tls.Config{} // system roots
if cfg.CAFile != "" {
pem, err := os.ReadFile(cfg.CAFile)
if err != nil {
return nil, fmt.Errorf("hub: reading ca_file: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, fmt.Errorf("hub: ca_file %q contained no usable certificates", cfg.CAFile)
}
tlsCfg.RootCAs = pool
}
hc := &http.Client{
Timeout: time.Duration(cfg.TimeoutSeconds) * time.Second,
// R-344, consistency only: this client is built ONCE per process, so it never accumulated
// and contributed nothing to the ep0 leak. It carried the same missing default, which over
// a tunnel is how one idle connection survives long enough to fail on next use.
Transport: httpx.NewTransport(tlsCfg, 0),
}
return newClient(cfg.URL, cfg.APIKey, cfg.HostID, hc, logger), nil
}
// newClient is the shared constructor (tests inject a mock-transport *http.Client).
func newClient(baseURL, apiKey, hostID string, hc *http.Client, logger *slog.Logger) *Client {
return &Client{baseURL: strings.TrimRight(baseURL, "/"), apiKey: apiKey, hostID: hostID, hc: hc, logger: logger}
}
// TransportError is a network/connection failure (no HTTP response). It never
// contains the bearer token.
type TransportError struct{ Err error }
func (e *TransportError) Error() string { return "hub: transport error: " + e.Err.Error() }
func (e *TransportError) Unwrap() error { return e.Err }
// HTTPError is a non-2xx response. BodyTail is a short, token-free excerpt.
type HTTPError struct {
StatusCode int
BodyTail string
}
func (e *HTTPError) Error() string {
return fmt.Sprintf("hub: HTTP %d: %s", e.StatusCode, e.BodyTail)
}
// Report POSTs the host-report and returns the parsed control envelope. The report
// IS the heartbeat (locked decision 1). Errors are typed (transport vs HTTP) and
// never include the bearer token.
func (c *Client) Report(ctx context.Context, r *HostReport) (*ControlEnvelope, error) {
body, err := json.Marshal(r)
if err != nil {
return nil, fmt.Errorf("hub: marshaling report: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+reportPath, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("hub: building request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err} // token is in the request header, never the error
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var env ControlEnvelope
if err := json.Unmarshal(raw, &env); err != nil {
return nil, fmt.Errorf("hub: decoding control envelope: %w", err)
}
return &env, nil
}
// FetchDesiredState GETs the host's authoritative desired-state (slice 10A — the "Down" channel's
// heavy payload). The agent calls this ONLY when the heartbeat envelope's DesiredGeneration has
// advanced past its cached one (the heartbeat stays light; the state moves on change). It is
// self-scoped server-side: the per-host key only ever reads ITS OWN host (the client uses its
// configured hostID). Errors are typed (transport vs HTTP) and never include the bearer token.
func (c *Client) FetchDesiredState(ctx context.Context) (*DesiredStateResponse, error) {
if c.hostID == "" {
return nil, fmt.Errorf("hub: FetchDesiredState requires a configured host_id")
}
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/desired-state"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("hub: building desired-state request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out DesiredStateResponse
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("hub: decoding desired-state: %w", err)
}
return &out, nil
}
// WGRegisterResponse is the hub's answer to a WG pubkey registration (S3; hub S2
// handleRegisterHostWG). Existed=true = idempotent re-register (nothing moved hub-side).
type WGRegisterResponse struct {
Pubkey string `json:"pubkey"`
AssignedIP string `json:"assigned_ip"` // "10.77.0.2/32"
Existed bool `json:"existed"`
Generation int64 `json:"generation"`
Sync string `json:"sync"` // hub→endpoint push status: ok | deferred:… | disabled | unchanged
}
// RegisterWG registers this host's WG public key with the hub (S3 — doc 06 §3.3 step 2; POST
// /hosts/{host_id}/wg, per-host key, self-scoped server-side). The hub allocates/keeps the /32,
// bumps the desired generation on real change, and pushes the peer to the endpoint. Errors are
// typed (transport vs HTTP: 403 auth, 404 unknown host, 409 conflict/endpoint-unset) and never
// include the bearer token. Only the PUBLIC key ever travels.
func (c *Client) RegisterWG(ctx context.Context, pubkey string) (*WGRegisterResponse, error) {
if c.hostID == "" {
return nil, fmt.Errorf("hub: RegisterWG requires a configured host_id")
}
body, err := json.Marshal(map[string]string{"pubkey": pubkey})
if err != nil {
return nil, fmt.Errorf("hub: marshaling wg registration: %w", err)
}
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/wg"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
if err != nil {
return nil, fmt.Errorf("hub: building wg-register request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out WGRegisterResponse
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("hub: decoding wg-register response: %w", err)
}
return &out, nil
}
// ErrNoPBSSecret is the typed "404: no unconsumed PBS token secret staged for this host" outcome
// (PBS DR slice 2). Absent-or-already-consumed are indistinguishable by design (consume-once).
var ErrNoPBSSecret = fmt.Errorf("hub: no unconsumed PBS token secret staged for this host")
// ConsumePBSToken fetches this host's one-time PBS token secret — EXACTLY ONCE (PBS DR slice 2;
// POST /api/v1/hosts/{host_id}/pbs/consume-token, per-host key, self-scoped; NOTE the PLURAL
// /hosts/ — the slice-1 route). A 200 burns the secret hub-side: the caller MUST apply it or
// surface a loud consumed-but-failed state (never silent-retry). The secret is returned to the
// caller only — never logged, never in an error.
func (c *Client) ConsumePBSToken(ctx context.Context) (string, error) {
if c.hostID == "" {
return "", fmt.Errorf("hub: ConsumePBSToken requires a configured host_id")
}
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/pbs/consume-token"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, nil)
if err != nil {
return "", err
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return "", &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
if resp.StatusCode == http.StatusNotFound {
return "", ErrNoPBSSecret
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out struct {
TokenSecret string `json:"token_secret"`
}
if err := json.Unmarshal(raw, &out); err != nil {
return "", fmt.Errorf("hub: decoding consume-token response (body withheld — secret channel)")
}
if out.TokenSecret == "" {
return "", fmt.Errorf("hub: consume-token returned an empty secret")
}
return out.TokenSecret, nil
}
// JobWire is one queued signed-op job as served by GET /hosts/{id}/jobs (slice 10A). The blob is
// OPAQUE to the hub — for slice 10B it is a base64 `SignedJobEnvelope` (op-blob + armored SSHSIG)
// the agent verifies before executing.
type JobWire struct {
JobID string `json:"job_id"`
BlobB64 string `json:"blob_b64"`
CreatedAt string `json:"created_at"`
}
// Jobs fetches this host's pending signed-op jobs (slice 10B). Self-scoped server-side (the
// per-host key only reads its own host). The agent verifies each before executing.
func (c *Client) Jobs(ctx context.Context) ([]JobWire, error) {
if c.hostID == "" {
return nil, fmt.Errorf("hub: Jobs requires a configured host_id")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/hosts/"+c.hostID+"/jobs", nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out struct {
Jobs []JobWire `json:"jobs"`
}
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("hub: decoding jobs: %w", err)
}
return out.Jobs, nil
}
// CompleteJob clears a processed job from the host's queue (slice 10B): DELETE
// /hosts/{id}/jobs/{job_id}, self-scoped. Called after a job is executed OR permanently rejected
// (the nonce is already durably burned on a passing verify, so re-processing is replay-safe).
func (c *Client) CompleteJob(ctx context.Context, jobID string) error {
if c.hostID == "" || jobID == "" {
return fmt.Errorf("hub: CompleteJob requires host_id + job_id")
}
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.baseURL+"/api/v1/hosts/"+c.hostID+"/jobs/"+jobID, nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
resp, err := c.hc.Do(req)
if err != nil {
return &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
return nil
}
func tail(b []byte, max int) string {
s := strings.TrimSpace(string(b))
if len(s) > max {
return s[:max] + "…"
}
return s
}
// IdentityEscrowResponse mirrors GET /api/v1/hosts/{host_id}/escrow (hub >= v0.94.0, R-199).
// Present=false is a CLEAN answer, not a fault: the host simply has no sealed bundle yet.
type IdentityEscrowResponse struct {
HostID string `json:"host_id"`
Present bool `json:"present"`
IdentityEscrowB64 string `json:"identity_escrow_b64"`
}
// FetchIdentityEscrow reads back THIS host's own opaque identity-escrow blob (R-199 link 6 — the
// mirror of UploadEscrow, self-scoped server-side by the per-host key). The bytes are ciphertext: they
// are useless without the customer's recovery code R, which neither the hub nor this agent ever holds.
//
// It is the ONLY retrieval this client performs, and it is deliberately narrow — no directive, no
// K-escrow, no key rotation. The operator-driven DR path (recovery-mode re-enroll) is a different
// endpoint with a different gate and is not reached from here.
//
// Errors are typed (transport vs HTTP) and never include the bearer token. The BLOB is never logged —
// only its length.
func (c *Client) FetchIdentityEscrow(ctx context.Context) (*IdentityEscrowResponse, error) {
if c.hostID == "" {
return nil, fmt.Errorf("hub: FetchIdentityEscrow requires a configured host_id")
}
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/escrow"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("hub: building escrow-fetch request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out IdentityEscrowResponse
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("hub: decoding escrow fetch: %w", err)
}
return &out, nil
}
// RetainedEscrowPackage is one RETAINED (superseded) sealed identity package. The blob is ciphertext
// and is useless without R. `SupersededAt` is the only thing here a human ever sees — it is what lets
// the recovery screen name WHICH earlier package a code belongs to.
type RetainedEscrowPackage struct {
Index int `json:"index"`
SupersededAt string `json:"superseded_at"`
KeyFingerprint string `json:"key_fingerprint"`
IdentityEscrowB64 string `json:"identity_escrow_b64"`
}
// RetainedEscrowResponse mirrors GET /api/v1/hosts/{host_id}/escrow/retained (hub >= v0.103.0, R-311).
//
// UnopenableCount is NOT noise. It counts retained packages the hub holds whose key material is absent
// (every pre-v0.93.0 row): on a box with those and nothing else, a perfectly correct old recovery code
// opens nothing, and the reason is a defect of ours. A caller that ignores this number will tell such a
// customer their code is wrong — the exact failure this whole chain exists to stop.
type RetainedEscrowResponse struct {
HostID string `json:"host_id"`
Count int `json:"count"`
UnopenableCount int `json:"unopenable_count"`
TruncatedCount int `json:"truncated_count"`
Packages []RetainedEscrowPackage `json:"packages"`
}
// FetchRetainedIdentityEscrow reads back THIS host's RETAINED sealed identity packages (R-311 —
// the retained siblings of FetchIdentityEscrow, self-scoped server-side by the same per-host key).
//
// SEPARATE FROM FetchIdentityEscrow ON PURPOSE. The ordinary recovery must not pay for this call, and
// must not fail because of it: the current package is tried first and alone, and this is reached only
// after that has refused. A hub too old to know this route answers 404, which is a CLEAN "none" here
// and must never be reported as a failed recovery.
func (c *Client) FetchRetainedIdentityEscrow(ctx context.Context) (*RetainedEscrowResponse, error) {
if c.hostID == "" {
return nil, fmt.Errorf("hub: FetchRetainedIdentityEscrow requires a configured host_id")
}
url := c.baseURL + "/api/v1/hosts/" + c.hostID + "/escrow/retained"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("hub: building retained-escrow request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+c.apiKey)
req.Header.Set("Accept", "application/json")
resp, err := c.hc.Do(req)
if err != nil {
return nil, &TransportError{Err: err}
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode == http.StatusNotFound {
// A hub older than v0.103.0 has no such route. That is "no retained packages", not a fault —
// returning an error here would turn an old hub into a failed recovery on a box whose current
// package simply did not open.
return &RetainedEscrowResponse{HostID: c.hostID}, nil
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, &HTTPError{StatusCode: resp.StatusCode, BodyTail: tail(raw, 256)}
}
var out RetainedEscrowResponse
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("hub: decoding retained escrow fetch: %w", err)
}
return &out, nil
}