Files
felhom-controller/controller/internal/agentapi/client.go
T

1053 lines
44 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Package agentapi is the in-guest controller's client for the host agent's per-guest local
// API (doc 03 §6, slice 8A). It reaches the agent over the bridge, pinning the agent's
// self-signed leaf by SHA-256 (the same pin convention the agent uses for the Proxmox/PBS host
// certs), and authenticates with the per-guest bearer token. In 8A it exercises GET /storage
// (connectivity + the controller learning its mounts); the full surface (the /backup/due
// quiesce loop) lands in 8B.
package agentapi
import (
"bytes"
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"regexp"
"strings"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/logx"
)
// Client talks to one agent local-API endpoint with a pinned leaf + bearer token.
type Client struct {
baseURL string
token string
hc *http.Client
// features caches capability-probe verdicts for Supports (features.go).
features SupportCache
// verMu guards lastAgentVersion — the most recent STRICTLY-VALIDATED X-Felhom-Agent-Version
// seen on any agent response (v0.82.0 version channel). "" = never seen (pre-0.82 agent) →
// Supports falls back to the route probe.
verMu sync.Mutex
lastAgentVersion string
// logger is the optional per-call DEBUG trace sink (v0.116.0 observability — the
// capture ring holds these even at logging.level=info). nil = silent (unchanged).
logger *log.Logger
}
// SetLogger wires the optional per-call DEBUG trace logger (method, path, status,
// duration + agent-version changes — never bodies or tokens).
func (c *Client) SetLogger(l *log.Logger) { c.logger = l }
// reAgentVersion is the bare-semver shape the publish pipeline enforces (publish-agent.sh) — the
// ONLY header values trusted for capability comparison. Anything else (garbage, "dev", suffixes)
// is ignored and the probe fallback stays in charge.
var reAgentVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+$`)
// noteAgentVersion records a response's version header (passive capture — called on EVERY response
// path). Invalid/absent headers never overwrite a previously-seen valid version.
func (c *Client) noteAgentVersion(resp *http.Response) {
v := strings.TrimSpace(resp.Header.Get("X-Felhom-Agent-Version"))
if v == "" || !reAgentVersion.MatchString(v) {
return
}
c.verMu.Lock()
prev := c.lastAgentVersion
c.lastAgentVersion = v
c.verMu.Unlock()
if prev != v {
logx.Debugf(c.logger, "[agentapi] agent version seen: %s (was %q)", v, prev)
}
}
// AgentVersion returns the last strictly-validated agent version seen on this client's traffic
// ("" = unknown — header-less agent or no traffic yet). This is the Supports comparison source.
func (c *Client) AgentVersion() string {
c.verMu.Lock()
defer c.verMu.Unlock()
return c.lastAgentVersion
}
// MountInfo mirrors the agent's GET /storage mount entry (doc 03 §6).
type MountInfo struct {
Key string `json:"key"`
Storage string `json:"storage"`
MountPoint string `json:"mount_point"`
Class string `json:"class"` // fast | slow | ""
Backup bool `json:"backup"`
}
// StorageResponse mirrors the agent's GET /storage data payload.
type StorageResponse struct {
VMID int `json:"vmid"`
Mounts []MountInfo `json:"mounts"`
}
// apiResponse is the agent's {ok,data,error} envelope.
type apiResponse struct {
OK bool `json:"ok"`
Data json.RawMessage `json:"data"`
Error string `json:"error"`
}
// New builds a pinned client for endpoint ("host:port") with the given per-guest token and the
// agent leaf-cert SHA-256 fingerprint (hex, ':'-separators tolerated). The pin is the trust
// anchor — the agent serves a self-signed cert, so chain verification is replaced by an exact
// leaf-DER SHA-256 match (fails closed on any mismatch).
func New(endpoint, token, fingerprintHex string) (*Client, error) {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return nil, fmt.Errorf("agentapi: endpoint required")
}
if token == "" {
return nil, fmt.Errorf("agentapi: token required")
}
want, err := normalizeFingerprint(fingerprintHex)
if err != nil {
return nil, err
}
tlsCfg := &tls.Config{
InsecureSkipVerify: true, // self-signed leaf — the pin below is the real check
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return fmt.Errorf("agentapi: TLS pin: peer presented no certificate")
}
got := sha256.Sum256(rawCerts[0]) // leaf DER
if hex.EncodeToString(got[:]) != want {
return fmt.Errorf("agentapi: TLS pin mismatch: agent leaf SHA-256 does not match the bootstrap fingerprint")
}
return nil
},
MinVersion: tls.VersionTLS12,
}
return &Client{
baseURL: "https://" + endpoint,
token: token,
hc: &http.Client{
Timeout: 15 * time.Second,
// Bound + expire the idle-conn pool. With the controller reusing one Client (so the pool
// stays ~2), IdleConnTimeout also lets idle conns to a RESTARTED agent drain instead of
// lingering as stale ESTABLISHED entries, and caps any future per-call misuse. (The earlier
// bare Transport had IdleConnTimeout:0 = idle keep-alives never expire → the leak.)
Transport: &http.Transport{
TLSClientConfig: tlsCfg,
MaxIdleConns: 4,
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
},
},
}, nil
}
// Close releases the client's idle keep-alive connections. Optional hygiene for any caller that builds
// a short-lived client; the controller reuses one long-lived client, so it relies on the bounded,
// expiring idle pool (above) rather than calling this.
func (c *Client) Close() { c.hc.CloseIdleConnections() }
// Storage calls GET /storage and returns this guest's mounts (connectivity + placement view).
func (c *Client) Storage(ctx context.Context) (StorageResponse, error) {
var out StorageResponse
body, err := c.get(ctx, "/storage")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /storage: %w", err)
}
return out, nil
}
// ---- slice 8B: app-consistent backup (quiesce loop) -------------------------------------
// DueResponse mirrors the agent's GET /backup/due payload. AgeSecs is the age of the newest
// successful backup (nil when none has run yet).
type DueResponse struct {
VMID int `json:"vmid"`
Due bool `json:"due"`
Reason string `json:"reason"`
AgeSecs *int64 `json:"age_seconds"`
}
// BackupResponse mirrors the agent's POST /backup payload.
type BackupResponse struct {
VMID int `json:"vmid"`
JobID string `json:"job_id"`
Phase string `json:"phase"`
}
// StatusResponse mirrors the agent's GET /backup/status payload. Backup is the latest RECORDED
// whole-guest backup (nil until one has run), surfaced to the controller's backup page for visibility.
type StatusResponse struct {
VMID int `json:"vmid"`
Phase string `json:"phase"` // idle | running | snapshotted | done | failed
JobID string `json:"job_id"`
Error string `json:"error"`
Backup *BackupRecord `json:"backup,omitempty"`
}
// BackupRecord mirrors the agent's hub.Backup — one whole-guest vzdump/PBS backup result. The
// controller renders it read-only (it does NOT own whole-guest backup; the agent does).
type BackupRecord struct {
TargetID string `json:"target_id"` // backup storage name (e.g. "local", "felhom-pbs")
VMID int `json:"vmid"`
Archive string `json:"archive"` // produced vzdump volid (e.g. "local:backup/vzdump-lxc-…")
Mode string `json:"mode"` // snapshot | stop
CrashConsistent bool `json:"crash_consistent"`
SizeBytes int64 `json:"size_bytes"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
StartedAt string `json:"started_at"` // RFC3339
DurationSeconds float64 `json:"duration_seconds"`
}
// RestoreTestRecord mirrors the agent's hub.RestoreTest — the latest self-restore-test (the "backup
// verified restorable" trust signal). Nil when none has run yet.
type RestoreTestRecord struct {
SourceArchive string `json:"source_archive"`
SourceTier string `json:"source_tier"` // "local" (pbs = Phase B)
Pass bool `json:"pass"`
Verified string `json:"verified"` // "boot+running" this slice
Error string `json:"error,omitempty"`
TestedAt string `json:"tested_at"` // RFC3339
DurationSeconds float64 `json:"duration_seconds"`
Warnings []string `json:"warnings,omitempty"`
}
// Backup status phases (mirror the agent's vocabulary).
const (
PhaseIdle = "idle"
PhaseRunning = "running"
PhaseDone = "done"
PhaseFailed = "failed"
)
// BackupDue reports whether a policy-scheduled backup is due for this guest (the quiesce trigger).
func (c *Client) BackupDue(ctx context.Context) (DueResponse, error) {
var out DueResponse
body, err := c.get(ctx, "/backup/due")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /backup/due: %w", err)
}
return out, nil
}
// StartBackup enqueues a backup of this guest (the agent vzdump) and returns the job to poll.
func (c *Client) StartBackup(ctx context.Context) (BackupResponse, error) {
var out BackupResponse
body, err := c.post(ctx, "/backup", struct{}{})
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode POST /backup: %w", err)
}
return out, nil
}
// BackupStatus reports the current/last backup job phase for this guest.
func (c *Client) BackupStatus(ctx context.Context) (StatusResponse, error) {
var out StatusResponse
body, err := c.get(ctx, "/backup/status")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /backup/status: %w", err)
}
return out, nil
}
// RestoreTestStatus calls GET /restore-test/status and returns the latest self-restore-test result
// (nil when none has run yet — the agent payload is {"restore_test": {...}|null}).
func (c *Client) RestoreTestStatus(ctx context.Context) (*RestoreTestRecord, error) {
body, err := c.get(ctx, "/restore-test/status")
if err != nil {
return nil, err
}
var out struct {
RestoreTest *RestoreTestRecord `json:"restore_test"`
}
if err := json.Unmarshal(body, &out); err != nil {
return nil, fmt.Errorf("agentapi: decode /restore-test/status: %w", err)
}
return out.RestoreTest, nil
}
// ---- slice 8C: disk management (execution is the agent's) --------------------------------
// DiskInfo mirrors the agent's GET /disks entry.
type DiskInfo struct {
Name string `json:"name"`
Type string `json:"type"`
State string `json:"state"`
BackingDevice string `json:"backing_device"`
MountPath string `json:"mount_path"`
Class string `json:"class"`
// Role is the agent's AUTHORITATIVE protection tier: "system" | "backup" | "user-data". The UI
// is driven from it — system/backup get a lock badge and NO destructive controls; user-data is
// customer-manageable (eject/wipe with type-to-confirm).
Role string `json:"role"`
DataBearing bool `json:"data_bearing"`
DataReason string `json:"data_reason"`
TotalBytes int64 `json:"total_bytes"`
UsedBytes int64 `json:"used_bytes"`
UsedFraction float64 `json:"used_fraction"`
// DurableID is the target's stable identity (e.g. "uuid:<fs-uuid>" for usb/local-dir). The
// fs UUID (strip the "uuid:" prefix) is the key the controller passes to AssignDisk — it's the
// only way the de-privileged controller learns a mount key it cannot read off the device itself.
DurableID string `json:"durable_id"`
// WipeDurableID is the device's wipe-binding id in the gate's scheme (byid:<wwn>/byuuid:<uuid>) —
// the id a customer-confirmed data-bearing wipe must carry (F20-BUG2). DISTINCT from DurableID
// (uuid:, used for assign): confirming a wipe with DurableID was rejected (binding_mismatch).
WipeDurableID string `json:"wipe_durable_id,omitempty"`
// GuestAttached reports whether the drive is actually bound into THIS guest (usable in-guest), as
// opposed to merely present on the host (F9) — the signal whose absence let the HDD look available
// when it wasn't attached. LEGACY (per-drive mp model); the intermediary model uses BoundUnderParent.
GuestAttached bool `json:"guest_attached"`
// GuestPath is the drive's STABLE in-guest path in the intermediary-mount model
// (/mnt/felhom-drives/<name>) — what the controller registers + repoints HDD_PATH to. Distinct from
// MountPath (the raw /mnt/<name> host PVE mount the agent ops on). "" for non-user-data drives.
GuestPath string `json:"guest_path,omitempty"`
// BoundUnderParent reports whether the drive's felhom-data is currently bound under the shared parent
// (live + usable in the guest). The controller's drive-absent gate keys on this + State.
BoundUnderParent bool `json:"bound_under_parent"`
}
// FSUUID returns the raw filesystem UUID from a "uuid:<…>" DurableID, or "" if this disk's identity
// is not a filesystem UUID (network/lvm targets — not assignable as a host mount).
func (d DiskInfo) FSUUID() string {
if rest, ok := strings.CutPrefix(d.DurableID, "uuid:"); ok {
return rest
}
return ""
}
// DisksResponse mirrors GET /disks.
type DisksResponse struct {
VMID int `json:"vmid"`
Disks []DiskInfo `json:"disks"`
// GuestBootID changes on every guest boot (host or guest reboot) but is stable across a
// controller-only restart — the deterministic signal the controller recreates drive-backed apps on.
GuestBootID string `json:"guest_boot_id,omitempty"`
}
// FormatResult mirrors POST /disks/format (the success/refusal payload).
type FormatResult struct {
VMID int `json:"vmid"`
Device string `json:"device"`
Formatted bool `json:"formatted"`
DataBearing bool `json:"data_bearing"`
Reason string `json:"reason"`
// Role is the agent's tier for this device (system | backup | user-data).
Role string `json:"role,omitempty"`
// NeedsConfirmation is set on a USER-DATA data-bearing refusal: re-submit with confirmed=true +
// DurableID after the type-to-confirm UI (NOT an operator signature).
NeedsConfirmation bool `json:"needs_confirmation,omitempty"`
DurableID string `json:"durable_id,omitempty"`
// PendingOp is set on a SYSTEM/BACKUP data-bearing refusal — the operator-signature op.
PendingOp *PendingOp `json:"pending_op,omitempty"`
}
// PendingOp mirrors the agent's bound destructive intent on a data-bearing refusal. The controller
// surfaces the exact `felhom-opsign` command from it — it CANNOT complete a destructive format itself.
type PendingOp struct {
Op string `json:"op"` // e.g. "storage_wipe"
HostScope string `json:"host_scope"` // the agent's host id (anti-retarget)
DurableID string `json:"durable_id"` // byid:…|byuuid:… — the device's stable identity
FSType string `json:"fstype"` // the filesystem to mkfs after the wipe
}
// OpsignCommand returns the literal command the operator must run offline to authorize the wipe.
func (p PendingOp) OpsignCommand() string {
return fmt.Sprintf("felhom-opsign -op %s -host %s -durable-id %s", p.Op, p.HostScope, p.DurableID)
}
// ErrFormatRefused is returned by FormatDisk when the agent refuses a data-bearing format on a
// SYSTEM/BACKUP device (operator signature required). The UI surfaces the pending opsign command.
var ErrFormatRefused = fmt.Errorf("agentapi: format refused — system/backup device (operator authorization required)")
// ErrNeedsConfirmation is returned by FormatDisk when the agent refuses a data-bearing format on a
// USER-DATA device pending the CUSTOMER's informed-confirmation (bound to FormatResult.DurableID).
// The UI surfaces the type-to-confirm flow, then re-submits with confirmed=true + that durable id.
var ErrNeedsConfirmation = fmt.Errorf("agentapi: format needs customer confirmation — user-data device")
// Disks lists the host drives the agent manages, with a data-bearing flag per drive.
func (c *Client) Disks(ctx context.Context) (DisksResponse, error) {
var out DisksResponse
body, err := c.get(ctx, "/disks")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /disks: %w", err)
}
return out, nil
}
// DiskCandidate mirrors one entry from the agent's GET /disks/candidates (Impl-2a candidates.go) —
// a host disk the agent's unclaimed-disk filter proved is FREE for Felhom to enroll.
type DiskCandidate struct {
Device string `json:"device"`
SizeBytes int64 `json:"size_bytes"`
Model string `json:"model,omitempty"`
FSType string `json:"fstype,omitempty"`
DataBearing bool `json:"data_bearing"`
Mountable bool `json:"mountable"`
MountSource string `json:"mount_source,omitempty"`
DurableID string `json:"durable_id,omitempty"`
}
// CandidatesResult mirrors GET /disks/candidates: disks free to enroll, split into initialize (all
// unclaimed) and attach (the mountable-FS subset).
type CandidatesResult struct {
VMID int `json:"vmid"`
Initialize []DiskCandidate `json:"initialize"`
Attach []DiskCandidate `json:"attach"`
}
// ListCandidates fetches the host disks free for Felhom to enroll (Impl-2b wizard source).
func (c *Client) ListCandidates(ctx context.Context) (CandidatesResult, error) {
var out CandidatesResult
body, err := c.get(ctx, "/disks/candidates")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /disks/candidates: %w", err)
}
return out, nil
}
// AssignDisk attaches a drive (by fs-UUID) as a host mount (benign, self-serve).
func (c *Client) AssignDisk(ctx context.Context, uuid, where, fstype, options string) error {
_, err := c.post(ctx, "/disks/assign", map[string]string{
"uuid": uuid, "where": where, "fstype": fstype, "options": options,
})
return err
}
// GuestAttach binds an enrolled drive's felhom-data namespace into THIS guest (slice 10 P2, Model A).
// The drive must already be host-mounted at `where` (the enroll flow's assign did that). Idempotent on
// the agent side (returns the existing slot if already bound). Returns nil on success.
func (c *Client) GuestAttach(ctx context.Context, where string) error {
_, err := c.post(ctx, "/disks/guest-attach", map[string]string{"where": where})
return err
}
// GuestReboot reboots THIS guest to activate persisted-but-inactive drive binds (slice 10 P2
// activation — the host-side live inject is blocked on an unprivileged guest, so a drive enrolled into
// a running guest activates only at the next boot). The agent runs the reboot detached + returns 202;
// this guest (and the controller) restarts shortly after. User-triggered ("Újraindítás most").
func (c *Client) GuestReboot(ctx context.Context) error {
_, err := c.post(ctx, "/guest/reboot", struct{}{})
return err
}
// SwapResult mirrors the agent's 202 from POST /controller/swap (agentic controller update, Phase 1).
type SwapResult struct {
Status string `json:"status"` // "swapping"
PreviousImage string `json:"previous_image"` // image before the swap (for the UI/log)
TargetImage string `json:"target_image"`
}
// SwapController asks the agent to swap the in-guest controller to `image` (which the controller has
// already pulled into the guest's docker storage). The agent responds 202 and performs the swap+verify
// +rollback asynchronously, EXTERNALLY to this controller container (so it survives this process being
// killed by the swap). Latest-only is enforced by the caller (queryRegistry); the agent re-validates
// the ref shape.
func (c *Client) SwapController(ctx context.Context, image string) (SwapResult, error) {
var out SwapResult
body, err := c.post(ctx, "/controller/swap", map[string]string{"image": image})
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /controller/swap: %w", err)
}
return out, nil
}
// SwapStatus mirrors the agent's GET /controller/swap/status (observability for the post-restart UI).
type SwapStatus struct {
State string `json:"state"` // none | swapping | done | failed
InFlight bool `json:"in_flight"`
Current string `json:"current"`
Previous string `json:"previous"`
Target string `json:"target"`
Error string `json:"error"`
}
// SwapStatus reads the last/in-flight swap outcome for this guest.
func (c *Client) SwapStatus(ctx context.Context) (SwapStatus, error) {
var out SwapStatus
body, err := c.get(ctx, "/controller/swap/status")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /controller/swap/status: %w", err)
}
return out, nil
}
// EjectResult mirrors POST /disks/eject (the dependent-guest warning).
type EjectResult struct {
VMID int `json:"vmid"`
Ejected string `json:"ejected"`
DependentGuests []int `json:"dependent_guests"`
}
// EjectDisk safe-unmounts a host mount (data preserved) and returns the dependent guests.
// Status-aware POST (campaign F2 evidence gap): the agent's refusal body carries the reason
// (e.g. "…eject refused (role: system)") — surface it instead of a bare "HTTP 403".
// StageEscrowSecret pushes the offsite restic repo password to the agent (fork-4), which stages it
// transiently for the escrow-create ceremony to wrap under the customer recovery code R. The value is
// sent over the authenticated pinned local-API channel; the CALLER must never log it.
func (c *Client) StageEscrowSecret(ctx context.Context, resticRepoPassword string) error {
env, status, err := c.postWithStatus(ctx, "/escrow/stage-secret", map[string]string{"restic_repo_password": resticRepoPassword})
if err != nil {
return err
}
return refusalError("/escrow/stage-secret", status, env)
}
// WipeStagedEscrowSecret removes the agent-staged offsite repo password (fork-4 hygiene) — called whenever
// EscrowState flips to escrowed, so the transient 0600 staging file doesn't outlive its purpose. Idempotent
// on the agent side (an absent file is a clean 200). Requires agent >= v0.78.0 (older agents 404 — the
// caller logs loudly and moves on).
func (c *Client) WipeStagedEscrowSecret(ctx context.Context) error {
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.baseURL+"/escrow/stage-secret", nil)
if err != nil {
return err
}
req.Header.Set("Authorization", "Bearer "+c.token)
resp, err := c.hc.Do(req)
if err != nil {
return fmt.Errorf("agentapi: DELETE /escrow/stage-secret: %w", err)
}
defer resp.Body.Close()
c.noteAgentVersion(resp) // v0.82.0 version channel: passive capture on EVERY response
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
var env apiResponse
if err := json.Unmarshal(raw, &env); err != nil {
return fmt.Errorf("agentapi: DELETE /escrow/stage-secret: HTTP %d, bad envelope: %w", resp.StatusCode, err)
}
return refusalError("/escrow/stage-secret", resp.StatusCode, env)
}
func (c *Client) EjectDisk(ctx context.Context, where string) (EjectResult, error) {
var out EjectResult
env, status, err := c.postWithStatus(ctx, "/disks/eject", map[string]string{"where": where})
if err != nil {
return out, err
}
if err := refusalError("/disks/eject", status, env); err != nil {
return out, err
}
if err := json.Unmarshal(env.Data, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /disks/eject: %w", err)
}
return out, nil
}
// DecommissionResult mirrors POST /disks/decommission.
type DecommissionResult struct {
VMID int `json:"vmid"`
Decommissioned string `json:"decommissioned"`
DependentGuests []int `json:"dependent_guests"`
}
// Decommission permanently removes a user-data drive (self-serve, non-destructive — the agent records
// IntentDecommissioned, prunes the bind record, and unmounts; it NEVER formats). Data stays on the
// drive. The agent role-gates to user-data and refuses a system/backup mount regardless.
// Status-aware POST (campaign F2 evidence gap): the agent's refusal body carries the reason
// (e.g. "…decommission refused (role: system)") — surface it instead of a bare "HTTP 403".
func (c *Client) Decommission(ctx context.Context, where string) (DecommissionResult, error) {
var out DecommissionResult
env, status, err := c.postWithStatus(ctx, "/disks/decommission", map[string]string{"where": where})
if err != nil {
return out, err
}
if err := refusalError("/disks/decommission", status, env); err != nil {
return out, err
}
if err := json.Unmarshal(env.Data, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /disks/decommission: %w", err)
}
return out, nil
}
// refusalError converts a non-2xx status or an ok:false envelope into an error that CARRIES the
// agent's reason (truncated; never request bodies or secrets). nil on an accepted 200/202+ok=true.
func refusalError(path string, status int, env apiResponse) error {
accepted := status == http.StatusOK || status == http.StatusAccepted
if accepted && env.OK {
return nil
}
reason := truncateErr(env.Error, 300)
if reason == "" {
reason = "(no reason in agent response)"
}
if accepted { // 2xx but ok:false — business refusal without an HTTP error code
return fmt.Errorf("agentapi: POST %s: %s", path, reason)
}
return fmt.Errorf("agentapi: POST %s: HTTP %d: %s", path, status, reason)
}
// truncateErr mirrors stacks.truncateStr for agent refusal reasons.
func truncateErr(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
}
// FormatDisk asks the agent to format a device. The AGENT inspects the device and tiers it by ROLE
// (its own classification, never the controller's claim):
// - blank device → formatted.
// - user-data, data-bearing, NOT confirmed → ErrNeedsConfirmation (out.DurableID = the id to confirm).
// - user-data, data-bearing, confirmed + matching durable id → formatted.
// - system/backup, data-bearing → ErrFormatRefused (out.PendingOp = the operator opsign command).
//
// confirmed + durableID authorize a user-data wipe (the durable id the agent gave on the prior
// ErrNeedsConfirmation); they are inert for system/backup.
func (c *Client) FormatDisk(ctx context.Context, device, fstype string, confirmed bool, durableID string) (FormatResult, error) {
var out FormatResult
// Status-aware POST: the agent returns the FULL FormatResponse (incl. pending_op / durable_id)
// even on the 403 refusal, so we must read the body on non-2xx rather than discarding it.
env, status, err := c.postWithStatus(ctx, "/disks/format", map[string]any{
"device": device, "fstype": fstype, "confirmed": confirmed, "durable_id": durableID,
})
if err != nil {
return out, err
}
// env.Data is the envelope's {data:…} payload (present on both success and the 403 refusal).
if len(env.Data) > 0 {
_ = json.Unmarshal(env.Data, &out) // best-effort; fields default on a missing/partial body
}
if out.Formatted {
return out, nil
}
if out.NeedsConfirmation {
out.DataBearing = true
return out, ErrNeedsConfirmation // user-data: surface the type-to-confirm flow
}
if status == http.StatusForbidden || (out.DataBearing && !out.Formatted) {
out.DataBearing = true
return out, ErrFormatRefused // system/backup: surface the opsign command
}
// F20-BUG1: a non-2xx response (or ok:false) that is NOT one of the recognized refusals above is a
// real failure (e.g. the agent's 502 on a mkfs error: "device is mounted"). Returning the zero-value
// result with a nil error here made a failed destructive format read as a silent SUCCESS in the web
// layer. Surface it as an error so the caller (and the dashboard) report the failure.
if status < 200 || status >= 300 || !env.OK {
msg := strings.TrimSpace(env.Error)
if msg == "" {
msg = "format failed"
}
return out, fmt.Errorf("agentapi: format: HTTP %d: %s", status, msg)
}
return out, nil
}
// FormatStatusResult mirrors GET /disks/format/status (F20-BUG3): the most-recent / in-flight format
// job on the host. Phase ∈ idle | running | done | failed. The drive-init flow polls this to follow a
// mkfs that outran the 15 s client timeout — the agent runs the mkfs DETACHED and keeps the record, so
// the client can learn the real outcome instead of assuming failure (F6).
type FormatStatusResult struct {
Phase string `json:"phase"`
Device string `json:"device"`
FSType string `json:"fstype"`
Error string `json:"error"`
}
// FormatStatus fetches the agent's most-recent format-job record.
func (c *Client) FormatStatus(ctx context.Context) (FormatStatusResult, error) {
var out FormatStatusResult
body, err := c.get(ctx, "/disks/format/status")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /disks/format/status: %w", err)
}
return out, nil
}
// ---- NAS network storage (Part A2 → agent A1 /netstorage/*) ------------------------------
//
// A NAS share is a DISTINCT storage class from a drive: the controller proxies add/list/remove to the
// agent (which owns the host-side automount), holds NO mount authority, and persists NO SMB password
// (it passes the credentials straight through to the agent's add request — the agent writes the 0600
// creds file). There is NO eject/decommission/migrate/wipe/SMART here — those are drive-only.
// NetworkMountStatus mirrors the agent's GET /netstorage entry (A1). health ∈ {ok, idle, unreachable}:
// `idle` (reachable + automount idle-unmounted) is BENIGN, not a fault; only `unreachable` is degraded.
type NetworkMountStatus struct {
Name string `json:"name"`
Protocol string `json:"protocol"`
Server string `json:"server"`
Export string `json:"export"`
Where string `json:"where"`
Configured bool `json:"configured"`
Mounted bool `json:"mounted"`
Reachable bool `json:"reachable"`
Health string `json:"health"` // ok | idle | unreachable
}
// Unreachable reports the degraded state (NAS not reachable). `idle` is explicitly NOT unreachable — an
// idle-unmounted automount is the normal steady state, never a warning.
func (n NetworkMountStatus) Unreachable() bool { return n.Health == "unreachable" }
// AddNetStorageRequest is the controller→agent POST /netstorage/add body (A1). Username/Password are SMB
// only and flow STRAIGHT THROUGH to the agent (which writes the 0600 creds file) — the controller never
// stores the password at rest.
type AddNetStorageRequest struct {
Name string `json:"name"`
Protocol string `json:"protocol"` // nfs | smb
Server string `json:"server"`
Export string `json:"export"`
MappedUID int `json:"mapped_uid"`
MappedGID int `json:"mapped_gid"`
IdleTimeoutSec int `json:"idle_timeout_sec,omitempty"`
Username string `json:"username,omitempty"` // SMB secret — pass-through, never persisted
Password string `json:"password,omitempty"` // SMB secret — pass-through, never persisted
}
// NetStorageAddResult mirrors the agent's add response (the in-guest path the media app's data dir
// points at). Since agent v0.81.0 (verify-before-commit) a successful add means "units installed,
// verify STARTED" — Verify/JobID carry the detached verify job the caller polls via NetVerifyStatus.
type NetStorageAddResult struct {
Name string `json:"name"`
Protocol string `json:"protocol"`
Where string `json:"where"`
GuestPath string `json:"guest_path"`
HostUID int `json:"host_uid"`
HostGID int `json:"host_gid"`
Verify string `json:"verify"` // "started" on a verify-before-commit agent (v0.81.0+)
JobID string `json:"job_id"`
Code string `json:"code"` // set on a categorized SYNC refusal (e.g. "unreachable", "busy")
}
// NetAddRefusedError is the agent's CATEGORIZED sync refusal of a netstorage add (the 2 s TCP
// pre-probe "unreachable", or "busy" single-flight). Code is the verify-category vocabulary the UI
// maps to Hungarian; nothing was installed agent-side.
type NetAddRefusedError struct {
Code string
Msg string
}
func (e *NetAddRefusedError) Error() string {
return "agentapi: netstorage add refused (" + e.Code + "): " + e.Msg
}
// NetVerifyStatus mirrors the agent's GET /netstorage/verify-status: the single verify slot.
// Phase "none" is a REAL signal — after an agent restart the in-memory job is gone; the caller
// treats none-after-install as verify-lost and rolls the add back (Scenario F).
type NetVerifyStatus struct {
Phase string `json:"phase"` // none | running | done | failed
Name string `json:"name"`
Where string `json:"where"`
Protocol string `json:"protocol"`
Code string `json:"code"` // failure category (the agent's classifier vocabulary)
Detail string `json:"detail"` // operator hint + raw journal fragment
JobID string `json:"job_id"`
}
// AddNetStorage installs a NAS share host-side and starts the agent's detached verify job (agent
// v0.81.0 verify-before-commit). A categorized sync refusal returns *NetAddRefusedError carrying
// the category code (the result body also carries it); other failures are plain errors.
func (c *Client) AddNetStorage(ctx context.Context, req AddNetStorageRequest) (NetStorageAddResult, error) {
var out NetStorageAddResult
env, status, err := c.postWithStatus(ctx, "/netstorage/add", req)
if err != nil {
return out, err
}
if len(env.Data) > 0 {
_ = json.Unmarshal(env.Data, &out) // best-effort; the refusal body carries {code}
}
if rerr := refusalError("/netstorage/add", status, env); rerr != nil {
if out.Code != "" {
return out, &NetAddRefusedError{Code: out.Code, Msg: truncateErr(env.Error, 300)}
}
return out, rerr
}
return out, nil
}
// NetVerifyStatus polls the agent's verify slot (short GET — fits the client's global 15 s timeout;
// the LONG wait lives in the caller's poll loop, never in one HTTP call).
func (c *Client) NetVerifyStatus(ctx context.Context) (NetVerifyStatus, error) {
var out NetVerifyStatus
body, err := c.get(ctx, "/netstorage/verify-status")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /netstorage/verify-status: %w", err)
}
return out, nil
}
// ListNetStorage returns the configured NAS shares + per-share liveness.
func (c *Client) ListNetStorage(ctx context.Context) ([]NetworkMountStatus, error) {
body, err := c.get(ctx, "/netstorage")
if err != nil {
return nil, err
}
var wrap struct {
NetworkMounts []NetworkMountStatus `json:"network_mounts"`
}
if err := json.Unmarshal(body, &wrap); err != nil {
return nil, fmt.Errorf("agentapi: decode /netstorage: %w", err)
}
return wrap.NetworkMounts, nil
}
// RemoveNetStorage unmounts + removes a NAS share (the agent drops the mount + creds file). This is NOT a
// drive decommission/migrate — a NAS has no device lifecycle.
func (c *Client) RemoveNetStorage(ctx context.Context, name string) error {
_, err := c.post(ctx, "/netstorage/remove", map[string]string{"name": name})
return err
}
// postWithStatus issues an authenticated JSON POST and returns the envelope's data payload + the HTTP
// status, even on a non-2xx (so callers like FormatDisk can read a 403 refusal body). A transport or
// envelope-parse failure is still an error; an `ok:false` business refusal is NOT (the data carries it).
func (c *Client) postWithStatus(ctx context.Context, path string, body any) (apiResponse, int, error) {
var env apiResponse
buf, err := json.Marshal(body)
if err != nil {
return env, 0, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(buf))
if err != nil {
return env, 0, err
}
req.Header.Set("Authorization", "Bearer "+c.token)
req.Header.Set("Content-Type", "application/json")
start := time.Now()
resp, err := c.hc.Do(req)
if err != nil {
logx.Debugf(c.logger, "[agentapi] POST %s failed after %dms: %v", path, time.Since(start).Milliseconds(), err)
return env, 0, fmt.Errorf("agentapi: POST %s: %w", path, err)
}
defer resp.Body.Close()
c.noteAgentVersion(resp) // v0.82.0 version channel: passive capture on EVERY response
logx.Debugf(c.logger, "[agentapi] POST %s -> %d (%dms)", path, resp.StatusCode, time.Since(start).Milliseconds())
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err := json.Unmarshal(raw, &env); err != nil {
return env, resp.StatusCode, fmt.Errorf("agentapi: POST %s: HTTP %d, bad envelope: %w", path, resp.StatusCode, err)
}
return env, resp.StatusCode, nil
}
// ---- v0.116.0: agent debug-log ring (the Debug page agent tab) ----------------------------
// AgentLogEntry mirrors the agent's GET /debug/logs entry (agent ≥ 0.83.0).
type AgentLogEntry struct {
Timestamp time.Time `json:"timestamp"`
Level string `json:"level"`
Message string `json:"message"`
}
// AgentLogsResponse mirrors the agent's GET /debug/logs data payload.
type AgentLogsResponse struct {
VMID int `json:"vmid"`
Entries []AgentLogEntry `json:"entries"`
Total int `json:"total"`
}
// DebugLogs fetches the agent's always-DEBUG capture ring. Against a pre-0.83
// agent the route is absent → a typed *StatusError with Code 404 (the caller
// renders the "available after the agent's next update" notice — S6).
func (c *Client) DebugLogs(ctx context.Context) (AgentLogsResponse, error) {
var out AgentLogsResponse
data, err := c.get(ctx, "/debug/logs")
if err != nil {
return out, err
}
if err := json.Unmarshal(data, &out); err != nil {
return out, fmt.Errorf("agentapi: parsing debug logs: %w", err)
}
return out, nil
}
// ---- slice 9: host metrics (the customer host-health view) -------------------------------
// HostMetrics mirrors the agent's GET /host/metrics `host` block (shared HostMetrics wire shape).
// CPUTempC is a pointer so a host with no temp sensor is null ("n/a"), distinct from a real 0.
type HostMetrics struct {
Node string `json:"node"`
CPUPercent float64 `json:"cpu_percent"` // 0100
MemoryTotalBytes int64 `json:"memory_total_bytes"`
MemoryUsedBytes int64 `json:"memory_used_bytes"`
MemoryPercent float64 `json:"memory_percent"`
DiskTotalBytes int64 `json:"disk_total_bytes"` // host root fs
DiskUsedBytes int64 `json:"disk_used_bytes"`
DiskPercent float64 `json:"disk_percent"`
LoadAvg []string `json:"loadavg"`
UptimeSeconds int64 `json:"uptime_seconds"`
CPUTempC *int `json:"cpu_temp_c"` // °C or null ("n/a")
}
// ThinPoolFill mirrors the agent's lvmthin pool fill (a full thin-pool corrupts every guest on it).
type ThinPoolFill struct {
DataUsedFraction float64 `json:"data_used_fraction"`
MetadataUsedFraction *float64 `json:"metadata_used_fraction"`
}
// SmartSummary mirrors the agent's per-disk SMART health (only the fields the UI renders). Pointers
// are null when the device type does not expose that attribute.
type SmartSummary struct {
Health string `json:"health"` // PASSED | FAILING | UNKNOWN
TemperatureC *int `json:"temperature_c"`
PercentageUsed *int `json:"percentage_used"` // NVMe wear (%); null for SATA/USB
}
// StorageTarget mirrors the agent's GET /host/metrics storage_targets entry (the per-storage
// capacity + health the monitoring view renders). It is a SUBSET of the agent's wire shape — only
// the fields the UI reads; unknown JSON keys are ignored.
type StorageTarget struct {
Name string `json:"name"`
Type string `json:"type"`
State string `json:"state"`
Reachable bool `json:"reachable"`
TotalBytes int64 `json:"total_bytes"`
UsedBytes int64 `json:"used_bytes"`
AvailBytes int64 `json:"avail_bytes"`
UsedFraction float64 `json:"used_fraction"`
Content string `json:"content"`
MountPath string `json:"mount_path"`
ClassHint string `json:"class_hint"`
ThinPool *ThinPoolFill `json:"thin_pool,omitempty"`
Smart SmartSummary `json:"smart"`
// Label and Purpose are controller-side display enrichment (NOT from the agent): a friendly
// Hungarian name + one-line purpose so the customer understands what each storage holds. The
// raw PVE storage id stays in Name (display-only labels — we never rename the actual storage).
Label string `json:"label,omitempty"`
Purpose string `json:"purpose,omitempty"`
}
// HostMetricsResponse mirrors the agent's GET /host/metrics payload (host-wide health + per-storage
// capacity). Host-wide and token-authed (one-customer-per-host); a fresh collect, not a snapshot.
type HostMetricsResponse struct {
VMID int `json:"vmid"`
Host HostMetrics `json:"host"`
StorageTargets []StorageTarget `json:"storage_targets"`
}
// HostMetrics calls GET /host/metrics and returns the host's live health + per-storage capacity.
func (c *Client) HostMetrics(ctx context.Context) (HostMetricsResponse, error) {
var out HostMetricsResponse
body, err := c.get(ctx, "/host/metrics")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /host/metrics: %w", err)
}
return out, nil
}
// StatusError is a non-2xx agent HTTP status surfaced as a TYPED error (same text the old
// fmt.Errorf produced). errors.As-able — the capability probe (features.go) keys on Code 404 to
// distinguish "this agent predates the route" from every other failure. Never match the string.
type StatusError struct {
Path string
Code int
}
func (e *StatusError) Error() string {
return fmt.Sprintf("agentapi: GET %s: HTTP %d", e.Path, e.Code)
}
// get issues an authenticated GET and unwraps the {ok,data,error} envelope.
func (c *Client) get(ctx context.Context, path string) (json.RawMessage, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+c.token)
start := time.Now()
resp, err := c.hc.Do(req)
if err != nil {
logx.Debugf(c.logger, "[agentapi] GET %s failed after %dms: %v", path, time.Since(start).Milliseconds(), err)
return nil, fmt.Errorf("agentapi: GET %s: %w", path, err)
}
defer resp.Body.Close()
c.noteAgentVersion(resp) // v0.82.0 version channel: passive capture on EVERY response
logx.Debugf(c.logger, "[agentapi] GET %s -> %d (%dms)", path, resp.StatusCode, time.Since(start).Milliseconds())
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode != http.StatusOK {
return nil, &StatusError{Path: path, Code: resp.StatusCode}
}
var env apiResponse
if err := json.Unmarshal(raw, &env); err != nil {
return nil, fmt.Errorf("agentapi: GET %s: bad envelope: %w", path, err)
}
if !env.OK {
return nil, fmt.Errorf("agentapi: GET %s: %s", path, env.Error)
}
return env.Data, nil
}
// post issues an authenticated JSON POST and unwraps the {ok,data,error} envelope. The agent
// returns 200 or 202 for accepted requests.
func (c *Client) post(ctx context.Context, path string, body any) (json.RawMessage, error) {
buf, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, bytes.NewReader(buf))
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer "+c.token)
req.Header.Set("Content-Type", "application/json")
start := time.Now()
resp, err := c.hc.Do(req)
if err != nil {
logx.Debugf(c.logger, "[agentapi] POST %s failed after %dms: %v", path, time.Since(start).Milliseconds(), err)
return nil, fmt.Errorf("agentapi: POST %s: %w", path, err)
}
defer resp.Body.Close()
c.noteAgentVersion(resp) // v0.82.0 version channel: passive capture on EVERY response
logx.Debugf(c.logger, "[agentapi] POST %s -> %d (%dms)", path, resp.StatusCode, time.Since(start).Milliseconds())
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return nil, fmt.Errorf("agentapi: POST %s: HTTP %d", path, resp.StatusCode)
}
var env apiResponse
if err := json.Unmarshal(raw, &env); err != nil {
return nil, fmt.Errorf("agentapi: POST %s: bad envelope: %w", path, err)
}
if !env.OK {
return nil, fmt.Errorf("agentapi: POST %s: %s", path, env.Error)
}
return env.Data, nil
}
// normalizeFingerprint lowercases and strips ':'/' ' separators, requiring a 64-hex SHA-256.
func normalizeFingerprint(fp string) (string, error) {
s := strings.ToLower(strings.NewReplacer(":", "", " ", "", "\t", "").Replace(strings.TrimSpace(fp)))
if len(s) != 64 {
return "", fmt.Errorf("agentapi: fingerprint must be a SHA-256 (64 hex chars), got %d", len(s))
}
if _, err := hex.DecodeString(s); err != nil {
return "", fmt.Errorf("agentapi: fingerprint is not valid hex: %w", err)
}
return s, nil
}