b527430ec7
The guest-level backup layer + the journaled self-restore-test (restore→boot→verify→ teardown) that closes "a backup you haven't restored isn't a backup". All benign (reuses the slice-4 classifier/gate/journal; no new destructive class/crypto). Local target only; PBS = Phase B. Restore to a NEW guest only. Backups crash-consistent. - proxmox: DestroyLXC, VzdumpOptions.Notes (notes-template), LatestBackupVolID. - reconcile: Engine.RunRestoreTest (journal Scratch entry BEFORE mutation; net link-down pre-boot; defer teardown always; benign gated destroy) + Recover extended to reap a leaked scratch guest (Scratch flag, special-cased before the UPID path; idempotent). - internal/backup: runner (vzdump + archive resolve + bulk-gap = backup!=1) + cadence scheduler (4th daemon goroutine, default 24h) + in-memory report store. - hub: Backup/RestoreTest filled; collector seams; cross-repo golden byte-identical + bidirectional key-set tests; hub handler logs a FAILED restore-test prominently. - config BackupConfig (band 990000-990009 default); --selftest=backup / restore-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
216 lines
10 KiB
Go
216 lines
10 KiB
Go
package hub
|
||
|
||
// HostReport is the wire contract shared with the hub's ingest
|
||
// (felhom.eu TASK-slice3-hub-ingest). Field NAMES must match the hub
|
||
// field-for-field. Encoding is ordinary encoding/json (no canonicalization —
|
||
// nothing signs this report; canonical JSON is a slice-10 signing concern).
|
||
//
|
||
// The report IS the heartbeat: one periodic POST /api/v1/host-report, whose
|
||
// server-side received_at is the hub's dead-man's-switch liveness signal. There is
|
||
// no separate heartbeat endpoint.
|
||
type HostReport struct {
|
||
HostID string `json:"host_id"` // echoes config.Hub.HostID
|
||
ReportedAt string `json:"reported_at"` // RFC3339, agent clock
|
||
AgentVersion string `json:"agent_version"`
|
||
|
||
Host HostMetrics `json:"host"`
|
||
Guests []Guest `json:"guests"`
|
||
|
||
// Defined now as the stable contract; emitted EMPTY (non-nil) this slice.
|
||
StorageTargets []StorageTarget `json:"storage_targets"` // slice 5 (storage manifest)
|
||
Backups []Backup `json:"backups"` // slice 6
|
||
RestoreTests []RestoreTest `json:"restore_tests"` // slice 6
|
||
PBSSnapshots []PBSSnapshot `json:"pbs_snapshots"` // slice 6
|
||
|
||
Cloudflared Cloudflared `json:"cloudflared"`
|
||
AuditTail []AuditEntry `json:"audit_tail"` // populated by a later slice
|
||
}
|
||
|
||
// HostMetrics is the host block, sourced from proxmox NodeStatus.
|
||
type HostMetrics struct {
|
||
Node string `json:"node"`
|
||
CPUPercent float64 `json:"cpu_percent"` // 0–100
|
||
MemoryTotalBytes int64 `json:"memory_total_bytes"`
|
||
MemoryUsedBytes int64 `json:"memory_used_bytes"`
|
||
MemoryPercent float64 `json:"memory_percent"`
|
||
DiskTotalBytes int64 `json:"disk_total_bytes"` // host root fs
|
||
DiskUsedBytes int64 `json:"disk_used_bytes"`
|
||
DiskPercent float64 `json:"disk_percent"`
|
||
LoadAvg []string `json:"loadavg"` // array of STRINGS (PVE shape)
|
||
UptimeSeconds int64 `json:"uptime_seconds"`
|
||
}
|
||
|
||
// Guest is one LXC. The agent reports vmid; the hub derives the guest PK
|
||
// "<host_id>/<vmid>" (keeping the id scheme hub-side — locked decision 4).
|
||
type Guest struct {
|
||
VMID int `json:"vmid"`
|
||
Name string `json:"name"`
|
||
Status string `json:"status"` // running | stopped | unknown
|
||
ControllerVersion string `json:"controller_version"` // "" this slice (slice 8 fills)
|
||
Spec *GuestSpec `json:"spec,omitempty"` // omitted when status unknown
|
||
}
|
||
|
||
// GuestSpec is the provisioned guest sizing.
|
||
type GuestSpec struct {
|
||
Cores int `json:"cores"`
|
||
MemoryBytes int64 `json:"memory_bytes"`
|
||
DiskBytes int64 `json:"disk_bytes"`
|
||
}
|
||
|
||
// Cloudflared is the tunnel service health (read-only probe this slice).
|
||
type Cloudflared struct {
|
||
Status string `json:"status"` // active | inactive | failed | unknown
|
||
}
|
||
|
||
// The following element types are declared now so the empty collections above are
|
||
// typed and slices 5/6 only fill them.
|
||
|
||
// StorageTarget is one observed host storage target (doc 03 §7). It is the REPORTED
|
||
// shape — what the agent observes and tells the hub. The hub holds the AUTHORITATIVE
|
||
// manifest (desired class/role/policy/creds, slice 10); the agent reports what it sees
|
||
// and reconciles toward the hub's manifest. So `class_hint` is a rotational HINT, never
|
||
// authoritative class, and `role` is set only when derivable from an existing Proxmox
|
||
// storage definition (else empty — the hub owns it).
|
||
//
|
||
// This is a cross-repo contract DUPLICATED in felhom.eu/hub (no shared module yet);
|
||
// testdata/host-report.golden.json must stay byte-identical with the hub's copy and the
|
||
// bidirectional key-set test (contract_test.go) guards drift.
|
||
type StorageTarget struct {
|
||
Name string `json:"name"` // the Proxmox storage id (its name)
|
||
Type string `json:"type"` // local-dir | lvmthin | usb | nfs | cifs | pbs | local
|
||
DurableID string `json:"durable_id"` // fs-UUID (usb/local-dir) | server:export (nfs/cifs) | repo+fingerprint (pbs)
|
||
// State is the observed lifecycle state. attached (present & usable) | disconnected
|
||
// (a KNOWN target whose backing device/mount/reachability dropped) | decommissioned
|
||
// (hub-manifest state — built but not served until slice 10).
|
||
State string `json:"state"`
|
||
Reachable bool `json:"reachable"` // backing device present + mounted (local) / reachable (network)
|
||
|
||
TotalBytes int64 `json:"total_bytes"`
|
||
UsedBytes int64 `json:"used_bytes"`
|
||
AvailBytes int64 `json:"avail_bytes"`
|
||
UsedFraction float64 `json:"used_fraction"`
|
||
Content string `json:"content"` // Proxmox content list, e.g. "rootdir,images" / "backup,vztmpl"
|
||
|
||
MountPath string `json:"mount_path"` // host mountpoint (dir/usb); "" for network/lvm
|
||
BackingDevice string `json:"backing_device"` // resolved block device (e.g. /dev/sdb1); "" for network
|
||
// ClassHint is a fast|slow HINT derived from the backing disk's rotational flag — a
|
||
// hint only; the authoritative class is hub-owned (locked decision). "" when not
|
||
// derivable (network targets have no local rotational flag).
|
||
ClassHint string `json:"class_hint"`
|
||
// Role is primary|vzdump-target|pbs-offsite|bulk-data when derivable from the existing
|
||
// storage definition; else "" (the manifest role is hub-owned, slice 10).
|
||
Role string `json:"role"`
|
||
|
||
// ThinPool carries the lvmthin DATA fill prominently — a full thin-pool corrupts every
|
||
// guest on it (the storage analog of single-node OOM). Present ONLY for lvmthin targets;
|
||
// metadata fill is null until Phase B's privileged `lvs` read.
|
||
ThinPool *ThinPoolFill `json:"thin_pool,omitempty"`
|
||
|
||
// Smart is the disk-health summary, populated in Phase B via smartctl (sudoers). In
|
||
// Phase A it is {health:"UNKNOWN", <counters null>} — the keys are committed now so the
|
||
// contract is stable across both phases.
|
||
Smart SmartSummary `json:"smart"`
|
||
}
|
||
|
||
// ThinPoolFill is the lvmthin pool fill (doc 03 §7). DataUsedFraction is the live data
|
||
// fill (used/total of the lvmthin store); MetadataUsedFraction needs the privileged
|
||
// `lvs` read (Phase B) and is null until then.
|
||
type ThinPoolFill struct {
|
||
DataUsedFraction float64 `json:"data_used_fraction"`
|
||
MetadataUsedFraction *float64 `json:"metadata_used_fraction"`
|
||
}
|
||
|
||
// SmartSummary is a read-only disk-health summary. Health is PASSED|FAILING|UNKNOWN.
|
||
// Counters are pointers so "unknown / not-applicable for this device type" (e.g. a USB
|
||
// bridge that exposes no SMART, or NVMe counters on a SATA disk) is null, distinct from a
|
||
// real zero. The SATA set (reallocated/pending/offline-uncorrectable) and the NVMe set
|
||
// (critical_warning/media_errors/percentage_used) are both carried; a device populates
|
||
// only its own set. Filled in Phase B.
|
||
type SmartSummary struct {
|
||
Health string `json:"health"`
|
||
|
||
TemperatureC *int `json:"temperature_c"`
|
||
PowerOnHours *int `json:"power_on_hours"`
|
||
|
||
// SATA attributes.
|
||
ReallocatedSectors *int `json:"reallocated_sectors"`
|
||
PendingSectors *int `json:"pending_sectors"`
|
||
OfflineUncorrectable *int `json:"offline_uncorrectable"`
|
||
|
||
// NVMe attributes.
|
||
CriticalWarning *int `json:"critical_warning"`
|
||
MediaErrors *int `json:"media_errors"`
|
||
PercentageUsed *int `json:"percentage_used"`
|
||
}
|
||
|
||
// SMART health constants (the reported vocabulary).
|
||
const (
|
||
SmartPassed = "PASSED"
|
||
SmartFailing = "FAILING"
|
||
SmartUnknown = "UNKNOWN"
|
||
)
|
||
|
||
// Storage target type + state constants (the reported vocabulary; doc 03 §7).
|
||
const (
|
||
StorageTypeLocalDir = "local-dir"
|
||
StorageTypeLVMThin = "lvmthin"
|
||
StorageTypeUSB = "usb"
|
||
StorageTypeNFS = "nfs"
|
||
StorageTypeCIFS = "cifs"
|
||
StorageTypePBS = "pbs"
|
||
StorageTypeLocal = "local" // builtin dir storage (PVE "local")
|
||
|
||
StorageStateAttached = "attached"
|
||
StorageStateDisconnected = "disconnected"
|
||
StorageStateDecommissioned = "decommissioned"
|
||
)
|
||
|
||
// Backup is the latest guest-vzdump result per target (doc 03 §8, slice 6 Phase A). An
|
||
// agent-initiated vzdump is CRASH-CONSISTENT only (no fsfreeze; app-consistency needs the
|
||
// controller quiesce, slice 8) — marked so here. UncoveredVolumes lists the guest's
|
||
// backup=0 (or backup-unset) mountpoints excluded from the vzdump — the bulk-volume DR gap
|
||
// (the bulk-backup mechanism is slice 10). Cross-repo contract: keep byte-identical with
|
||
// felhom.eu/hub and the bidirectional golden key-set test.
|
||
type Backup struct {
|
||
TargetID string `json:"target_id"` // backup storage name (e.g. "local")
|
||
VMID int `json:"vmid"` // source guest
|
||
Archive string `json:"archive"` // produced vzdump volid
|
||
Mode string `json:"mode"` // snapshot | stop
|
||
CrashConsistent bool `json:"crash_consistent"` // always true this slice
|
||
SizeBytes int64 `json:"size_bytes"`
|
||
Success bool `json:"success"`
|
||
Error string `json:"error,omitempty"`
|
||
StartedAt string `json:"started_at"` // RFC3339
|
||
DurationSeconds float64 `json:"duration_seconds"`
|
||
UncoveredVolumes []string `json:"uncovered_volumes"` // backup=0/unset mountpoints (bulk gap)
|
||
}
|
||
|
||
// RestoreTest is the latest self-restore-test result (doc 03 §8). This slice verifies
|
||
// boot+running only (deep app-health is slice 8). SourceTier is "local" here; PBS is Phase B.
|
||
type RestoreTest struct {
|
||
SourceArchive string `json:"source_archive"`
|
||
SourceTier string `json:"source_tier"` // "local" (pbs = Phase B)
|
||
ScratchVMID int `json:"scratch_vmid"`
|
||
Pass bool `json:"pass"`
|
||
Verified string `json:"verified"` // "boot+running" this slice
|
||
Error string `json:"error,omitempty"`
|
||
TestedAt string `json:"tested_at"` // RFC3339
|
||
DurationSeconds float64 `json:"duration_seconds"`
|
||
}
|
||
|
||
type PBSSnapshot struct{} // slice 6 Phase B: PBS snapshot inventory fields TBD
|
||
type AuditEntry struct{} // audit-log tail entry fields TBD
|
||
|
||
// ControlEnvelope is the hub's 200 response to a host-report. This slice the agent
|
||
// adopts ONLY PollIntervalSeconds; the rest are reserved/forward-compat fields it
|
||
// logs at most and never acts on (reconcile, slice 4, consumes them).
|
||
type ControlEnvelope struct {
|
||
Status string `json:"status"`
|
||
// PollIntervalSeconds is a pointer so a missing field (keep current interval) is
|
||
// distinguishable from an explicit 0.
|
||
PollIntervalSeconds *int `json:"poll_interval_seconds"`
|
||
Blocked bool `json:"blocked"` // reserved — ignored (slice 4)
|
||
DesiredGeneration int64 `json:"desired_generation"` // reserved — ignored (slice 4)
|
||
HasSignedOps bool `json:"has_signed_ops"` // reserved — ignored (slice 4)
|
||
}
|