slice 8B (agent half): /backup/due cadence policy + /backup/status phases (v0.11.0)
internal/localapi: real /backup/due (cadence; due when no successful backup or newest older than backup.backup_cadence_seconds; false in-window after success; failed doesn't count) + /backup/status phases (idle|running|done|failed + job id) + POST /backup single-flight with job id. Drives the controller quiesce loop. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,32 @@
|
|||||||
All notable changes to **felhom-agent** are recorded here. Update on every code
|
All notable changes to **felhom-agent** are recorded here. Update on every code
|
||||||
change that gets pushed.
|
change that gets pushed.
|
||||||
|
|
||||||
|
## v0.11.0 — slice 8B: app-consistent backup — /backup/due policy + /backup/status phases (2026-06-10)
|
||||||
|
|
||||||
|
The agent half of slice 8B (doc 03 §8). Turns the 8A thin backup stubs into the real policy the
|
||||||
|
in-guest controller's quiesce loop drives (controller half: `felhom-controller` v0.36.0). No hub
|
||||||
|
change. The downtime optimization (`vzdump --mode snapshot` + a `snapshotted` phase) is the 8B.2
|
||||||
|
fast-follow; the hub-served per-guest policy is slice 10.
|
||||||
|
|
||||||
|
### Changed (`internal/localapi`)
|
||||||
|
- **`GET /backup/due`** — real **cadence** policy (replaces the 8A "never backed up" stub): a guest
|
||||||
|
is due when no **successful** backup is recorded OR the newest one is older than the agent-local
|
||||||
|
cadence (`backup.backup_cadence_seconds`, default 24h). A successful `POST /backup` flips due to
|
||||||
|
**false** for the window, so the controller won't re-quiesce in a loop. A failed backup does not
|
||||||
|
satisfy the cadence. Returns `age_seconds` for diagnosis.
|
||||||
|
- **`GET /backup/status`** — real **phases** `idle | running | done | failed` + the job id, so the
|
||||||
|
controller can poll a backup to completion (was: just the latest stored backup).
|
||||||
|
- **`POST /backup`** — returns a **job id** + `running` phase; tracks the in-flight job and is
|
||||||
|
**single-flight per guest** (a second POST while one runs returns the same job — no concurrent
|
||||||
|
vzdump). On completion the job transitions done/failed and the result is recorded to the store.
|
||||||
|
- Config: `backup.backup_cadence_seconds` + `BackupCadence()`; the local-API server takes the cadence.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
- `/backup/due`: due when stale / no backup, **not due within the window after a success**, due again
|
||||||
|
past the cadence, **a failed backup does not count**. `/backup/status`: running→done and
|
||||||
|
running→failed (gated fake to observe the running phase). `POST /backup` single-flight (one vzdump
|
||||||
|
for concurrent POSTs). All still self-scoped (token→guest).
|
||||||
|
|
||||||
## v0.10.0 — slice 8A: agent local-API server + provisioning back-half (2026-06-10)
|
## v0.10.0 — slice 8A: agent local-API server + provisioning back-half (2026-06-10)
|
||||||
|
|
||||||
The host-agent half of slice 8A (doc 03 §6). Adds the per-guest **local API** the in-guest
|
The host-agent half of slice 8A (doc 03 §6). Adds the per-guest **local API** the in-guest
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ import (
|
|||||||
|
|
||||||
// version is the agent version. Overridable at build time with
|
// version is the agent version. Overridable at build time with
|
||||||
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
// -ldflags "-X main.version=<v>"; defaults to the in-repo CHANGELOG version.
|
||||||
var version = "0.10.0"
|
var version = "0.11.0"
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
var (
|
var (
|
||||||
@@ -485,14 +485,15 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
|||||||
logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
logger.Info("local-api leaf ready", "fingerprint_sha256", fp, "cert", cfg.LocalAPI.CertPath())
|
||||||
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger)
|
runner := backup.NewBackupRunner(px, cfg.Backup.LocalBackupTarget, "", "felhom local-api", logger)
|
||||||
srv, err := localapi.NewServer(localapi.Options{
|
srv, err := localapi.NewServer(localapi.Options{
|
||||||
ListenAddr: cfg.LocalAPI.ListenAddr,
|
ListenAddr: cfg.LocalAPI.ListenAddr,
|
||||||
Cert: cert,
|
Cert: cert,
|
||||||
Guests: px,
|
Guests: px,
|
||||||
Backups: runner,
|
Backups: runner,
|
||||||
Store: store,
|
Store: store,
|
||||||
Storage: observer,
|
Storage: observer,
|
||||||
Tokens: tokens,
|
Tokens: tokens,
|
||||||
Logger: logger,
|
BackupCadence: cfg.Backup.BackupCadence(),
|
||||||
|
Logger: logger,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn("daemon: local-api disabled (server build)", "err", err)
|
logger.Warn("daemon: local-api disabled (server build)", "err", err)
|
||||||
|
|||||||
@@ -43,7 +43,8 @@
|
|||||||
"scratch_vmid_min": 990000,
|
"scratch_vmid_min": 990000,
|
||||||
"scratch_vmid_max": 990009,
|
"scratch_vmid_max": 990009,
|
||||||
"pbs_verify_cadence_seconds": 0,
|
"pbs_verify_cadence_seconds": 0,
|
||||||
"pbs_secret_dir": "/etc/pve/priv/storage"
|
"pbs_secret_dir": "/etc/pve/priv/storage",
|
||||||
|
"backup_cadence_seconds": 0
|
||||||
},
|
},
|
||||||
"local_api": {
|
"local_api": {
|
||||||
"enable": true,
|
"enable": true,
|
||||||
|
|||||||
@@ -139,6 +139,19 @@ type BackupConfig struct {
|
|||||||
// PBSSecretDir holds the per-storage PBS token secret files (<id>.pw). Default
|
// PBSSecretDir holds the per-storage PBS token secret files (<id>.pw). Default
|
||||||
// /etc/pve/priv/storage (PVE-managed, 0600). The agent reads it at runtime; never logged.
|
// /etc/pve/priv/storage (PVE-managed, 0600). The agent reads it at runtime; never logged.
|
||||||
PBSSecretDir string `json:"pbs_secret_dir"`
|
PBSSecretDir string `json:"pbs_secret_dir"`
|
||||||
|
|
||||||
|
// BackupCadenceSeconds drives the local-API GET /backup/due (slice 8B): a guest is "due" when
|
||||||
|
// its newest successful backup is older than this (or none exists). 0 → default (24h). The
|
||||||
|
// hub-served per-guest policy is slice 10; this is the agent-local cadence.
|
||||||
|
BackupCadenceSeconds int `json:"backup_cadence_seconds"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackupCadence returns the per-guest /backup/due window: positive as-is, else 24h default.
|
||||||
|
func (b BackupConfig) BackupCadence() time.Duration {
|
||||||
|
if b.BackupCadenceSeconds > 0 {
|
||||||
|
return time.Duration(b.BackupCadenceSeconds) * time.Second
|
||||||
|
}
|
||||||
|
return 24 * time.Hour
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default scratch VMID band + restore-test cadence.
|
// Default scratch VMID band + restore-test cadence.
|
||||||
|
|||||||
+157
-20
@@ -11,6 +11,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||||
@@ -60,7 +61,33 @@ type Options struct {
|
|||||||
Store BackupStore
|
Store BackupStore
|
||||||
Storage StorageView
|
Storage StorageView
|
||||||
Tokens TokenAuthority
|
Tokens TokenAuthority
|
||||||
Logger *slog.Logger
|
// BackupCadence is the per-guest backup interval driving GET /backup/due (slice 8B). A guest
|
||||||
|
// is "due" when no successful backup is recorded OR the newest one is older than this. 0 → a
|
||||||
|
// safe default (24h). The hub-served policy is slice 10; this is the agent-local cadence.
|
||||||
|
BackupCadence time.Duration
|
||||||
|
Logger *slog.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultBackupCadence is the fallback /backup/due window when none is configured.
|
||||||
|
const defaultBackupCadence = 24 * time.Hour
|
||||||
|
|
||||||
|
// Backup phase vocabulary reported by GET /backup/status (slice 8B). The 8B.2 fast-follow adds a
|
||||||
|
// `snapshotted` phase (vzdump --mode snapshot) so the controller can unquiesce at snapshot-taken.
|
||||||
|
const (
|
||||||
|
PhaseIdle = "idle"
|
||||||
|
PhaseRunning = "running"
|
||||||
|
PhaseDone = "done"
|
||||||
|
PhaseFailed = "failed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// backupJob is the in-flight/last backup job for one guest (drives /backup/status phases).
|
||||||
|
type backupJob struct {
|
||||||
|
JobID string
|
||||||
|
Phase string
|
||||||
|
StartedAt time.Time
|
||||||
|
FinishedAt time.Time
|
||||||
|
Archive string
|
||||||
|
Error string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Server is the per-guest local API (doc 03 §6). It serves the agent's pinned self-signed leaf
|
// Server is the per-guest local API (doc 03 §6). It serves the agent's pinned self-signed leaf
|
||||||
@@ -73,7 +100,12 @@ type Server struct {
|
|||||||
store BackupStore
|
store BackupStore
|
||||||
storage StorageView
|
storage StorageView
|
||||||
tokens TokenAuthority
|
tokens TokenAuthority
|
||||||
|
cadence time.Duration
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
|
now func() time.Time
|
||||||
|
|
||||||
|
jobsMu sync.Mutex
|
||||||
|
jobs map[int]*backupJob // per-guest backup job state (slice 8B)
|
||||||
|
|
||||||
baseCtx context.Context // for fire-and-forget backups; set in Run
|
baseCtx context.Context // for fire-and-forget backups; set in Run
|
||||||
}
|
}
|
||||||
@@ -89,6 +121,10 @@ func NewServer(o Options) (*Server, error) {
|
|||||||
if o.Logger == nil {
|
if o.Logger == nil {
|
||||||
o.Logger = slog.Default()
|
o.Logger = slog.Default()
|
||||||
}
|
}
|
||||||
|
cadence := o.BackupCadence
|
||||||
|
if cadence <= 0 {
|
||||||
|
cadence = defaultBackupCadence
|
||||||
|
}
|
||||||
return &Server{
|
return &Server{
|
||||||
addr: o.ListenAddr,
|
addr: o.ListenAddr,
|
||||||
cert: o.Cert,
|
cert: o.Cert,
|
||||||
@@ -97,7 +133,10 @@ func NewServer(o Options) (*Server, error) {
|
|||||||
store: o.Store,
|
store: o.Store,
|
||||||
storage: o.Storage,
|
storage: o.Storage,
|
||||||
tokens: o.Tokens,
|
tokens: o.Tokens,
|
||||||
|
cadence: cadence,
|
||||||
logger: o.Logger,
|
logger: o.Logger,
|
||||||
|
now: func() time.Time { return time.Now().UTC() },
|
||||||
|
jobs: map[int]*backupJob{},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,6 +352,13 @@ type backupRequest struct {
|
|||||||
VMID int `json:"vmid"`
|
VMID int `json:"vmid"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BackupResponse is POST /backup. The controller polls GET /backup/status on job_id to completion.
|
||||||
|
type BackupResponse struct {
|
||||||
|
VMID int `json:"vmid"`
|
||||||
|
JobID string `json:"job_id"`
|
||||||
|
Phase string `json:"phase"`
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) {
|
func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||||
// Body is optional; if present its vmid must match the token's guest.
|
// Body is optional; if present its vmid must match the token's guest.
|
||||||
if r.ContentLength != 0 {
|
if r.ContentLength != 0 {
|
||||||
@@ -324,10 +370,24 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int)
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Enqueue: a vzdump runs for minutes, so we do not block the request. The backup runs on
|
// Single-flight per guest: if a backup is already running for this guest, return that job
|
||||||
// the server's base context (cancelled on daemon shutdown) and records into the store; the
|
// (don't start a second concurrent vzdump). The controller polls /backup/status on it.
|
||||||
// controller polls GET /backup/status. This is the crash-consistent path (8A); the
|
s.jobsMu.Lock()
|
||||||
// app-consistent quiesce-then-backup loop is 8B.
|
if cur := s.jobs[vmid]; cur != nil && cur.Phase == PhaseRunning {
|
||||||
|
job := *cur
|
||||||
|
s.jobsMu.Unlock()
|
||||||
|
writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: job.JobID, Phase: job.Phase}, "")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
jobID := "backup-" + strconv.Itoa(vmid) + "-" + strconv.FormatInt(s.now().UnixNano(), 10)
|
||||||
|
s.jobs[vmid] = &backupJob{JobID: jobID, Phase: PhaseRunning, StartedAt: s.now()}
|
||||||
|
s.jobsMu.Unlock()
|
||||||
|
|
||||||
|
// Enqueue: a vzdump runs for minutes, so we do not block the request. The backup runs on the
|
||||||
|
// server's base context (cancelled on daemon shutdown), updates the job phase, and records
|
||||||
|
// into the store; the controller polls GET /backup/status. This is the host-side half of the
|
||||||
|
// 8B app-consistent path — the controller quiesces (stops its stacks) BEFORE calling this, so
|
||||||
|
// the vzdump captures a clean-shutdown-consistent state.
|
||||||
base := s.baseCtx
|
base := s.baseCtx
|
||||||
if base == nil {
|
if base == nil {
|
||||||
base = context.Background()
|
base = context.Background()
|
||||||
@@ -342,36 +402,94 @@ func (s *Server) handleBackup(w http.ResponseWriter, r *http.Request, vmid int)
|
|||||||
if b.Error == "" {
|
if b.Error == "" {
|
||||||
b.Error = err.Error()
|
b.Error = err.Error()
|
||||||
}
|
}
|
||||||
s.logger.Error("local-api: enqueued backup failed", "vmid", vmid, "err", err)
|
s.logger.Error("local-api: backup job failed", "vmid", vmid, "job", jobID, "err", err)
|
||||||
} else {
|
} else {
|
||||||
s.logger.Info("local-api: enqueued backup complete", "vmid", vmid, "archive", b.Archive)
|
s.logger.Info("local-api: backup job complete", "vmid", vmid, "job", jobID, "archive", b.Archive)
|
||||||
}
|
}
|
||||||
s.store.RecordBackup(b)
|
s.store.RecordBackup(b)
|
||||||
|
s.finishJob(vmid, jobID, b)
|
||||||
}()
|
}()
|
||||||
writeStatus(w, http.StatusAccepted, true, map[string]any{"vmid": vmid, "enqueued": true}, "")
|
writeStatus(w, http.StatusAccepted, true, BackupResponse{VMID: vmid, JobID: jobID, Phase: PhaseRunning}, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
// BackupDueResponse is GET /backup/due. Thin in 8A: a guest with no successful backup recorded
|
// finishJob transitions the guest's job to done/failed (only if it is still the current job — a
|
||||||
// is "due"; otherwise not. Policy-scheduled cadence (hub manifest) lands in slice 10, and the
|
// later job started after a single-flight gap must not be overwritten by an older one's result).
|
||||||
// quiesce-on-due consumer is 8B — both noted in the response.
|
func (s *Server) finishJob(vmid int, jobID string, b hub.Backup) {
|
||||||
|
s.jobsMu.Lock()
|
||||||
|
defer s.jobsMu.Unlock()
|
||||||
|
cur := s.jobs[vmid]
|
||||||
|
if cur == nil || cur.JobID != jobID {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cur.FinishedAt = s.now()
|
||||||
|
if b.Success {
|
||||||
|
cur.Phase = PhaseDone
|
||||||
|
cur.Archive = b.Archive
|
||||||
|
} else {
|
||||||
|
cur.Phase = PhaseFailed
|
||||||
|
cur.Error = b.Error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobSnapshot returns a copy of the guest's current job (ok=false if none).
|
||||||
|
func (s *Server) jobSnapshot(vmid int) (backupJob, bool) {
|
||||||
|
s.jobsMu.Lock()
|
||||||
|
defer s.jobsMu.Unlock()
|
||||||
|
if j := s.jobs[vmid]; j != nil {
|
||||||
|
return *j, true
|
||||||
|
}
|
||||||
|
return backupJob{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackupDueResponse is GET /backup/due (slice 8B). A guest is due when no successful backup is
|
||||||
|
// recorded OR the newest successful one is older than the agent-local cadence. A successful
|
||||||
|
// POST /backup flips this to false for the window, so the controller won't re-quiesce in a loop.
|
||||||
|
// The hub-served policy is slice 10.
|
||||||
type BackupDueResponse struct {
|
type BackupDueResponse struct {
|
||||||
VMID int `json:"vmid"`
|
VMID int `json:"vmid"`
|
||||||
Due bool `json:"due"`
|
Due bool `json:"due"`
|
||||||
Reason string `json:"reason"`
|
Reason string `json:"reason"`
|
||||||
|
AgeSecs *int64 `json:"age_seconds,omitempty"` // age of the newest successful backup; null if none
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid int) {
|
func (s *Server) handleBackupDue(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||||
latest := s.latestBackupFor(r.Context(), vmid)
|
latest := s.latestSuccessfulBackupFor(r.Context(), vmid)
|
||||||
if latest == nil || !latest.Success {
|
if latest == nil {
|
||||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet"})
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "no successful backup recorded yet"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeOK(w, BackupDueResponse{VMID: vmid, Due: false, Reason: "policy-scheduled cadence lands in slice 10"})
|
age, ok := backupAge(latest.StartedAt, s.now())
|
||||||
|
if !ok {
|
||||||
|
// Unparseable timestamp: fail safe toward "due" so a backup still happens.
|
||||||
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "last backup time unparseable — treating as due"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ageSecs := int64(age.Seconds())
|
||||||
|
if age >= s.cadence {
|
||||||
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: true, Reason: "older than cadence", AgeSecs: &ageSecs})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeOK(w, BackupDueResponse{VMID: vmid, Due: false, Reason: "within cadence window", AgeSecs: &ageSecs})
|
||||||
|
}
|
||||||
|
|
||||||
|
// BackupStatusResponse is GET /backup/status (slice 8B): the current/last job phase + the latest
|
||||||
|
// recorded backup. Phase is idle when no job has run this process lifetime.
|
||||||
|
type BackupStatusResponse struct {
|
||||||
|
VMID int `json:"vmid"`
|
||||||
|
Phase string `json:"phase"` // idle | running | done | failed
|
||||||
|
JobID string `json:"job_id,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
Backup *hub.Backup `json:"backup,omitempty"` // latest recorded backup for this guest
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
func (s *Server) handleBackupStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||||
latest := s.latestBackupFor(r.Context(), vmid)
|
resp := BackupStatusResponse{VMID: vmid, Phase: PhaseIdle, Backup: s.latestBackupFor(r.Context(), vmid)}
|
||||||
writeOK(w, map[string]any{"vmid": vmid, "backup": latest})
|
if job, ok := s.jobSnapshot(vmid); ok {
|
||||||
|
resp.Phase = job.Phase
|
||||||
|
resp.JobID = job.JobID
|
||||||
|
resp.Error = job.Error
|
||||||
|
}
|
||||||
|
writeOK(w, resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||||
@@ -389,9 +507,19 @@ func (s *Server) handleRestoreTestStatus(w http.ResponseWriter, r *http.Request,
|
|||||||
|
|
||||||
// latestBackupFor returns this guest's most recent backup from the store (nil if none).
|
// latestBackupFor returns this guest's most recent backup from the store (nil if none).
|
||||||
func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup {
|
func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup {
|
||||||
|
return s.pickLatestBackup(ctx, vmid, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// latestSuccessfulBackupFor returns this guest's most recent SUCCESSFUL backup (nil if none) —
|
||||||
|
// the basis for /backup/due (a failed backup must not satisfy the cadence).
|
||||||
|
func (s *Server) latestSuccessfulBackupFor(ctx context.Context, vmid int) *hub.Backup {
|
||||||
|
return s.pickLatestBackup(ctx, vmid, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) pickLatestBackup(ctx context.Context, vmid int, successOnly bool) *hub.Backup {
|
||||||
var latest *hub.Backup
|
var latest *hub.Backup
|
||||||
for _, b := range s.store.Backups(ctx) {
|
for _, b := range s.store.Backups(ctx) {
|
||||||
if b.VMID != vmid {
|
if b.VMID != vmid || (successOnly && !b.Success) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
bb := b
|
bb := b
|
||||||
@@ -402,6 +530,15 @@ func (s *Server) latestBackupFor(ctx context.Context, vmid int) *hub.Backup {
|
|||||||
return latest
|
return latest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// backupAge parses an RFC3339 backup start time and returns its age relative to now.
|
||||||
|
func backupAge(startedAt string, now time.Time) (time.Duration, bool) {
|
||||||
|
t, err := time.Parse(time.RFC3339, startedAt)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return now.Sub(t), true
|
||||||
|
}
|
||||||
|
|
||||||
// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
|
// classByStorage builds a storage-id → class(fast|slow|"") map from the host storage view. A
|
||||||
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
|
// view error is logged and yields an empty map (class falls back to "" — a hint, not load-bearing).
|
||||||
func (s *Server) classByStorage(ctx context.Context) map[string]string {
|
func (s *Server) classByStorage(ctx context.Context) map[string]string {
|
||||||
|
|||||||
@@ -64,14 +64,24 @@ func (f *fakeGuests) WaitTask(_ context.Context, _ string, _ proxmox.WaitOptions
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fakeBackups struct {
|
type fakeBackups struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
vmids []int
|
vmids []int
|
||||||
|
gate chan struct{} // if non-nil, Backup blocks until it is closed (observe the running phase)
|
||||||
|
failErr string // if set, Backup returns this error (drives the failed phase)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeBackups) Backup(_ context.Context, vmid int) (hub.Backup, error) {
|
func (f *fakeBackups) Backup(_ context.Context, vmid int) (hub.Backup, error) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
f.vmids = append(f.vmids, vmid)
|
f.vmids = append(f.vmids, vmid)
|
||||||
|
gate := f.gate
|
||||||
|
failErr := f.failErr
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
|
if gate != nil {
|
||||||
|
<-gate
|
||||||
|
}
|
||||||
|
if failErr != "" {
|
||||||
|
return hub.Backup{VMID: vmid}, fmt.Errorf("%s", failErr)
|
||||||
|
}
|
||||||
return hub.Backup{VMID: vmid, Success: true, Archive: "local:backup/vzdump-x", StartedAt: "2026-06-10T00:00:00Z"}, nil
|
return hub.Backup{VMID: vmid, Success: true, Archive: "local:backup/vzdump-x", StartedAt: "2026-06-10T00:00:00Z"}, nil
|
||||||
}
|
}
|
||||||
func (f *fakeBackups) called() []int { f.mu.Lock(); defer f.mu.Unlock(); return append([]int(nil), f.vmids...) }
|
func (f *fakeBackups) called() []int { f.mu.Lock(); defer f.mu.Unlock(); return append([]int(nil), f.vmids...) }
|
||||||
@@ -106,25 +116,35 @@ func (m staticTokens) Lookup(tok string) (int, bool) { v, ok := m[tok]; return v
|
|||||||
|
|
||||||
// ---- harness ----------------------------------------------------------------------------
|
// ---- harness ----------------------------------------------------------------------------
|
||||||
|
|
||||||
func newTestServer(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) http.Handler {
|
// testNow is the fixed clock the test server uses, so /backup/due cadence math is deterministic.
|
||||||
|
var testNow = time.Date(2026, 6, 10, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func newTestServerS(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) *Server {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
if sv == nil {
|
if sv == nil {
|
||||||
sv = fakeStorage{}
|
sv = fakeStorage{}
|
||||||
}
|
}
|
||||||
srv, err := NewServer(Options{
|
srv, err := NewServer(Options{
|
||||||
ListenAddr: "127.0.0.1:0",
|
ListenAddr: "127.0.0.1:0",
|
||||||
Guests: g,
|
Guests: g,
|
||||||
Backups: b,
|
Backups: b,
|
||||||
Store: st,
|
Store: st,
|
||||||
Storage: sv,
|
Storage: sv,
|
||||||
Tokens: staticTokens{"A": 8200, "B": 9300},
|
Tokens: staticTokens{"A": 8200, "B": 9300},
|
||||||
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
BackupCadence: 24 * time.Hour,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(io.Discard, nil)),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("new server: %v", err)
|
t.Fatalf("new server: %v", err)
|
||||||
}
|
}
|
||||||
srv.baseCtx = context.Background()
|
srv.baseCtx = context.Background()
|
||||||
return srv.Handler()
|
srv.now = func() time.Time { return testNow }
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestServer(t *testing.T, g *fakeGuests, b *fakeBackups, st *fakeStore, sv StorageView) http.Handler {
|
||||||
|
t.Helper()
|
||||||
|
return newTestServerS(t, g, b, st, sv).Handler()
|
||||||
}
|
}
|
||||||
|
|
||||||
func do(t *testing.T, h http.Handler, method, path, token, body string) *httptest.ResponseRecorder {
|
func do(t *testing.T, h http.Handler, method, path, token, body string) *httptest.ResponseRecorder {
|
||||||
@@ -304,26 +324,126 @@ func TestBackup_EnqueuesForTokenGuest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBackupDue_ThinHeuristic(t *testing.T) {
|
func dueOf(t *testing.T, h http.Handler) BackupDueResponse {
|
||||||
st := &fakeStore{}
|
t.Helper()
|
||||||
h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil)
|
|
||||||
|
|
||||||
// no backup recorded → due
|
|
||||||
w := do(t, h, "GET", "/backup/due", "A", "")
|
w := do(t, h, "GET", "/backup/due", "A", "")
|
||||||
var resp struct {
|
var resp struct {
|
||||||
Data BackupDueResponse `json:"data"`
|
Data BackupDueResponse `json:"data"`
|
||||||
}
|
}
|
||||||
_ = json.Unmarshal(w.Body.Bytes(), &resp)
|
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||||
if !resp.Data.Due {
|
t.Fatalf("decode due: %v", err)
|
||||||
|
}
|
||||||
|
return resp.Data
|
||||||
|
}
|
||||||
|
|
||||||
|
// testNow is 2026-06-10T12:00:00Z, cadence 24h.
|
||||||
|
func TestBackupDue_Cadence(t *testing.T) {
|
||||||
|
st := &fakeStore{}
|
||||||
|
h := newTestServer(t, &fakeGuests{}, &fakeBackups{}, st, nil)
|
||||||
|
|
||||||
|
// no backup recorded → due
|
||||||
|
if d := dueOf(t, h); !d.Due {
|
||||||
t.Fatal("expected due=true with no backup recorded")
|
t.Fatal("expected due=true with no backup recorded")
|
||||||
}
|
}
|
||||||
// a successful backup for this guest → not due
|
// a successful backup 1h ago → NOT due (within the 24h window)
|
||||||
st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-10T00:00:00Z"}}
|
st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-10T11:00:00Z"}}
|
||||||
w = do(t, h, "GET", "/backup/due", "A", "")
|
if d := dueOf(t, h); d.Due {
|
||||||
_ = json.Unmarshal(w.Body.Bytes(), &resp)
|
t.Fatalf("expected due=false 1h after a successful backup; reason=%q", d.Reason)
|
||||||
if resp.Data.Due {
|
|
||||||
t.Fatal("expected due=false after a successful backup")
|
|
||||||
}
|
}
|
||||||
|
// a successful backup >24h ago → due again
|
||||||
|
st.backups = []hub.Backup{{VMID: 8200, Success: true, StartedAt: "2026-06-08T00:00:00Z"}}
|
||||||
|
if d := dueOf(t, h); !d.Due {
|
||||||
|
t.Fatal("expected due=true when the newest backup is older than the cadence")
|
||||||
|
}
|
||||||
|
// a FAILED backup 1h ago does NOT satisfy the cadence → still due
|
||||||
|
st.backups = []hub.Backup{{VMID: 8200, Success: false, StartedAt: "2026-06-10T11:00:00Z"}}
|
||||||
|
if d := dueOf(t, h); !d.Due {
|
||||||
|
t.Fatal("expected due=true: a failed backup must not count as a successful one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST /backup returns a running phase; the job transitions running→done; a successful backup then
|
||||||
|
// flips /backup/due to false (the controller won't re-quiesce in a loop).
|
||||||
|
func TestBackupStatus_RunningToDone(t *testing.T) {
|
||||||
|
st := &fakeStore{}
|
||||||
|
b := &fakeBackups{}
|
||||||
|
srv := newTestServerS(t, &fakeGuests{}, b, st, nil)
|
||||||
|
h := srv.Handler()
|
||||||
|
|
||||||
|
// gate the backup goroutine so we can observe the running phase deterministically
|
||||||
|
b.gate = make(chan struct{})
|
||||||
|
|
||||||
|
w := do(t, h, "POST", "/backup", "A", "")
|
||||||
|
if w.Code != http.StatusAccepted {
|
||||||
|
t.Fatalf("POST /backup: got %d want 202", w.Code)
|
||||||
|
}
|
||||||
|
var br struct {
|
||||||
|
Data BackupResponse `json:"data"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(w.Body.Bytes(), &br)
|
||||||
|
if br.Data.Phase != PhaseRunning || br.Data.JobID == "" {
|
||||||
|
t.Fatalf("expected running phase + job id, got %+v", br.Data)
|
||||||
|
}
|
||||||
|
if ph := statusPhase(t, h); ph != PhaseRunning {
|
||||||
|
t.Fatalf("status while running: got %q want running", ph)
|
||||||
|
}
|
||||||
|
close(b.gate) // let the backup complete
|
||||||
|
// wait for done
|
||||||
|
deadline := time.Now().Add(time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if statusPhase(t, h) == PhaseDone {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if ph := statusPhase(t, h); ph != PhaseDone {
|
||||||
|
t.Fatalf("status after completion: got %q want done", ph)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupStatus_RunningToFailed(t *testing.T) {
|
||||||
|
b := &fakeBackups{failErr: "vzdump exploded"}
|
||||||
|
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
|
||||||
|
h := srv.Handler()
|
||||||
|
|
||||||
|
if do(t, h, "POST", "/backup", "A", "").Code != http.StatusAccepted {
|
||||||
|
t.Fatal("POST /backup not accepted")
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(time.Second)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
if statusPhase(t, h) == PhaseFailed {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(5 * time.Millisecond)
|
||||||
|
}
|
||||||
|
if ph := statusPhase(t, h); ph != PhaseFailed {
|
||||||
|
t.Fatalf("status after a failed backup: got %q want failed", ph)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second POST /backup while one is running does NOT start a second vzdump (single-flight).
|
||||||
|
func TestBackup_SingleFlight(t *testing.T) {
|
||||||
|
b := &fakeBackups{gate: make(chan struct{})}
|
||||||
|
srv := newTestServerS(t, &fakeGuests{}, b, &fakeStore{}, nil)
|
||||||
|
h := srv.Handler()
|
||||||
|
|
||||||
|
do(t, h, "POST", "/backup", "A", "")
|
||||||
|
do(t, h, "POST", "/backup", "A", "") // should be coalesced onto the running job
|
||||||
|
close(b.gate)
|
||||||
|
time.Sleep(30 * time.Millisecond)
|
||||||
|
if c := b.called(); len(c) != 1 {
|
||||||
|
t.Fatalf("expected a single vzdump for concurrent POSTs, got %d", len(c))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func statusPhase(t *testing.T, h http.Handler) string {
|
||||||
|
t.Helper()
|
||||||
|
w := do(t, h, "GET", "/backup/status", "A", "")
|
||||||
|
var resp struct {
|
||||||
|
Data BackupStatusResponse `json:"data"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(w.Body.Bytes(), &resp)
|
||||||
|
return resp.Data.Phase
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBackupStatus_FiltersToThisGuest(t *testing.T) {
|
func TestBackupStatus_FiltersToThisGuest(t *testing.T) {
|
||||||
|
|||||||
Reference in New Issue
Block a user