felhom-controller/controller/internal/quiesce/tiers.go

package quiesce

import (
	"context"
	"errors"
)

// R-82 Slice B — one quiesce window, two tiers.
//
// The agent gained per-target backup tiers in v0.97.0 ("local daily + PBS weekly"). The controller
// owns quiescing, so the multi-tier schedule has to be reconciled HERE: on the weekly night both
// tiers come due at once, and running two quiesce cycles would mean **two app outages for one
// night's work** — which would undo the entire argument for weekly-over-daily.
//
// THE DEDUP RULE (specified, not emergent):
//
//	local due | PBS due | result
//	----------+---------+---------------------------------------------------------------
//	yes       | no      | one quiesce, local backup
//	no        | yes     | one quiesce, PBS backup
//	yes       | yes     | ONE quiesce window, BOTH backups inside it — never two cycles
//	no        | no      | no quiesce
//
// ErrTiersUnsupported is returned by TieredBackend.Tiers when the agent does not serve
// GET /backup/tiers — i.e. it predates R-82 (the endpoint 404s). It is the DESIGNED capability
// probe, not an error condition: the loop degrades to the single untargeted tier and logs it once.
//
// It must NEVER be treated as "nothing to do". A new controller meeting an old agent must still
// back up; concluding "not due" from an unrecognised response would silently stop backups
// fleet-wide during a rollout — the exact failure this project has hit before (controller v0.154.0,
// agent v0.91.0, the hub allowedEventTypes 400 in R-77).
var ErrTiersUnsupported = errors.New("quiesce: agent does not serve /backup/tiers (pre-R-82)")

// BackupTier is one tier as advertised by the agent, primary first.
type BackupTier struct {
	Target  string
	Primary bool
}

// TieredBackend is the OPTIONAL R-82 extension to Backend. A backend that does not implement it
// (or whose Tiers returns ErrTiersUnsupported) drives the pre-R-82 single-tier path unchanged.
//
// The untargeted Backend methods are NOT redundant: they remain the single-tier path, and the agent
// guarantees they keep their exact pre-R-82 meaning and response bytes.
type TieredBackend interface {
	Backend
	// Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent.
	Tiers(ctx context.Context) ([]BackupTier, error)
	DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, err error)
	StartBackupFor(ctx context.Context, target string) (jobID string, err error)
	BackupStatusFor(ctx context.Context, target string) (phase string, err error)
}

// dueTier is a tier this cycle must back up.
type dueTier struct {
	target  string // "" = the untargeted single-tier path (pre-R-82 agent)
	ageSecs *int64
}

// resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place.
//
// Returns the due tiers IN AGENT ORDER (primary first). That order is deliberate and it is a
// downtime decision, not cosmetics: tiers run SEQUENTIALLY because vzdump holds a guest lock, and
// the app stays stopped until the LAST tier has snapshotted. Running the fast local tier first and
// the slow WAN/PBS tier last makes downtime ≈ (local backup) + (PBS snapshot); the reverse order
// would make it ≈ (PBS backup) + (local snapshot), which is far worse.
//
// degraded is true when the agent is pre-R-82 and the caller must use the untargeted path.
func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded bool, err error) {
	tb, ok := l.backend.(TieredBackend)
	if !ok {
		// Backend built without the tiered surface — the pre-R-82 path, no probe needed.
		return l.resolveUntargeted(ctx)
	}
	tiers, terr := tb.Tiers(ctx)
	if errors.Is(terr, ErrTiersUnsupported) {
		// A new controller meeting an OLD agent. Degrade — and SAY SO, once.
		l.logTierDegradeOnce()
		return l.resolveUntargeted(ctx)
	}
	if terr != nil {
		return nil, false, terr
	}
	if len(tiers) == 0 {
		// An agent that advertises no tiers cannot be backed up per-tier, but it can still be
		// backed up untargeted. Fail toward DOING the backup, never toward skipping it.
		l.logger.Printf("[WARN] [quiesce] agent advertised ZERO backup tiers — falling back to the untargeted path")
		return l.resolveUntargeted(ctx)
	}
	for _, t := range tiers {
		isDue, age, derr := tb.DueFor(ctx, t.Target)
		if derr != nil {
			// One tier's due-check failing must not silently drop the OTHER tier's backup.
			l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr)
			continue
		}
		if isDue {
			due = append(due, dueTier{target: t.Target, ageSecs: age})
		}
	}
	return due, false, nil
}

// resolveUntargeted is the pre-R-82 single-tier resolution.
func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) {
	isDue, age, err := l.backend.Due(ctx)
	if err != nil {
		return nil, true, err
	}
	if !isDue {
		return nil, true, nil
	}
	return []dueTier{{target: "", ageSecs: age}}, true, nil
}

// logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a
// steady state during a rollout and would otherwise log every poll; but never zero times, because a
// silent degrade is indistinguishable from multi-tier working.
func (l *Loop) logTierDegradeOnce() {
	l.degradeOnce.Do(func() {
		l.logger.Printf("[INFO] [quiesce] agent predates R-82 (no /backup/tiers) — using the single untargeted backup tier; per-tier scheduling is inactive until the agent is upgraded")
	})
}

// startBackupOn starts a backup on one tier (untargeted when target is "").
func (l *Loop) startBackupOn(ctx context.Context, target string) (string, error) {
	if target == "" {
		return l.backend.StartBackup(ctx)
	}
	return l.backend.(TieredBackend).StartBackupFor(ctx, target)
}

// backupStatusOn reads one tier's job phase (untargeted when target is "").
func (l *Loop) backupStatusOn(ctx context.Context, target string) (string, error) {
	if target == "" {
		return l.backend.BackupStatus(ctx)
	}
	return l.backend.(TieredBackend).BackupStatusFor(ctx, target)
}

// tierLabel renders a tier for logs ("" → the untargeted tier).
func tierLabel(target string) string {
	if target == "" {
		return "(untargeted)"
	}
	return target
}