felhom-controller/controller/internal/quiesce/tiers.go

package quiesce

import (
	"context"
	"errors"
)

// R-82 Slice B — one quiesce window, two tiers.
//
// The agent gained per-target backup tiers in v0.97.0 ("local daily + PBS weekly"). The controller
// owns quiescing, so the multi-tier schedule has to be reconciled HERE: on the weekly night both
// tiers come due at once, and running two quiesce cycles would mean **two app outages for one
// night's work** — which would undo the entire argument for weekly-over-daily.
//
// THE DEDUP RULE (specified, not emergent):
//
//	local due | PBS due | result
//	----------+---------+---------------------------------------------------------------
//	yes       | no      | one quiesce, local backup
//	no        | yes     | one quiesce, PBS backup
//	yes       | yes     | ONE quiesce window, BOTH backups inside it — never two cycles
//	no        | no      | no quiesce
//
// ErrTiersUnsupported is returned by TieredBackend.Tiers when the agent does not serve
// GET /backup/tiers — i.e. it predates R-82 (the endpoint 404s). It is the DESIGNED capability
// probe, not an error condition: the loop degrades to the single untargeted tier and logs it once.
//
// It must NEVER be treated as "nothing to do". A new controller meeting an old agent must still
// back up; concluding "not due" from an unrecognised response would silently stop backups
// fleet-wide during a rollout — the exact failure this project has hit before (controller v0.154.0,
// agent v0.91.0, the hub allowedEventTypes 400 in R-77).
var ErrTiersUnsupported = errors.New("quiesce: agent does not serve /backup/tiers (pre-R-82)")

// BackupTier is one tier as advertised by the agent, primary first.
type BackupTier struct {
	Target  string
	Primary bool
}

// TieredBackend is the OPTIONAL R-82 extension to Backend. A backend that does not implement it
// (or whose Tiers returns ErrTiersUnsupported) drives the pre-R-82 single-tier path unchanged.
//
// The untargeted Backend methods are NOT redundant: they remain the single-tier path, and the agent
// guarantees they keep their exact pre-R-82 meaning and response bytes.
type TieredBackend interface {
	Backend
	// Tiers lists the agent's backup tiers, primary first. ErrTiersUnsupported ⇒ pre-R-82 agent.
	Tiers(ctx context.Context) ([]BackupTier, error)
	// DueFor returns due-ness, the age (nil when unavailable) and the R-88 Part 2 age STATE as the
	// agent sent it ("" = legacy agent, which is NOT the same as "unknown").
	DueFor(ctx context.Context, target string) (due bool, ageSecs *int64, ageState string, err error)
	StartBackupFor(ctx context.Context, target string) (jobID string, err error)
	BackupStatusFor(ctx context.Context, target string) (phase string, err error)
}

// dueTier is a tier this cycle must back up.
// AgeState (R-88 Part 2) is why a tier's age is nil. It mirrors the agent's `age_state` wire field.
//
// THE ZERO VALUE IS LEGACY, NOT UNKNOWN, and that is the whole point of the type. An agent older than
// v0.105.0 omits the field entirely; reading that silence as "unknown" would stop the controller
// firing its first-backup safety valve on un-upgraded boxes, so a genuinely new box would never back
// up outside its window and nobody would notice for weeks. Preserving the KNOWN behaviour is correct;
// the MinAgent floor is what drives the upgrade.
type AgeState string

const (
	// AgeStateLegacy — the agent did not send the field. Behave exactly as before R-88 Part 2.
	AgeStateLegacy AgeState = ""
	// AgeStateKnown — the age is real.
	AgeStateKnown AgeState = "known"
	// AgeStateAbsent — a POSITIVE determination of "never backed up". The ONLY state (besides legacy)
	// that may fire the window-gate safety valve.
	AgeStateAbsent AgeState = "absent"
	// AgeStateUnknown — the agent could not tell. Still due, but it must NOT bypass the window gate.
	AgeStateUnknown AgeState = "unknown"
)

// ageStateFromWire maps the agent's string to an AgeState, mapping anything unrecognised to LEGACY.
//
// An unknown FUTURE value is treated as legacy on purpose: a newer agent inventing a fourth state
// must not accidentally acquire "unknown" semantics from a controller that has never heard of it.
// Fail toward the behaviour we already understand.
func ageStateFromWire(s string) AgeState {
	switch AgeState(s) {
	case AgeStateKnown, AgeStateAbsent, AgeStateUnknown:
		return AgeState(s)
	default:
		return AgeStateLegacy
	}
}

type dueTier struct {
	target  string // "" = the untargeted single-tier path (pre-R-82 agent)
	ageSecs *int64
	// state (R-88 Part 2) disambiguates a nil ageSecs. Empty = legacy agent.
	state AgeState
}

// resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place.
//
// Returns the due tiers IN AGENT ORDER (primary first). That order is deliberate and it is a
// downtime decision, not cosmetics: tiers run SEQUENTIALLY because vzdump holds a guest lock, and
// the app stays stopped until the LAST tier has snapshotted. Running the fast local tier first and
// the slow WAN/PBS tier last makes downtime ≈ (local backup) + (PBS snapshot); the reverse order
// would make it ≈ (PBS backup) + (local snapshot), which is far worse.
//
// degraded is true when the agent is pre-R-82 and the caller must use the untargeted path.
func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded bool, err error) {
	tb, ok := l.backend.(TieredBackend)
	if !ok {
		// Backend built without the tiered surface — the pre-R-82 path, no probe needed.
		return l.resolveUntargeted(ctx)
	}
	tiers, terr := tb.Tiers(ctx)
	if errors.Is(terr, ErrTiersUnsupported) {
		// A new controller meeting an OLD agent. Degrade — and SAY SO, once.
		l.logTierDegradeOnce()
		return l.resolveUntargeted(ctx)
	}
	if terr != nil {
		return nil, false, terr
	}
	if len(tiers) == 0 {
		// An agent that advertises no tiers cannot be backed up per-tier, but it can still be
		// backed up untargeted. Fail toward DOING the backup, never toward skipping it.
		l.logger.Printf("[WARN] [quiesce] agent advertised ZERO backup tiers — falling back to the untargeted path")
		return l.resolveUntargeted(ctx)
	}
	for _, t := range tiers {
		isDue, age, wireState, derr := tb.DueFor(ctx, t.Target)
		if derr != nil {
			// One tier's due-check failing must not silently drop the OTHER tier's backup.
			l.logger.Printf("[ERROR] [quiesce] due-check failed for tier %q: %v (other tiers still evaluated)", t.Target, derr)
			continue
		}
		if isDue {
			st := ageStateFromWire(wireState)
			if st == AgeStateLegacy {
				// A pre-v0.105.0 agent. Say so ONCE — the same shape as the pre-R-82 tier degrade,
				// because a silent behaviour difference between boxes is how a fleet drifts unnoticed.
				l.logAgeStateDegradeOnce()
			}
			due = append(due, dueTier{target: t.Target, ageSecs: age, state: st})
		}
	}
	return due, false, nil
}

// resolveUntargeted is the pre-R-82 single-tier resolution.
func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) {
	isDue, age, err := l.backend.Due(ctx)
	if err != nil {
		return nil, true, err
	}
	if !isDue {
		return nil, true, nil
	}
	return []dueTier{{target: "", ageSecs: age}}, true, nil
}

// logAgeStateDegradeOnce reports a pre-v0.105.0 agent exactly once per process (R-88 Part 2), the
// same shape and for the same reason as logTierDegradeOnce: a rollout is a steady state, so logging
// every poll would bury it, but logging zero times makes a real behaviour difference between boxes
// invisible.
//
// The behaviour on such an agent is DELIBERATELY today's: a nil age still fires the window-gate
// safety valve. Treating the missing field as "unknown" would look safer and would regress the
// first-backup guarantee on every un-upgraded box.
func (l *Loop) logAgeStateDegradeOnce() {
	l.ageStateDegradeOnce.Do(func() {
		l.logger.Printf("[WARN] [quiesce] agent does not report backup age_state (pre-v0.105.0) — " +
			"an unreadable storage cannot be told apart from 'never backed up', so a nil age still " +
			"bypasses the backup window as before. Upgrade the agent to close R-88.")
	})
}

// logTierDegradeOnce reports the pre-R-82 fallback exactly once per process. Once, because it is a
// steady state during a rollout and would otherwise log every poll; but never zero times, because a
// silent degrade is indistinguishable from multi-tier working.
func (l *Loop) logTierDegradeOnce() {
	l.degradeOnce.Do(func() {
		l.logger.Printf("[INFO] [quiesce] agent predates R-82 (no /backup/tiers) — using the single untargeted backup tier; per-tier scheduling is inactive until the agent is upgraded")
	})
}

// startBackupOn starts a backup on one tier (untargeted when target is "").
func (l *Loop) startBackupOn(ctx context.Context, target string) (string, error) {
	if target == "" {
		return l.backend.StartBackup(ctx)
	}
	return l.backend.(TieredBackend).StartBackupFor(ctx, target)
}

// backupStatusOn reads one tier's job phase (untargeted when target is "").
func (l *Loop) backupStatusOn(ctx context.Context, target string) (string, error) {
	if target == "" {
		return l.backend.BackupStatus(ctx)
	}
	return l.backend.(TieredBackend).BackupStatusFor(ctx, target)
}

// tierLabel renders a tier for logs ("" → the untargeted tier).
func tierLabel(target string) string {
	if target == "" {
		return "(untargeted)"
	}
	return target
}