116 lines
5.8 KiB
Go
116 lines
5.8 KiB
Go
package quiesce
|
|
|
|
import "context"
|
|
|
|
// R-921 — CHECK FIRST, STOP SECOND.
|
|
//
|
|
// MEASURED 2026-10-08 night on demo-hp (felhom.eu documentation/audits/dooplex-survival-2026-10-09/partE/
|
|
// R-518.txt): the controller stopped every app for the off-site tier, the agent refused the backup
|
|
// („a heavy operation is already in flight", busy=backup:local — the night OS step that follows the local
|
|
// copy), and the apps were down about a minute for no copy.
|
|
//
|
|
// What the agent lets the controller see BEFORE a stop (felhom-agent v0.154.0, read 2026-10-09): the agent
|
|
// refuses POST /backup for two reasons. (1) A job of ANOTHER tier of this guest is in flight
|
|
// (localapi otherTierInFlight) — that IS readable, per tier, from GET /backup/status?target=…. (2) Its
|
|
// host-wide heavy-operation gate is held (backup.InFlight: the OS step after the night's local copy, a
|
|
// restore-test, fstrim) — that is served by NO endpoint (InFlight.Busy() exists and nothing exposes it).
|
|
// So the pre-check below covered reason (1) only. Reason (2) — the measured instance — was met by the
|
|
// BUSY path in quiesceAndPollTiers, which resumes the apps at once (pinned by
|
|
// TestR921_BusyRefusalResumesAtOnce).
|
|
//
|
|
// UPDATED 2026-10-19 (controller 0.335.0, operator answer „add one field"): the agent now serves its gate —
|
|
// GET /backup/status `host_busy` (felhom-agent, Unreleased on 2026-10-19; it reaches a box with the next agent
|
|
// release). HostBusyProber reads it; a held gate is handled exactly like a 409 (contention: the tier stays due,
|
|
// is skipped for contentionRetryAfter, alarms after contentionAlarmAfter) — but with NO app stopped. An agent
|
|
// that does not serve the field keeps the old order (stop, ask, resume at once on 409). Pinned by
|
|
// TestR921_NoStopWhileTheHostIsBusy.
|
|
//
|
|
// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again — one short stop,
|
|
// resumed at once, as before. The check is a read, never a reservation.
|
|
|
|
// InFlightProber is the OPTIONAL pre-check surface (R-921). An adapter that does not implement it keeps
|
|
// the pre-R-921 behaviour exactly: stop, ask, and on refusal resume at once.
|
|
type InFlightProber interface {
|
|
// BackupJobsInFlight returns the tiers whose backup job the agent holds in flight right now
|
|
// (phase running or snapshotted). An agent without per-tier jobs (pre-R-82) answers nil, nil.
|
|
BackupJobsInFlight(ctx context.Context) ([]string, error)
|
|
}
|
|
|
|
// HostBusyProber is the OPTIONAL read of the agent's host-wide heavy-operation gate (R-921, agent with
|
|
// `host_busy`). known=false: the agent does not serve it — no information.
|
|
type HostBusyProber interface {
|
|
HostBusy(ctx context.Context) (busy bool, what string, known bool, err error)
|
|
}
|
|
|
|
// precheckSkips is the whole check-first rule: true = the agent would refuse this window, so no app is stopped.
|
|
func (l *Loop) precheckSkips(ctx context.Context, tiers []dueTier) bool {
|
|
first := tiers[0].target
|
|
skip, ownInFlight := l.anotherTierInFlight(ctx, first)
|
|
if skip {
|
|
return true
|
|
}
|
|
if ownInFlight || first == "" {
|
|
// The window's own tier in flight: the agent answers its start with the running job (202) whatever its
|
|
// gate says (the gate may be held by that very job) — attach as before. An untargeted window is a
|
|
// pre-R-82 agent, which serves no gate read.
|
|
return false
|
|
}
|
|
return l.hostBusyRefuses(ctx, tiers)
|
|
}
|
|
|
|
// hostBusyRefuses: the agent says its host-wide gate is held, so every tier of this window would be refused (409).
|
|
// Recorded as CONTENTION for each tier — the same verdict a 409 gets (retry after contentionRetryAfter, one alarm
|
|
// after contentionAlarmAfter), so a gate that never lets go still pages the operator — and no app is stopped.
|
|
// Fail toward backing up: an error, an older agent or an adapter without the surface → false.
|
|
func (l *Loop) hostBusyRefuses(ctx context.Context, tiers []dueTier) bool {
|
|
p, ok := l.backend.(HostBusyProber)
|
|
if !ok {
|
|
return false
|
|
}
|
|
busy, what, known, err := p.HostBusy(ctx)
|
|
if err != nil {
|
|
l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent whether its host is busy (%v) — stopping the apps and asking as before (R-921)", err)
|
|
return false
|
|
}
|
|
if !known || !busy {
|
|
return false
|
|
}
|
|
l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent's host is busy (%s) — it would refuse the backup, so no app is stopped (R-921)",
|
|
tierLabel(tiers[0].target), what)
|
|
for _, t := range tiers {
|
|
l.noteTierContention(t.target, tierLabel(t.target))
|
|
}
|
|
return true
|
|
}
|
|
|
|
// anotherTierInFlight reports whether the agent would refuse the window's first tier because a job of a
|
|
// DIFFERENT tier is in flight — so the window must not stop any app. The window's OWN tier in flight is
|
|
// not a refusal (the agent answers its start with the running job, 202), and that path is left as it was.
|
|
//
|
|
// Fail toward backing up: an unanswerable pre-check, an untargeted window (pre-R-82 agent) or an adapter
|
|
// without the surface all return false — the window runs as before.
|
|
// ownInFlight: the window's own tier has a job in flight (the agent will answer its start with that job).
|
|
func (l *Loop) anotherTierInFlight(ctx context.Context, first string) (skip, ownInFlight bool) {
|
|
if first == "" {
|
|
return false, false
|
|
}
|
|
p, ok := l.backend.(InFlightProber)
|
|
if !ok {
|
|
return false, false
|
|
}
|
|
busy, err := p.BackupJobsInFlight(ctx)
|
|
if err != nil {
|
|
l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent which backup jobs are in flight (%v) — stopping the apps and asking as before (R-921)", err)
|
|
return false, false
|
|
}
|
|
for _, t := range busy {
|
|
if t != first {
|
|
l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent still holds a backup job on tier %s — no app is stopped; the tier stays due and is asked again at the next poll (R-921)",
|
|
tierLabel(first), tierLabel(t))
|
|
return true, false
|
|
}
|
|
ownInFlight = true
|
|
}
|
|
return false, ownInFlight
|
|
}
|