package quiesce import "context" // R-921 — CHECK FIRST, STOP SECOND. // // MEASURED 2026-10-08 night on demo-hp (felhom.eu documentation/audits/dooplex-survival-2026-10-09/partE/ // R-518.txt): the controller stopped every app for the off-site tier, the agent refused the backup // („a heavy operation is already in flight", busy=backup:local — the night OS step that follows the local // copy), and the apps were down about a minute for no copy. // // What the agent lets the controller see BEFORE a stop (felhom-agent v0.154.0, read 2026-10-09): the agent // refuses POST /backup for two reasons. (1) A job of ANOTHER tier of this guest is in flight // (localapi otherTierInFlight) — that IS readable, per tier, from GET /backup/status?target=…. (2) Its // host-wide heavy-operation gate is held (backup.InFlight: the OS step after the night's local copy, a // restore-test, fstrim) — that is served by NO endpoint (InFlight.Busy() exists and nothing exposes it). // So the pre-check below covered reason (1) only. Reason (2) — the measured instance — was met by the // BUSY path in quiesceAndPollTiers, which resumes the apps at once (pinned by // TestR921_BusyRefusalResumesAtOnce). // // UPDATED 2026-10-19 (controller 0.335.0, operator answer „add one field"): the agent now serves its gate — // GET /backup/status `host_busy` (felhom-agent, Unreleased on 2026-10-19; it reaches a box with the next agent // release). HostBusyProber reads it; a held gate is handled exactly like a 409 (contention: the tier stays due, // is skipped for contentionRetryAfter, alarms after contentionAlarmAfter) — but with NO app stopped. An agent // that does not serve the field keeps the old order (stop, ask, resume at once on 409). Pinned by // TestR921_NoStopWhileTheHostIsBusy. // // The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again — one short stop, // resumed at once, as before. The check is a read, never a reservation. // InFlightProber is the OPTIONAL pre-check surface (R-921). An adapter that does not implement it keeps // the pre-R-921 behaviour exactly: stop, ask, and on refusal resume at once. type InFlightProber interface { // BackupJobsInFlight returns the tiers whose backup job the agent holds in flight right now // (phase running or snapshotted). An agent without per-tier jobs (pre-R-82) answers nil, nil. BackupJobsInFlight(ctx context.Context) ([]string, error) } // HostBusyProber is the OPTIONAL read of the agent's host-wide heavy-operation gate (R-921, agent with // `host_busy`). known=false: the agent does not serve it — no information. type HostBusyProber interface { HostBusy(ctx context.Context) (busy bool, what string, known bool, err error) } // precheckSkips is the whole check-first rule: true = the agent would refuse this window, so no app is stopped. func (l *Loop) precheckSkips(ctx context.Context, tiers []dueTier) bool { first := tiers[0].target skip, ownInFlight := l.anotherTierInFlight(ctx, first) if skip { return true } if ownInFlight || first == "" { // The window's own tier in flight: the agent answers its start with the running job (202) whatever its // gate says (the gate may be held by that very job) — attach as before. An untargeted window is a // pre-R-82 agent, which serves no gate read. return false } return l.hostBusyRefuses(ctx, tiers) } // hostBusyRefuses: the agent says its host-wide gate is held, so every tier of this window would be refused (409). // Recorded as CONTENTION for each tier — the same verdict a 409 gets (retry after contentionRetryAfter, one alarm // after contentionAlarmAfter), so a gate that never lets go still pages the operator — and no app is stopped. // Fail toward backing up: an error, an older agent or an adapter without the surface → false. func (l *Loop) hostBusyRefuses(ctx context.Context, tiers []dueTier) bool { p, ok := l.backend.(HostBusyProber) if !ok { return false } busy, what, known, err := p.HostBusy(ctx) if err != nil { l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent whether its host is busy (%v) — stopping the apps and asking as before (R-921)", err) return false } if !known || !busy { return false } l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent's host is busy (%s) — it would refuse the backup, so no app is stopped (R-921)", tierLabel(tiers[0].target), what) for _, t := range tiers { l.noteTierContention(t.target, tierLabel(t.target)) } return true } // anotherTierInFlight reports whether the agent would refuse the window's first tier because a job of a // DIFFERENT tier is in flight — so the window must not stop any app. The window's OWN tier in flight is // not a refusal (the agent answers its start with the running job, 202), and that path is left as it was. // // Fail toward backing up: an unanswerable pre-check, an untargeted window (pre-R-82 agent) or an adapter // without the surface all return false — the window runs as before. // ownInFlight: the window's own tier has a job in flight (the agent will answer its start with that job). func (l *Loop) anotherTierInFlight(ctx context.Context, first string) (skip, ownInFlight bool) { if first == "" { return false, false } p, ok := l.backend.(InFlightProber) if !ok { return false, false } busy, err := p.BackupJobsInFlight(ctx) if err != nil { l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent which backup jobs are in flight (%v) — stopping the apps and asking as before (R-921)", err) return false, false } for _, t := range busy { if t != first { l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent still holds a backup job on tier %s — no app is stopped; the tier stays due and is asked again at the next poll (R-921)", tierLabel(first), tierLabel(t)) return true, false } ownInFlight = true } return false, ownInFlight }