felhom-controller/controller/internal/quiesce/precheck.go

package quiesce

import "context"

// R-921 — CHECK FIRST, STOP SECOND.
//
// MEASURED 2026-10-08 night on demo-hp (felhom.eu documentation/audits/dooplex-survival-2026-10-09/partE/
// R-518.txt): the controller stopped every app for the off-site tier, the agent refused the backup
// („a heavy operation is already in flight", busy=backup:local — the night OS step that follows the local
// copy), and the apps were down about a minute for no copy.
//
// What the agent lets the controller see BEFORE a stop (felhom-agent v0.154.0, read 2026-10-09): the agent
// refuses POST /backup for two reasons. (1) A job of ANOTHER tier of this guest is in flight
// (localapi otherTierInFlight) — that IS readable, per tier, from GET /backup/status?target=…. (2) Its
// host-wide heavy-operation gate is held (backup.InFlight: the OS step after the night's local copy, a
// restore-test, fstrim) — that is served by NO endpoint (InFlight.Busy() exists and nothing exposes it).
// So the pre-check below covered reason (1) only. Reason (2) — the measured instance — was met by the
// BUSY path in quiesceAndPollTiers, which resumes the apps at once (pinned by
// TestR921_BusyRefusalResumesAtOnce).
//
// UPDATED 2026-10-19 (controller 0.335.0, operator answer „add one field"): the agent now serves its gate —
// GET /backup/status `host_busy` (felhom-agent, Unreleased on 2026-10-19; it reaches a box with the next agent
// release). HostBusyProber reads it; a held gate is handled exactly like a 409 (contention: the tier stays due,
// is skipped for contentionRetryAfter, alarms after contentionAlarmAfter) — but with NO app stopped. An agent
// that does not serve the field keeps the old order (stop, ask, resume at once on 409). Pinned by
// TestR921_NoStopWhileTheHostIsBusy.
//
// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again — one short stop,
// resumed at once, as before. The check is a read, never a reservation.

// InFlightProber is the OPTIONAL pre-check surface (R-921). An adapter that does not implement it keeps
// the pre-R-921 behaviour exactly: stop, ask, and on refusal resume at once.
type InFlightProber interface {
	// BackupJobsInFlight returns the tiers whose backup job the agent holds in flight right now
	// (phase running or snapshotted). An agent without per-tier jobs (pre-R-82) answers nil, nil.
	BackupJobsInFlight(ctx context.Context) ([]string, error)
}

// HostBusyProber is the OPTIONAL read of the agent's host-wide heavy-operation gate (R-921, agent with
// `host_busy`). known=false: the agent does not serve it — no information.
type HostBusyProber interface {
	HostBusy(ctx context.Context) (busy bool, what string, known bool, err error)
}

// precheckSkips is the whole check-first rule: true = the agent would refuse this window, so no app is stopped.
func (l *Loop) precheckSkips(ctx context.Context, tiers []dueTier) bool {
	first := tiers[0].target
	skip, ownInFlight := l.anotherTierInFlight(ctx, first)
	if skip {
		return true
	}
	if ownInFlight || first == "" {
		// The window's own tier in flight: the agent answers its start with the running job (202) whatever its
		// gate says (the gate may be held by that very job) — attach as before. An untargeted window is a
		// pre-R-82 agent, which serves no gate read.
		return false
	}
	return l.hostBusyRefuses(ctx, tiers)
}

// hostBusyRefuses: the agent says its host-wide gate is held, so every tier of this window would be refused (409).
// Recorded as CONTENTION for each tier — the same verdict a 409 gets (retry after contentionRetryAfter, one alarm
// after contentionAlarmAfter), so a gate that never lets go still pages the operator — and no app is stopped.
// Fail toward backing up: an error, an older agent or an adapter without the surface → false.
func (l *Loop) hostBusyRefuses(ctx context.Context, tiers []dueTier) bool {
	p, ok := l.backend.(HostBusyProber)
	if !ok {
		return false
	}
	busy, what, known, err := p.HostBusy(ctx)
	if err != nil {
		l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent whether its host is busy (%v) — stopping the apps and asking as before (R-921)", err)
		return false
	}
	if !known || !busy {
		return false
	}
	l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent's host is busy (%s) — it would refuse the backup, so no app is stopped (R-921)",
		tierLabel(tiers[0].target), what)
	for _, t := range tiers {
		l.noteTierContention(t.target, tierLabel(t.target))
	}
	return true
}

// anotherTierInFlight reports whether the agent would refuse the window's first tier because a job of a
// DIFFERENT tier is in flight — so the window must not stop any app. The window's OWN tier in flight is
// not a refusal (the agent answers its start with the running job, 202), and that path is left as it was.
//
// Fail toward backing up: an unanswerable pre-check, an untargeted window (pre-R-82 agent) or an adapter
// without the surface all return false — the window runs as before.
// ownInFlight: the window's own tier has a job in flight (the agent will answer its start with that job).
func (l *Loop) anotherTierInFlight(ctx context.Context, first string) (skip, ownInFlight bool) {
	if first == "" {
		return false, false
	}
	p, ok := l.backend.(InFlightProber)
	if !ok {
		return false, false
	}
	busy, err := p.BackupJobsInFlight(ctx)
	if err != nil {
		l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent which backup jobs are in flight (%v) — stopping the apps and asking as before (R-921)", err)
		return false, false
	}
	for _, t := range busy {
		if t != first {
			l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent still holds a backup job on tier %s — no app is stopped; the tier stays due and is asked again at the next poll (R-921)",
				tierLabel(first), tierLabel(t))
			return true, false
		}
		ownInFlight = true
	}
	return false, ownInFlight
}