v0.335.0: no app stop for a tier the agent will refuse because its host is busy (host_busy pre-check, R-921)

controller/internal/agentapi/backup_tiers.go

@@ -132,7 +132,7 @@ func (c *Client) BackupStatusFor(ctx context.Context, target string) (StatusResp
 // tier is the agent's first reason to refuse POST /backup (HTTP 409), so the controller asks this BEFORE it
 // stops any app. It reads only GET /backup/tiers and GET /backup/status?target= (both agent >= v0.97.0); a
 // pre-R-82 agent (no /backup/tiers) answers nil, nil. Any other error is returned — the caller fails toward
-// backing up. It does NOT see the agent's host-wide heavy-operation gate: no endpoint serves it.
+// backing up. It does NOT see the agent's host-wide heavy-operation gate; HostBusy does (newer agents).
 func (c *Client) BackupJobsInFlight(ctx context.Context) ([]string, error) {
 	tiers, err := c.BackupTiers(ctx)
 	if errors.Is(err, ErrTiersUnsupported) {
@@ -154,6 +154,21 @@ func (c *Client) BackupJobsInFlight(ctx context.Context) ([]string, error) {
 	return out, nil
 }
 
+// HostBusy (R-921) reads GET /backup/status's host_busy: whether the agent's host-wide heavy-operation gate is held
+// right now (another guest's backup, the restore test, a night OS step …) — the agent's second reason to refuse
+// POST /backup. known=false: the agent does not serve the field (older than the agent release that adds it) — no
+// information, never „free". An error is returned; the caller fails toward backing up.
+func (c *Client) HostBusy(ctx context.Context) (busy bool, what string, known bool, err error) {
+	st, err := c.BackupStatusFor(ctx, "")
+	if err != nil {
+		return false, "", false, err
+	}
+	if st.HostBusy == nil {
+		return false, "", false, nil
+	}
+	return st.HostBusy.Busy, st.HostBusy.What, true, nil
+}
+
 // SetBackupTargetResponse mirrors POST /backup/target (agent >= v0.113.0).
 type SetBackupTargetResponse struct {
 	Target string `json:"target"`

controller/internal/agentapi/client.go

@@ -202,6 +202,15 @@ type StatusResponse struct {
 	// Tiers (agent >= v0.131.0, R-517) — per tier: newest success, last attempt, storage presence.
 	// Absent on an older agent; the page then falls back to the single latest record.
 	Tiers []TierBackupState `json:"tiers,omitempty"`
+	// HostBusy (R-921, agent Unreleased 2026-10-19) — whether the agent's host-wide heavy-operation gate is held
+	// right now (then POST /backup answers 409). nil = an older agent that does not serve it.
+	HostBusy *HostBusyState `json:"host_busy,omitempty"`
+}
+
+// HostBusyState mirrors the agent's localapi.HostBusyState.
+type HostBusyState struct {
+	Busy bool   `json:"busy"`
+	What string `json:"what,omitempty"`
 }
 
 // TierBackupState mirrors the agent's localapi.TierBackupState.

controller/internal/agentapi/r921_inflight_test.go

@@ -73,3 +73,35 @@ func TestBackupJobsInFlight_StatusErrorIsReturned(t *testing.T) {
 		t.Fatal("a failed status read was swallowed — the caller could not tell it from 'nothing in flight'")
 	}
 }
+
+// R-921 part 2: HostBusy reads the agent's real wire shape (untargeted GET /backup/status, `host_busy`); an agent
+// without the field is „not known" — never „free".
+func TestHostBusy_ReadsTheAgentsGate(t *testing.T) {
+	for _, tc := range []struct {
+		name, body  string
+		busy, known bool
+		what        string
+	}{
+		{"held", `{"ok":true,"data":{"vmid":9201,"phase":"idle","host_busy":{"busy":true,"what":"os-pve-step"}}}`, true, true, "os-pve-step"},
+		{"free", `{"ok":true,"data":{"vmid":9201,"phase":"idle","host_busy":{"busy":false}}}`, false, true, ""},
+		{"older agent", `{"ok":true,"data":{"vmid":9201,"phase":"idle"}}`, false, false, ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			mux := http.NewServeMux()
+			mux.HandleFunc("GET /backup/status", func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.RawQuery != "" {
+					http.Error(w, "want the untargeted status", http.StatusBadRequest)
+					return
+				}
+				_, _ = w.Write([]byte(tc.body))
+			})
+			s := httptest.NewTLSServer(mux)
+			defer s.Close()
+			c := clientFor(t, s, strings.TrimPrefix(s.URL, "https://"))
+			busy, what, known, err := c.HostBusy(context.Background())
+			if err != nil || busy != tc.busy || known != tc.known || what != tc.what {
+				t.Fatalf("HostBusy = %v %q known=%v err=%v, want %v %q known=%v", busy, what, known, err, tc.busy, tc.what, tc.known)
+			}
+		})
+	}
+}

controller/internal/quiesce/precheck.go

@@ -14,11 +14,19 @@ import "context"
 // (localapi otherTierInFlight) — that IS readable, per tier, from GET /backup/status?target=…. (2) Its
 // host-wide heavy-operation gate is held (backup.InFlight: the OS step after the night's local copy, a
 // restore-test, fstrim) — that is served by NO endpoint (InFlight.Busy() exists and nothing exposes it).
-// So the pre-check below covers reason (1) only. Reason (2) — the measured instance — is met by the
+// So the pre-check below covered reason (1) only. Reason (2) — the measured instance — was met by the
 // BUSY path in quiesceAndPollTiers, which resumes the apps at once (pinned by
-// TestR921_BusyRefusalResumesAtOnce); closing it before the stop needs the agent to serve its gate.
+// TestR921_BusyRefusalResumesAtOnce).
 //
-// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again.
+// UPDATED 2026-10-19 (controller 0.335.0, operator answer „add one field"): the agent now serves its gate —
+// GET /backup/status `host_busy` (felhom-agent, Unreleased on 2026-10-19; it reaches a box with the next agent
+// release). HostBusyProber reads it; a held gate is handled exactly like a 409 (contention: the tier stays due,
+// is skipped for contentionRetryAfter, alarms after contentionAlarmAfter) — but with NO app stopped. An agent
+// that does not serve the field keeps the old order (stop, ask, resume at once on 409). Pinned by
+// TestR921_NoStopWhileTheHostIsBusy.
+//
+// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again — one short stop,
+// resumed at once, as before. The check is a read, never a reservation.
 
 // InFlightProber is the OPTIONAL pre-check surface (R-921). An adapter that does not implement it keeps
 // the pre-R-921 behaviour exactly: stop, ask, and on refusal resume at once.
@@ -28,31 +36,80 @@ type InFlightProber interface {
 	BackupJobsInFlight(ctx context.Context) ([]string, error)
 }
 
+// HostBusyProber is the OPTIONAL read of the agent's host-wide heavy-operation gate (R-921, agent with
+// `host_busy`). known=false: the agent does not serve it — no information.
+type HostBusyProber interface {
+	HostBusy(ctx context.Context) (busy bool, what string, known bool, err error)
+}
+
+// precheckSkips is the whole check-first rule: true = the agent would refuse this window, so no app is stopped.
+func (l *Loop) precheckSkips(ctx context.Context, tiers []dueTier) bool {
+	first := tiers[0].target
+	skip, ownInFlight := l.anotherTierInFlight(ctx, first)
+	if skip {
+		return true
+	}
+	if ownInFlight || first == "" {
+		// The window's own tier in flight: the agent answers its start with the running job (202) whatever its
+		// gate says (the gate may be held by that very job) — attach as before. An untargeted window is a
+		// pre-R-82 agent, which serves no gate read.
+		return false
+	}
+	return l.hostBusyRefuses(ctx, tiers)
+}
+
+// hostBusyRefuses: the agent says its host-wide gate is held, so every tier of this window would be refused (409).
+// Recorded as CONTENTION for each tier — the same verdict a 409 gets (retry after contentionRetryAfter, one alarm
+// after contentionAlarmAfter), so a gate that never lets go still pages the operator — and no app is stopped.
+// Fail toward backing up: an error, an older agent or an adapter without the surface → false.
+func (l *Loop) hostBusyRefuses(ctx context.Context, tiers []dueTier) bool {
+	p, ok := l.backend.(HostBusyProber)
+	if !ok {
+		return false
+	}
+	busy, what, known, err := p.HostBusy(ctx)
+	if err != nil {
+		l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent whether its host is busy (%v) — stopping the apps and asking as before (R-921)", err)
+		return false
+	}
+	if !known || !busy {
+		return false
+	}
+	l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent's host is busy (%s) — it would refuse the backup, so no app is stopped (R-921)",
+		tierLabel(tiers[0].target), what)
+	for _, t := range tiers {
+		l.noteTierContention(t.target, tierLabel(t.target))
+	}
+	return true
+}
+
 // anotherTierInFlight reports whether the agent would refuse the window's first tier because a job of a
 // DIFFERENT tier is in flight — so the window must not stop any app. The window's OWN tier in flight is
 // not a refusal (the agent answers its start with the running job, 202), and that path is left as it was.
 //
 // Fail toward backing up: an unanswerable pre-check, an untargeted window (pre-R-82 agent) or an adapter
 // without the surface all return false — the window runs as before.
-func (l *Loop) anotherTierInFlight(ctx context.Context, first string) bool {
+// ownInFlight: the window's own tier has a job in flight (the agent will answer its start with that job).
+func (l *Loop) anotherTierInFlight(ctx context.Context, first string) (skip, ownInFlight bool) {
 	if first == "" {
-		return false
+		return false, false
 	}
 	p, ok := l.backend.(InFlightProber)
 	if !ok {
-		return false
+		return false, false
 	}
 	busy, err := p.BackupJobsInFlight(ctx)
 	if err != nil {
 		l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent which backup jobs are in flight (%v) — stopping the apps and asking as before (R-921)", err)
-		return false
+		return false, false
 	}
 	for _, t := range busy {
 		if t != first {
 			l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent still holds a backup job on tier %s — no app is stopped; the tier stays due and is asked again at the next poll (R-921)",
 				tierLabel(first), tierLabel(t))
-			return true
+			return true, false
 		}
+		ownInFlight = true
 	}
-	return false
+	return false, ownInFlight
 }

controller/internal/quiesce/quiesce.go

@@ -556,9 +556,10 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error {
 	if len(tiers) == 0 {
 		return nil
 	}
-	// R-921: check first, stop second — a tier the agent will refuse (another tier's job in flight) stops
-	// no app. Pinned by TestR921_NoStopWhileAnotherTiersJobIsInFlight (precheck.go says what it cannot see).
-	if l.anotherTierInFlight(ctx, tiers[0].target) {
+	// R-921: check first, stop second — a tier the agent will refuse (another tier's job in flight, or its
+	// host-wide gate held) stops no app. Pinned by TestR921_NoStopWhileAnotherTiersJobIsInFlight and
+	// TestR921_NoStopWhileTheHostIsBusy (precheck.go).
+	if l.precheckSkips(ctx, tiers) {
 		return nil
 	}
 	running := l.stacks.RunningAppStacks()

controller/internal/quiesce/r921_precheck_test.go

@@ -34,6 +34,10 @@ type r921Backend struct {
 	probeErr error
 	busyOn   map[string]bool // StartBackupFor answers ErrTierBusy for these targets
 	starts   []string
+	// R-921 part 2: the agent's host_busy answer (hostKnown=false = an older agent without the field).
+	hostKnown, hostBusy bool
+	hostWhat            string
+	hostErr             error
 }
 
 func (b *r921Backend) log(e string) {
@@ -68,6 +72,13 @@ func (b *r921Backend) BackupJobsInFlight(context.Context) ([]string, error) {
 	}
 	return append([]string(nil), b.inFlight...), nil
 }
+func (b *r921Backend) HostBusy(context.Context) (bool, string, bool, error) {
+	b.log("hostbusy")
+	if b.hostErr != nil {
+		return false, "", false, b.hostErr
+	}
+	return b.hostBusy, b.hostWhat, b.hostKnown, nil
+}
 func (b *r921Backend) Due(context.Context) (bool, *int64, error)   { return false, nil, nil }
 func (b *r921Backend) StartBackup(context.Context) (string, error) { return "", errors.New("unused") }
 func (b *r921Backend) BackupStatus(context.Context) (string, error) {
@@ -284,3 +295,114 @@ func (n *noProbeBackend) StartBackup(ctx context.Context) (string, error) {
 func (n *noProbeBackend) BackupStatus(ctx context.Context) (string, error) {
 	return n.b.BackupStatus(ctx)
 }
+
+// R-921 part 2 — THE MEASURED CASE, now seen before the stop. The agent's host-wide gate is held (the night OS step
+// after the local copy: `os-pve-step`), the off-site tier is due. With the agent's host_busy answer the loop stops NO
+// app and asks for NO backup, and records contention for the tier (skipped for contentionRetryAfter, alarmed after
+// contentionAlarmAfter) — the verdict a 409 gets today, minus the stop.
+//
+// COMPANION RED-PROOF (observed 2026-10-19): with precheckSkips back to the pre-fix shape (the host_busy branch
+// removed), it fails with
+//
+//	r921_precheck_test.go: R-921: 3 app(s) stopped while the agent says its host is busy (os-pve-step); events=[probe stop:immich … start:felhom-pbs=BUSY startstack:immich …]
+//
+// Restored.
+func TestR921_NoStopWhileTheHostIsBusy(t *testing.T) {
+	be, _, l, events, logs := r921Setup(t)
+	be.due["felhom-pbs"] = true
+	be.busyOn["felhom-pbs"] = true // what the real agent answers while its gate is held
+	be.hostKnown, be.hostBusy, be.hostWhat = true, true, "os-pve-step"
+
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("runOnce: %v", err)
+	}
+	if n := countPrefix(*events, "stop:"); n != 0 {
+		t.Fatalf("R-921: %d app(s) stopped while the agent says its host is busy (os-pve-step); events=%v", n, *events)
+	}
+	if len(be.starts) != 0 {
+		t.Fatalf("a backup was requested while the host is busy: %v", be.starts)
+	}
+	if _, blocked := l.contention.blocked("felhom-pbs", l.now()); !blocked {
+		t.Errorf("the busy tier was not recorded as contention — the next poll would ask again at once, and a stuck gate would never alarm")
+	}
+	if got := l.breaker.failuresFor("felhom-pbs"); got != 0 {
+		t.Errorf("a busy host armed the failure breaker (%d) — nothing failed", got)
+	}
+	if !strings.Contains(logs.String(), "os-pve-step") || !strings.Contains(logs.String(), "R-921") {
+		t.Errorf("the skip does not name the holder and R-921: %q", logs.String())
+	}
+
+	// The gate lets go and the skip window passes → the next poll backs the tier up as normal.
+	be.hostBusy, be.hostWhat, be.busyOn["felhom-pbs"] = false, "", false
+	l.contention.clear("felhom-pbs")
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("runOnce 2: %v", err)
+	}
+	if len(be.starts) != 1 || be.starts[0] != "felhom-pbs" {
+		t.Fatalf("with the host free the tier was not backed up: starts=%v", be.starts)
+	}
+}
+
+// A gate that never lets go must still page the operator — the skip path must not be a silence path.
+func TestR921_HostBusySkipsStillAlarmAfterTheCeiling(t *testing.T) {
+	be, _, l, events, _ := r921Setup(t)
+	n := &recordingNotifier{}
+	l.SetTierNotifier(n)
+	base := time.Now()
+	now := base
+	l.now = func() time.Time { return now }
+	be.due["felhom-pbs"] = true
+	be.hostKnown, be.hostBusy, be.hostWhat = true, true, "restore-test"
+	for _, step := range []time.Duration{0, time.Hour, 2 * time.Hour, contentionAlarmAfter + time.Minute} {
+		now = base.Add(step)
+		if st, ok := l.contention.states["felhom-pbs"]; ok {
+			st.until = time.Time{} // let each poll through the skip window; the run (since, alarmed) is kept
+			l.contention.states["felhom-pbs"] = st
+		}
+		if err := l.runOnce(context.Background()); err != nil {
+			t.Fatalf("runOnce at +%s: %v", step, err)
+		}
+	}
+	if countPrefix(*events, "stop:") != 0 {
+		t.Fatalf("an app was stopped while the host was busy: %v", *events)
+	}
+	if len(n.failed) != 1 || !contains(n.failed[0], "BLOCKED") {
+		t.Fatalf("after %s of a busy host: alarms %v, want exactly one BLOCKED alarm", contentionAlarmAfter, n.failed)
+	}
+}
+
+// The other answers keep today's order: a FREE host backs up; an OLDER agent (no host_busy), or a host_busy read
+// that fails, stops the apps and asks as before (fail toward backing up); the window's OWN tier in flight attaches
+// to its job whatever the gate says (the gate may be held by that very job).
+func TestR921_HostBusyOtherAnswersKeepTheOldOrder(t *testing.T) {
+	for _, tc := range []struct {
+		name                string
+		known, busy         bool
+		err                 error
+		ownInFlight         bool
+		wantStarts, wantStp bool
+	}{
+		{"host free", true, false, nil, false, true, true},
+		{"older agent (no host_busy)", false, false, nil, false, true, true},
+		{"host_busy read failed", false, false, errors.New("agentapi: GET /backup/status: connection refused"), false, true, true},
+		{"own tier in flight, gate held by it", true, true, nil, true, true, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			be, _, l, events, _ := r921Setup(t)
+			be.due["felhom-pbs"] = true
+			be.hostKnown, be.hostBusy, be.hostErr = tc.known, tc.busy, tc.err
+			if tc.ownInFlight {
+				be.inFlight = []string{"felhom-pbs"}
+			}
+			if err := l.runOnce(context.Background()); err != nil {
+				t.Fatalf("runOnce: %v", err)
+			}
+			if got := len(be.starts) == 1; got != tc.wantStarts {
+				t.Fatalf("starts=%v, want a start: %v; events=%v", be.starts, tc.wantStarts, *events)
+			}
+			if tc.wantStp && countPrefix(*events, "stop:") == 0 {
+				t.Fatalf("no app stopped — the old order (stop, then ask) was not kept; events=%v", *events)
+			}
+		})
+	}
+}