v0.335.0: no app stop for a tier the agent will refuse because its host is busy (host_busy pre-check, R-921)

controller/internal/quiesce/r921_precheck_test.go

@@ -34,6 +34,10 @@ type r921Backend struct {
 	probeErr error
 	busyOn   map[string]bool // StartBackupFor answers ErrTierBusy for these targets
 	starts   []string
+	// R-921 part 2: the agent's host_busy answer (hostKnown=false = an older agent without the field).
+	hostKnown, hostBusy bool
+	hostWhat            string
+	hostErr             error
 }
 
 func (b *r921Backend) log(e string) {
@@ -68,6 +72,13 @@ func (b *r921Backend) BackupJobsInFlight(context.Context) ([]string, error) {
 	}
 	return append([]string(nil), b.inFlight...), nil
 }
+func (b *r921Backend) HostBusy(context.Context) (bool, string, bool, error) {
+	b.log("hostbusy")
+	if b.hostErr != nil {
+		return false, "", false, b.hostErr
+	}
+	return b.hostBusy, b.hostWhat, b.hostKnown, nil
+}
 func (b *r921Backend) Due(context.Context) (bool, *int64, error)   { return false, nil, nil }
 func (b *r921Backend) StartBackup(context.Context) (string, error) { return "", errors.New("unused") }
 func (b *r921Backend) BackupStatus(context.Context) (string, error) {
@@ -284,3 +295,114 @@ func (n *noProbeBackend) StartBackup(ctx context.Context) (string, error) {
 func (n *noProbeBackend) BackupStatus(ctx context.Context) (string, error) {
 	return n.b.BackupStatus(ctx)
 }
+
+// R-921 part 2 — THE MEASURED CASE, now seen before the stop. The agent's host-wide gate is held (the night OS step
+// after the local copy: `os-pve-step`), the off-site tier is due. With the agent's host_busy answer the loop stops NO
+// app and asks for NO backup, and records contention for the tier (skipped for contentionRetryAfter, alarmed after
+// contentionAlarmAfter) — the verdict a 409 gets today, minus the stop.
+//
+// COMPANION RED-PROOF (observed 2026-10-19): with precheckSkips back to the pre-fix shape (the host_busy branch
+// removed), it fails with
+//
+//	r921_precheck_test.go: R-921: 3 app(s) stopped while the agent says its host is busy (os-pve-step); events=[probe stop:immich … start:felhom-pbs=BUSY startstack:immich …]
+//
+// Restored.
+func TestR921_NoStopWhileTheHostIsBusy(t *testing.T) {
+	be, _, l, events, logs := r921Setup(t)
+	be.due["felhom-pbs"] = true
+	be.busyOn["felhom-pbs"] = true // what the real agent answers while its gate is held
+	be.hostKnown, be.hostBusy, be.hostWhat = true, true, "os-pve-step"
+
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("runOnce: %v", err)
+	}
+	if n := countPrefix(*events, "stop:"); n != 0 {
+		t.Fatalf("R-921: %d app(s) stopped while the agent says its host is busy (os-pve-step); events=%v", n, *events)
+	}
+	if len(be.starts) != 0 {
+		t.Fatalf("a backup was requested while the host is busy: %v", be.starts)
+	}
+	if _, blocked := l.contention.blocked("felhom-pbs", l.now()); !blocked {
+		t.Errorf("the busy tier was not recorded as contention — the next poll would ask again at once, and a stuck gate would never alarm")
+	}
+	if got := l.breaker.failuresFor("felhom-pbs"); got != 0 {
+		t.Errorf("a busy host armed the failure breaker (%d) — nothing failed", got)
+	}
+	if !strings.Contains(logs.String(), "os-pve-step") || !strings.Contains(logs.String(), "R-921") {
+		t.Errorf("the skip does not name the holder and R-921: %q", logs.String())
+	}
+
+	// The gate lets go and the skip window passes → the next poll backs the tier up as normal.
+	be.hostBusy, be.hostWhat, be.busyOn["felhom-pbs"] = false, "", false
+	l.contention.clear("felhom-pbs")
+	if err := l.runOnce(context.Background()); err != nil {
+		t.Fatalf("runOnce 2: %v", err)
+	}
+	if len(be.starts) != 1 || be.starts[0] != "felhom-pbs" {
+		t.Fatalf("with the host free the tier was not backed up: starts=%v", be.starts)
+	}
+}
+
+// A gate that never lets go must still page the operator — the skip path must not be a silence path.
+func TestR921_HostBusySkipsStillAlarmAfterTheCeiling(t *testing.T) {
+	be, _, l, events, _ := r921Setup(t)
+	n := &recordingNotifier{}
+	l.SetTierNotifier(n)
+	base := time.Now()
+	now := base
+	l.now = func() time.Time { return now }
+	be.due["felhom-pbs"] = true
+	be.hostKnown, be.hostBusy, be.hostWhat = true, true, "restore-test"
+	for _, step := range []time.Duration{0, time.Hour, 2 * time.Hour, contentionAlarmAfter + time.Minute} {
+		now = base.Add(step)
+		if st, ok := l.contention.states["felhom-pbs"]; ok {
+			st.until = time.Time{} // let each poll through the skip window; the run (since, alarmed) is kept
+			l.contention.states["felhom-pbs"] = st
+		}
+		if err := l.runOnce(context.Background()); err != nil {
+			t.Fatalf("runOnce at +%s: %v", step, err)
+		}
+	}
+	if countPrefix(*events, "stop:") != 0 {
+		t.Fatalf("an app was stopped while the host was busy: %v", *events)
+	}
+	if len(n.failed) != 1 || !contains(n.failed[0], "BLOCKED") {
+		t.Fatalf("after %s of a busy host: alarms %v, want exactly one BLOCKED alarm", contentionAlarmAfter, n.failed)
+	}
+}
+
+// The other answers keep today's order: a FREE host backs up; an OLDER agent (no host_busy), or a host_busy read
+// that fails, stops the apps and asks as before (fail toward backing up); the window's OWN tier in flight attaches
+// to its job whatever the gate says (the gate may be held by that very job).
+func TestR921_HostBusyOtherAnswersKeepTheOldOrder(t *testing.T) {
+	for _, tc := range []struct {
+		name                string
+		known, busy         bool
+		err                 error
+		ownInFlight         bool
+		wantStarts, wantStp bool
+	}{
+		{"host free", true, false, nil, false, true, true},
+		{"older agent (no host_busy)", false, false, nil, false, true, true},
+		{"host_busy read failed", false, false, errors.New("agentapi: GET /backup/status: connection refused"), false, true, true},
+		{"own tier in flight, gate held by it", true, true, nil, true, true, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			be, _, l, events, _ := r921Setup(t)
+			be.due["felhom-pbs"] = true
+			be.hostKnown, be.hostBusy, be.hostErr = tc.known, tc.busy, tc.err
+			if tc.ownInFlight {
+				be.inFlight = []string{"felhom-pbs"}
+			}
+			if err := l.runOnce(context.Background()); err != nil {
+				t.Fatalf("runOnce: %v", err)
+			}
+			if got := len(be.starts) == 1; got != tc.wantStarts {
+				t.Fatalf("starts=%v, want a start: %v; events=%v", be.starts, tc.wantStarts, *events)
+			}
+			if tc.wantStp && countPrefix(*events, "stop:") == 0 {
+				t.Fatalf("no app stopped — the old order (stop, then ask) was not kept; events=%v", *events)
+			}
+		})
+	}
+}