diff --git a/CHANGELOG.md b/CHANGELOG.md index bde61b8..82e2288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ +## v0.335.0 — no app stop for an off-site tier the agent will refuse because its host is busy (R-921) (2026-10-19) + +**MinAgent: 0.158.1** — the floor's, carried. The new check needs the agent's `host_busy` answer (felhom-agent, +Unreleased on 2026-10-19, `56aabc7`); an agent without it keeps the old order, so no agent is required. No golden +(`09` §3 decision 250). Task: the evening/night brief of 2026-10-19, Part 1 (operator answer: „add one field"). + +- **Check first, stop second — now for the agent's host-wide gate too (R-921).** Measured 2026-10-08 on demo-hp: the + night OS step held the agent's heavy-operation gate, the controller stopped all 15 apps for the off-site tier, the + agent refused (409), the apps came back a minute later with no copy. Before it stops any app the loop now reads + `host_busy` on `GET /backup/status` (`agentapi.HostBusy`, `quiesce.HostBusyProber`, wired in `main.go` with a + compile-time pin). Held → no app is stopped, no backup is asked for, and each due tier gets the verdict a 409 gets + (contention: skipped for 15 minutes, one „BLOCKED" alarm after 3 hours, no failure breaker). Free, an older agent, a + failed read → the old order (stop, ask, resume at once on 409). The window's own tier in flight attaches to its job + as before. **The race stays:** the check is a read, not a reservation — a gate taken between the read and the start + costs one short stop, resumed at once (pinned since 0.305.0 by `TestR921_BusyRefusalResumesAtOnce`). Tests + `TestR921_NoStopWhileTheHostIsBusy` (red-proved: without the branch, 3 apps stopped and the start refused), + `TestR921_HostBusySkipsStillAlarmAfterTheCeiling`, `TestR921_HostBusyOtherAnswersKeepTheOldOrder` (4 rows), + `TestHostBusy_ReadsTheAgentsGate` (the wire shape; an agent without the field is „not known", never „free"). + ## v0.334.0 — a live box refreshes its config with its own key (box-config, R-1159, operator decision D19) (2026-10-19) **MinAgent: 0.158.1** — the floor's, carried; no agent change. No golden (`09` §3 decision 250). Needs hub ≥ 0.161.0 diff --git a/controller/README.md b/controller/README.md index cbc463b..8acb8f6 100644 --- a/controller/README.md +++ b/controller/README.md @@ -1410,6 +1410,12 @@ The nightly backup has two phases that run sequentially. All paths are **per-dri > the three daily legs via `scheduler.UpdateDaily` and takes effect **without a restart**. Precedence: > settings > controller.yaml `db_dump_schedule` > "02:30". See `internal/backupwindow`. +> **Check first, stop second (v0.305.0 / v0.335.0, R-921).** Before any app is stopped for a whole-guest tier the loop +> asks the agent whether it would refuse: another tier's job in flight (`GET /backup/status?target=…`), or — with an +> agent that serves it — its host-wide heavy-operation gate held (`host_busy`: another guest's backup, the restore +> test, the night OS step). Then no app is stopped; a held gate is recorded as contention (retry in 15 min, one +> „BLOCKED" alarm after 3 h). An agent without `host_busy` keeps the old order (stop, ask, resume at once on 409). + > **Absent-storage tier skip (v0.243.0, R-518).** A tier whose storage the agent (≥ 0.131.0) reports `absent` is dropped from the manual and the scheduled run before anything is stopped (`quiesce.skipAbsentTiers`), logged, and reported once as `backup_tier_skipped`. `unknown` or a legacy agent is never skipped. > **OOM visibility (v0.243.0, R-514).** The 30 s dead-app check also reads `State.OOMKilled` for running app containers (`Manager.ScanOOMKilled`); the dashboard shows „Memória elfogyott" and the hub gets `app_oom` once per container run. The controller does not restart the app. diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 038703a..64ee6c1 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -3642,6 +3642,14 @@ func (b quiesceBackend) BackupJobsInFlight(ctx context.Context) ([]string, error return b.c.BackupJobsInFlight(ctx) } +// HostBusy satisfies quiesce.HostBusyProber (R-921, controller 0.335.0): the agent's host-wide gate, read before +// any app is stopped. The assertion is the wiring pin, as above. +var _ quiesce.HostBusyProber = quiesceBackend{} + +func (b quiesceBackend) HostBusy(ctx context.Context) (bool, string, bool, error) { + return b.c.HostBusy(ctx) +} + // quiesceTierNotifier adapts *notify.Notifier to quiesce.TierNotifier (R-97a). // // The whole-guest tier had NO route to the hub at all — `internal/quiesce` did not import diff --git a/controller/internal/agentapi/backup_tiers.go b/controller/internal/agentapi/backup_tiers.go index 8ef90e6..71629c9 100644 --- a/controller/internal/agentapi/backup_tiers.go +++ b/controller/internal/agentapi/backup_tiers.go @@ -132,7 +132,7 @@ func (c *Client) BackupStatusFor(ctx context.Context, target string) (StatusResp // tier is the agent's first reason to refuse POST /backup (HTTP 409), so the controller asks this BEFORE it // stops any app. It reads only GET /backup/tiers and GET /backup/status?target= (both agent >= v0.97.0); a // pre-R-82 agent (no /backup/tiers) answers nil, nil. Any other error is returned — the caller fails toward -// backing up. It does NOT see the agent's host-wide heavy-operation gate: no endpoint serves it. +// backing up. It does NOT see the agent's host-wide heavy-operation gate; HostBusy does (newer agents). func (c *Client) BackupJobsInFlight(ctx context.Context) ([]string, error) { tiers, err := c.BackupTiers(ctx) if errors.Is(err, ErrTiersUnsupported) { @@ -154,6 +154,21 @@ func (c *Client) BackupJobsInFlight(ctx context.Context) ([]string, error) { return out, nil } +// HostBusy (R-921) reads GET /backup/status's host_busy: whether the agent's host-wide heavy-operation gate is held +// right now (another guest's backup, the restore test, a night OS step …) — the agent's second reason to refuse +// POST /backup. known=false: the agent does not serve the field (older than the agent release that adds it) — no +// information, never „free". An error is returned; the caller fails toward backing up. +func (c *Client) HostBusy(ctx context.Context) (busy bool, what string, known bool, err error) { + st, err := c.BackupStatusFor(ctx, "") + if err != nil { + return false, "", false, err + } + if st.HostBusy == nil { + return false, "", false, nil + } + return st.HostBusy.Busy, st.HostBusy.What, true, nil +} + // SetBackupTargetResponse mirrors POST /backup/target (agent >= v0.113.0). type SetBackupTargetResponse struct { Target string `json:"target"` diff --git a/controller/internal/agentapi/client.go b/controller/internal/agentapi/client.go index 1012ed7..8ea8299 100644 --- a/controller/internal/agentapi/client.go +++ b/controller/internal/agentapi/client.go @@ -202,6 +202,15 @@ type StatusResponse struct { // Tiers (agent >= v0.131.0, R-517) — per tier: newest success, last attempt, storage presence. // Absent on an older agent; the page then falls back to the single latest record. Tiers []TierBackupState `json:"tiers,omitempty"` + // HostBusy (R-921, agent Unreleased 2026-10-19) — whether the agent's host-wide heavy-operation gate is held + // right now (then POST /backup answers 409). nil = an older agent that does not serve it. + HostBusy *HostBusyState `json:"host_busy,omitempty"` +} + +// HostBusyState mirrors the agent's localapi.HostBusyState. +type HostBusyState struct { + Busy bool `json:"busy"` + What string `json:"what,omitempty"` } // TierBackupState mirrors the agent's localapi.TierBackupState. diff --git a/controller/internal/agentapi/r921_inflight_test.go b/controller/internal/agentapi/r921_inflight_test.go index 88d07c0..71efd98 100644 --- a/controller/internal/agentapi/r921_inflight_test.go +++ b/controller/internal/agentapi/r921_inflight_test.go @@ -73,3 +73,35 @@ func TestBackupJobsInFlight_StatusErrorIsReturned(t *testing.T) { t.Fatal("a failed status read was swallowed — the caller could not tell it from 'nothing in flight'") } } + +// R-921 part 2: HostBusy reads the agent's real wire shape (untargeted GET /backup/status, `host_busy`); an agent +// without the field is „not known" — never „free". +func TestHostBusy_ReadsTheAgentsGate(t *testing.T) { + for _, tc := range []struct { + name, body string + busy, known bool + what string + }{ + {"held", `{"ok":true,"data":{"vmid":9201,"phase":"idle","host_busy":{"busy":true,"what":"os-pve-step"}}}`, true, true, "os-pve-step"}, + {"free", `{"ok":true,"data":{"vmid":9201,"phase":"idle","host_busy":{"busy":false}}}`, false, true, ""}, + {"older agent", `{"ok":true,"data":{"vmid":9201,"phase":"idle"}}`, false, false, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + mux := http.NewServeMux() + mux.HandleFunc("GET /backup/status", func(w http.ResponseWriter, r *http.Request) { + if r.URL.RawQuery != "" { + http.Error(w, "want the untargeted status", http.StatusBadRequest) + return + } + _, _ = w.Write([]byte(tc.body)) + }) + s := httptest.NewTLSServer(mux) + defer s.Close() + c := clientFor(t, s, strings.TrimPrefix(s.URL, "https://")) + busy, what, known, err := c.HostBusy(context.Background()) + if err != nil || busy != tc.busy || known != tc.known || what != tc.what { + t.Fatalf("HostBusy = %v %q known=%v err=%v, want %v %q known=%v", busy, what, known, err, tc.busy, tc.what, tc.known) + } + }) + } +} diff --git a/controller/internal/quiesce/precheck.go b/controller/internal/quiesce/precheck.go index 745a9ce..fd5ec1a 100644 --- a/controller/internal/quiesce/precheck.go +++ b/controller/internal/quiesce/precheck.go @@ -14,11 +14,19 @@ import "context" // (localapi otherTierInFlight) — that IS readable, per tier, from GET /backup/status?target=…. (2) Its // host-wide heavy-operation gate is held (backup.InFlight: the OS step after the night's local copy, a // restore-test, fstrim) — that is served by NO endpoint (InFlight.Busy() exists and nothing exposes it). -// So the pre-check below covers reason (1) only. Reason (2) — the measured instance — is met by the +// So the pre-check below covered reason (1) only. Reason (2) — the measured instance — was met by the // BUSY path in quiesceAndPollTiers, which resumes the apps at once (pinned by -// TestR921_BusyRefusalResumesAtOnce); closing it before the stop needs the agent to serve its gate. +// TestR921_BusyRefusalResumesAtOnce). // -// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again. +// UPDATED 2026-10-19 (controller 0.335.0, operator answer „add one field"): the agent now serves its gate — +// GET /backup/status `host_busy` (felhom-agent, Unreleased on 2026-10-19; it reaches a box with the next agent +// release). HostBusyProber reads it; a held gate is handled exactly like a 409 (contention: the tier stays due, +// is skipped for contentionRetryAfter, alarms after contentionAlarmAfter) — but with NO app stopped. An agent +// that does not serve the field keeps the old order (stop, ask, resume at once on 409). Pinned by +// TestR921_NoStopWhileTheHostIsBusy. +// +// The race (free at the check, busy at the start) needs nothing new: it is the BUSY path again — one short stop, +// resumed at once, as before. The check is a read, never a reservation. // InFlightProber is the OPTIONAL pre-check surface (R-921). An adapter that does not implement it keeps // the pre-R-921 behaviour exactly: stop, ask, and on refusal resume at once. @@ -28,31 +36,80 @@ type InFlightProber interface { BackupJobsInFlight(ctx context.Context) ([]string, error) } +// HostBusyProber is the OPTIONAL read of the agent's host-wide heavy-operation gate (R-921, agent with +// `host_busy`). known=false: the agent does not serve it — no information. +type HostBusyProber interface { + HostBusy(ctx context.Context) (busy bool, what string, known bool, err error) +} + +// precheckSkips is the whole check-first rule: true = the agent would refuse this window, so no app is stopped. +func (l *Loop) precheckSkips(ctx context.Context, tiers []dueTier) bool { + first := tiers[0].target + skip, ownInFlight := l.anotherTierInFlight(ctx, first) + if skip { + return true + } + if ownInFlight || first == "" { + // The window's own tier in flight: the agent answers its start with the running job (202) whatever its + // gate says (the gate may be held by that very job) — attach as before. An untargeted window is a + // pre-R-82 agent, which serves no gate read. + return false + } + return l.hostBusyRefuses(ctx, tiers) +} + +// hostBusyRefuses: the agent says its host-wide gate is held, so every tier of this window would be refused (409). +// Recorded as CONTENTION for each tier — the same verdict a 409 gets (retry after contentionRetryAfter, one alarm +// after contentionAlarmAfter), so a gate that never lets go still pages the operator — and no app is stopped. +// Fail toward backing up: an error, an older agent or an adapter without the surface → false. +func (l *Loop) hostBusyRefuses(ctx context.Context, tiers []dueTier) bool { + p, ok := l.backend.(HostBusyProber) + if !ok { + return false + } + busy, what, known, err := p.HostBusy(ctx) + if err != nil { + l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent whether its host is busy (%v) — stopping the apps and asking as before (R-921)", err) + return false + } + if !known || !busy { + return false + } + l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent's host is busy (%s) — it would refuse the backup, so no app is stopped (R-921)", + tierLabel(tiers[0].target), what) + for _, t := range tiers { + l.noteTierContention(t.target, tierLabel(t.target)) + } + return true +} + // anotherTierInFlight reports whether the agent would refuse the window's first tier because a job of a // DIFFERENT tier is in flight — so the window must not stop any app. The window's OWN tier in flight is // not a refusal (the agent answers its start with the running job, 202), and that path is left as it was. // // Fail toward backing up: an unanswerable pre-check, an untargeted window (pre-R-82 agent) or an adapter // without the surface all return false — the window runs as before. -func (l *Loop) anotherTierInFlight(ctx context.Context, first string) bool { +// ownInFlight: the window's own tier has a job in flight (the agent will answer its start with that job). +func (l *Loop) anotherTierInFlight(ctx context.Context, first string) (skip, ownInFlight bool) { if first == "" { - return false + return false, false } p, ok := l.backend.(InFlightProber) if !ok { - return false + return false, false } busy, err := p.BackupJobsInFlight(ctx) if err != nil { l.logger.Printf("[WARN] [quiesce] pre-check: could not ask the agent which backup jobs are in flight (%v) — stopping the apps and asking as before (R-921)", err) - return false + return false, false } for _, t := range busy { if t != first { l.logger.Printf("[INFO] [quiesce] tier %s is due, but the agent still holds a backup job on tier %s — no app is stopped; the tier stays due and is asked again at the next poll (R-921)", tierLabel(first), tierLabel(t)) - return true + return true, false } + ownInFlight = true } - return false + return false, ownInFlight } diff --git a/controller/internal/quiesce/quiesce.go b/controller/internal/quiesce/quiesce.go index b59654e..a2cb05e 100644 --- a/controller/internal/quiesce/quiesce.go +++ b/controller/internal/quiesce/quiesce.go @@ -556,9 +556,10 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error { if len(tiers) == 0 { return nil } - // R-921: check first, stop second — a tier the agent will refuse (another tier's job in flight) stops - // no app. Pinned by TestR921_NoStopWhileAnotherTiersJobIsInFlight (precheck.go says what it cannot see). - if l.anotherTierInFlight(ctx, tiers[0].target) { + // R-921: check first, stop second — a tier the agent will refuse (another tier's job in flight, or its + // host-wide gate held) stops no app. Pinned by TestR921_NoStopWhileAnotherTiersJobIsInFlight and + // TestR921_NoStopWhileTheHostIsBusy (precheck.go). + if l.precheckSkips(ctx, tiers) { return nil } running := l.stacks.RunningAppStacks() diff --git a/controller/internal/quiesce/r921_precheck_test.go b/controller/internal/quiesce/r921_precheck_test.go index a3f7aab..4e176d8 100644 --- a/controller/internal/quiesce/r921_precheck_test.go +++ b/controller/internal/quiesce/r921_precheck_test.go @@ -34,6 +34,10 @@ type r921Backend struct { probeErr error busyOn map[string]bool // StartBackupFor answers ErrTierBusy for these targets starts []string + // R-921 part 2: the agent's host_busy answer (hostKnown=false = an older agent without the field). + hostKnown, hostBusy bool + hostWhat string + hostErr error } func (b *r921Backend) log(e string) { @@ -68,6 +72,13 @@ func (b *r921Backend) BackupJobsInFlight(context.Context) ([]string, error) { } return append([]string(nil), b.inFlight...), nil } +func (b *r921Backend) HostBusy(context.Context) (bool, string, bool, error) { + b.log("hostbusy") + if b.hostErr != nil { + return false, "", false, b.hostErr + } + return b.hostBusy, b.hostWhat, b.hostKnown, nil +} func (b *r921Backend) Due(context.Context) (bool, *int64, error) { return false, nil, nil } func (b *r921Backend) StartBackup(context.Context) (string, error) { return "", errors.New("unused") } func (b *r921Backend) BackupStatus(context.Context) (string, error) { @@ -284,3 +295,114 @@ func (n *noProbeBackend) StartBackup(ctx context.Context) (string, error) { func (n *noProbeBackend) BackupStatus(ctx context.Context) (string, error) { return n.b.BackupStatus(ctx) } + +// R-921 part 2 — THE MEASURED CASE, now seen before the stop. The agent's host-wide gate is held (the night OS step +// after the local copy: `os-pve-step`), the off-site tier is due. With the agent's host_busy answer the loop stops NO +// app and asks for NO backup, and records contention for the tier (skipped for contentionRetryAfter, alarmed after +// contentionAlarmAfter) — the verdict a 409 gets today, minus the stop. +// +// COMPANION RED-PROOF (observed 2026-10-19): with precheckSkips back to the pre-fix shape (the host_busy branch +// removed), it fails with +// +// r921_precheck_test.go: R-921: 3 app(s) stopped while the agent says its host is busy (os-pve-step); events=[probe stop:immich … start:felhom-pbs=BUSY startstack:immich …] +// +// Restored. +func TestR921_NoStopWhileTheHostIsBusy(t *testing.T) { + be, _, l, events, logs := r921Setup(t) + be.due["felhom-pbs"] = true + be.busyOn["felhom-pbs"] = true // what the real agent answers while its gate is held + be.hostKnown, be.hostBusy, be.hostWhat = true, true, "os-pve-step" + + if err := l.runOnce(context.Background()); err != nil { + t.Fatalf("runOnce: %v", err) + } + if n := countPrefix(*events, "stop:"); n != 0 { + t.Fatalf("R-921: %d app(s) stopped while the agent says its host is busy (os-pve-step); events=%v", n, *events) + } + if len(be.starts) != 0 { + t.Fatalf("a backup was requested while the host is busy: %v", be.starts) + } + if _, blocked := l.contention.blocked("felhom-pbs", l.now()); !blocked { + t.Errorf("the busy tier was not recorded as contention — the next poll would ask again at once, and a stuck gate would never alarm") + } + if got := l.breaker.failuresFor("felhom-pbs"); got != 0 { + t.Errorf("a busy host armed the failure breaker (%d) — nothing failed", got) + } + if !strings.Contains(logs.String(), "os-pve-step") || !strings.Contains(logs.String(), "R-921") { + t.Errorf("the skip does not name the holder and R-921: %q", logs.String()) + } + + // The gate lets go and the skip window passes → the next poll backs the tier up as normal. + be.hostBusy, be.hostWhat, be.busyOn["felhom-pbs"] = false, "", false + l.contention.clear("felhom-pbs") + if err := l.runOnce(context.Background()); err != nil { + t.Fatalf("runOnce 2: %v", err) + } + if len(be.starts) != 1 || be.starts[0] != "felhom-pbs" { + t.Fatalf("with the host free the tier was not backed up: starts=%v", be.starts) + } +} + +// A gate that never lets go must still page the operator — the skip path must not be a silence path. +func TestR921_HostBusySkipsStillAlarmAfterTheCeiling(t *testing.T) { + be, _, l, events, _ := r921Setup(t) + n := &recordingNotifier{} + l.SetTierNotifier(n) + base := time.Now() + now := base + l.now = func() time.Time { return now } + be.due["felhom-pbs"] = true + be.hostKnown, be.hostBusy, be.hostWhat = true, true, "restore-test" + for _, step := range []time.Duration{0, time.Hour, 2 * time.Hour, contentionAlarmAfter + time.Minute} { + now = base.Add(step) + if st, ok := l.contention.states["felhom-pbs"]; ok { + st.until = time.Time{} // let each poll through the skip window; the run (since, alarmed) is kept + l.contention.states["felhom-pbs"] = st + } + if err := l.runOnce(context.Background()); err != nil { + t.Fatalf("runOnce at +%s: %v", step, err) + } + } + if countPrefix(*events, "stop:") != 0 { + t.Fatalf("an app was stopped while the host was busy: %v", *events) + } + if len(n.failed) != 1 || !contains(n.failed[0], "BLOCKED") { + t.Fatalf("after %s of a busy host: alarms %v, want exactly one BLOCKED alarm", contentionAlarmAfter, n.failed) + } +} + +// The other answers keep today's order: a FREE host backs up; an OLDER agent (no host_busy), or a host_busy read +// that fails, stops the apps and asks as before (fail toward backing up); the window's OWN tier in flight attaches +// to its job whatever the gate says (the gate may be held by that very job). +func TestR921_HostBusyOtherAnswersKeepTheOldOrder(t *testing.T) { + for _, tc := range []struct { + name string + known, busy bool + err error + ownInFlight bool + wantStarts, wantStp bool + }{ + {"host free", true, false, nil, false, true, true}, + {"older agent (no host_busy)", false, false, nil, false, true, true}, + {"host_busy read failed", false, false, errors.New("agentapi: GET /backup/status: connection refused"), false, true, true}, + {"own tier in flight, gate held by it", true, true, nil, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + be, _, l, events, _ := r921Setup(t) + be.due["felhom-pbs"] = true + be.hostKnown, be.hostBusy, be.hostErr = tc.known, tc.busy, tc.err + if tc.ownInFlight { + be.inFlight = []string{"felhom-pbs"} + } + if err := l.runOnce(context.Background()); err != nil { + t.Fatalf("runOnce: %v", err) + } + if got := len(be.starts) == 1; got != tc.wantStarts { + t.Fatalf("starts=%v, want a start: %v; events=%v", be.starts, tc.wantStarts, *events) + } + if tc.wantStp && countPrefix(*events, "stop:") == 0 { + t.Fatalf("no app stopped — the old order (stop, then ask) was not kept; events=%v", *events) + } + }) + } +}