From 6d07ca2be46ff55b710457aacada1174db97c704 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Thu, 8 Oct 2026 07:34:03 +0200 Subject: [PATCH] R-899: a daytime press never cancels the night's whole-guest backup (operator ruling 2026-10-08, option A); press sends trigger=manual Unreleased; ships with tomorrow's release. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS --- CHANGELOG.md | 18 ++ REUSE.md | 1 + controller/cmd/controller/main.go | 7 +- controller/internal/agentapi/backup_tiers.go | 24 ++- controller/internal/agentapi/client.go | 10 +- .../internal/agentapi/r899_trigger_test.go | 47 +++++ controller/internal/quiesce/nightowed.go | 191 +++++++++++++++++ controller/internal/quiesce/nightowed_test.go | 192 ++++++++++++++++++ controller/internal/quiesce/quiesce.go | 10 +- controller/internal/quiesce/tiers.go | 14 ++ controller/internal/quiesce/tiers_test.go | 5 +- 11 files changed, 504 insertions(+), 15 deletions(-) create mode 100644 controller/internal/agentapi/r899_trigger_test.go create mode 100644 controller/internal/quiesce/nightowed.go create mode 100644 controller/internal/quiesce/nightowed_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb126e..0e6c438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## Unreleased (2026-10-08) — a daytime press never cancels the night's whole-guest backup (R-899; operator ruling 2026-10-08, option A) — ships with tomorrow's release + +**MinAgent: 0.131.0** (unchanged — the new `trigger=manual` query is ignored by an older agent, which keeps running the +OS leg after a press exactly as before; the night itself is fixed by the controller alone). + +- **R-899:** the agent answers `/backup/due` from the newest archive on the tier, and a „Mentés most" press is an archive + like any other. So a press at 08:49 made the 24 h tier due at 08:49 the next day — after the gate window + [W+2h, W+6h) closed — and that night had no whole-guest backup, no OS leg and no kernel step (demo-hp, 2026-10-07 → 08). + Now the quiesce loop keeps a durable ledger beside its marker (`whole-guest-ledger.json`: the last successful press and + the last successful scheduled run per tier). A tier the agent calls „not due" is still due on the scheduled path when + a press succeeded after its last scheduled success and that success is older than tonight's gate opening. Such an + „owed" tier never fires the window gate's safety valve (it waits for the window). A scheduled success inside tonight's + window ends it; a failed one does not. `internal/quiesce/nightowed.go`; tests `TestR899_*` (6; red-proved: with the + rule off, the 2026-10-07 replay started nothing on night 2 — `started=[local local]`, and two more failed). +- The press now reaches the agent as `POST /backup?…trigger=manual` (`agentapi.StartBackupForTrigger`); an agent that + knows it runs no OS leg after a press (agent, same day). Test `TestR899_PressCarriesTriggerManual` asserts the query the + agent receives for each of the four shapes. + ## v0.303.0 — after a restart, a backup capture waits for the drive (R-897; `09` §3 decision 176) (2026-10-07) **MinAgent: 0.131.0** (unchanged — no agent route is read; the signal is the controller's own mount table). diff --git a/REUSE.md b/REUSE.md index c4ce646..5245cf3 100644 --- a/REUSE.md +++ b/REUSE.md @@ -125,6 +125,7 @@ | `sambaWriteAtomic` | controller/internal/stacks/samba.go | `(path, data, mode) error` | samba smb.conf/compose writes | tmp+**fsync**+rename (the only one of these that fsyncs). Fourth atomic-write helper in the tree — see §6 | | `Loop.writeMarker` / `Recover` | controller/internal/quiesce/quiesce.go | `(m Marker)` / `()` | Quiesce crash-safety | Marker written BEFORE stopping stacks; Recover restarts stranded stacks at boot | | `quiesce.TieredBackend` + `Loop.resolveDueTiers` / `quiesceAndPollTiers` | controller/internal/quiesce/tiers.go, quiesce.go | `Tiers/DueFor/StartBackupFor/BackupStatusFor`; `resolveDueTiers(ctx) ([]dueTier,bool,error)` | THE R-82 multi-tier backup schedule — several whole-guest tiers (local daily + PBS weekly) reconciled into ONE quiesce window | **Both tiers due ⇒ ONE stop/start pair**, never two (two = two app outages for one night). Tiers run SEQUENTIALLY (vzdump holds a guest lock) and the app stays down until the LAST tier snapshots — resuming earlier loses app-consistency on the DR tier. Order is fast-first (agent advertises primary first) or downtime blows up. `ErrTiersUnsupported` (route 404) ⇒ pre-R-82 agent ⇒ degrade to the untargeted path and **STILL BACK UP** — never read it as "nothing due". | +| `quiesce` whole-guest ledger — `Loop.pressOwesNight` / `recordWholeGuestSuccess`, `WithManualTrigger` / `IsManualTrigger` (R-899) | controller/internal/quiesce/nightowed.go | `pressOwesNight(led, target) bool`; `WithManualTrigger(ctx) ctx` | **A household press never cancels the night's whole-guest backup** (operator ruling 2026-10-08) | The ledger (`whole-guest-ledger.json` beside the marker) records SUCCESSES only, per tier, press vs scheduled — it decides due-ness and is never evidence that a copy exists (the agent's storage answers that). An „owed" tier never fires the window gate's safety valve (`withoutOwed`/`gateAge`). The press mark rides the context to the agent adapter (`agentapi.StartBackupForTrigger` → `trigger=manual`), so the agent runs no OS leg after a press. | | `quiesce.failureBreaker` + `Loop.dropBackedOffTiers` / `noteTierFailure` / `noteTierSuccess` | controller/internal/quiesce/breaker.go, quiesce.go | `blocked/recordFailure/recordSuccess(target, now)`; `backoffFor(n) time.Duration` | **R-88** — a tier whose backups keep failing stops re-quiescing. Backoff 15m→30m→1h→2h→4h (cap), reset on success | **It gates the QUIESCE, not the backup** — the harm was never the failing backup, it was the app outage taken to attempt it, so backed-off tiers are dropped from the due set BEFORE any stack is stopped. **Per TARGET** — a broken offsite tier must never suppress a healthy local one (`TestBreaker_OneFailingTierDoesNotSuppressAHealthyOne`). **Never permanent** — the cap bounds the retry INTERVAL, it never stops retrying; a latched breaker is a silent backup outage, worse than the loop it replaces. **`TriggerNow` is never gated** (it already bypasses due-ness and the window gate), though a manual run still RECORDS its outcome. **`stillRunning` is NOT a failure** — a first full offsite snapshot legitimately runs for hours. State is **in-memory on purpose**: a restart forgets the backoff and re-attempts, which is the cheap direction to fail. Log the deferral ONCE when armed, never per tick. | | `quiesce.TierNotifier` + `Loop.SetTierNotifier` / `noteTierFailure` / `noteTierSuccess` | controller/internal/quiesce/breaker.go, quiesce.go | `BackupFailed(tier,msg,err)` / `BackupRecovered(tier,msg)`; `SetTierNotifier(n)` INIT-ONLY | **R-97a** — the whole-guest backup tier reports its outcome to the hub | A **seam, not an import** — quiesce keeps no dependency on `internal/notify` (same reason `windowStartFn` is injected). Wired by a setter because main.go builds the notifier AFTER the loop; `nil` = unprovisioned guest, not an error. **Edge-triggered:** failure fires only when the breaker ARMS (`n == 1`), never per retry — the cadence is 15m/30m/1h/2h/4h and an event per attempt is an inbox nobody reads. Recovery rides `recordSuccess`'s existing bool. **Event types are OPERATOR-ONLY** (`whole_guest_backup_failed`/`_recovered`, hub >= v0.78.0) — NOT `backup_failed`, which has a customerMessages entry AND sits in live `enabled_events`, so it would email the CUSTOMER about a backup they cannot act on. `WholeGuestBackupDetails.Tier` is load-bearing: the hub keys its per-tier cooldown on it. | | `quiesce.Loop.SuppressedStacks` + `markQuiesced` / `markUnquiesced` | controller/internal/quiesce/suppress.go | `() map[string]bool` (nil-safe on a nil *Loop) | **R-97b** — an app THIS controller stopped for a backup is not a fault | Consumed at the SINGLE derivation point `classifyRunStates` (which computes both the banner dead-list and the notifier Down-set — keep it one place). **Cycle-keyed, not state-based:** v0.164.0's `!= StateStopped` filter cannot see an app caught MID-RESTART (`starting`/`unhealthy`), which is how BookStack alarmed on 2026-07-27. The window (`quiesceAlarmGrace` = 180 s, derived from the deploy flow's 120 s health timeout and Mealie's 60 s start_period) **EXPIRES** — permanent suppression turns a loud false alarm into a silent real one. Open-ended while the cycle runs (a first offsite snapshot legitimately takes hours). **This set alone is NOT the whole answer** — see `AppStopGuard.SuppressedStacks` (R-330) for the per-app operations; `classifyRunStates` consumes the union of both. | diff --git a/controller/cmd/controller/main.go b/controller/cmd/controller/main.go index 35236dd..f889487 100644 --- a/controller/cmd/controller/main.go +++ b/controller/cmd/controller/main.go @@ -3471,8 +3471,9 @@ func (b quiesceBackend) Due(ctx context.Context) (bool, *int64, error) { return r.Due, r.AgeSecs, err } func (b quiesceBackend) StartBackup(ctx context.Context) (string, error) { - r, err := b.c.StartBackup(ctx) - return r.JobID, mapBusy(err) // F-A1: the untargeted (pre-R-82) path can 409 identically + // R-899: a press says so. F-A1: the untargeted (pre-R-82) path can 409 identically. + r, err := b.c.StartBackupForTrigger(ctx, "", quiesce.IsManualTrigger(ctx)) + return r.JobID, mapBusy(err) } func (b quiesceBackend) BackupStatus(ctx context.Context) (string, error) { r, err := b.c.BackupStatus(ctx) @@ -3508,7 +3509,7 @@ func (b quiesceBackend) DueFor(ctx context.Context, target string) (bool, *int64 return r.Due, r.AgeSecs, r.AgeState, err } func (b quiesceBackend) StartBackupFor(ctx context.Context, target string) (string, error) { - r, err := b.c.StartBackupFor(ctx, target) + r, err := b.c.StartBackupForTrigger(ctx, target, quiesce.IsManualTrigger(ctx)) // R-899: a press says so // F-A1: translate the agent's HTTP 409 into the loop's vocabulary, exactly as Tiers translates a // 404 into ErrTiersUnsupported. 409 means the agent's R-85 single-flight gate refused because a // restore-test (or another backup) holds it — CONTENTION, not failure. `internal/quiesce` keeps diff --git a/controller/internal/agentapi/backup_tiers.go b/controller/internal/agentapi/backup_tiers.go index 50ac3d7..31bd080 100644 --- a/controller/internal/agentapi/backup_tiers.go +++ b/controller/internal/agentapi/backup_tiers.go @@ -64,6 +64,21 @@ func targetQuery(target string) string { return "?target=" + url.QueryEscape(target) } +// backupStartQuery is targetQuery plus `trigger=manual` for a household press (R-899). +func backupStartQuery(target string, manual bool) string { + q := url.Values{} + if target != "" { + q.Set("target", target) + } + if manual { + q.Set("trigger", "manual") + } + if len(q) == 0 { + return "" + } + return "?" + q.Encode() +} + // BackupDueFor reports whether THIS TIER is due. A fresh backup on another tier must not satisfy it // — that filtering happens agent-side (latestSuccessfulBackupForTarget); this just asks per tier. func (c *Client) BackupDueFor(ctx context.Context, target string) (DueResponse, error) { @@ -80,8 +95,15 @@ func (c *Client) BackupDueFor(ctx context.Context, target string) (DueResponse, // StartBackupFor enqueues a backup of this guest ON THE GIVEN TIER. func (c *Client) StartBackupFor(ctx context.Context, target string) (BackupResponse, error) { + return c.StartBackupForTrigger(ctx, target, false) +} + +// StartBackupForTrigger is StartBackupFor that also says whether this is a household press (R-899): a press +// carries `trigger=manual`, and an agent that knows it does not start the night's OS leg after it. An older +// agent ignores the parameter (it reads only `target`), so the request is safe against any agent. +func (c *Client) StartBackupForTrigger(ctx context.Context, target string, manual bool) (BackupResponse, error) { var out BackupResponse - body, err := c.post(ctx, "/backup"+targetQuery(target), struct{}{}) + body, err := c.post(ctx, "/backup"+backupStartQuery(target, manual), struct{}{}) if err != nil { return out, err } diff --git a/controller/internal/agentapi/client.go b/controller/internal/agentapi/client.go index d967653..5983e50 100644 --- a/controller/internal/agentapi/client.go +++ b/controller/internal/agentapi/client.go @@ -272,15 +272,7 @@ func (c *Client) BackupDue(ctx context.Context) (DueResponse, error) { // StartBackup enqueues a backup of this guest (the agent vzdump) and returns the job to poll. func (c *Client) StartBackup(ctx context.Context) (BackupResponse, error) { - var out BackupResponse - body, err := c.post(ctx, "/backup", struct{}{}) - if err != nil { - return out, err - } - if err := json.Unmarshal(body, &out); err != nil { - return out, fmt.Errorf("agentapi: decode POST /backup: %w", err) - } - return out, nil + return c.StartBackupForTrigger(ctx, "", false) } // BackupStatus reports the current/last backup job phase for this guest. diff --git a/controller/internal/agentapi/r899_trigger_test.go b/controller/internal/agentapi/r899_trigger_test.go new file mode 100644 index 0000000..28d7606 --- /dev/null +++ b/controller/internal/agentapi/r899_trigger_test.go @@ -0,0 +1,47 @@ +package agentapi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// R-899: a household press tells the agent so (`trigger=manual`), and nothing else does — the agent runs the +// night's OS leg only after a backup that is not a press. The request the agent RECEIVES is asserted. +func TestR899_PressCarriesTriggerManual(t *testing.T) { + var got []string + mux := http.NewServeMux() + mux.HandleFunc("POST /backup", func(w http.ResponseWriter, r *http.Request) { + got = append(got, r.URL.RawQuery) + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(`{"ok":true,"data":{"vmid":9201,"job_id":"backup-9201-2","phase":"running"}}`)) + }) + s := httptest.NewTLSServer(mux) + defer s.Close() + c := clientFor(t, s, strings.TrimPrefix(s.URL, "https://")) + ctx := context.Background() + + if _, err := c.StartBackup(ctx); err != nil { + t.Fatal(err) + } + if _, err := c.StartBackupFor(ctx, "local"); err != nil { + t.Fatal(err) + } + if _, err := c.StartBackupForTrigger(ctx, "local", true); err != nil { + t.Fatal(err) + } + if _, err := c.StartBackupForTrigger(ctx, "", true); err != nil { + t.Fatal(err) + } + want := []string{"", "target=local", "target=local&trigger=manual", "trigger=manual"} + if len(got) != len(want) { + t.Fatalf("queries = %q, want %q", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("query %d = %q, want %q (all: %q)", i, got[i], want[i], got) + } + } +} diff --git a/controller/internal/quiesce/nightowed.go b/controller/internal/quiesce/nightowed.go new file mode 100644 index 0000000..d95e695 --- /dev/null +++ b/controller/internal/quiesce/nightowed.go @@ -0,0 +1,191 @@ +package quiesce + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow" +) + +// R-899 (operator ruling 2026-10-08, option A): a daytime whole-guest backup never moves the night's backup. +// Every night takes its own. +// +// The agent answers /backup/due from the newest archive on the tier's storage, and a household's „Mentés most" +// press makes an archive like any other. So a press at 08:49 made the 24 h tier due again at 08:49 the next day — +// after the gate window [W+2h, W+6h) had closed — and that night had no whole-guest backup, no OS leg and no +// kernel step (measured on demo-hp, 2026-10-07 → 08, `audits/kernel-night-2026-10-07/readback/`). +// +// The rule, kept here and nowhere else: a press keeps its own record (it is a real copy, and the agent counts it +// for everything else), but it does not count for „has tonight's backup run". A tier is OWED tonight when +// +// a press succeeded on it after its last SCHEDULED success, and +// that last scheduled success is older than the opening of the current gate window (W+2h). +// +// An owed tier is due on the scheduled path even when the agent says it is not. The window gate still decides +// WHEN (its age is the press's, so the safety valve never fires for it): outside the window it waits, inside it +// runs. A scheduled success inside tonight's window ends the debt, so a press inside the window after tonight's +// backup forces nothing. A failed scheduled run does not end it (the breaker still spaces the retries). +// +// Without a window function (the pre-v0.168.0 shape) nothing is owed: there is no night to protect. +// +// The ledger is durable (beside the quiesce marker) so a controller restart between the press and the night +// does not forget the press. It is an attempt-free record of SUCCESSES only, and it is read only to decide +// due-ness — never as evidence that a backup exists (the agent's storage answers that). +// +// Pinned by TestR899_* (nightowed_test.go). + +// wholeGuestLedger is the per-tier record of the last successful press and the last successful scheduled run. +type wholeGuestLedger struct { + Tiers map[string]ledgerTier `json:"tiers"` +} + +type ledgerTier struct { + PressOK time.Time `json:"press_ok,omitempty"` + ScheduledOK time.Time `json:"scheduled_ok,omitempty"` +} + +// manualCtxKey marks the context of a household press, so the agent adapter can tell the agent (R-899: the agent +// runs the night's OS leg only after a scheduled backup). +type manualCtxKey struct{} + +// WithManualTrigger marks ctx as a household press. +func WithManualTrigger(ctx context.Context) context.Context { + return context.WithValue(ctx, manualCtxKey{}, true) +} + +// IsManualTrigger reports whether ctx belongs to a household press. +func IsManualTrigger(ctx context.Context) bool { + v, _ := ctx.Value(manualCtxKey{}).(bool) + return v +} + +func (l *Loop) ledgerPath() string { + if l.markerPath == "" { + return "" + } + return filepath.Join(filepath.Dir(l.markerPath), "whole-guest-ledger.json") +} + +// loadLedger reads the ledger (in memory when there is no marker path). A missing or unreadable file is an +// empty ledger: nothing owed, the agent's own answer stands — the pre-R-899 behaviour, never a skipped backup. +func (l *Loop) loadLedger() wholeGuestLedger { + l.ledgerMu.Lock() + defer l.ledgerMu.Unlock() + return l.loadLedgerLocked() +} + +func (l *Loop) loadLedgerLocked() wholeGuestLedger { + led := wholeGuestLedger{Tiers: map[string]ledgerTier{}} + p := l.ledgerPath() + if p == "" { + for k, v := range l.memLedger { + led.Tiers[k] = v + } + return led + } + data, err := os.ReadFile(p) + if err != nil { + if !os.IsNotExist(err) { + l.logger.Printf("[WARN] [quiesce] whole-guest ledger unreadable (%v) — no night is owed by a press this poll (R-899)", err) + } + return led + } + if err := json.Unmarshal(data, &led); err != nil { + l.logger.Printf("[WARN] [quiesce] whole-guest ledger corrupt (%v) — starting a new one (R-899)", err) + return wholeGuestLedger{Tiers: map[string]ledgerTier{}} + } + if led.Tiers == nil { + led.Tiers = map[string]ledgerTier{} + } + return led +} + +// recordWholeGuestSuccess notes a successful backup on a tier, as a press or as a scheduled run. +func (l *Loop) recordWholeGuestSuccess(target string, manual bool) { + l.ledgerMu.Lock() + defer l.ledgerMu.Unlock() + led := l.loadLedgerLocked() + t := led.Tiers[target] + if manual { + t.PressOK = l.now() + } else { + t.ScheduledOK = l.now() + } + led.Tiers[target] = t + p := l.ledgerPath() + if p == "" { + if l.memLedger == nil { + l.memLedger = map[string]ledgerTier{} + } + l.memLedger[target] = t + return + } + data, err := json.MarshalIndent(led, "", " ") + if err == nil { + tmp := p + ".tmp" + if err = os.WriteFile(tmp, data, 0o600); err == nil { + err = os.Rename(tmp, p) + } + } + if err != nil { + l.logger.Printf("[WARN] [quiesce] could not save the whole-guest ledger (%v) — a press may still move the next night (R-899)", err) + } +} + +// pressOwesNight reports whether a press made this tier's agent answer „not due" while tonight's scheduled +// backup has not run (the rule at the top of this file). +func (l *Loop) pressOwesNight(led wholeGuestLedger, target string) bool { + if l.windowStartFn == nil { + return false + } + t, ok := led.Tiers[target] + if !ok || t.PressOK.IsZero() || !t.PressOK.After(t.ScheduledOK) { + return false + } + open, ok := lastGateOpen(l.now(), l.windowStartFn()) + if !ok { + return false + } + return t.ScheduledOK.Before(open) +} + +// lastGateOpen returns the most recent opening of the gate window (W+2h, Budapest wall clock) at or before now. +func lastGateOpen(now time.Time, windowStart string) (time.Time, bool) { + startMin, err := backupwindow.ParseHHMM(windowStart) + if err != nil { + return time.Time{}, false + } + openMin := mod1440(startMin + gateOpenOffsetMin) + loc := budapestLocation() + n := now.In(loc) + open := time.Date(n.Year(), n.Month(), n.Day(), openMin/60, openMin%60, 0, 0, loc) + if open.After(n) { + open = open.AddDate(0, 0, -1) + } + return open, true +} + +// withoutOwed drops the R-899 owed tiers: they never license the window gate's safety valve. +func withoutOwed(tiers []dueTier) []dueTier { + out := make([]dueTier, 0, len(tiers)) + for _, t := range tiers { + if !t.owed { + out = append(out, t) + } + } + return out +} + +// gateAge is the age the window gate judges: the oldest of the tiers that are due by the agent. When every due tier +// is owed by a press, it is zero — „just backed up" — so outside the window the gate waits and inside it runs. +func gateAge(tiers []dueTier) *int64 { + agentDue := withoutOwed(tiers) + if len(agentDue) == 0 { + zero := int64(0) + return &zero + } + return oldestAge(agentDue) +} diff --git a/controller/internal/quiesce/nightowed_test.go b/controller/internal/quiesce/nightowed_test.go new file mode 100644 index 0000000..b44e447 --- /dev/null +++ b/controller/internal/quiesce/nightowed_test.go @@ -0,0 +1,192 @@ +package quiesce + +import ( + "bytes" + "log" + "path/filepath" + "strings" + "testing" + "time" +) + +// R-899 (operator ruling 2026-10-08, option A): a daytime whole-guest backup never moves the night's backup. +// These assert the CONSEQUENCE — is tonight's local backup started? — not the ledger's mechanism. + +func budapest(t *testing.T, y int, m time.Month, d, hh, mm int) time.Time { + t.Helper() + return time.Date(y, m, d, hh, mm, 0, 0, budapestLocation()) +} + +// r899Loop is a tiered loop with the window gate on (W = 02:30 → gate [04:30, 08:30)) and a settable clock. +func r899Loop(t *testing.T, be *tierBackend, st *fakeStacks, markerPath string, now *time.Time, logs *bytes.Buffer) *Loop { + t.Helper() + l := New(Options{ + Backend: be, Stacks: st, MarkerPath: markerPath, + StatusPoll: time.Millisecond, MaxQuiesce: 30 * time.Second, + Logger: log.New(logs, "", 0), + }) + l.windowStartFn = func() string { return "02:30" } + l.now = func() time.Time { return *now } + return l +} + +func press(t *testing.T, l *Loop) { + t.Helper() + if err := l.TriggerNow(); err != nil { + t.Fatalf("TriggerNow: %v", err) + } + l.mu.Lock() + l.mu.Unlock() //nolint:staticcheck // wait for the async cycle +} + +// The 2026-10-07 case, replayed: last night's backup at 04:35, a press at 08:49, and the agent then answers +// „not due" at 04:35 the next night (its newest archive is the press, 19.7 h old). Tonight must still back up. +// Before R-899 the cycle started nothing and the night had no OS leg and no kernel step. +func TestR899_DaytimePressDoesNotCancelTheNight(t *testing.T) { + st := &fakeStacks{running: []string{"opengist"}} + be := newTierBackend() + be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}} + var logs bytes.Buffer + now := budapest(t, 2026, 10, 7, 4, 35) + l := r899Loop(t, be, st, filepath.Join(t.TempDir(), "quiesce-state.json"), &now, &logs) + + be.setDue("local", true) + if err := l.runOnce(t.Context()); err != nil { + t.Fatalf("night 1: %v", err) + } + be.setDue("local", false) + + now = budapest(t, 2026, 10, 7, 8, 49) + press(t, l) + + now = budapest(t, 2026, 10, 8, 4, 35) // agent: local NOT due (the press is 19.7 h old) + if err := l.runOnce(t.Context()); err != nil { + t.Fatalf("night 2: %v", err) + } + got := be.startedTargets() + if len(got) != 3 || got[2] != "local" { + t.Fatalf("night 2 must take its own local backup despite the morning press; started=%v\nlogs:\n%s", got, logs.String()) + } + if !strings.Contains(logs.String(), "R-899") { + t.Fatalf("the forced night must say why in the log; logs:\n%s", logs.String()) + } + // The press was marked as a press; the two scheduled runs were not. + if m := be.manualStarts; len(m) != 3 || m[0] || !m[1] || m[2] { + t.Fatalf("press mark per start = %v, want [false true false]", m) + } + + // And once tonight's backup ran, the debt is paid: a later poll in the same window starts nothing. + now = budapest(t, 2026, 10, 8, 4, 45) + if err := l.runOnce(t.Context()); err != nil { + t.Fatalf("night 2, later poll: %v", err) + } + if got := be.startedTargets(); len(got) != 3 { + t.Fatalf("tonight's backup already ran — no second one; started=%v", got) + } +} + +// Outside the window the owed night waits for the window (the press's own age never fires the valve). +func TestR899_OwedNightWaitsForTheWindow(t *testing.T) { + st := &fakeStacks{running: []string{"opengist"}} + be := newTierBackend() + be.tiers = []BackupTier{{Target: "local", Primary: true}} + var logs bytes.Buffer + now := budapest(t, 2026, 10, 7, 8, 49) + l := r899Loop(t, be, st, filepath.Join(t.TempDir(), "quiesce-state.json"), &now, &logs) + press(t, l) + + now = budapest(t, 2026, 10, 7, 15, 0) + if err := l.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + if got := be.startedTargets(); len(got) != 1 { + t.Fatalf("outside the window nothing may start; started=%v", got) + } + now = budapest(t, 2026, 10, 8, 4, 31) + if err := l.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + if got := be.startedTargets(); len(got) != 2 || got[1] != "local" { + t.Fatalf("inside the window the owed night runs; started=%v", got) + } +} + +// A press INSIDE the window after tonight's backup forces nothing more tonight. +func TestR899_PressAfterTonightsBackupForcesNothing(t *testing.T) { + st := &fakeStacks{running: []string{"opengist"}} + be := newTierBackend() + be.tiers = []BackupTier{{Target: "local", Primary: true}} + var logs bytes.Buffer + now := budapest(t, 2026, 10, 8, 4, 35) + l := r899Loop(t, be, st, filepath.Join(t.TempDir(), "quiesce-state.json"), &now, &logs) + be.setDue("local", true) + if err := l.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + be.setDue("local", false) + now = budapest(t, 2026, 10, 8, 5, 0) + press(t, l) + now = budapest(t, 2026, 10, 8, 5, 10) + if err := l.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + if got := be.startedTargets(); len(got) != 2 { + t.Fatalf("tonight's scheduled backup already ran before the press — no third backup; started=%v", got) + } +} + +// Without a press, an agent „not due" stands (no extra backups for a box nobody pressed). +func TestR899_NoPressNoExtraBackup(t *testing.T) { + st := &fakeStacks{running: []string{"opengist"}} + be := newTierBackend() + be.tiers = []BackupTier{{Target: "local", Primary: true}} + var logs bytes.Buffer + now := budapest(t, 2026, 10, 8, 4, 35) + l := r899Loop(t, be, st, filepath.Join(t.TempDir(), "quiesce-state.json"), &now, &logs) + if err := l.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + if got := be.startedTargets(); len(got) != 0 { + t.Fatalf("no press, agent not due → nothing; started=%v", got) + } +} + +// The press is remembered across a controller restart (the ledger is on disk beside the marker). +func TestR899_PressSurvivesARestart(t *testing.T) { + st := &fakeStacks{running: []string{"opengist"}} + be := newTierBackend() + be.tiers = []BackupTier{{Target: "local", Primary: true}} + var logs bytes.Buffer + marker := filepath.Join(t.TempDir(), "quiesce-state.json") + now := budapest(t, 2026, 10, 7, 8, 49) + press(t, r899Loop(t, be, st, marker, &now, &logs)) + + now = budapest(t, 2026, 10, 8, 4, 35) + l2 := r899Loop(t, be, st, marker, &now, &logs) // a new process + if err := l2.runOnce(t.Context()); err != nil { + t.Fatal(err) + } + if got := be.startedTargets(); len(got) != 2 || got[1] != "local" { + t.Fatalf("after a restart the press still does not count for tonight; started=%v", got) + } +} + +// The gate-open instant, across midnight and a DST change. +func TestR899_LastGateOpen(t *testing.T) { + cases := []struct { + now time.Time + win string + want time.Time + }{ + {budapest(t, 2026, 10, 8, 4, 35), "02:30", budapest(t, 2026, 10, 8, 4, 30)}, + {budapest(t, 2026, 10, 8, 4, 29), "02:30", budapest(t, 2026, 10, 7, 4, 30)}, + {budapest(t, 2026, 10, 8, 1, 0), "23:30", budapest(t, 2026, 10, 8, 1, 30).AddDate(0, 0, -1)}, + {budapest(t, 2026, 10, 25, 5, 0), "02:30", budapest(t, 2026, 10, 25, 4, 30)}, // DST ends that night + } + for _, c := range cases { + got, ok := lastGateOpen(c.now, c.win) + if !ok || !got.Equal(c.want) { + t.Errorf("lastGateOpen(%s, %s) = %s, want %s", c.now, c.win, got, c.want) + } + } +} diff --git a/controller/internal/quiesce/quiesce.go b/controller/internal/quiesce/quiesce.go index 5944bef..0bb826a 100644 --- a/controller/internal/quiesce/quiesce.go +++ b/controller/internal/quiesce/quiesce.go @@ -131,6 +131,10 @@ type Loop struct { // restartFailed (F-CRIT-1) is the set of stacks this loop stopped and could NOT restart. Guarded // by suppressMu — same concern, same lock. See suppress.go. restartFailed map[string]struct{} + // R-899: the whole-guest ledger (nightowed.go) — the last successful press and scheduled run per tier. + // ledgerMu guards the file and memLedger (the in-memory copy used when there is no marker path). + ledgerMu sync.Mutex + memLedger map[string]ledgerTier } // SetTierNotifier wires the hub-event seam. INIT-ONLY — call once at startup, before Run. @@ -256,7 +260,7 @@ func (l *Loop) runOnce(ctx context.Context) error { // cadence+24h" — cannot be suppressed by a fresher sibling tier. if l.windowStartFn != nil { window := l.windowStartFn() - if !scheduledRunAllowed(l.now().In(budapestLocation()), window, oldestAge(dueTiers), valveLicensed(dueTiers), l.cadence) { + if !scheduledRunAllowed(l.now().In(budapestLocation()), window, gateAge(dueTiers), valveLicensed(withoutOwed(dueTiers)), l.cadence) { from, to := gateBounds(window) l.logger.Printf("[DEBUG] [quiesce] scheduled backup due but outside the backup window [%s–%s) — deferring to the next poll inside it", from, to) return nil @@ -432,6 +436,9 @@ func (l *Loop) TriggerNow() error { // The page says so in both languages (templates/backups.html, keys // backups.a_mentes_alatt_az_alkalmazasok + backups.elinditod_a_teljes_rendszermentest_most, // pinned by TestR518_BackupButtonStatesTheShortLocalOnlyStop). + // R-899: the press is marked, so its success is recorded as a press (it does not count for + // tonight's backup) and the agent is told not to start the night's OS leg after it. + ctx = WithManualTrigger(ctx) if err := l.quiesceAndPollTiers(ctx, l.manualRunTiers(ctx)); err != nil { l.logger.Printf("[ERROR] [quiesce] manual backup cycle error: %v", err) } @@ -641,6 +648,7 @@ func (l *Loop) quiesceAndPollTiers(ctx context.Context, tiers []dueTier) error { // must NOT count as a failure, or a slow-but-healthy tier would back itself off. default: l.noteTierSuccess(t.target, label) + l.recordWholeGuestSuccess(t.target, IsManualTrigger(ctx)) // R-899 (nightowed.go) } if stillRunning { // The max-quiesce guard fired while THIS tier's backup is still going (a first full diff --git a/controller/internal/quiesce/tiers.go b/controller/internal/quiesce/tiers.go index be7f2b2..c41df6b 100644 --- a/controller/internal/quiesce/tiers.go +++ b/controller/internal/quiesce/tiers.go @@ -137,6 +137,9 @@ type dueTier struct { // primary is the agent's Primary flag (the local tier). Set on the manual path, which backs up // only the primary (`09` §3 decision 156). primary bool + // owed (R-899) marks a tier the agent called „not due" only because of a household press. It runs inside the + // window and never fires the safety valve (nightowed.go). + owed bool } // resolveDueTiers answers "what must this cycle back up?" — the dedup rule above, in one place. @@ -175,6 +178,7 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo l.logger.Printf("[WARN] [quiesce] agent advertised ZERO backup tiers — falling back to the untargeted path") return l.resolveUntargeted(ctx) } + led := l.loadLedger() // R-899: read once per cycle for _, t := range tiers { isDue, age, wireState, derr := tb.DueFor(ctx, t.Target) if derr != nil { @@ -190,6 +194,12 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo l.logAgeStateDegradeOnce() } due = append(due, dueTier{target: t.Target, ageSecs: age, state: st}) + continue + } + if l.pressOwesNight(led, t.Target) { + // R-899: the agent says „not due" only because of a household press — tonight still takes its own. + l.logger.Printf("[INFO] [quiesce] tier %s: not due by the agent, but its newest copy is a manual press and tonight's scheduled backup has not run — due tonight (R-899)", tierLabel(t.Target)) + due = append(due, dueTier{target: t.Target, ageSecs: age, state: ageStateFromWire(wireState), owed: true}) } } return due, false, nil @@ -202,6 +212,10 @@ func (l *Loop) resolveUntargeted(ctx context.Context) ([]dueTier, bool, error) { return nil, true, err } if !isDue { + if l.pressOwesNight(l.loadLedger(), "") { + l.logger.Printf("[INFO] [quiesce] not due by the agent, but its newest copy is a manual press and tonight's scheduled backup has not run — due tonight (R-899)") + return []dueTier{{target: "", ageSecs: age, owed: true}}, true, nil + } return nil, true, nil } return []dueTier{{target: "", ageSecs: age}}, true, nil diff --git a/controller/internal/quiesce/tiers_test.go b/controller/internal/quiesce/tiers_test.go index f30a1ec..760c3bf 100644 --- a/controller/internal/quiesce/tiers_test.go +++ b/controller/internal/quiesce/tiers_test.go @@ -47,6 +47,8 @@ type tierBackend struct { // statusRestarts[target] samples the restart count at EACH status poll of that tier, so a test can // say "the apps were already started when the Nth poll answered" (R-518, decision 156). statusRestarts map[string][]int + // manualStarts records, per start, whether the context carried the R-899 press mark. + manualStarts []bool } func newTierBackend() *tierBackend { @@ -70,13 +72,14 @@ func (b *tierBackend) DueFor(_ context.Context, target string) (bool, *int64, st defer b.mu.Unlock() return b.dueSet[target], nil, "", nil } -func (b *tierBackend) StartBackupFor(_ context.Context, target string) (string, error) { +func (b *tierBackend) StartBackupFor(ctx context.Context, target string) (string, error) { b.mu.Lock() defer b.mu.Unlock() if b.startErrOn == target { return "", fmt.Errorf("simulated start failure on %s", target) } b.started = append(b.started, target) + b.manualStarts = append(b.manualStarts, IsManualTrigger(ctx)) if b.stacks != nil { b.startsAtStart = append(b.startsAtStart, len(b.stacks.startedNames())) b.stopsAtStart = append(b.stopsAtStart, len(b.stacks.stoppedNames()))