package backup import ( "encoding/json" "errors" "fmt" "log" "os" "path/filepath" "sort" "time" ) // ── The app-stop marker (R-166 part 2, decision D-b "in-flight operations") ─────────────────────── // // Several operations stop a customer's app, do something to its data, and start it again. Between // the stop and the start, NOTHING ON DISK RECORDED THAT AN APP WAS OWED A RESTART. A controller that // died in that window left the app down with no explanation anywhere — and because a stopped app has // zero containers, the boot reconciler read it as a deliberate customer stop and deliberately left // it alone. Silently, indefinitely. // // A `defer` is NOT the fix and must never be described as one. Campaign 8 fault 10 established this // on live hardware: a SIGKILL runs no deferred function, and what brought the quiesce loop's stacks // back was its persisted marker read by Recover() one second after restart. The defer covers the // graceful exits; the marker covers the hard crash and the power cut. This file is that marker for // the app-data path, modelled directly on internal/quiesce's. // // WHY ITS OWN FILE, not quiesce's: one file, one writer. Quiesce's marker records a whole-guest // backup window and is written by the quiesce loop; this one records an app-data operation and is // written by the backup manager and the exporter. Sharing the file would give it two writers with // two lifetimes, and one clearing the other's record is a stranded app by a different route. // // SAFETY (D-b's binding rule): losing this file must never be worse than not having it. A lost or // corrupt marker means the app is not auto-restarted by THIS mechanism — which is precisely the // pre-v0.189.0 position, not a new hazard. It never deletes, restores, or touches a backup artifact. // AppStopReason names WHY an app was stopped, so the recovery log tells an operator which operation // was interrupted rather than merely that something was. type AppStopReason string const ( // ReasonVolumeDump — DumpAppVolumesSafe: stop, tar the volumes consistently, start. ReasonVolumeDump AppStopReason = "volume_dump" // ReasonOffboxReconstitute — a full offsite restore overwriting the app's files. ReasonOffboxReconstitute AppStopReason = "offbox_reconstitute" // ReasonAppExport — a .fab export taken with "stop the app first". ReasonAppExport AppStopReason = "app_export" ) // humanReason is the operator-facing phrasing for each reason. func (r AppStopReason) humanReason() string { switch r { case ReasonVolumeDump: return "an app-data backup (volume dump)" case ReasonOffboxReconstitute: return "an off-site restore" case ReasonAppExport: return "an app export" default: return string(r) } } // AppStopMarker is the persisted "these apps were stopped by an operation that has not reported // finishing — they are owed a restart" note. type AppStopMarker struct { Active bool `json:"active"` OpID string `json:"op_id"` Reason AppStopReason `json:"reason"` Stacks []string `json:"stacks"` StartedAt time.Time `json:"started_at"` } // AppStopStarter is the one thing recovery needs: the ability to start a stack. StartStack must be // idempotent (it is — `compose up -d` on a running stack is a no-op). // // R-174: production MUST pass a GATED starter, never the raw stack manager. Recover runs at STARTUP — // exactly when an external drive may not have come back — and `Manager.StartStack` has no drive gate // of its own. See `gatedAppStopStarter` in cmd/controller/main.go. type AppStopStarter interface { StartStack(name string) error } // ErrStartRefused is what a gated starter returns when a DELIBERATE HOLDER — today the drive gate — // says an app must not be started. Wrap it (`fmt.Errorf("%w: …", ErrStartRefused)`) so the reason // survives; Recover matches with errors.Is. // // IT IS NOT A FAILURE, AND THE DISTINCTION IS THE WHOLE POINT OF THE TYPE. A refusal means the // holder is doing its job and owns the restart; a failure means the restart was attempted and broke. // Collapsing the two would put a deliberately-held app into `Failed`, which main.go reports through // `NotifyBackupFailed` — a type that is customer-enabled by default (`settings.DefaultEnabledEvents`) // and carries the Hungarian "A biztonsági mentés sikertelen!". That is R-171's defect one path over: // a false alarm about an app the drive gate is deliberately holding. Both buckets keep the marker; // only `Failed` alarms. var ErrStartRefused = errors.New("start refused by a deliberate holder") // AppStopGuard owns one marker file. Construct with NewAppStopGuard; the zero value is inert (every // method is a no-op on a nil guard), so a caller that was never wired degrades to pre-v0.189.0 // behaviour instead of panicking. type AppStopGuard struct { path string logger *log.Logger now func() time.Time // starter is only needed by Recover; Begin/End work without one. starter AppStopStarter } // AppStopRecovery is what Recover found and did. Returned rather than pushed through a notifier // seam, because of a hard ordering constraint: Recover must COMPLETE before the boot reconciler is // launched (§8.4, main.go:236) and the hub notifier is not constructed until main.go:307. A seam // wired after the fact would be a seam that never fires — the "built but never wired" shape this // project has now hit four times. Returning the outcome lets main.go report it the moment the // notifier exists, and makes the reporting decision visible at the call site instead of buried here. type AppStopRecovery struct { Reason AppStopReason OpID string StartedAt time.Time Restarted []string // apps started again by this recovery Failed []string // apps whose restart was ATTEMPTED and broke (the marker was kept for these) // Refused are apps a deliberate holder said must not start — today, an absent data drive // (R-174). The marker is kept for these too, but they are NOT a fault and MUST NOT alarm: the // holder owns the restart. Separate from Failed for the reason recorded on ErrStartRefused. Refused []string } // Alarming reports whether this recovery is worth paging an operator about. A recovery that only // REFUSED starts is the drive gate working as designed, and reporting it through the customer-enabled // `backup_failed` type would be the R-171 false alarm one path over. func (r *AppStopRecovery) Alarming() bool { if r == nil { return false } return len(r.Failed) > 0 || len(r.Restarted) > 0 } // Message is the operator-facing headline for an interrupted operation. func (r *AppStopRecovery) Message() string { if r == nil { return "" } if len(r.Failed) > 0 { m := fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted", r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed)) if len(r.Refused) > 0 { m += fmt.Sprintf(" (a further %d are held by an absent drive and are not counted as failures)", len(r.Refused)) } return m } if len(r.Refused) > 0 && len(r.Restarted) == 0 { return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are left stopped and HELD: their data drive is not available, so the drive gate restarts them when it returns", r.Reason.humanReason(), len(r.Refused)) } m := fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) were left stopped and have been restarted", r.Reason.humanReason(), len(r.Restarted)) if len(r.Refused) > 0 { m += fmt.Sprintf("; %d more are held by an absent drive", len(r.Refused)) } return m } // Detail is the machine-readable tail. App/stack NAMES only — never env values (§9.5). func (r *AppStopRecovery) Detail() string { if r == nil { return "" } d := fmt.Sprintf("op=%s reason=%s started_at=%s restarted=%v", r.OpID, r.Reason, r.StartedAt.UTC().Format(time.RFC3339), r.Restarted) if len(r.Failed) > 0 { d += fmt.Sprintf(" restart_failed=%v", r.Failed) } if len(r.Refused) > 0 { d += fmt.Sprintf(" held_by_drive=%v", r.Refused) } return d } // NewAppStopGuard builds a guard over the given marker path. func NewAppStopGuard(path string, logger *log.Logger) *AppStopGuard { if logger == nil { logger = log.Default() } return &AppStopGuard{path: path, logger: logger, now: time.Now} } // SetStarter wires the stack-start seam used by Recover. INIT-ONLY — call once at startup, before // Recover. Separate from the constructor because the guard is built alongside the backup manager, // which learns its stack provider later (the same shape as SetStackProvider). func (g *AppStopGuard) SetStarter(s AppStopStarter) { if g == nil { return } g.starter = s } // Begin records that `stacks` are about to be stopped by `reason`. It MUST be called BEFORE the // first stop — an error here means the marker could not be written, and the caller must not proceed // to stop an app it cannot promise to restart. func (g *AppStopGuard) Begin(opID string, reason AppStopReason, stackNames []string) error { if g == nil || g.path == "" { return nil // not wired — pre-v0.189.0 behaviour, never a hard failure } if len(stackNames) == 0 { return nil } return g.write(AppStopMarker{ Active: true, OpID: opID, Reason: reason, Stacks: append([]string(nil), stackNames...), StartedAt: g.now(), }) } // End clears the marker after a successful restart. Best-effort by contract: a failure to clear is // logged, never returned as the operation's error — a stale marker costs one idempotent StartStack // on the next boot, which is exactly D-b's "worst acceptable outcome" and far cheaper than failing // a backup that actually succeeded. func (g *AppStopGuard) End() { if g == nil || g.path == "" { return } if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) { g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v (a stale marker costs one idempotent restart at next startup)", g.path, err) } } // Recover restarts any apps left stopped by an operation that died before restarting them, then // clears the marker. Call ONCE at startup, and — critically — call it to COMPLETION before the boot // reconciler is launched, so an app this marker explains is not also reported as an unexplained boot // orphan (§8.4). // // Idempotent: StartStack on a running stack is tolerated, and an absent or inactive marker is a // no-op. On a restart FAILURE the marker is deliberately LEFT IN PLACE — the next startup retries, // and in the meantime the app is down with desired_state:running, so the boot reconciler sees it as // an orphan and the dead-app alarm owns it. Clearing a marker whose restart failed would erase the // only durable record that an app is owed one. // // Returns nil when there was nothing to recover — so "no interrupted operation" and "the recovery // never ran" are distinguishable to the caller, not only in a log (standing rule 3). func (g *AppStopGuard) Recover() *AppStopRecovery { if g == nil || g.path == "" { return nil } m, ok := g.read() if !ok || !m.Active || len(m.Stacks) == 0 { return nil } if g.starter == nil { g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) were stopped by %s and are owed a restart, but no stack starter is wired — leaving the marker for the next startup: %v", len(m.Stacks), m.Reason.humanReason(), m.Stacks) return nil } g.logger.Printf("[WARN] [appstop] crash recovery: %s (op %q) was interrupted and left %d app(s) stopped — restarting them: %v", m.Reason.humanReason(), m.OpID, len(m.Stacks), m.Stacks) res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt} for _, name := range m.Stacks { if err := g.starter.StartStack(name); err != nil { // R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be // started (an absent data drive), so the app is left down deliberately and the holder // owns the restart. Logged at WARN with the reason, and kept out of Failed so it never // reaches the customer-enabled backup_failed alarm — see ErrStartRefused. if errors.Is(err, ErrStartRefused) { g.logger.Printf("[WARN] [appstop] crash recovery: NOT restarting %s — %v; the marker is KEPT and the holder owns the restart", name, err) res.Refused = append(res.Refused, name) continue } g.logger.Printf("[ERROR] [appstop] crash recovery: restart %s failed: %v", name, err) res.Failed = append(res.Failed, name) continue } g.logger.Printf("[INFO] [appstop] crash recovery: restarted %s after the interrupted %s", name, m.Reason.humanReason()) res.Restarted = append(res.Restarted, name) } sort.Strings(res.Failed) sort.Strings(res.Refused) sort.Strings(res.Restarted) // The marker is kept for BOTH unfinished outcomes, for the same reason and with different // urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished // until its drive returns. Clearing it in either case would erase the only durable record that // an app is owed a restart. if len(res.Failed) > 0 { g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v", len(res.Failed), res.Failed) return res } if len(res.Refused) > 0 { g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent) — KEEPING the marker; this is the gate working, not a fault: %v", len(res.Refused), res.Refused) return res } g.End() return res } // HeldStacks returns the stacks an app-data operation is CURRENTLY holding down, or nil. // // Read-only and nil-safe. It exists for the boot reconciler (§8.2): once R-157 mechanism A widened // the boot window, the sweep could overlap a running volume dump or export and "recover" an app that // is deliberately stopped mid-operation — restarting it under a tar, which is the inconsistency the // stop was taken to avoid. Recover() has already run to completion by then, so a marker seen through // this method belongs to an operation running NOW, not to a crashed one. func (g *AppStopGuard) HeldStacks() []string { if g == nil || g.path == "" { return nil } m, ok := g.read() if !ok || !m.Active { return nil } return append([]string(nil), m.Stacks...) } // ---- marker persistence (atomic, 0600) — the quiesce shape ------------------------------------ func (g *AppStopGuard) write(m AppStopMarker) error { data, err := json.MarshalIndent(m, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(g.path), 0o755); err != nil { return err } tmp := g.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return err } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) return err } // fsync before rename: the whole point is surviving a power cut, and a rename that lands ahead // of the bytes it points at is a marker that reads as corrupt at exactly the wrong moment. if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Close(); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, g.path) } func (g *AppStopGuard) read() (AppStopMarker, bool) { data, err := os.ReadFile(g.path) if err != nil { return AppStopMarker{}, false } var m AppStopMarker if err := json.Unmarshal(data, &m); err != nil { // Never a silent skip (§9.4): a corrupt marker is LOUD and the bad file is quarantined, so a // genuinely interrupted operation leaves a trace instead of vanishing. Still returns false — // "no usable marker ⇒ no recovery" is the correct contract, and matches quiesce's. g.logger.Printf("[WARN] [appstop] the app-stop marker at %s is corrupt (%v) — quarantining; apps are NOT auto-restarted from it", g.path, err) _ = os.Rename(g.path, fmt.Sprintf("%s.corrupt-%d", g.path, g.now().Unix())) return AppStopMarker{}, false } return m, true }