package backup import ( "encoding/json" "fmt" "log" "os" "path/filepath" "sort" "time" ) // ── The app-stop marker (R-166 part 2, decision D-b "in-flight operations") ─────────────────────── // // Several operations stop a customer's app, do something to its data, and start it again. Between // the stop and the start, NOTHING ON DISK RECORDED THAT AN APP WAS OWED A RESTART. A controller that // died in that window left the app down with no explanation anywhere — and because a stopped app has // zero containers, the boot reconciler read it as a deliberate customer stop and deliberately left // it alone. Silently, indefinitely. // // A `defer` is NOT the fix and must never be described as one. Campaign 8 fault 10 established this // on live hardware: a SIGKILL runs no deferred function, and what brought the quiesce loop's stacks // back was its persisted marker read by Recover() one second after restart. The defer covers the // graceful exits; the marker covers the hard crash and the power cut. This file is that marker for // the app-data path, modelled directly on internal/quiesce's. // // WHY ITS OWN FILE, not quiesce's: one file, one writer. Quiesce's marker records a whole-guest // backup window and is written by the quiesce loop; this one records an app-data operation and is // written by the backup manager and the exporter. Sharing the file would give it two writers with // two lifetimes, and one clearing the other's record is a stranded app by a different route. // // SAFETY (D-b's binding rule): losing this file must never be worse than not having it. A lost or // corrupt marker means the app is not auto-restarted by THIS mechanism — which is precisely the // pre-v0.189.0 position, not a new hazard. It never deletes, restores, or touches a backup artifact. // AppStopReason names WHY an app was stopped, so the recovery log tells an operator which operation // was interrupted rather than merely that something was. type AppStopReason string const ( // ReasonVolumeDump — DumpAppVolumesSafe: stop, tar the volumes consistently, start. ReasonVolumeDump AppStopReason = "volume_dump" // ReasonOffboxReconstitute — a full offsite restore overwriting the app's files. ReasonOffboxReconstitute AppStopReason = "offbox_reconstitute" // ReasonAppExport — a .fab export taken with "stop the app first". ReasonAppExport AppStopReason = "app_export" ) // humanReason is the operator-facing phrasing for each reason. func (r AppStopReason) humanReason() string { switch r { case ReasonVolumeDump: return "an app-data backup (volume dump)" case ReasonOffboxReconstitute: return "an off-site restore" case ReasonAppExport: return "an app export" default: return string(r) } } // AppStopMarker is the persisted "these apps were stopped by an operation that has not reported // finishing — they are owed a restart" note. type AppStopMarker struct { Active bool `json:"active"` OpID string `json:"op_id"` Reason AppStopReason `json:"reason"` Stacks []string `json:"stacks"` StartedAt time.Time `json:"started_at"` } // AppStopStarter is the one thing recovery needs: the ability to start a stack. StartStack must be // idempotent (it is — `compose up -d` on a running stack is a no-op). type AppStopStarter interface { StartStack(name string) error } // AppStopGuard owns one marker file. Construct with NewAppStopGuard; the zero value is inert (every // method is a no-op on a nil guard), so a caller that was never wired degrades to pre-v0.189.0 // behaviour instead of panicking. type AppStopGuard struct { path string logger *log.Logger now func() time.Time // starter is only needed by Recover; Begin/End work without one. starter AppStopStarter } // AppStopRecovery is what Recover found and did. Returned rather than pushed through a notifier // seam, because of a hard ordering constraint: Recover must COMPLETE before the boot reconciler is // launched (§8.4, main.go:236) and the hub notifier is not constructed until main.go:307. A seam // wired after the fact would be a seam that never fires — the "built but never wired" shape this // project has now hit four times. Returning the outcome lets main.go report it the moment the // notifier exists, and makes the reporting decision visible at the call site instead of buried here. type AppStopRecovery struct { Reason AppStopReason OpID string StartedAt time.Time Restarted []string // apps started again by this recovery Failed []string // apps that could NOT be restarted (the marker was kept for these) } // Message is the operator-facing headline for an interrupted operation. func (r *AppStopRecovery) Message() string { if r == nil { return "" } if len(r.Failed) > 0 { return fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted", r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed)) } return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) were left stopped and have been restarted", r.Reason.humanReason(), len(r.Restarted)) } // Detail is the machine-readable tail. App/stack NAMES only — never env values (§9.5). func (r *AppStopRecovery) Detail() string { if r == nil { return "" } d := fmt.Sprintf("op=%s reason=%s started_at=%s restarted=%v", r.OpID, r.Reason, r.StartedAt.UTC().Format(time.RFC3339), r.Restarted) if len(r.Failed) > 0 { d += fmt.Sprintf(" restart_failed=%v", r.Failed) } return d } // NewAppStopGuard builds a guard over the given marker path. func NewAppStopGuard(path string, logger *log.Logger) *AppStopGuard { if logger == nil { logger = log.Default() } return &AppStopGuard{path: path, logger: logger, now: time.Now} } // SetStarter wires the stack-start seam used by Recover. INIT-ONLY — call once at startup, before // Recover. Separate from the constructor because the guard is built alongside the backup manager, // which learns its stack provider later (the same shape as SetStackProvider). func (g *AppStopGuard) SetStarter(s AppStopStarter) { if g == nil { return } g.starter = s } // Begin records that `stacks` are about to be stopped by `reason`. It MUST be called BEFORE the // first stop — an error here means the marker could not be written, and the caller must not proceed // to stop an app it cannot promise to restart. func (g *AppStopGuard) Begin(opID string, reason AppStopReason, stackNames []string) error { if g == nil || g.path == "" { return nil // not wired — pre-v0.189.0 behaviour, never a hard failure } if len(stackNames) == 0 { return nil } return g.write(AppStopMarker{ Active: true, OpID: opID, Reason: reason, Stacks: append([]string(nil), stackNames...), StartedAt: g.now(), }) } // End clears the marker after a successful restart. Best-effort by contract: a failure to clear is // logged, never returned as the operation's error — a stale marker costs one idempotent StartStack // on the next boot, which is exactly D-b's "worst acceptable outcome" and far cheaper than failing // a backup that actually succeeded. func (g *AppStopGuard) End() { if g == nil || g.path == "" { return } if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) { g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v (a stale marker costs one idempotent restart at next startup)", g.path, err) } } // Recover restarts any apps left stopped by an operation that died before restarting them, then // clears the marker. Call ONCE at startup, and — critically — call it to COMPLETION before the boot // reconciler is launched, so an app this marker explains is not also reported as an unexplained boot // orphan (§8.4). // // Idempotent: StartStack on a running stack is tolerated, and an absent or inactive marker is a // no-op. On a restart FAILURE the marker is deliberately LEFT IN PLACE — the next startup retries, // and in the meantime the app is down with desired_state:running, so the boot reconciler sees it as // an orphan and the dead-app alarm owns it. Clearing a marker whose restart failed would erase the // only durable record that an app is owed one. // // Returns nil when there was nothing to recover — so "no interrupted operation" and "the recovery // never ran" are distinguishable to the caller, not only in a log (standing rule 3). func (g *AppStopGuard) Recover() *AppStopRecovery { if g == nil || g.path == "" { return nil } m, ok := g.read() if !ok || !m.Active || len(m.Stacks) == 0 { return nil } if g.starter == nil { g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) were stopped by %s and are owed a restart, but no stack starter is wired — leaving the marker for the next startup: %v", len(m.Stacks), m.Reason.humanReason(), m.Stacks) return nil } g.logger.Printf("[WARN] [appstop] crash recovery: %s (op %q) was interrupted and left %d app(s) stopped — restarting them: %v", m.Reason.humanReason(), m.OpID, len(m.Stacks), m.Stacks) res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt} for _, name := range m.Stacks { if err := g.starter.StartStack(name); err != nil { g.logger.Printf("[ERROR] [appstop] crash recovery: restart %s failed: %v", name, err) res.Failed = append(res.Failed, name) continue } g.logger.Printf("[INFO] [appstop] crash recovery: restarted %s after the interrupted %s", name, m.Reason.humanReason()) res.Restarted = append(res.Restarted, name) } sort.Strings(res.Failed) sort.Strings(res.Restarted) if len(res.Failed) > 0 { g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v", len(res.Failed), res.Failed) return res } g.End() return res } // HeldStacks returns the stacks an app-data operation is CURRENTLY holding down, or nil. // // Read-only and nil-safe. It exists for the boot reconciler (§8.2): once R-157 mechanism A widened // the boot window, the sweep could overlap a running volume dump or export and "recover" an app that // is deliberately stopped mid-operation — restarting it under a tar, which is the inconsistency the // stop was taken to avoid. Recover() has already run to completion by then, so a marker seen through // this method belongs to an operation running NOW, not to a crashed one. func (g *AppStopGuard) HeldStacks() []string { if g == nil || g.path == "" { return nil } m, ok := g.read() if !ok || !m.Active { return nil } return append([]string(nil), m.Stacks...) } // ---- marker persistence (atomic, 0600) — the quiesce shape ------------------------------------ func (g *AppStopGuard) write(m AppStopMarker) error { data, err := json.MarshalIndent(m, "", " ") if err != nil { return err } if err := os.MkdirAll(filepath.Dir(g.path), 0o755); err != nil { return err } tmp := g.path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return err } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) return err } // fsync before rename: the whole point is surviving a power cut, and a rename that lands ahead // of the bytes it points at is a marker that reads as corrupt at exactly the wrong moment. if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Close(); err != nil { os.Remove(tmp) return err } return os.Rename(tmp, g.path) } func (g *AppStopGuard) read() (AppStopMarker, bool) { data, err := os.ReadFile(g.path) if err != nil { return AppStopMarker{}, false } var m AppStopMarker if err := json.Unmarshal(data, &m); err != nil { // Never a silent skip (§9.4): a corrupt marker is LOUD and the bad file is quarantined, so a // genuinely interrupted operation leaves a trace instead of vanishing. Still returns false — // "no usable marker ⇒ no recovery" is the correct contract, and matches quiesce's. g.logger.Printf("[WARN] [appstop] the app-stop marker at %s is corrupt (%v) — quarantining; apps are NOT auto-restarted from it", g.path, err) _ = os.Rename(g.path, fmt.Sprintf("%s.corrupt-%d", g.path, g.now().Unix())) return AppStopMarker{}, false } return m, true }