package backup import ( "errors" "os" "path/filepath" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/util" ) // ── R-1270: the verdict follows the app (brief 2026-10-25, Part 3) ───────────────────────────────────────────────── // // MEASURED BEFORE (catalog sweep 2026-10-24; again on 9202 2026-10-25, `audits/restore-truth-2026-10-25/before/`): the // controller logged „tandoor restored but health check failed: stack tandoor did not reach running state within 1m30s" // at WARN, the restore function returned success, and the page the household reads said `ok: true` — „…visszaállítva — // az alkalmazás újraindult." The app was `starting` and never came up. Every restore path had the same WARN-and-succeed // shape: the unit restore, the off-site restore, the second drive's file restore and the no-unit fallback. // // THE RULE. A restore whose app does not reach running in time NEVER reports success. It returns ErrRestoreNotStarted // and the household reads „Az adatokat visszatöltöttük, de az alkalmazás még nem indult el. Várj pár percet, majd nyisd // meg újra. Ha így marad, szólj nekünk." The operator gets one alarm (backup_run_failures, leg `restore-not-started`). // // THE SAFEST READING (brief 2026-10-25 Part 3.2 — the operator may change it). A health timeout does NOT pour the old // data back: a slow first start (a database migration) is common, and a put-back would throw away a correct restore. // The data stays restored and the restore's undo copy is KEPT (the same „kept for support" state a failed put-back // leaves, `07` §8 — no new retention rule). A watcher looks for the app for lateStartWatch more: when it runs, the // record turns to success („…elindult") and the copy is released by the normal rule (removed, as after any successful // restore). If it never runs, the copy stays for support, as a failed put-back's does. // ErrRestoreNotStarted is the KIND of a restore whose data is back but whose app did not reach running in time. var ErrRestoreNotStarted = errors.New("the data is restored but the app did not start in time") // lateStartWatch / lateStartPoll bound the watcher after a not-started verdict (tests shorten them). var ( lateStartWatch = 30 * time.Minute lateStartPoll = 30 * time.Second ) // SetRestoreNotStartedNotify wires the operator alarm for a restore whose app did not start (main.go). func (m *Manager) SetRestoreNotStartedNotify(fn func(stack, kind string, cause error)) { m.restoreNotStartedNotify = fn } // restoreNotStarted is the verdict of a restore whose health check failed: the undo copy is kept, the operator is told // once, the watcher starts, and the household's sentence is returned as the error. kind names the path (log + alarm). func (m *Manager) restoreNotStarted(stack, kind string, healthErr error, undo *liveVolumeCopy) error { copyDir := "" if undo != nil && undo.dir != "" { undo.keep = true copyDir = undo.dir } m.logger.Printf("[ERROR] [backup] %s: the %s put the data back, but the app did not start: %v — NOT reported as a success; the restore copy is kept (%s); watching %s for a late start (R-1270)", stack, kind, healthErr, orNone(copyDir), lateStartWatch) if m.restoreNotStartedNotify != nil { m.restoreNotStartedNotify(stack, kind, healthErr) } go m.watchLateStart(stack, copyDir, time.Now()) return util.MsgErrorf(ErrRestoreNotStarted, "err.backup.restore_not_started", stack) } func orNone(s string) string { if s == "" { return "no copy" } return s } // watchLateStart waits for the app after a not-started verdict. When it runs: the record says so and the kept copy is // removed (the normal rule after a successful restore). A new restore of the app ends the watch untouched. func (m *Manager) watchLateStart(stack, copyDir string, since time.Time) { deadline := since.Add(lateStartWatch) for time.Now().Before(deadline) { time.Sleep(lateStartPoll) if m.restoreRunningFor(stack) { return // a new restore of this app owns it now } if m.stackProvider == nil || !m.stackProvider.RefreshAndIsRunning(stack) { continue } if copyDir != "" { if err := os.RemoveAll(copyDir); err != nil { m.logger.Printf("[WARN] [backup] %s: started late; the kept restore copy %s could not be removed: %v", stack, copyDir, err) } else { _ = os.Remove(filepath.Dir(copyDir)) } } m.markLateStart(stack, m.note("note.restore.late_started", stack)) m.logger.Printf("[INFO] [backup] %s: the app started %s after its restore — the restore is a success now; the restore copy is released (R-1270)", stack, time.Since(since).Round(time.Second)) return } m.logger.Printf("[WARN] [backup] %s: still not running %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir)) }