94 lines
4.8 KiB
Go
94 lines
4.8 KiB
Go
package backup
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
|
)
|
|
|
|
// ── R-1270: the verdict follows the app (brief 2026-10-25, Part 3) ─────────────────────────────────────────────────
|
|
//
|
|
// MEASURED BEFORE (catalog sweep 2026-10-24; again on 9202 2026-10-25, `audits/restore-truth-2026-10-25/before/`): the
|
|
// controller logged „tandoor restored but health check failed: stack tandoor did not reach running state within 1m30s"
|
|
// at WARN, the restore function returned success, and the page the household reads said `ok: true` — „…visszaállítva —
|
|
// az alkalmazás újraindult." The app was `starting` and never came up. Every restore path had the same WARN-and-succeed
|
|
// shape: the unit restore, the off-site restore, the second drive's file restore and the no-unit fallback.
|
|
//
|
|
// THE RULE. A restore whose app does not reach running in time NEVER reports success. It returns ErrRestoreNotStarted
|
|
// and the household reads „Az adatokat visszatöltöttük, de az alkalmazás még nem indult el. Várj pár percet, majd nyisd
|
|
// meg újra. Ha így marad, szólj nekünk." The operator gets one alarm (backup_run_failures, leg `restore-not-started`).
|
|
//
|
|
// THE SAFEST READING (brief 2026-10-25 Part 3.2 — the operator may change it). A health timeout does NOT pour the old
|
|
// data back: a slow first start (a database migration) is common, and a put-back would throw away a correct restore.
|
|
// The data stays restored and the restore's undo copy is KEPT (the same „kept for support" state a failed put-back
|
|
// leaves, `07` §8 — no new retention rule). A watcher looks for the app for lateStartWatch more: when it runs, the
|
|
// record turns to success („…elindult") and the copy is released by the normal rule (removed, as after any successful
|
|
// restore). If it never runs, the copy stays for support, as a failed put-back's does.
|
|
|
|
// ErrRestoreNotStarted is the KIND of a restore whose data is back but whose app did not reach running in time.
|
|
var ErrRestoreNotStarted = errors.New("the data is restored but the app did not start in time")
|
|
|
|
// lateStartWatch / lateStartPoll bound the watcher after a not-started verdict (tests shorten them).
|
|
var (
|
|
lateStartWatch = 30 * time.Minute
|
|
lateStartPoll = 30 * time.Second
|
|
)
|
|
|
|
// SetRestoreNotStartedNotify wires the operator alarm for a restore whose app did not start (main.go).
|
|
func (m *Manager) SetRestoreNotStartedNotify(fn func(stack, kind string, cause error)) {
|
|
m.restoreNotStartedNotify = fn
|
|
}
|
|
|
|
// restoreNotStarted is the verdict of a restore whose health check failed: the undo copy is kept, the operator is told
|
|
// once, the watcher starts, and the household's sentence is returned as the error. kind names the path (log + alarm).
|
|
func (m *Manager) restoreNotStarted(stack, kind string, healthErr error, undo *liveVolumeCopy) error {
|
|
copyDir := ""
|
|
if undo != nil && undo.dir != "" {
|
|
undo.keep = true
|
|
copyDir = undo.dir
|
|
}
|
|
m.logger.Printf("[ERROR] [backup] %s: the %s put the data back, but the app did not start: %v — NOT reported as a success; the restore copy is kept (%s); watching %s for a late start (R-1270)",
|
|
stack, kind, healthErr, orNone(copyDir), lateStartWatch)
|
|
if m.restoreNotStartedNotify != nil {
|
|
m.restoreNotStartedNotify(stack, kind, healthErr)
|
|
}
|
|
go m.watchLateStart(stack, copyDir, time.Now())
|
|
return util.MsgErrorf(ErrRestoreNotStarted, "err.backup.restore_not_started", stack)
|
|
}
|
|
|
|
func orNone(s string) string {
|
|
if s == "" {
|
|
return "no copy"
|
|
}
|
|
return s
|
|
}
|
|
|
|
// watchLateStart waits for the app after a not-started verdict. When it runs: the record says so and the kept copy is
|
|
// removed (the normal rule after a successful restore). A new restore of the app ends the watch untouched.
|
|
func (m *Manager) watchLateStart(stack, copyDir string, since time.Time) {
|
|
deadline := since.Add(lateStartWatch)
|
|
for time.Now().Before(deadline) {
|
|
time.Sleep(lateStartPoll)
|
|
if m.restoreRunningFor(stack) {
|
|
return // a new restore of this app owns it now
|
|
}
|
|
if m.stackProvider == nil || !m.stackProvider.RefreshAndIsRunning(stack) {
|
|
continue
|
|
}
|
|
if copyDir != "" {
|
|
if err := os.RemoveAll(copyDir); err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: started late; the kept restore copy %s could not be removed: %v", stack, copyDir, err)
|
|
} else {
|
|
_ = os.Remove(filepath.Dir(copyDir))
|
|
}
|
|
}
|
|
m.markLateStart(stack, m.note("note.restore.late_started", stack))
|
|
m.logger.Printf("[INFO] [backup] %s: the app started %s after its restore — the restore is a success now; the restore copy is released (R-1270)", stack, time.Since(since).Round(time.Second))
|
|
return
|
|
}
|
|
m.logger.Printf("[WARN] [backup] %s: still not running %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir))
|
|
}
|