package backup import ( "fmt" "os" "sync" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // ── R-1251: a pending put-back blocks every start (review round 2, HUNT-1 H1) ───────────────────────────────────── // // D22 pours a cut restore's copy back at start (restore_crash_putback.go). For a DRIVE-backed app the copy lives on // the app's drive (restoreUndoDir under the drive's namespace), and at start the drive is often not bound yet. Until // this file Recover read the missing folder as „gone", went on to the gated start (refused: no drive) and kept the // marker; the drive then bound, the boot recreate started the app on the cut restore's data (it asked only „held?"), // the household wrote, and the NEXT controller start found the copy and poured it over everything written since. // // Now: // - a copy that cannot be seen while its drive is not bound is WAITING, never gone. Recover writes a restore hold // (settings.HoldReasonPutBackPending, the copy's folder in SafetyDump) and starts nothing. The hold is the ONE // storage every start path already honours — the household's Start/Restart/Update, the boot recreate, the drive // return, the media restart, the boot sweep, the night chain, the update leg — so no path needs its own check; // - when the drive binds, ResumePutBack pours the copy back FIRST and lifts the hold; only then may the app start; // - a put-back that FAILS (at start or on resume) leaves the app HELD with the D20 hold (HoldReasonRestoreMixed, the // household's „stopped half-way" sentence) and the copy on disk — never stopped-and-unheld, which Start or the boot // recreate would run and a later start would pour an old copy over (H2). The marker then lets go of that app: the // hold owns it, exactly as holdAppAfterMixedRestore does in-process; // - Stop pressed during the restore no longer skips the put-back after a cut (H7): the data is put back, the app // stays stopped as the household asked. // // The hold — not the marker — is the durable record of a waiting put-back, because the marker has one slot and the // next operation's Begin replaces it (R-1253). // // Pinned by r1251_putback_waits_test.go. // AppStopHolds is the hold store the guard writes (settings.Settings in production). type AppStopHolds interface { SetRestoreHold(settings.RestoreHold) error GetRestoreHold(stack string) (settings.RestoreHold, bool) ClearRestoreHold(stack string) (bool, error) ListRestoreHolds() []settings.RestoreHold } // SetHolds wires the hold store. INIT-ONLY, before Recover. nil = the pre-R-1251 behaviour (no hold is written). func (g *AppStopGuard) SetHolds(h AppStopHolds) { if g != nil { g.holds = h } } // SetUndoReachable wires the question „is the drive this app's restore copy lives on bound right now?" (main.go: // the app's HDD_PATH is a live mountpoint). nil = every folder counts as reachable, so a missing copy is „gone". func (g *AppStopGuard) SetUndoReachable(f func(stack, dir string) bool) { if g != nil { g.undoReachable = f } } type undoCopyState int const ( undoPresent undoCopyState = iota undoGone // its drive is bound and the folder is not there undoWaiting // its drive is not bound (or the folder cannot be read): it may come back ) func (s undoCopyState) String() string { switch s { case undoPresent: return "present" case undoGone: return "gone" default: return "waiting" } } // copyState decides present / gone / waiting for one recorded copy. func (g *AppStopGuard) copyState(stack, dir string) (undoCopyState, error) { _, err := os.Stat(dir) if err == nil { return undoPresent, nil } if !os.IsNotExist(err) { return undoWaiting, err // an unreadable folder is not proof that it is gone } if g.undoReachable != nil && !g.undoReachable(stack, dir) { return undoWaiting, err } return undoGone, err } // holdPutBackPending writes the „waiting for the drive" hold. Returns whether the app is now held by SOME hold. func (g *AppStopGuard) holdPutBackPending(stack, dir string) bool { if g.holds == nil { g.logger.Printf("[ERROR] [appstop] %s: its restore copy %s waits for its drive, but no hold store is wired — nothing durable refuses a start (R-1251)", stack, dir) return false } if h, ok := g.holds.GetRestoreHold(stack); ok && h.Reason != settings.HoldReasonPutBackPending { // Another hold already refuses every start, and its sentence names the harder problem. The marker keeps the // copy's folder, so the next start puts it back once the drive is there. g.logger.Printf("[WARN] [appstop] %s: its restore copy %s waits for its drive; the app is already held (%q, set %s) — that hold stays", stack, dir, h.Reason, h.At) return true } h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending, SafetyDump: dir} if err := g.holds.SetRestoreHold(h); err != nil { g.logger.Printf("[ERROR] [appstop] %s: the waiting put-back could not be recorded as a hold: %v — the marker is kept", stack, err) return false } g.logger.Printf("[WARN] [appstop] %s: the cut restore's copy %s is on a drive that is not bound — the app is HELD until the copy is put back (R-1251)", stack, dir) return true } // holdPutBackFailed writes the D20 hold for a put-back that could not be done (H2). Returns whether it was written. func (g *AppStopGuard) holdPutBackFailed(stack, dir string, cause error) bool { if g.holds == nil { return false } h := settings.RestoreHold{Stack: stack, At: g.now().UTC().Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed, ReplayError: "interrupted restore, put-back failed: " + cause.Error(), SafetyDump: dir} if err := g.holds.SetRestoreHold(h); err != nil { g.logger.Printf("[ERROR] [appstop] %s: the failed put-back could not be recorded as a hold: %v — the marker is kept", stack, err) return false } g.logger.Printf("[ERROR] [appstop] %s: the interrupted restore could NOT be put back (%v) — the app is HELD for support, the copy stays at %s (R-1251 H2)", stack, cause, dir) return true } // clearPutBackPending lifts this app's waiting-put-back hold, and only that kind. func (g *AppStopGuard) clearPutBackPending(stack string) { if g.holds == nil { return } if h, ok := g.holds.GetRestoreHold(stack); !ok || h.Reason != settings.HoldReasonPutBackPending { return } if _, err := g.holds.ClearRestoreHold(stack); err != nil { g.logger.Printf("[ERROR] [appstop] %s: the put-back ran, but its hold could not be cleared: %v — the app stays held", stack, err) } } // dropFromMarker lets the active marker forget these apps (and their copies): a hold now owns them, or their put-back // is done. A marker left with no app is cleared. A marker that belongs to another operation (it does not hold them) // is not touched. func (g *AppStopGuard) dropFromMarker(names ...string) { if g == nil || g.path == "" || len(names) == 0 { return } m, ok := g.read() if !ok || !m.Active { return } drop := map[string]bool{} for _, n := range names { drop[n] = true } var keep []string for _, s := range m.Stacks { if !drop[s] { keep = append(keep, s) } } if len(keep) == len(m.Stacks) { return } if len(keep) == 0 { if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) { g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v", g.path, err) } return } m.Stacks = keep for n := range drop { delete(m.Undo, n) } if err := g.write(m); err != nil { g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without %v: %v", names, err) } } // forgetUndo removes these apps' copy folders from the active marker (their put-back is done; the copy is removed), // so a later start never tries to pour back a folder that no longer holds the app's data. func (g *AppStopGuard) forgetUndo(names ...string) { if g == nil || g.path == "" || len(names) == 0 { return } m, ok := g.read() if !ok || !m.Active || len(m.Undo) == 0 { return } changed := false for _, n := range names { if _, ok := m.Undo[n]; ok { delete(m.Undo, n) changed = true } } if !changed { return } if err := g.write(m); err != nil { g.logger.Printf("[ERROR] [appstop] could not rewrite the app-stop marker without the used copies of %v: %v", names, err) } } // resumeMu serialises ResumePutBack: the boot recreate and the drive-return sweep can reach the same app at once, and // a copy must be poured back exactly once. var resumeMu sync.Mutex // ResumePutBack runs a put-back that was WAITING for its drive. It returns true when nothing is owed for stack any // more — there was no waiting put-back, or it ran now — and the caller may go on to start the app (its other holds // still apply). false: the app stays held (the drive is still not bound, or the put-back failed and the app is now // held for support). Nil-safe. func (g *AppStopGuard) ResumePutBack(stack string) bool { if g == nil || g.holds == nil { return true } resumeMu.Lock() defer resumeMu.Unlock() h, ok := g.holds.GetRestoreHold(stack) if !ok || h.Reason != settings.HoldReasonPutBackPending { return true } dir := h.SafetyDump if m, ok := g.read(); ok && m.Active && m.Undo[stack] != "" { dir = m.Undo[stack] } if dir == "" || g.putBacker == nil { g.logger.Printf("[ERROR] [appstop] %s: a put-back waits, but its copy is not named (or no put-back is wired) — the app stays HELD", stack) return false } state, serr := g.copyState(stack, dir) switch state { case undoWaiting: g.logger.Printf("[INFO] [appstop] %s: the restore copy %s is still not reachable (%v) — the app stays held until its drive is bound", stack, dir, serr) return false case undoGone: g.holdPutBackFailed(stack, dir, fmt.Errorf("the restore copy %s is gone while its drive is bound", dir)) g.dropFromMarker(stack) return false } if err := g.putBacker.PutBackInterruptedRestore(stack, dir); err != nil { if g.holdPutBackFailed(stack, dir, err) { g.dropFromMarker(stack) } return false } g.clearPutBackPending(stack) g.dropFromMarker(stack) g.logger.Printf("[WARN] [appstop] %s: the drive is bound — the interrupted restore is put back from %s BEFORE the app starts; its data is as before the restore (R-1251)", stack, dir) return true } // ResumeWaitingPutBacks runs every waiting put-back whose drive is bound now, and starts each app afterwards through // the guard's gated starter — unless the household stopped it. Called at start (after Recover) and when a drive // returns. Returns the apps started. func (g *AppStopGuard) ResumeWaitingPutBacks() []string { if g == nil || g.holds == nil { return nil } var started []string for _, h := range g.holds.ListRestoreHolds() { if h.Reason != settings.HoldReasonPutBackPending { continue } if !g.ResumePutBack(h.Stack) { continue } if ws, ok := g.starter.(interface{ WantsStopped(string) bool }); ok && ws.WantsStopped(h.Stack) { g.logger.Printf("[INFO] [appstop] %s: put back; NOT starting it — the household stopped it", h.Stack) continue } if g.starter == nil { continue } if err := g.starter.StartStack(h.Stack); err != nil { g.logger.Printf("[WARN] [appstop] %s: put back, but the start was not done: %v", h.Stack, err) continue } started = append(started, h.Stack) } return started }