package backup import ( "context" "errors" "fmt" "gitea.dooplex.hu/admin/felhom-controller/internal/util" "os" "path/filepath" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // ── The backup side of the guarded update (update arc slice 4, controller v0.237.0) ───────────── // // 09-update-architecture.md §3 decision 1 (operator ruling 2026-09-02): the safety copy for an update // is a VERIFIED RECENT BACKUP as a PRECONDITION — not a new copy mechanism invented for the update // path. So everything in this file composes machinery that already exists and is proven live: // the Tier-2 unit restore's own predicate (R-102/R-103), the nightly legs (DB dump, volume dump, // unit capture, Tier-2 mirror), the pre-restore safety dump (R-361) and the R-379 hold. // // The stacks package cannot import this one, so the update job reaches all of it through the // stacks.UpdateGuards interface, implemented by an adapter in cmd/controller/main.go. // Tier2RestorePoint is the answer to "could this app be restored from its Tier-2 copy, and from // when?" — the predicate the destructive „Teljes visszaállítás" action is gated on. // // EXTRACTED, NOT DUPLICATED (slice 4). Until v0.237.0 this computation lived inline in the backups // page handler (buildAppBackupRows). The update path needs exactly the same question answered, and // a second copy of a predicate is how this project's two copies of `namespaceRoot` came to differ // (R-203). So there is one function, and the page and the update both call it. type Tier2RestorePoint struct { // Restorable — the copy holds an OPENABLE recovery unit (Tier2Coverage.CanRestoreUnit). Restorable bool // CopyDate — the date the unit restore NAMES: the package's own manifest date, falling back to the // copy date (Tier2Coverage.UnitRestoreDate, R-403). RFC3339 as recorded, "" when unknown. CopyDate string // CopyDateProven — a copy actually SUCCEEDED (LastSuccess is set), never merely an attempt (R-101). CopyDateProven bool // PackagePreserved — the newest run PRESERVED an older package instead of refreshing it (R-403). PackagePreserved bool // CopyLastSuccess — the RFC3339 time of the last Tier-2 copy that succeeded. CopyLastSuccess string } // restorePointFromCoverage is the pure half of the predicate. func restorePointFromCoverage(cov Tier2Coverage) Tier2RestorePoint { pkgDate, preserved := cov.UnitRestoreDate() return Tier2RestorePoint{ Restorable: cov.CanRestoreUnit(), CopyDate: pkgDate, CopyDateProven: cov.CopyLastSuccess != "", PackagePreserved: preserved, CopyLastSuccess: cov.CopyLastSuccess, } } // Tier2UnitRestorePoint resolves the app's recorded Tier-2 copy and returns the restore point. The // error is the same refusal Tier2RestoreCoverage raises (no copy, drive gone, pre-v2 layout). func (m *Manager) Tier2UnitRestorePoint(stackName string) (Tier2RestorePoint, error) { cov, err := m.Tier2RestoreCoverage(stackName) if err != nil { return Tier2RestorePoint{}, err } return restorePointFromCoverage(cov), nil } // ProvenCopyTime returns WHEN the data this copy would restore was last proven copied, and false when // there is no proven, restorable copy at all. // // WHY NOT CopyDate, measured rather than assumed. CopyDate is the unit MANIFEST's created_at, and a // capture rewrites the manifest only when the app's DEFINITION changes (compose, app.yaml, controller // version) — a nightly DB dump keeps the same file name, so it does not move it. Measured on demo-hp // 2026-09-13: bookstack's Tier-2 mirror held `bookstack-mariadb.sql` written 2026-09-13T00:30Z while // its manifest still read 2026-09-12T02:15:29Z. Judging "recent" by that date would call a fresh copy // stale — and, worse, a "back up first" run would not move it either on a quiet app, so the update // would be refused forever. // // So the age is the last SUCCESSFUL copy (LastSuccess), which the Tier-2 run records only when it // actually mirrored the unit — EXCEPT when the run preserved an older package (R-403), in which case // the package date is the honest one, because that is what the copy really holds. func (p Tier2RestorePoint) ProvenCopyTime() (time.Time, bool) { if !p.Restorable || !p.CopyDateProven { return time.Time{}, false } src := p.CopyLastSuccess if p.PackagePreserved { src = p.CopyDate } t, err := time.Parse(time.RFC3339, src) if err != nil { return time.Time{}, false } return t, true } // ── R-475: any backup tier lets an app update (operator ruling 2026-09-13, controller v0.239.0) ──── // // Until v0.239.0 the update's precondition was Tier2UnitRestorePoint alone, so an app with no second // drive could never be updated — even with a fresh recovery unit on its own drive and an off-site // snapshot from last night. The ruling: every backup counts. Tier2UnitRestorePoint itself is NOT // changed; the backups page still calls it for the „Teljes visszaállítás" action, which really does // restore from the second drive only. // Backup tiers, as the update precondition and the hold sentence name them. const ( UpdateTierLocal = 1 // the app's own recovery unit on its drive — „helyi" on the restore page UpdateTierSecondDrive = 2 // the Tier-2 mirror on another drive UpdateTierOffsite = 3 // the off-site restic repository ) // updateTierOrder is the preference order the ruling set: the second drive, then the app's own unit, // then off-site. The first tier holding a copy the caller ACCEPTS is chosen. var updateTierOrder = []int{UpdateTierSecondDrive, UpdateTierLocal, UpdateTierOffsite} // updateTierOrderBindData (R-479, operator ruling 2026-09-13, v0.241.0) is the order for an app whose // DATA lives in bind-mounted files outside its recovery unit: second drive, OFF-SITE, own unit. The own // unit then holds the definition and the database dumps but not the files, so a route back that names // it would restore settings and not data — measured on demo-hp with gokapi (v0.239.0: „a beállítások // visszaálltak … adatot nem"). Off-site carries the mandatory file legs; it comes before the unit. var updateTierOrderBindData = []int{UpdateTierSecondDrive, UpdateTierOffsite, UpdateTierLocal} // DataOutsideUnit reports whether the app keeps data in bind-mounted files that the recovery unit does // not hold — i.e. the app has classified binds. Nil provider or no binds → false (the unit holds the // data: named volumes and database dumps). Pinned by TestR479_. func (m *Manager) DataOutsideUnit(stackName string) bool { if m == nil || m.stackProvider == nil { return false } binds, has := m.stackProvider.GetStackClassifiedBinds(stackName) return has && len(binds) > 0 } // UpdateTierOrderFor is the tier order the update walks for this app (R-475 / R-479). func (m *Manager) UpdateTierOrderFor(stackName string) []int { if m.DataOutsideUnit(stackName) { return updateTierOrderBindData } return updateTierOrder } // UpdateCopyHolds is the customer phrase for what a copy on `tier` holds for this app — the second half // of the R-479 ruling: the hold sentence names WHAT the chosen copy holds, not only where it is. func (m *Manager) UpdateCopyHolds(stackName string, tier int) string { outside := m.DataOutsideUnit(stackName) switch tier { case UpdateTierLocal: if outside { return "csak a beállításokat és az adatbázist tartalmazza, a fájlokat nem" } return "a beállításokat, az adatbázist és az adatköteteket tartalmazza" case UpdateTierSecondDrive: if outside { return "a beállításokat, az adatbázist és a fájlokat tartalmazza" } return "a beállításokat, az adatbázist és az adatköteteket tartalmazza" case UpdateTierOffsite: if outside { return "a beállításokat, az adatbázist és a fájlokat tartalmazza" } return "a beállításokat, az adatbázist és az adatköteteket tartalmazza" } return "" } // UpdateTierLabel is a tier's name in the customer's hold sentence. "" for an unknown tier. func UpdateTierLabel(tier int) string { switch tier { case UpdateTierSecondDrive: return "második meghajtó" case UpdateTierLocal: return "saját meghajtó" case UpdateTierOffsite: return "távoli mentés" } return "" } // updateOffsiteCheckTimeout bounds the off-site lookup. An update must not stall on an unreachable // Storage Box: past this the off-site copy counts as ABSENT (with a WARN), and the update carries on // with backing up first. A var only so a test can shorten it. var updateOffsiteCheckTimeout = 15 * time.Second // UpdateTierPoint is one proven, restorable copy of an app on one tier. type UpdateTierPoint struct { Tier int // At is when the data in that copy was last proven written: Tier 2 ProvenCopyTime, Tier 1 the // newest artifact of the unit (ListRestorePoints), Tier 3 the newest snapshot for the app. At time.Time } // UpdateRestorePoints walks the tiers in preference order (2, 1, 3) and returns the FIRST copy that // accept admits (nil accepts any), whether one was found, and every copy it looked at on the way. // // It stops at the first accepted copy, so a box with a fresh second-drive copy never touches the // network. The AGE rule is the caller's (stacks applies backup_max_age through accept) — that is what // makes "the age rule applies to whichever tier is chosen" one rule, not three (R-475 Scenario M). func (m *Manager) UpdateRestorePoints(ctx context.Context, stackName string, accept func(UpdateTierPoint) bool) (UpdateTierPoint, bool, []UpdateTierPoint) { var seen []UpdateTierPoint for _, tier := range m.UpdateTierOrderFor(stackName) { p, ok := m.updateTierPoint(ctx, stackName, tier) if !ok { continue } seen = append(seen, p) if accept == nil || accept(p) { return p, true, seen } } return UpdateTierPoint{}, false, seen } func (m *Manager) updateTierPoint(ctx context.Context, stackName string, tier int) (UpdateTierPoint, bool) { switch tier { case UpdateTierSecondDrive: get := m.updateTier2PointFn if get == nil { get = m.Tier2UnitRestorePoint } rp, err := get(stackName) if err != nil { if m.isDebug() { m.logger.Printf("[DEBUG] [backup] update precondition for %s: no Tier-2 copy (%v)", stackName, err) } return UpdateTierPoint{}, false } at, ok := rp.ProvenCopyTime() return UpdateTierPoint{Tier: tier, At: at}, ok case UpdateTierLocal: list := m.updateTier1PointsFn if list == nil { list = m.ListRestorePoints } pts, _ := list(stackName) for _, rp := range pts { if at, err := time.Parse(time.RFC3339, rp.Time); err == nil { return UpdateTierPoint{Tier: tier, At: at}, true } } return UpdateTierPoint{}, false case UpdateTierOffsite: times := m.updateOffsiteTimesFn if times == nil { if m.settings == nil || !m.OffboxConfigured() { return UpdateTierPoint{}, false } times = m.OffsiteSnapshotTimes } cctx, cancel := context.WithTimeout(ctx, updateOffsiteCheckTimeout) defer cancel() got, err := times(cctx) if err != nil { if !errors.Is(err, errNoOffsiteTarget) { m.logger.Printf("[WARN] [backup] update precondition for %s: the off-site copy could not be checked within %s (%v) — counted as ABSENT", stackName, updateOffsiteCheckTimeout, err) } return UpdateTierPoint{}, false } if at, ok := got[stackName]; ok && !at.IsZero() { return UpdateTierPoint{Tier: tier, At: at}, true } } return UpdateTierPoint{}, false } // CanBackUpApp reports whether "back up first" can run for this app at all right now — the second // half of R-475 Scenario L: an app with no copy anywhere is refused only when this is false too. // Cheap and read-only; RunAppBackupNow re-checks everything when it actually runs. func (m *Manager) CanBackUpApp(stackName string) (bool, string) { if m == nil { return false, "backup is not enabled on this box" } if m.stackProvider == nil { return false, "stack provider not configured" } if m.migrationActive() { return false, "a data migration is running" } drivePath := m.GetAppDrivePath(stackName) if drivePath == "" || !filepath.IsAbs(drivePath) { return false, "the app's drive cannot be resolved" } if m.settings != nil && (m.settings.IsDisconnected(drivePath) || m.settings.IsDecommissioned(drivePath)) { return false, fmt.Sprintf("the app's drive %s is not available", drivePath) } return true, "" } // UpdateBusy reports whether something else is ALREADY touching this app's data, which refuses an // update before anything moves (slice 4 Scenario D). The reason is operator-English; the customer // sentence is chosen by the caller. // // IsRunning is box-wide, deliberately: the backup/restore single-flight is box-wide, and an update's // "back up first" leg needs that same flag — an update started beside a running backup would either // wait on it invisibly or fail half-way. func (m *Manager) UpdateBusy(stackName string) (bool, string) { if m == nil { return false, "" } if m.IsRunning() { return true, "a backup or restore is running (single-flight held)" } if st := m.RestoreStatus(); st.Running { return true, fmt.Sprintf("restore op %q is running for %q", st.Op, st.Stack) } for _, held := range m.appStop.HeldStacks() { if held == stackName { return true, "an app-data operation (volume dump / export / reconstitute) is holding it" } } return false, "" } // ErrUpdateBackupNoUnit is returned when a "back up first" run completed but the app still has no // openable Tier-2 unit — typically Tier 2 is switched off for the app or has no second target. var ErrUpdateBackupNoUnit = util.MsgError("err.backup.a_frissites_elotti_mentes_lefutott_de") // RunAppBackupNow runs THIS app's backup legs now, in the nightly order, and then its Tier-2 copy: // database dump(s) → volume dump (if the app has named volumes) → recovery-unit capture → Tier-2 // mirror. It is the "back up first" of slice 4 Scenario B. // // Composed, not reinvented: every leg is the one runDBDumpsInternal and RunAllTier2 already run, // including the R-181 reserve (admitApp) before the first write and the R-166 app-stop marker inside // DumpAppVolumesSafe. What differs is only the scope — one app instead of all of them — because an // update must not bounce every other app on the box to back up one. func (m *Manager) RunAppBackupNow(ctx context.Context, stackName string) error { if m.stackProvider == nil { return fmt.Errorf("stack provider not configured") } if m.migrationActive() { return util.MsgError("err.backup.adatathelyezes_folyamatban_a_mentes_most_nem") } if err := m.acquireRunning(); err != nil { return err } m.logger.Printf("[INFO] [backup] update pre-backup for %s: starting (DB dump → volume dump → unit capture → Tier 2)", stackName) start := time.Now() var nsRoot string legErr := func() error { defer m.releaseRunning() defer m.beginAdmissionRun()() drivePath := m.GetAppDrivePath(stackName) if drivePath == "" || !filepath.IsAbs(drivePath) { return util.MsgError("err.backup.az_alkalmazas_meghajtoja_nem_hatarozhato_meg") } if m.settings != nil && (m.settings.IsDisconnected(drivePath) || m.settings.IsDecommissioned(drivePath)) { return util.MsgError("err.backup.az_alkalmazas_meghajtoja_nem_elerheto", drivePath) } if !m.admitApp(stackName) { return util.MsgError("err.backup.nincs_eleg_szabad_hely_a_menteshez", drivePath) } nsRoot = m.namespaceRoot(drivePath) discover := m.discoverDBs if discover == nil { discover = func(ctx context.Context) ([]DiscoveredDB, error) { return DiscoverDatabases(ctx, m.logger, m.isDebug(), m.knownStackNames()) } } dbs, err := discover(ctx) if err != nil { return util.MsgError("err.backup.adatbazis_felderites_sikertelen", err) } dumped := 0 for _, db := range dbs { if db.StackName != stackName { continue } res := DumpOne(ctx, db, AppDBDumpPath(nsRoot, stackName), m.logger, m.isDebug()) if res.Error != nil { return util.MsgError("err.backup.adatbazis_mentes_sikertelen", db.ContainerName, res.Error) } dumped++ m.logger.Printf("[INFO] [backup] update pre-backup for %s: database dump OK (%s, %s)", stackName, db.ContainerName, humanizeBytes(res.Size)) } if len(m.stackProvider.GetDockerVolumes(stackName)) > 0 { dump := m.dumpVolumesSafe if dump == nil { dump = m.DumpAppVolumesSafe } if err := dump(stackName); err != nil { return util.MsgError("err.backup.kotetmentes_sikertelen", err) } m.logger.Printf("[INFO] [backup] update pre-backup for %s: volume dump OK", stackName) } if err := m.CaptureRecoveryUnit(stackName); err != nil { return util.MsgError("err.backup.a_mentesi_egyseg_rogzitese_sikertelen", err) } m.logger.Printf("[INFO] [backup] update pre-backup for %s: recovery unit captured (%d database dump(s))", stackName, dumped) return nil }() if legErr != nil { m.logger.Printf("[ERROR] [backup] update pre-backup for %s FAILED after %s: %v", stackName, time.Since(start).Round(time.Millisecond), legErr) return legErr } m.updatePreBackupTail(stackName, nsRoot, time.Now()) m.logger.Printf("[INFO] [backup] update pre-backup for %s: complete in %s", stackName, time.Since(start).Round(time.Millisecond)) return nil } // updatePreBackupTail is what "back up first" does after the capture succeeded (R-475). // // 1. It marks the app's OWN unit as proven current NOW. CaptureRecoveryUnit leaves the manifest alone // when nothing changed (the checksum skip), and Tier 1's age is the newest artifact's mtime — so on // an app with no database and no named volume a fresh "back up first" would leave Tier 1 as old as // its last definition change, and the update would be refused forever. That is the trap // ProvenCopyTime documents for Tier 2, one tier down. The capture has just compared the unit with // the live definition, so "current as of now" is exactly what it established. // 2. It runs the Tier-2 copy, and a Tier-2 failure is a WARN, not a failure: the update may lean on // any tier, and the Tier-1 unit it can lean on was just written. Pinned by // TestR475_PreBackupTail_Tier2FailureIsAWarnAndTheOwnUnitIsFresh. func (m *Manager) updatePreBackupTail(stackName, nsRoot string, now time.Time) { if nsRoot != "" { if err := os.Chtimes(RecoveryUnitManifestPath(nsRoot, stackName), now, now); err != nil { m.logger.Printf("[WARN] [backup] update pre-backup for %s: could not mark the recovery unit as proven current (%v) — its own-unit copy may read older than it is", stackName, err) } } runOne := m.perAppTier2 if runOne == nil { runOne = m.RunTier2 } if err := runOne(stackName); err != nil { m.logger.Printf("[WARN] [backup] update pre-backup for %s: Tier 2 copy FAILED: %v — not fatal: the app's own recovery unit was just captured, and an update may lean on any tier (R-475)", stackName, err) } } // WriteUpdateSafetyDump takes the last-minute database copy an update makes just before it moves the // pin: "the state the customer was in a minute ago". It is writeSafetyDump (R-361) unchanged — the // same `pre-restore-` undo naming, the same pruning to three, the same never-the-canonical-name rule // — so it is also picked up by the same exclusions (it never enters a manifest's db_dumps). // // Returns the paths written; an app with no database returns (nil, nil), which is a no-op and never a // failure (measured in writeSafetyDump: `len(mine) == 0` returns an empty set). func (m *Manager) WriteUpdateSafetyDump(ctx context.Context, stackName string) ([]string, error) { nsRoot := m.AppNamespaceRoot(stackName) if nsRoot == "" { return nil, util.MsgError("err.backup.az_alkalmazas_mentesi_helye_nem_hatarozhato") } set, err := m.writeSafetyDump(ctx, stackName, nsRoot) if err != nil { return nil, err } var paths []string for _, f := range set.Files { paths = append(paths, f.Path) } if len(paths) == 0 { m.logger.Printf("[INFO] [backup] update safety dump for %s: the app has no database — nothing to copy (no-op)", stackName) } else { m.logger.Printf("[INFO] [backup] update safety dump for %s: %d file(s) %v", stackName, len(paths), paths) } return paths, nil } // UpdateHoldFmt is the customer sentence for an app held after a failed update. Arguments: the app, // the time of the failure, the TIER of the copy it can be restored from (UpdateTierLabel), and that // copy's PROVEN date. One named string so a test asserts it verbatim instead of retyping Hungarian // (R-364). Since v0.239.0 (R-475) it names the tier: the copy may be on any of three, and each is // restored from a different place on the Mentések page. const UpdateHoldFmt = "A(z) %s frissítése %s-kor nem sikerült, és az alkalmazás nem indult el az új verzióval. " + "Az alkalmazás biztonsági okból leállítva marad, hogy az adatai ne sérüljenek. " + "Visszaállítható a Mentések oldalon ebből a biztonsági mentésből: %s, %s — ez a másolat %s." // UpdateHoldTierFmt is the v0.239.0–v0.240.0 sentence, kept for a hold that recorded a tier but not // what the copy holds (CopyHolds empty). const UpdateHoldTierFmt = "A(z) %s frissítése %s-kor nem sikerült, és az alkalmazás nem indult el az új verzióval. " + "Az alkalmazás biztonsági okból leállítva marad, hogy az adatai ne sérüljenek. " + "Visszaállítható a Mentések oldalon ebből a biztonsági mentésből: %s, %s." // UpdateHoldLegacyFmt is the v0.237.0–v0.238.1 sentence, kept for a hold written before the tier was // recorded (CopyTier 0) — every such hold named a Tier-2 copy, but it did not SAY so, and rewriting // it now would state a fact the record does not hold. const UpdateHoldLegacyFmt = "A(z) %s frissítése %s-kor nem sikerült, és az alkalmazás nem indult el az új verzióval. " + "Az alkalmazás biztonsági okból leállítva marad, hogy az adatai ne sérüljenek. " + "Visszaállítható a(z) %s-i biztonsági mentésből a Mentések oldalon." // holdTimeZone is where the customer-facing hold sentence renders its times. The same zone the web // layer renders the Mentések page's copy dates in (web.getTimezone), so the date in the hold text and // the date on the page it points at are the same string. func holdTimeZone() *time.Location { if loc, err := time.LoadLocation("Europe/Budapest"); err == nil { return loc } return time.UTC } func fmtHoldTime(rfc3339 string) string { t, err := time.Parse(time.RFC3339, rfc3339) if err != nil { return rfc3339 } return t.In(holdTimeZone()).Format("2006-01-02 15:04") } // HoldAfterFailedUpdate records that an app is held stopped because its new version did not come up // healthy. Same storage and same gate as the R-379 hold — every start path that already refuses a // restore hold refuses this one without being touched. // // It returns the error rather than only logging it, unlike holdAppAfterFailedRollback: the update job // has just stopped the app on the strength of this record, and an unrecorded hold is a stopped app // that the next restart button will quietly start again. The caller logs it at ERROR and keeps the // failure on the page. func (m *Manager) HoldAfterFailedUpdate(stackName string, at time.Time, copyDate time.Time, copyTier int) error { return m.HoldAfterFailedUpdateHolding(stackName, at, copyDate, copyTier, "", "") } // HoldAfterFailedUpdateHolding is HoldAfterFailedUpdate with the R-479 phrase for what the copy holds; // "" records none (the tier-only sentence). The adapter in main.go computes the phrase with // UpdateCopyHolds at hold time. // // undoState (v0.263.0) is what a FAILED undo left the data as (stacks.UndoState*), "" when no undo was // attempted. RestoreHoldFor puts it in front of the sentence, so the household reads that the box // already tried to put the app back, and in what state that left the data. func (m *Manager) HoldAfterFailedUpdateHolding(stackName string, at time.Time, copyDate time.Time, copyTier int, copyHolds, undoState string) error { if m == nil || m.settings == nil { return fmt.Errorf("no settings wired — the update hold for %s cannot be persisted", stackName) } h := settings.RestoreHold{ Stack: stackName, At: at.UTC().Format(time.RFC3339), Reason: settings.HoldReasonUpdateFailed, UndoState: undoState, } if !copyDate.IsZero() { h.CopyDate = copyDate.UTC().Format(time.RFC3339) h.CopyTier = copyTier h.CopyHolds = copyHolds } if err := m.settings.SetRestoreHold(h); err != nil { return fmt.Errorf("persisting the update hold for %s: %w", stackName, err) } m.logger.Printf("[WARN] [backup] %s is HELD STOPPED after a failed update (restore point: tier %d %q, %s; holds: %q; undo: %q)", stackName, h.CopyTier, UpdateTierLabel(h.CopyTier), h.CopyDate, h.CopyHolds, h.UndoState) return nil } // isHeld reports whether the nightly legs must leave an app alone: it carries ANY hold, OR a guarded // update is moving it right now. // // THE SECOND HALF WAS FOUND LIVE, v0.238.0 Scenario F on demo-hp 2026-09-13. During the update's // 5-minute health wait the app is not yet held, and the periodic capture ran at 10:17:09 and wrote // the NEW definition (alpine:3.20, which never started) into the app's PRIMARY unit, 53 s before the // hold landed at 10:18:02. The Tier-2 mirror the hold names was intact only because the Tier-2 run is // daily — a nightly Tier-2 falling inside a verify window would have mirrored the broken definition // over the very copy the customer is told to restore from. An app mid-update has a restore point that // must not move, exactly like a held one. func (m *Manager) isHeld(stackName string) bool { held, _ := m.RestoreHoldFor(stackName) if held { return true } return m.updatingCheck != nil && m.updatingCheck(stackName) } // SetUpdatingCheck wires the "is a guarded update moving this app" question (stacks.Manager.IsUpdating). // INIT-ONLY, in main.go — pinned by TestSlice4_UpdatingCheckIsWiredAtStartup. The backup package cannot // import stacks, which is why it is a seam. func (m *Manager) SetUpdatingCheck(fn func(stackName string) bool) { m.updatingCheck = fn } // clearUpdateHoldAfterRestore lifts an UPDATE hold once a person has restored the app successfully. // // "A person clears it by restoring" — slice 4 Part 3. The restore just put the app back on the // definition and data of its recovery unit and started it, which is the exact route back the hold // text names; leaving the hold in place would refuse the next restart of an app that is now fine. // // A RESTORE hold (R-379) is deliberately NOT cleared here: that hold means a previous restore already // left the database in an unknown state, and it stays operator-cleared (`-clear-restore-hold`). func (m *Manager) clearUpdateHoldAfterRestore(stackName string) { if m.settings == nil { return } h, ok := m.settings.GetRestoreHold(stackName) if !ok || h.Reason != settings.HoldReasonUpdateFailed { return } if _, err := m.settings.ClearRestoreHold(stackName); err != nil { m.logger.Printf("[ERROR] [backup] %s was restored, but its update hold could not be cleared: %v", stackName, err) return } m.logger.Printf("[INFO] [backup] %s: restore completed — the update hold (set %s) is CLEARED", stackName, h.At) }