R-379/R-380: put the customer's undo copy back when a database restore fails
gates / gates (push) Successful in 11s
gates / gates (push) Successful in 11s
R-379 and R-380 were one failure. Both ended with a half-restored database; the only difference was whether it looked broken. Postgres emptied and crash-looped; MariaDB applied part of the dump and reported health=healthy with a zero-row schema-version table. Measured live on demo-hp 2026-08-22. The undo copy was already taken and already good - proven by hand that day on both engines. Nothing in the product could apply it. Now it does, with the same ImportDump call, before any restart and inside the DB-only window. The WHOLE undo set, matched on this run's stamp. writeSafetyDump returned one path for an app with two databases; a rollback on that would restore one and leave the other half-written. When the rollback also fails the app is HELD STOPPED (operator ruling): a running app on a half-written database lets the customer make the damage permanent. Every start path refuses it - customer button, appstop Recover, boot sweep - via the shared driveStartGate, checked ABOVE its driveless early return because these apps have no drive. The marker is ended so nothing auto-restarts it. The row goes red. Cleared with --clear-restore-hold, an operator CLI route. --single-transaction is a belt on Postgres only; MariaDB DDL is not transactional and that is why the rollback is the fix. R-381: the engine's stderr stops reaching the customer (615 bytes on MariaDB, its middle rows out of their own database) and starts reaching the operator log, which never had it. R-382: the summary log prints the volume count it already held. Undo copies resolve to their own app, are marked IsUndo, and are capped at 3 per app, pruned from the capture side. The reported render-as-an-app symptom did NOT reproduce - the live page was read first and had zero occurrences. Tests 1468 -> 1483. Eight red-proofs; ONE PASSED and is reported: the R-381 behavioural test injected below ImportDump. A guard at that layer now convicts.
This commit is contained in:
@@ -84,6 +84,8 @@ func main() {
|
||||
abandonStatus := flag.Bool("abandon-status", false, "R-241: print the state of this box's off-site abandonment countdown (set-aside path, due date, days left) and exit. Read-only.")
|
||||
abandonExtend := flag.Int("abandon-extend", 0, "R-241 (operator): extend a running abandonment countdown by N days from now, then exit. Refuses when no countdown is running.")
|
||||
abandonStop := flag.Bool("abandon-stop", false, "R-241 (operator): stop a running abandonment countdown, then exit. The set-aside history is kept and nothing is deleted. Refuses when no countdown is running.")
|
||||
listRestoreHolds := flag.Bool("restore-holds", false, "R-379 (operator): list apps this controller is deliberately holding stopped after a failed database restore whose rollback also failed, then exit. Read-only.")
|
||||
clearRestoreHold := flag.String("clear-restore-hold", "", "R-379 (operator): clear the restore hold on APP so it can start again, then exit. Clearing does NOT repair the database — check the app's data first; the undo copies are in its unit's db-dumps dir.")
|
||||
flag.Parse()
|
||||
|
||||
if *showVersion {
|
||||
@@ -160,6 +162,48 @@ func main() {
|
||||
|
||||
// §7.5 (R-241) — the operator's abandonment levers. Grouped in one block, before the server
|
||||
// starts, exactly like the other CLI subcommands: each loads config + settings, acts, and exits.
|
||||
// R-379 (operator): the way OUT of a hold. A CLI subcommand rather than an API route, matching
|
||||
// --abandon-stop above: the controller's HTTP surface is authenticated as the CUSTOMER, and
|
||||
// clearing a hold is a decision that should follow an operator looking at the database. Reaching
|
||||
// this needs `docker exec` into the guest, which is the operator gate this project already uses.
|
||||
if *listRestoreHolds || *clearRestoreHold != "" {
|
||||
cfg, err := config.LoadPermissive(*configPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: loading config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
lg := log.New(os.Stderr, "", 0)
|
||||
sett, err := settings.Load(cfg.Paths.DataDir+"/settings.json", lg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: loading settings: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *listRestoreHolds {
|
||||
holds := sett.ListRestoreHolds()
|
||||
if len(holds) == 0 {
|
||||
fmt.Println("no restore holds in force")
|
||||
os.Exit(0)
|
||||
}
|
||||
for _, h := range holds {
|
||||
fmt.Printf("%s\theld since %s\n replay error : %s\n rollback err : %s\n", h.Stack, h.At, h.ReplayError, h.RollbackErr)
|
||||
}
|
||||
os.Exit(0)
|
||||
}
|
||||
cleared, err := sett.ClearRestoreHold(*clearRestoreHold)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "restore-hold: clearing %s: %v\n", *clearRestoreHold, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if !cleared {
|
||||
// "there was nothing to clear" is not success — a silent 0 here would let an operator
|
||||
// believe they had unblocked an app whose name they mistyped.
|
||||
fmt.Fprintf(os.Stderr, "no restore hold in force for %q — nothing was changed\n", *clearRestoreHold)
|
||||
os.Exit(2)
|
||||
}
|
||||
fmt.Printf("restore hold cleared for %s — the app may be started again. Check its data first: the undo copies are in its unit's db-dumps dir.\n", *clearRestoreHold)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if *abandonStatus || *abandonExtend > 0 || *abandonStop {
|
||||
cfg, err := config.LoadPermissive(*configPath)
|
||||
if err != nil {
|
||||
@@ -981,6 +1025,31 @@ func main() {
|
||||
}
|
||||
notifier.NotifyBackupRunFailures(rs.Message, d)
|
||||
})
|
||||
// R-379/R-380: an app HELD after a failed database restore whose rollback also failed.
|
||||
//
|
||||
// ROUTED THROUGH `backup_run_failures`, and the reuse is deliberate but imperfect — say so
|
||||
// rather than let a future reader assume it was the natural fit. That type is operator-only
|
||||
// (`notify.operatorOnlyEvents`), carries severity `error` (a VALID token — `warn` would
|
||||
// coerce to `info` and email nobody, which is R-329 and still live elsewhere), and its
|
||||
// payload is exactly {app, leg, reason}. What it is NOT is a restore event: it says "backup
|
||||
// run". **No operator-facing RESTORE event type exists**, and minting one is a two-repo
|
||||
// change with a hub deploy, which this task's baselines hold fixed. The leg is named
|
||||
// `restore-hold` so the record is unambiguous to whoever reads it.
|
||||
//
|
||||
// DELIBERATELY NOT `backup_failed`: customer-enabled by default, Hungarian "A biztonsági
|
||||
// mentés sikertelen!", and this is a deliberate hold rather than a backup failure. R-171 one
|
||||
// path over.
|
||||
backupMgr.SetRestoreHoldNotify(func(stack string, replayErr, rollbackErr error) {
|
||||
d := notify.BackupRunFailuresDetails{RunKind: "offsite-reconstitute", Failed: 1, Attempted: 1}
|
||||
reason := "rollback failed"
|
||||
if rollbackErr != nil {
|
||||
reason = rollbackErr.Error()
|
||||
}
|
||||
d.Apps = append(d.Apps, notify.RunFailureDetail{App: stack, Leg: "restore-hold", Reason: reason})
|
||||
msg := fmt.Sprintf("App %q is HELD STOPPED: its off-site database restore failed AND the rollback to the customer's own pre-restore copy also failed. "+
|
||||
"The app will not start from any path until the hold is cleared. Replay error: %v. Rollback error: %v", stack, replayErr, rollbackErr)
|
||||
notifier.NotifyBackupRunFailures(msg, d)
|
||||
})
|
||||
// 3a: the pre-push enlargement gate blocked an app's userdata push (config+DB still saved). Edge-
|
||||
// triggered by the engine (only NEW blocks notify), so the hub's per-event-type cooldown suffices —
|
||||
// no controller-side timer (the hub owns cooldown).
|
||||
@@ -1774,6 +1843,19 @@ type driveStartGate struct {
|
||||
}
|
||||
|
||||
func (g driveStartGate) MayStart(stackName string) (bool, string) {
|
||||
// R-379/R-380 HOLDER, checked FIRST and deliberately ABOVE the `HDD_PATH == ""` early return
|
||||
// below. The apps this hold exists for are DRIVELESS — a failed database restore is the case,
|
||||
// and 40 of the 53 catalogue apps have no drive at all — so a hold placed after that return
|
||||
// would never be consulted for the exact class it was built for.
|
||||
//
|
||||
// This is holder #4, and putting it here rather than in bootDriveGate is what gives it two
|
||||
// callers with one implementation: the boot sweep reaches it through bootDriveGate, and the
|
||||
// app-stop guard's Recover reaches it directly. A hold only one path honours is not a hold.
|
||||
if g.sett != nil {
|
||||
if h, ok := g.sett.GetRestoreHold(stackName); ok {
|
||||
return false, "held after a failed restore whose rollback also failed (" + h.At + ") — clear the hold to start it"
|
||||
}
|
||||
}
|
||||
if g.mgr == nil {
|
||||
return false, "no stack manager wired — the drive cannot be determined"
|
||||
}
|
||||
@@ -2444,7 +2526,9 @@ func (a *exportAdapter) GetImportRoot() string { return a.mgr.GetImportRoot() }
|
||||
// GetStackNamespaceRoot (R-203) — the felhom-data namespace root, NOT the drive path. The appbackup
|
||||
// path helpers all take this; passing HDD_PATH straight in is what made the export plan and the
|
||||
// off-site capture set describe different directories on the system-data fallback.
|
||||
func (a *exportAdapter) GetStackNamespaceRoot(name string) string { return a.mgr.StackNamespaceRoot(name) }
|
||||
func (a *exportAdapter) GetStackNamespaceRoot(name string) string {
|
||||
return a.mgr.StackNamespaceRoot(name)
|
||||
}
|
||||
|
||||
func (a *exportAdapter) GetStackHDDPath(name string) string {
|
||||
s, ok := a.mgr.GetStack(name)
|
||||
|
||||
Reference in New Issue
Block a user