package backup import ( "encoding/json" "fmt" "log" "os" "path/filepath" "time" ) // D22 (operator ruling 2026-10-21 20:05, R-1195): a restore cut by a power loss puts the app back on start. // // MEASURED (rehearsal 2026-10-20, audits/rehearsal-2026-10-20/B6-failures/s2-power-cut-mid-restore/): a power cut 14 s // into a full off-site restore left BookStack running on the backup's data — the household's newer page gone — while // the page said only „start it again". The undo copy taken before the change sat on disk, unused, because nothing that // runs at start knew it existed. // // Now every restore that takes the copy of the live volumes (offbox_restore_undo.go — the off-site restore and, since // D22, the local unit restore) also, once the copy is whole: // - copies the app's live definition files (docker-compose.yml, .felhom.yml, app.yaml — app.yaml keeps its secrets // encrypted, as on disk) into /definition/; // - writes /undo.json (which volumes were copied, which the backup adds); // - records the copy's folder in the app-stop marker (AppStopGuard.SetUndo). // // At start, Recover finds the marker still active (the restore never reached its end) and calls // PutBackInterruptedRestore BEFORE it restarts the app: stop, pour every copied volume back, remove the volumes the // backup added, write the live definition back, remove the copy. A cut BEFORE the copy was whole leaves no Undo entry, // and the old restart stays (nothing had moved yet). A put-back that fails is a failed recovery: the app is not // started, the marker stays and the next start tries again. // // Pinned by r1195_crash_putback_test.go. const ( undoJournalName = "undo.json" undoDefinitionIn = "definition" ) // undoJournal is what PutBackInterruptedRestore needs to know about a copy, written when the copy is whole. type undoJournal struct { Stack string `json:"stack"` Copied []string `json:"copied"` Absent []string `json:"absent,omitempty"` Definition bool `json:"definition"` At time.Time `json:"at"` } // undoDefinitionFiles are the files of an app's live definition the crash put-back writes back. var undoDefinitionFiles = []string{"docker-compose.yml", ".felhom.yml", "app.yaml"} // persistUndoForCrash makes the copy usable after a power cut (see the file comment). Best-effort: a failure is // logged and the restore goes on — the in-process put-back does not need any of this. func (m *Manager) persistUndoForCrash(stack string, c *liveVolumeCopy) { if c == nil || c.dir == "" { return } j := undoJournal{Stack: stack, Copied: c.copied, Absent: c.absent, At: time.Now().UTC()} if m.stackProvider != nil { if composePath, ok := m.stackProvider.GetStackComposePath(stack); ok && composePath != "" { defDir := filepath.Join(c.dir, undoDefinitionIn) if err := copyDefinitionFiles(filepath.Dir(composePath), defDir); err != nil { m.logger.Printf("[WARN] [backup] %s: the live definition could not be kept beside the restore copy: %v — a power cut would put back the volumes only", stack, err) } else { j.Definition = true } } } data, err := json.MarshalIndent(j, "", " ") if err == nil { err = atomicWriteFile(filepath.Join(c.dir, undoJournalName), data) } if err != nil { m.logger.Printf("[WARN] [backup] %s: the restore copy's journal could not be written: %v — a power cut would restart the app without putting it back", stack, err) return } // MEASURED 2026-10-24 on Tester 1 (audits/restore-holes-2026-10-24/P1-tester1/): a power cut 2.1 s after this // record left the copy's tar whole but its journal as undo.json.tmp — the rename lived only in the page cache, and // the marker (another filesystem, already synced) named a copy no start could read. The copy's folders are new // (copyLiveVolumes made them), so their own entries are not durable either. Every folder from the copy up to // backups/ is synced BEFORE the marker names the copy; a sync that fails is logged and the copy is still recorded // (an unreadable copy at start holds the app for support, R-1251 H2 — never a start on unknown data). if err := syncUndoDirs(c.dir); err != nil { m.logger.Printf("[WARN] [backup] %s: the restore copy's folders could not be synced to disk: %v — a power cut in the next seconds may leave a copy the start cannot read (the app is then held)", stack, err) } if err := m.appStop.SetUndo(stack, c.dir); err != nil { m.logger.Printf("[WARN] [backup] %s: the restore copy could not be recorded for a power cut: %v", stack, err) return } m.logger.Printf("[INFO] [backup] %s: the restore copy is recorded — a power cut from here on puts the app back on start (R-1195)", stack) } // copyDefinitionFiles copies the definition files that exist in stackDir into dst (0600). docker-compose.yml must exist. func copyDefinitionFiles(stackDir, dst string) error { if err := os.MkdirAll(dst, 0o700); err != nil { return err } for _, name := range undoDefinitionFiles { data, err := os.ReadFile(filepath.Join(stackDir, name)) if err != nil { if os.IsNotExist(err) && name != "docker-compose.yml" { continue } return err } if err := atomicWriteFile(filepath.Join(dst, name), data); err != nil { return err } } return nil } // syncUndoDirs makes a restore copy's folder entries durable: its definition/ folder, the copy folder, and each parent // up to and including backups/ (restoreUndoDir = /backups/restore-undo//). A seam for tests. var syncUndoDirs = func(copyDir string) error { dirs := []string{filepath.Join(copyDir, undoDefinitionIn), copyDir} for d, i := copyDir, 0; i < 3; i++ { // , restore-undo, backups d = filepath.Dir(d) dirs = append(dirs, d) } var first error for _, d := range dirs { if err := syncDir(d); err != nil && !os.IsNotExist(err) && first == nil { first = err } } return first } // syncDir fsyncs one directory, so the entries created or renamed in it survive a power cut. func syncDir(dir string) error { f, err := os.Open(dir) if err != nil { return err } if err := f.Sync(); err != nil { f.Close() return err } return f.Close() } // atomicWriteFile writes data to path through a synced temp file and a rename (0600), then syncs the folder so the // rename itself survives a power cut (R-1251: a journal left as .tmp was measured after a cut). func atomicWriteFile(path string, data []byte) error { tmp := path + ".tmp" f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return err } if _, err := f.Write(data); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Sync(); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Close(); err != nil { os.Remove(tmp) return err } if err := os.Rename(tmp, path); err != nil { return err } return syncDir(filepath.Dir(path)) } // CrashPutBack is the InterruptedRestorePutBacker the controller wires at start, before the backup manager exists. type CrashPutBack struct { stop func(stack string) error writeDef func(stack, defDir string) error // writes the kept definition back (nil = volumes only) logger *log.Logger // seams for tests; nil = the restore's own volume helpers (Docker) volReplay func(stack, dir string) (int, error) volRemove func(vol string) error } // NewCrashPutBack builds the put-back. stop stops the app; writeDef writes a kept definition folder back as the app's. func NewCrashPutBack(stop func(string) error, writeDef func(stack, defDir string) error, logger *log.Logger) *CrashPutBack { return &CrashPutBack{stop: stop, writeDef: writeDef, logger: logger} } // PutBackInterruptedRestore pours undoDir back for stack and removes it. The app is left stopped. func (p *CrashPutBack) PutBackInterruptedRestore(stack, undoDir string) error { data, err := os.ReadFile(filepath.Join(undoDir, undoJournalName)) if err != nil { return fmt.Errorf("reading the restore copy's journal: %w", err) } var j undoJournal if err := json.Unmarshal(data, &j); err != nil { return fmt.Errorf("reading the restore copy's journal: %w", err) } if j.Stack != stack { return fmt.Errorf("the restore copy at %s belongs to %q, not %q", undoDir, j.Stack, stack) } if p.stop != nil { if err := p.stop(stack); err != nil { p.logger.Printf("[WARN] [appstop] %s: stopping before the put-back failed: %v (continuing)", stack, err) } } mgr := &Manager{logger: p.logger, volumeReplayFrom: p.volReplay, undoVolumeRemoveFn: p.volRemove} if err := mgr.putBack(stack, &liveVolumeCopy{dir: undoDir, copied: j.Copied, absent: j.Absent}); err != nil { return err } if j.Definition && p.writeDef != nil { if err := p.writeDef(stack, filepath.Join(undoDir, undoDefinitionIn)); err != nil { return fmt.Errorf("writing the live definition back: %w", err) } } p.logger.Printf("[WARN] [appstop] %s: the interrupted restore is put back from %s — %d volume(s) poured back, %d removed, definition=%v (R-1195)", stack, undoDir, len(j.Copied), len(j.Absent), j.Definition) if err := os.RemoveAll(undoDir); err != nil { p.logger.Printf("[WARN] [appstop] %s: could not remove the used restore copy %s: %v", stack, undoDir, err) } else { _ = os.Remove(filepath.Dir(undoDir)) } return nil }