178 lines
7.1 KiB
Go
178 lines
7.1 KiB
Go
package backup
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
)
|
|
|
|
// D22 (operator ruling 2026-10-21 20:05, R-1195): a restore cut by a power loss puts the app back on start.
|
|
//
|
|
// MEASURED (rehearsal 2026-10-20, audits/rehearsal-2026-10-20/B6-failures/s2-power-cut-mid-restore/): a power cut 14 s
|
|
// into a full off-site restore left BookStack running on the backup's data — the household's newer page gone — while
|
|
// the page said only „start it again". The undo copy taken before the change sat on disk, unused, because nothing that
|
|
// runs at start knew it existed.
|
|
//
|
|
// Now every restore that takes the copy of the live volumes (offbox_restore_undo.go — the off-site restore and, since
|
|
// D22, the local unit restore) also, once the copy is whole:
|
|
// - copies the app's live definition files (docker-compose.yml, .felhom.yml, app.yaml — app.yaml keeps its secrets
|
|
// encrypted, as on disk) into <copy>/definition/;
|
|
// - writes <copy>/undo.json (which volumes were copied, which the backup adds);
|
|
// - records the copy's folder in the app-stop marker (AppStopGuard.SetUndo).
|
|
//
|
|
// At start, Recover finds the marker still active (the restore never reached its end) and calls
|
|
// PutBackInterruptedRestore BEFORE it restarts the app: stop, pour every copied volume back, remove the volumes the
|
|
// backup added, write the live definition back, remove the copy. A cut BEFORE the copy was whole leaves no Undo entry,
|
|
// and the old restart stays (nothing had moved yet). A put-back that fails is a failed recovery: the app is not
|
|
// started, the marker stays and the next start tries again.
|
|
//
|
|
// Pinned by r1195_crash_putback_test.go.
|
|
|
|
const (
|
|
undoJournalName = "undo.json"
|
|
undoDefinitionIn = "definition"
|
|
)
|
|
|
|
// undoJournal is what PutBackInterruptedRestore needs to know about a copy, written when the copy is whole.
|
|
type undoJournal struct {
|
|
Stack string `json:"stack"`
|
|
Copied []string `json:"copied"`
|
|
Absent []string `json:"absent,omitempty"`
|
|
Definition bool `json:"definition"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
// undoDefinitionFiles are the files of an app's live definition the crash put-back writes back.
|
|
var undoDefinitionFiles = []string{"docker-compose.yml", ".felhom.yml", "app.yaml"}
|
|
|
|
// persistUndoForCrash makes the copy usable after a power cut (see the file comment). Best-effort: a failure is
|
|
// logged and the restore goes on — the in-process put-back does not need any of this.
|
|
func (m *Manager) persistUndoForCrash(stack string, c *liveVolumeCopy) {
|
|
if c == nil || c.dir == "" {
|
|
return
|
|
}
|
|
j := undoJournal{Stack: stack, Copied: c.copied, Absent: c.absent, At: time.Now().UTC()}
|
|
if m.stackProvider != nil {
|
|
if composePath, ok := m.stackProvider.GetStackComposePath(stack); ok && composePath != "" {
|
|
defDir := filepath.Join(c.dir, undoDefinitionIn)
|
|
if err := copyDefinitionFiles(filepath.Dir(composePath), defDir); err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: the live definition could not be kept beside the restore copy: %v — a power cut would put back the volumes only", stack, err)
|
|
} else {
|
|
j.Definition = true
|
|
}
|
|
}
|
|
}
|
|
data, err := json.MarshalIndent(j, "", " ")
|
|
if err == nil {
|
|
err = atomicWriteFile(filepath.Join(c.dir, undoJournalName), data)
|
|
}
|
|
if err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: the restore copy's journal could not be written: %v — a power cut would restart the app without putting it back", stack, err)
|
|
return
|
|
}
|
|
if err := m.appStop.SetUndo(stack, c.dir); err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: the restore copy could not be recorded for a power cut: %v", stack, err)
|
|
return
|
|
}
|
|
m.logger.Printf("[INFO] [backup] %s: the restore copy is recorded — a power cut from here on puts the app back on start (R-1195)", stack)
|
|
}
|
|
|
|
// copyDefinitionFiles copies the definition files that exist in stackDir into dst (0600). docker-compose.yml must exist.
|
|
func copyDefinitionFiles(stackDir, dst string) error {
|
|
if err := os.MkdirAll(dst, 0o700); err != nil {
|
|
return err
|
|
}
|
|
for _, name := range undoDefinitionFiles {
|
|
data, err := os.ReadFile(filepath.Join(stackDir, name))
|
|
if err != nil {
|
|
if os.IsNotExist(err) && name != "docker-compose.yml" {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
if err := atomicWriteFile(filepath.Join(dst, name), data); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// atomicWriteFile writes data to path through a synced temp file and a rename (0600).
|
|
func atomicWriteFile(path string, data []byte) error {
|
|
tmp := path + ".tmp"
|
|
f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := f.Write(data); err != nil {
|
|
f.Close()
|
|
os.Remove(tmp)
|
|
return err
|
|
}
|
|
if err := f.Sync(); err != nil {
|
|
f.Close()
|
|
os.Remove(tmp)
|
|
return err
|
|
}
|
|
if err := f.Close(); err != nil {
|
|
os.Remove(tmp)
|
|
return err
|
|
}
|
|
return os.Rename(tmp, path)
|
|
}
|
|
|
|
// CrashPutBack is the InterruptedRestorePutBacker the controller wires at start, before the backup manager exists.
|
|
type CrashPutBack struct {
|
|
stop func(stack string) error
|
|
writeDef func(stack, defDir string) error // writes the kept definition back (nil = volumes only)
|
|
logger *log.Logger
|
|
// seams for tests; nil = the restore's own volume helpers (Docker)
|
|
volReplay func(stack, dir string) (int, error)
|
|
volRemove func(vol string) error
|
|
}
|
|
|
|
// NewCrashPutBack builds the put-back. stop stops the app; writeDef writes a kept definition folder back as the app's.
|
|
func NewCrashPutBack(stop func(string) error, writeDef func(stack, defDir string) error, logger *log.Logger) *CrashPutBack {
|
|
return &CrashPutBack{stop: stop, writeDef: writeDef, logger: logger}
|
|
}
|
|
|
|
// PutBackInterruptedRestore pours undoDir back for stack and removes it. The app is left stopped.
|
|
func (p *CrashPutBack) PutBackInterruptedRestore(stack, undoDir string) error {
|
|
data, err := os.ReadFile(filepath.Join(undoDir, undoJournalName))
|
|
if err != nil {
|
|
return fmt.Errorf("reading the restore copy's journal: %w", err)
|
|
}
|
|
var j undoJournal
|
|
if err := json.Unmarshal(data, &j); err != nil {
|
|
return fmt.Errorf("reading the restore copy's journal: %w", err)
|
|
}
|
|
if j.Stack != stack {
|
|
return fmt.Errorf("the restore copy at %s belongs to %q, not %q", undoDir, j.Stack, stack)
|
|
}
|
|
if p.stop != nil {
|
|
if err := p.stop(stack); err != nil {
|
|
p.logger.Printf("[WARN] [appstop] %s: stopping before the put-back failed: %v (continuing)", stack, err)
|
|
}
|
|
}
|
|
mgr := &Manager{logger: p.logger, volumeReplayFrom: p.volReplay, undoVolumeRemoveFn: p.volRemove}
|
|
if err := mgr.putBack(stack, &liveVolumeCopy{dir: undoDir, copied: j.Copied, absent: j.Absent}); err != nil {
|
|
return err
|
|
}
|
|
if j.Definition && p.writeDef != nil {
|
|
if err := p.writeDef(stack, filepath.Join(undoDir, undoDefinitionIn)); err != nil {
|
|
return fmt.Errorf("writing the live definition back: %w", err)
|
|
}
|
|
}
|
|
p.logger.Printf("[WARN] [appstop] %s: the interrupted restore is put back from %s — %d volume(s) poured back, %d removed, definition=%v (R-1195)",
|
|
stack, undoDir, len(j.Copied), len(j.Absent), j.Definition)
|
|
if err := os.RemoveAll(undoDir); err != nil {
|
|
p.logger.Printf("[WARN] [appstop] %s: could not remove the used restore copy %s: %v", stack, undoDir, err)
|
|
} else {
|
|
_ = os.Remove(filepath.Dir(undoDir))
|
|
}
|
|
return nil
|
|
}
|