Files
felhom-controller/controller/internal/backup/restore_record.go
T
admin 0fe315b759
gates / gates (push) Successful in 15s
v0.246.0: an interrupted restore is told; the recovery-code reminder waits until the box can take it
MinAgent: 0.131.0 (unchanged). Requires hub v0.117.0 for restore_interrupted.

R-550 (operator ruling: fix). A design reversed and recorded: the restore
op-status was in memory by choice. Now restore-status.json in DataDir, written
atomically at both ends of an op. At startup a record still marked running
becomes a failed, interrupted result kept per app until that app's next
restore, shown on /backups/restore and the off-site wizard, and raised once as
restore_interrupted. Cooldowns stay in memory.

R-546. The R-543 reminder bar consults the agent's own preflight ok (every
blocking item, not a copy of pbs_storage_id), cached 60 s, probed only while
paused. /backup/escrow shows a waiting card that polls and reloads instead of
red crosses and English diagnostics. POST /api/escrow/start refuses 409 before
staging or starting - the direct path chaos night used. Unknown readiness keeps
the bar.

Red-proofs (each seen failing): restore record across restart; main() calls
both startup functions; startup helper with loading skipped; restore page card;
bar held back; waiting card; start refusal. go build/vet/test ./... green, 28
packages; controller_gates --fast all OK.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-17 10:45:29 +02:00

134 lines
5.2 KiB
Go

package backup
import (
"encoding/json"
"os"
"sort"
"time"
)
// R-550 (operator ruling "fix", 2026-09-17) — the restore record survives a restart.
//
// A DESIGN REVERSED, AND RECORDED AS SUCH. opstatus.go was deliberately in-memory ("same precedent as
// notification cooldowns"). Chaos night round 10 measured the cost: a restore accepted at 23:26:08Z,
// the box hard-reset four seconds later, and afterwards the status answered the Go zero value — the
// household had pressed a button, been told it started, and could never learn whether it finished.
// The operator reversed the choice for the RESTORE RECORD ONLY; notification cooldowns stay in memory.
//
// Shape: one JSON file in the controller's state directory (DataDir, beside settings.json), written
// atomically (atomicWrite: tmp + rename) at BOTH ends of an op. At startup a record still marked
// running is, by construction, a restore nothing is running any more: it becomes a terminal failure
// (Interrupted, RestoreInterruptedMessage) and a per-app notice that stays until that app's next
// restore. LoadRestoreRecord reports the conversion ONCE, so the caller raises restore_interrupted once.
//
// No path set (tests that build a bare Manager, a box with backup disabled) = the old in-memory
// behaviour, silently: persistence is a property of the wired controller, not of every Manager.
// RestoreInterruptedMessage is what the household reads when the box stopped mid-restore.
const RestoreInterruptedMessage = "A visszaállítás megszakadt (a doboz újraindult) — indítsd el újra."
// restoreRecordFile is the on-disk shape.
type restoreRecordFile struct {
Running bool `json:"running"`
Op string `json:"op,omitempty"`
Stack string `json:"stack,omitempty"`
StartedAt time.Time `json:"started_at,omitempty"`
Last *RestoreOpResult `json:"last,omitempty"`
Interrupted map[string]RestoreOpResult `json:"interrupted,omitempty"`
}
// SetRestoreRecordPath wires persistence. Call before LoadRestoreRecord and before serving requests.
func (m *Manager) SetRestoreRecordPath(path string) {
m.mu.Lock()
defer m.mu.Unlock()
m.opRecordPath = path
}
// LoadRestoreRecord restores the record at startup. It returns the restore that was interrupted by the
// stop — non-nil exactly once per interruption — so the caller can log it and raise restore_interrupted.
func (m *Manager) LoadRestoreRecord() *RestoreOpResult {
m.mu.Lock()
defer m.mu.Unlock()
if m.opRecordPath == "" {
return nil
}
b, err := os.ReadFile(m.opRecordPath)
if err != nil {
if !os.IsNotExist(err) && m.logger != nil {
m.logger.Printf("[WARN] [backup] restore record unreadable (%v) — starting with no restore history", err)
}
return nil
}
var rec restoreRecordFile
if err := json.Unmarshal(b, &rec); err != nil {
if m.logger != nil {
m.logger.Printf("[WARN] [backup] restore record corrupt (%v) — starting with no restore history", err)
}
return nil
}
m.opLast = rec.Last
m.opInterrupted = rec.Interrupted
if !rec.Running || rec.Stack == "" {
if m.logger != nil && len(m.opInterrupted) > 0 {
m.logger.Printf("[INFO] [backup] restore record loaded: %d interrupted restore notice(s) still shown", len(m.opInterrupted))
}
return nil
}
res := RestoreOpResult{
Op: rec.Op, Stack: rec.Stack, OK: false,
Message: RestoreInterruptedMessage, FinishedAt: time.Now(), Interrupted: true,
}
m.opLast = &res
if m.opInterrupted == nil {
m.opInterrupted = map[string]RestoreOpResult{}
}
m.opInterrupted[rec.Stack] = res
m.opRunning = false
if m.logger != nil {
m.logger.Printf("[WARN] [backup] restore of %s (%s, started %s) was INTERRUPTED by a controller stop — recorded as failed; the household is told to run it again",
rec.Stack, rec.Op, rec.StartedAt.Format(time.RFC3339))
}
m.persistRestoreRecordLocked()
out := res
return &out
}
// InterruptedRestore reports the standing interrupted-restore notice for one app, if any.
func (m *Manager) InterruptedRestore(stack string) (RestoreOpResult, bool) {
m.mu.Lock()
defer m.mu.Unlock()
r, ok := m.opInterrupted[stack]
return r, ok
}
// InterruptedRestores lists every standing notice, sorted by app, for the restore page.
func (m *Manager) InterruptedRestores() []RestoreOpResult {
m.mu.Lock()
defer m.mu.Unlock()
out := make([]RestoreOpResult, 0, len(m.opInterrupted))
for _, r := range m.opInterrupted {
out = append(out, r)
}
sort.Slice(out, func(i, j int) bool { return out[i].Stack < out[j].Stack })
return out
}
// persistRestoreRecordLocked writes the current op-status. Caller holds m.mu. A failed write is logged
// and the in-memory status carries on — the page still works for this process's lifetime.
func (m *Manager) persistRestoreRecordLocked() {
if m.opRecordPath == "" {
return
}
rec := restoreRecordFile{
Running: m.opRunning, Op: m.opName, Stack: m.opStack, StartedAt: m.opStartedAt,
Last: m.opLast, Interrupted: m.opInterrupted,
}
b, err := json.Marshal(rec)
if err == nil {
err = atomicWrite(m.opRecordPath, b, 0o600)
}
if err != nil && m.logger != nil {
m.logger.Printf("[WARN] [backup] could not persist the restore record to %s: %v (kept in memory)", m.opRecordPath, err)
}
}