package backup import ( "bufio" "context" "encoding/json" "io" "os" "os/exec" "sync" "time" ) // Offsite backup progress (v0.147.0, feedback slice 4c). // // THE PROBLEM: „Távoli mentés most" started a background restic run and redirected with „A távoli // mentés elindult". After that the page polled a status field whose only values were running / ok / // error. For a first offsite push of tens of gigabytes over SFTP that is 20+ minutes of a spinner // with no total, no percentage and no indication of WHICH app is being pushed — indistinguishable // from a hang. // // restic already reports all of it: `backup --json` writes newline-delimited status objects to // stdout. We only had to stop throwing them away — the existing runner seam uses CombinedOutput(), // which buffers everything until exit. // // SCOPE: the MANUAL trigger only. The nightly scheduled run stays silent (nobody is watching a // progress bar at 03:00, and a sink left installed would keep publishing stale percentages into a // page that never asked). The sink is installed for the duration of a manual run and cleared after. // OffboxProgress is a snapshot of an in-flight manual offsite backup. type OffboxProgress struct { Active bool `json:"active"` CurrentApp string `json:"current_app"` Percent float64 `json:"percent"` // 0..100, restic's byte-based percent_done BytesDone int64 `json:"bytes_done"` TotalBytes int64 `json:"total_bytes"` DoneHuman string `json:"done_human"` TotalHuman string `json:"total_human"` // FilesDone/TotalFiles matter more than they look. On an INCREMENTAL run where nothing changed, // restic transfers no new bytes: bytes_done stays 0 (it is `omitempty`, so it is not even in the // JSON) and percent_done stays 0 for the whole run, while restic still walks every file. Measured // on the demo box: a 430MB immich push sat at 0% for 40+ seconds and then completed. A byte-only // bar is therefore indistinguishable from a hang precisely in the COMMON case. File counts move // in that case, so the page falls back to them. FilesDone int64 `json:"files_done"` TotalFiles int64 `json:"total_files"` } // resticStatusLine is the subset of restic's `--json` status object we consume. restic emits several // message_types (status, summary, error, verbose_status); anything that is not "status" is ignored // here rather than treated as garbage, because restic adds new types between versions and an unknown // type must never break the run. Schema captured from // restic 0.14.0 (the version in the controller image) via a live `backup --dry-run --json`: // // {"message_type":"status","percent_done":0,"total_files":1,"total_bytes":112} // {"message_type":"status","percent_done":0.558,"total_files":173,"files_done":87, // "total_bytes":166878,"bytes_done":93161,"current_files":[...]} // // Note every numeric field except percent_done is `omitempty` on restic's side: a zero simply is not // in the JSON. That is why an incremental run reports no bytes_done at all rather than an explicit 0. type resticStatusLine struct { MessageType string `json:"message_type"` PercentDone float64 `json:"percent_done"` // 0..1 TotalBytes int64 `json:"total_bytes"` BytesDone int64 `json:"bytes_done"` TotalFiles int64 `json:"total_files"` FilesDone int64 `json:"files_done"` } // resticProgress is one parsed status line. type resticProgress struct { Percent float64 // 0..100 BytesDone int64 TotalBytes int64 FilesDone int64 TotalFiles int64 } // parseResticStatus parses ONE line of restic --json output. // // Kept as a pure function precisely so it can be tested without restic, a network, or a repo — the // parser is the part that silently rots when restic changes its output, and a progress bar that // quietly stops moving is worse than no progress bar at all. func parseResticStatus(line string) (resticProgress, bool) { var s resticStatusLine if err := json.Unmarshal([]byte(line), &s); err != nil { return resticProgress{}, false } if s.MessageType != "status" { return resticProgress{}, false } pct := s.PercentDone * 100 // restic revises its total as the scan proceeds, so percent_done legitimately moves backwards // mid-run and has been seen slightly above 1 near completion. Clamp — a bar wider than its track // is a visible bug. if pct < 0 { pct = 0 } if pct > 100 { pct = 100 } return resticProgress{ Percent: pct, BytesDone: s.BytesDone, TotalBytes: s.TotalBytes, FilesDone: s.FilesDone, TotalFiles: s.TotalFiles, }, true } // offboxProgressState is the published snapshot, guarded independently of the Manager mutex so a // poll never blocks behind the running backup. type offboxProgressState struct { mu sync.Mutex cur OffboxProgress live bool } func (p *offboxProgressState) begin() { p.mu.Lock() p.cur = OffboxProgress{Active: true} p.live = true p.mu.Unlock() } func (p *offboxProgressState) end() { p.mu.Lock() p.cur = OffboxProgress{} p.live = false p.mu.Unlock() } func (p *offboxProgressState) setApp(app string) { p.mu.Lock() if p.live { // A new app resets the byte counters: restic's percentages are per-invocation, and carrying // the previous app's 100% into the next app's start would show a bar that jumps backwards. p.cur.CurrentApp = app p.cur.Percent, p.cur.BytesDone, p.cur.TotalBytes = 0, 0, 0 p.cur.FilesDone, p.cur.TotalFiles = 0, 0 p.cur.DoneHuman, p.cur.TotalHuman = "", "" } p.mu.Unlock() } func (p *offboxProgressState) update(r resticProgress) { p.mu.Lock() if p.live { p.cur.Percent, p.cur.BytesDone, p.cur.TotalBytes = r.Percent, r.BytesDone, r.TotalBytes p.cur.FilesDone, p.cur.TotalFiles = r.FilesDone, r.TotalFiles p.cur.DoneHuman, p.cur.TotalHuman = humanizeBytes(r.BytesDone), humanizeBytes(r.TotalBytes) } p.mu.Unlock() } func (p *offboxProgressState) snapshot() OffboxProgress { p.mu.Lock() defer p.mu.Unlock() return p.cur } // OffboxProgressSnapshot is the poll surface for the „Távoli mentés" page. func (m *Manager) OffboxProgressSnapshot() OffboxProgress { return m.offboxProgress.snapshot() } // offboxStreamRunner is the streaming restic-exec seam: like offboxRunner, but calls onLine for each // stdout line AS IT ARRIVES instead of only returning the buffered output at exit. Tests inject a // fake that emits canned `--json` status lines, so the whole progress path is exercised without // restic, a network or a repo. type offboxStreamRunner func(ctx context.Context, env []string, onLine func(string), args ...string) ([]byte, error) // SetOffboxStreamRunner installs the streaming seam (nil → the real streaming exec). func (m *Manager) SetOffboxStreamRunner(r offboxStreamRunner) { m.offboxStreamRunner = r } func (m *Manager) streamRunner() offboxStreamRunner { if m.offboxStreamRunner != nil { return m.offboxStreamRunner } return defaultOffboxStreamRunner } // defaultOffboxStreamRunner runs restic with stdout scanned line-by-line. stderr is captured whole // (restic's --json progress goes to stdout; errors go to stderr) and appended to the returned output // so callers keep the same error-diagnosis material CombinedOutput gave them. func defaultOffboxStreamRunner(ctx context.Context, env []string, onLine func(string), args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, "restic", args...) cmd.Env = append(os.Environ(), env...) stdout, err := cmd.StdoutPipe() if err != nil { return nil, err } var stderr syncBuf cmd.Stderr = &stderr if err := cmd.Start(); err != nil { return nil, err } var tail lineTail scanner := bufio.NewScanner(stdout) // restic status lines are small, but a --json summary listing many paths can exceed the 64KB // default; a scanner that dies mid-run would silently freeze the progress bar. scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := scanner.Text() tail.add(line) if onLine != nil { onLine(line) } } _, _ = io.Copy(io.Discard, stdout) werr := cmd.Wait() // Keep only the tail of stdout: the full --json stream of a large backup is megabytes of status // spam, and every caller uses this output for error diagnosis (and lock-pattern matching) only. out := append(tail.bytes(), stderr.bytes()...) return out, werr } // lineTail keeps the last N lines seen, so error diagnosis has context without buffering the whole // --json stream. type lineTail struct { lines []string } func (t *lineTail) add(s string) { const keep = 40 t.lines = append(t.lines, s) if len(t.lines) > keep { t.lines = t.lines[len(t.lines)-keep:] } } func (t *lineTail) bytes() []byte { var b []byte for _, l := range t.lines { b = append(b, l...) b = append(b, '\n') } return b } type syncBuf struct { mu sync.Mutex b []byte } func (s *syncBuf) Write(p []byte) (int, error) { s.mu.Lock() s.b = append(s.b, p...) s.mu.Unlock() return len(p), nil } func (s *syncBuf) bytes() []byte { s.mu.Lock() defer s.mu.Unlock() return append([]byte{}, s.b...) } // resticBackupStep is resticStep's streaming twin, used ONLY by the app-backup leg when a manual run // has a progress sink installed. It keeps resticStep's crash-lock self-heal semantics by delegating // the retry path to resticStep (a retry after an unlock is rare and does not need progress). func (m *Manager) resticBackupStep(ctx context.Context, env, base []string, label, app string, args ...string) ([]byte, error) { if !m.offboxProgress.snapshot().Active { return m.resticStep(ctx, env, base, label, args...) // nightly / no watcher: unchanged path } m.offboxProgress.setApp(app) full := append(append([]string{}, base...), args...) // --json turns on the machine-readable progress stream. It is added ONLY here, so the nightly // run's output format (and everything that greps it) is untouched. full = append(full, "--json") out, err := m.streamRunner()(ctx, env, func(line string) { if r, ok := parseResticStatus(line); ok { m.offboxProgress.update(r) } }, full...) if err == nil || !offboxLockRe.Match(out) { return out, err } // Lock collision: fall back to the non-streaming step, which owns the unlock --remove-all // self-heal. Progress stalls for that one retry; correctness beats a moving bar. m.logger.Printf("[WARN] [offbox] %s hit a lock during a manual run — retrying via the self-healing step", label) return m.resticStep(ctx, env, base, label, args...) } // beginManualProgress installs the progress sink for a manual run and returns the cleanup func. func (m *Manager) beginManualProgress() func() { m.offboxProgress.begin() started := time.Now() return func() { m.logger.Printf("[INFO] [offbox] manual run progress reporting ended after %s", time.Since(started).Round(time.Second)) m.offboxProgress.end() } }