feat(backup): async restore family — no proxy-timeout error page on a succeeding restore (v0.102.0)

Re-adjudicates F4: /backup/restore, /backup/tier2/restore, /backup/offbox/restore
blocked the HTTP request until completion, so through cloudflared's 100s cap a
customer got an error page while the restore succeeded (offbox worse — bounded
on r.Context(), canceling the SFTP restore mid-flight). Convert all three to the
offboxRun async shape: fast-path IsRunning refuse, background goroutine
(offbox ctx off r.Context() -> Background+30m), instant redirect. Add mutex-
guarded op-status (opstatus.go) + GET /api/backup/restore-status + a 3s-polling
backups.html banner (neutral running, red on failure). Restore single-flight
unchanged. Tests + red-proof (sync handler blocks indefinitely vs <500ms async).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PSK5g6qYLknKj8u3QAFEr6

controller/internal/web/async_restore_test.go

@@ -0,0 +1,157 @@
+package web
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/config"
+	"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
+)
+
+// blockProvider is a StackDataProvider whose StopStack blocks on a channel — so a Tier-2 restore
+// parks mid-flight while the test asserts the HANDLER already returned (async) and that a concurrent
+// POST is refused without launching a second restore.
+type blockProvider struct {
+	hdd     string
+	release chan struct{}
+	stops   int32 // atomic: number of StopStack calls == number of restores that actually launched
+}
+
+func (p *blockProvider) GetStackComposePath(string) (string, bool) { return "", false }
+func (p *blockProvider) ListDeployedStacks() []backup.StackSummary { return nil }
+func (p *blockProvider) GetStackHDDMounts(string) []string         { return nil }
+func (p *blockProvider) GetStackHDDPath(string) string             { return p.hdd }
+func (p *blockProvider) GetDockerVolumes(string) []string          { return nil }
+func (p *blockProvider) StopStack(string) error {
+	atomic.AddInt32(&p.stops, 1)
+	<-p.release // park here until the test releases it
+	return nil
+}
+func (p *blockProvider) StartStack(string) error         { return nil }
+func (p *blockProvider) RefreshAndIsRunning(string) bool { return true }
+func (p *blockProvider) GetStackRecoveryInfo(string) (backup.RecoveryInfo, bool) {
+	return backup.RecoveryInfo{}, false
+}
+func (p *blockProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
+func (p *blockProvider) RecreateStackFromUnit(string, string, map[string]string) error {
+	return nil
+}
+
+func newAsyncRestoreServer(t *testing.T) (*Server, *blockProvider, *backup.Manager) {
+	t.Helper()
+	tmp := t.TempDir()
+	live := filepath.Join(tmp, "usb")
+	dest := filepath.Join(tmp, "flash")
+	lg := log.New(io.Discard, "", 0)
+	sett, err := settings.Load(filepath.Join(tmp, "settings.json"), lg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range []string{live, dest} {
+		if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
+		Enabled: true, Method: "rsync", DestinationPath: dest, LastRun: "2026-07-06T03:30:00Z", LastStatus: "ok",
+	}); err != nil {
+		t.Fatal(err)
+	}
+	// a recorded Tier-2 copy dir so RestoreTier2Files proceeds to StopStack (where we block).
+	if err := os.MkdirAll(filepath.Join(dest, "backups", "secondary", "app", "appdata"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	cfg := &config.Config{}
+	cfg.Paths.DataDir = tmp
+	m := backup.NewManager(cfg, sett, lg)
+	prov := &blockProvider{hdd: live, release: make(chan struct{})}
+	m.SetStackProvider(prov)
+	s := &Server{cfg: cfg, backupMgr: m, logger: lg}
+	return s, prov, m
+}
+
+func postTier2(s *Server) *httptest.ResponseRecorder {
+	req := httptest.NewRequest(http.MethodPost, "/backup/tier2/restore", strings.NewReader("stack_name=app"))
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	w := httptest.NewRecorder()
+	s.backupTier2RestoreHandler(w, req)
+	return w
+}
+
+func waitFor(t *testing.T, cond func() bool, msg string) {
+	t.Helper()
+	// Budget past waitForHealthy's 3s post-restore settling sleep (+ copier).
+	for i := 0; i < 700; i++ {
+		if cond() {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatalf("timeout waiting for: %s", msg)
+}
+
+// B1 — the restore handler returns INSTANTLY (302) while the restore runs in the background, and the
+// op-status transitions running → terminal. RED-PROOF: on the pre-fix synchronous handler (calling
+// RestoreTier2Files inline) this POST blocks in StopStack until the channel is released, so the
+// sub-500ms response assertion FAILS.
+func TestBackupTier2Restore_AsyncReturnsInstantly(t *testing.T) {
+	s, prov, m := newAsyncRestoreServer(t)
+
+	start := time.Now()
+	w := postTier2(s)
+	elapsed := time.Since(start)
+
+	if elapsed > 500*time.Millisecond {
+		t.Fatalf("handler blocked %v — restore is not async (the F4 shape)", elapsed)
+	}
+	if w.Code != http.StatusFound {
+		t.Fatalf("want 302, got %d", w.Code)
+	}
+	if loc := w.Header().Get("Location"); !strings.Contains(loc, "elindult") {
+		t.Fatalf("redirect should carry the 'elindult' flash; got %q", loc)
+	}
+	// the background restore is now parked in StopStack → op-status shows running.
+	waitFor(t, func() bool { return m.RestoreStatus().Running }, "restore op running")
+	st := m.RestoreStatus()
+	if st.Op != "tier2-restore" || st.Stack != "app" {
+		t.Fatalf("running status = %+v", st)
+	}
+	// release → the restore completes → terminal status.
+	close(prov.release)
+	waitFor(t, func() bool { return !m.RestoreStatus().Running && m.RestoreStatus().Last != nil }, "restore terminal")
+	if got := atomic.LoadInt32(&prov.stops); got != 1 {
+		t.Fatalf("StopStack called %d times, want exactly 1", got)
+	}
+}
+
+// B2 — a concurrent restore POST while one runs is REFUSED (fast-path IsRunning) and does NOT launch a
+// second restore. RED-PROOF: removing the IsRunning() fast-path lets the second POST launch a second
+// goroutine → StopStack count becomes 2.
+func TestBackupTier2Restore_DoubleClickRefused(t *testing.T) {
+	s, prov, m := newAsyncRestoreServer(t)
+
+	// first restore → parks in StopStack (acquires the single-flight running flag).
+	_ = postTier2(s)
+	waitFor(t, func() bool { return m.IsRunning() }, "first restore holding the single-flight lock")
+
+	// second restore while the first runs → refused with the "már fut" flash, no second launch.
+	w2 := postTier2(s)
+	if loc := w2.Header().Get("Location"); !strings.Contains(loc, "m%C3%A1r+fut") && !strings.Contains(loc, "már fut") {
+		t.Fatalf("second POST should be refused with 'már fut'; got %q", loc)
+	}
+	if got := atomic.LoadInt32(&prov.stops); got != 1 {
+		t.Fatalf("double-click launched a second restore: StopStack count = %d, want 1", got)
+	}
+
+	close(prov.release)
+	waitFor(t, func() bool { return !m.IsRunning() }, "first restore done")
+}

controller/internal/web/handlers.go

@@ -871,29 +871,28 @@ func (s *Server) backupRestoreHandler(w http.ResponseWriter, r *http.Request) {
 		http.Redirect(w, r, "/backups?flash_error=Ment%C3%A9s+nincs+be%C3%A1ll%C3%ADtva", http.StatusFound)
 		return
 	}
-
-	s.logger.Printf("[WARN] [web] Restore requested: stack=%s, snapshot=%s from %s", stackName, snapshotID, r.RemoteAddr)
-
-	start := time.Now()
-	// Phase 2b: restore from the app's recovery unit (recovers secrets from the guest, fail-closed on
-	// an unrecoverable data-encrypting key; falls back to volume-only restore if no unit exists).
-	err := s.backupMgr.RestoreFromRecoveryUnit(stackName)
-	if err != nil {
-		s.logger.Printf("[ERROR] [web] Restore failed: %v", err)
-		if s.isDebug() {
-			s.logger.Printf("[DEBUG] [web] backupRestoreHandler: stack=%s failed after %s", stackName, time.Since(start))
-		}
-		errMsg := url.QueryEscape("Visszaállítás sikertelen: " + err.Error())
-		http.Redirect(w, r, "/backups?flash_error="+errMsg, http.StatusFound)
+	// Part B: restore is a long SYNCHRONOUS op (F4 — through cloudflared's hard 100s cap the customer
+	// got an error page while it silently succeeded). Fast-path refuse a concurrent op, then run it in
+	// a BACKGROUND goroutine (survives the request; the poll banner shows progress → result).
+	if s.backupMgr.IsRunning() {
+		http.Redirect(w, r, "/backups?flash_error="+url.QueryEscape("Egy mentési/visszaállítási művelet már fut."), http.StatusFound)
 		return
 	}
-
-	if s.isDebug() {
-		s.logger.Printf("[DEBUG] [web] backupRestoreHandler: stack=%s completed in %s", stackName, time.Since(start))
-	}
-
-	msg := url.QueryEscape(stackName + " visszaállítva (" + snapshotID + ").")
-	http.Redirect(w, r, "/backups?flash="+msg, http.StatusFound)
+	s.logger.Printf("[WARN] [web] Restore requested (async): stack=%s, snapshot=%s from %s", stackName, snapshotID, r.RemoteAddr)
+	s.backupMgr.BeginRestoreOp("restore", stackName)
+	go func() {
+		start := time.Now()
+		// Phase 2b: restore from the app's recovery unit (recovers secrets from the guest, fail-closed
+		// on an unrecoverable data-encrypting key; falls back to volume-only restore if no unit exists).
+		if err := s.backupMgr.RestoreFromRecoveryUnit(stackName); err != nil {
+			s.logger.Printf("[ERROR] [web] Restore failed (async): stack=%s: %v", stackName, err)
+			s.backupMgr.EndRestoreOp(false, "Visszaállítás sikertelen: "+err.Error())
+			return
+		}
+		s.logger.Printf("[INFO] [web] Restore completed (async): stack=%s in %s", stackName, time.Since(start))
+		s.backupMgr.EndRestoreOp(true, stackName+" visszaállítva ("+snapshotID+").")
+	}()
+	http.Redirect(w, r, "/backups?flash="+url.QueryEscape("Visszaállítás elindult — az állapot itt frissül."), http.StatusFound)
 }
 
 // backupTier2RestoreHandler (C2, closes F2) restores an app's MISSING user files in place from its
@@ -917,21 +916,28 @@ func (s *Server) backupTier2RestoreHandler(w http.ResponseWriter, r *http.Reques
 		http.Redirect(w, r, "/backups?flash_error=Ment%C3%A9s+nincs+be%C3%A1ll%C3%ADtva", http.StatusFound)
 		return
 	}
-
-	s.logger.Printf("[WARN] [web] Tier-2 file restore requested: stack=%s from %s", stackName, r.RemoteAddr)
-
-	n, err := s.backupMgr.RestoreTier2Files(stackName)
-	if err != nil {
-		s.logger.Printf("[ERROR] [web] Tier-2 file restore failed: %v", err)
-		http.Redirect(w, r, "/backups?flash_error="+url.QueryEscape("Fájl-visszaállítás sikertelen: "+err.Error()), http.StatusFound)
+	// Part B (same async shape as backupRestoreHandler): fast-path refuse, then background goroutine.
+	if s.backupMgr.IsRunning() {
+		http.Redirect(w, r, "/backups?flash_error="+url.QueryEscape("Egy mentési/visszaállítási művelet már fut."), http.StatusFound)
 		return
 	}
-
-	msg := "Nincs hiányzó fájl — minden fájl megvan a helyén."
-	if n > 0 {
-		msg = fmt.Sprintf("%s: %d fájl visszaállítva a másodlagos másolatból.", stackName, n)
-	}
-	http.Redirect(w, r, "/backups?flash="+url.QueryEscape(msg), http.StatusFound)
+	s.logger.Printf("[WARN] [web] Tier-2 file restore requested (async): stack=%s from %s", stackName, r.RemoteAddr)
+	s.backupMgr.BeginRestoreOp("tier2-restore", stackName)
+	go func() {
+		n, err := s.backupMgr.RestoreTier2Files(stackName)
+		if err != nil {
+			s.logger.Printf("[ERROR] [web] Tier-2 file restore failed (async): stack=%s: %v", stackName, err)
+			s.backupMgr.EndRestoreOp(false, "Fájl-visszaállítás sikertelen: "+err.Error())
+			return
+		}
+		msg := "Nincs hiányzó fájl — minden fájl megvan a helyén."
+		if n > 0 {
+			msg = fmt.Sprintf("%s: %d fájl visszaállítva a másodlagos másolatból.", stackName, n)
+		}
+		s.logger.Printf("[INFO] [web] Tier-2 file restore completed (async): stack=%s (%d files)", stackName, n)
+		s.backupMgr.EndRestoreOp(true, msg)
+	}()
+	http.Redirect(w, r, "/backups?flash="+url.QueryEscape("Fájl-visszaállítás elindult — az állapot itt frissül."), http.StatusFound)
 }
 
 // settingsBaseData is the shared identity block used by every settings-family subpage

controller/internal/web/offbox_handlers.go

@@ -134,13 +134,25 @@ func (s *Server) offboxRestoreHandler(w http.ResponseWriter, r *http.Request) {
 		offboxRedirect(w, r, "Hiányzó alkalmazás.", true)
 		return
 	}
-	dest := filepath.Join(s.cfg.Paths.DataDir, "offbox-restore", app)
-	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute)
-	defer cancel()
-	if err := s.backupMgr.RestoreOffbox(ctx, app, dest); err != nil {
-		s.logger.Printf("[ERROR] [web] off-box restore %s: %v", app, err)
-		offboxRedirect(w, r, "A visszaállítás sikertelen: "+err.Error(), true)
+	// Part B: fast-path refuse a concurrent op, then run async on a BACKGROUND context. The old code
+	// bounded on r.Context()+30m — a proxy read-timeout then CANCELED the SFTP restore mid-flight
+	// (worse than F4: not just an error page, an aborted restore). Background ctx fixes that.
+	if s.backupMgr.IsRunning() {
+		offboxRedirect(w, r, "Egy mentési/visszaállítási művelet már fut.", true)
 		return
 	}
-	offboxRedirect(w, r, "A(z) "+app+" visszaállítva ide (ellenőrzésre): "+dest, false)
+	dest := filepath.Join(s.cfg.Paths.DataDir, "offbox-restore", app)
+	s.backupMgr.BeginRestoreOp("offbox-restore", app)
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+		defer cancel()
+		if err := s.backupMgr.RestoreOffbox(ctx, app, dest); err != nil {
+			s.logger.Printf("[ERROR] [web] off-box restore %s (async): %v", app, err)
+			s.backupMgr.EndRestoreOp(false, "A visszaállítás sikertelen: "+err.Error())
+			return
+		}
+		s.logger.Printf("[INFO] [web] off-box restore %s completed (async) → %s", app, dest)
+		s.backupMgr.EndRestoreOp(true, "A(z) "+app+" visszaállítva ide (ellenőrzésre): "+dest)
+	}()
+	offboxRedirect(w, r, "A NAS-visszaállítás elindult — az állapot itt frissül.", false)
 }

controller/internal/web/templates/backups.html

@@ -13,6 +13,10 @@
 <div class="flash flash-error">{{.Backup.FlashError}}</div>
 {{end}}{{end}}
 
+<!-- Part B: async restore progress banner — polls /api/backup/restore-status; neutral while running,
+     red only on failure (exception-color principle). Hidden until a restore op is seen. -->
+<div id="restore-banner" class="flash" style="display:none"></div>
+
 {{if not .Backup}}
 <div class="backup-empty-state">
     <div class="backup-empty-icon">&#128737;</div>
@@ -597,6 +601,43 @@
 {{end}}
 
 <script>
+// Part B: restore-progress banner. Polls the async restore op-status every 3s. Shows a neutral
+// "in progress" while running (including on a fresh page load mid-op), success on completion, and the
+// error state ONLY on failure. Stops polling when idle after a terminal result was shown.
+(function(){
+    var banner = document.getElementById('restore-banner');
+    if (!banner) return;
+    var sawRunning = false;
+    function opLabel(op){ return op === 'tier2-restore' ? 'Fájl-visszaállítás'
+                        : op === 'offbox-restore' ? 'NAS-visszaállítás' : 'Visszaállítás'; }
+    function render(st){
+        if (st.running) {
+            sawRunning = true;
+            banner.className = 'flash';
+            banner.style.display = 'block';
+            banner.textContent = opLabel(st.op) + ' folyamatban' + (st.stack ? ': ' + st.stack : '') + '…';
+            return;
+        }
+        if (st.last && sawRunning) {
+            banner.style.display = 'block';
+            if (st.last.ok) {
+                banner.className = 'flash flash-success';
+                banner.textContent = st.last.message || (opLabel(st.last.op) + ' kész.');
+            } else {
+                banner.className = 'flash flash-error';
+                banner.textContent = st.last.message || (opLabel(st.last.op) + ' sikertelen.');
+            }
+        }
+    }
+    function poll(){
+        fetch('/api/backup/restore-status', {headers: {'Accept':'application/json'}})
+            .then(function(r){ return r.json(); })
+            .then(function(j){ if (j && j.data) render(j.data); })
+            .catch(function(){});
+    }
+    poll();
+    setInterval(poll, 3000);
+})();
 function toggleBackupDetail(header) {
     var detail = header.nextElementSibling;
     var icon = header.querySelector('.expand-icon');