From 753cd83456f4806012aef66312128add39433847 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 13 Jul 2026 09:47:10 +0200 Subject: [PATCH] =?UTF-8?q?controller:=20.fab=20browser=20download=20?= =?UTF-8?q?=E2=80=94=20the=20existing=20export=20pipeline=20staged=20under?= =?UTF-8?q?=20the=20data=20dir=20+=20a=20guarded=20streaming=20exit=20(est?= =?UTF-8?q?imate-first,=20io.Copy,=20post-stream=20cleanup,=201h=20TTL=20s?= =?UTF-8?q?weep);=20portability=20framing=20(decision=203);=20traversal=20?= =?UTF-8?q?guard=20red-proven,=20export=E2=86=92import=20round-trip=20+=20?= =?UTF-8?q?corrupt-bundle=20refusal=20unit-proven?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01GzammAMzsJTgpQHqxwM2bC --- .../internal/appexport/roundtrip_test.go | 177 ++++++++++++++++ .../internal/web/export_download_test.go | 137 +++++++++++++ controller/internal/web/handler_export.go | 9 + .../internal/web/handler_export_download.go | 189 ++++++++++++++++++ controller/internal/web/server.go | 6 + .../web/templates/backups_restore.html | 89 +++++++++ 6 files changed, 607 insertions(+) create mode 100644 controller/internal/appexport/roundtrip_test.go create mode 100644 controller/internal/web/export_download_test.go create mode 100644 controller/internal/web/handler_export_download.go diff --git a/controller/internal/appexport/roundtrip_test.go b/controller/internal/appexport/roundtrip_test.go new file mode 100644 index 0000000..e86fb45 --- /dev/null +++ b/controller/internal/appexport/roundtrip_test.go @@ -0,0 +1,177 @@ +package appexport + +import ( + "io" + "log" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// v0.124.0 Part 3 (§7D) — the .fab loop's proof at unit level: a real export through +// executeExport produces a bundle that a real executeImport restores to identical content, and +// a corrupted downloaded copy is REFUSED (the .fab's integrity is the gzip CRC + the manifest +// segment validation — there is no per-file checksum; corruption breaks the extract, and the +// import must fail loudly, not restore garbage). + +// rtProvider is a filesystem-only fake: a config-only app (no HDD, no volumes, no DB) so the +// whole loop runs without docker. +type rtProvider struct { + stackDir string + stacksDir string + deployed bool + started bool + savedEnv map[string]string +} + +func (p *rtProvider) GetStackDir(string) (string, bool) { return p.stackDir, true } +func (p *rtProvider) GetStackComposePath(string) (string, bool) { + return filepath.Join(p.stackDir, "docker-compose.yml"), true +} +func (p *rtProvider) GetStackHDDMounts(string) []string { return nil } +func (p *rtProvider) GetStackHDDPath(string) string { return "" } +func (p *rtProvider) IsStackRunning(string) bool { return false } +func (p *rtProvider) StopStack(string) error { return nil } +func (p *rtProvider) StartStack(string) error { p.started = true; return nil } +func (p *rtProvider) GetStackDisplayName(n string) string { return "RT " + n } +func (p *rtProvider) GetStackNeedsHDD(string) bool { return false } +func (p *rtProvider) GetDockerVolumes(string) []string { return nil } +func (p *rtProvider) IsStackDeployed(string) bool { return p.deployed } +func (p *rtProvider) GetDecryptedEnv(string) map[string]string { return nil } +func (p *rtProvider) GetStacksBaseDir() string { return p.stacksDir } +func (p *rtProvider) RefreshStacks() error { return nil } +func (p *rtProvider) RemoveStackVolumes(string) error { return nil } +func (p *rtProvider) SaveEncryptedAppConfig(stackDir string, env map[string]string) error { + p.savedEnv = env + return nil +} + +func waitJob(t *testing.T, e *Exporter) *Job { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + job := e.GetActiveJob() + if job != nil { + job.mu.RLock() + done := job.Done + job.mu.RUnlock() + if done { + return job + } + } + time.Sleep(50 * time.Millisecond) + } + t.Fatal("job did not finish in time") + return nil +} + +func jobErr(j *Job) string { + j.mu.RLock() + defer j.mu.RUnlock() + if j.Error != "" { + return j.Error + } + for _, s := range j.Steps { + if s.Status == "failed" { + return s.Error + } + } + return "" +} + +func TestFabRoundTrip_ExportImportContentEquality(t *testing.T) { + const stack = "rt-app" + lg := log.New(io.Discard, "", 0) + + // Source stack: a compose file + a marker config with known content. + srcStack := t.TempDir() + compose := "services:\n rt-app:\n image: alpine\n" + marker := "MARKER-CONTENT-42\n" + os.WriteFile(filepath.Join(srcStack, "docker-compose.yml"), []byte(compose), 0644) + os.WriteFile(filepath.Join(srcStack, "settings.conf"), []byte(marker), 0644) + + drive := t.TempDir() + prov := &rtProvider{stackDir: srcStack, stacksDir: t.TempDir(), deployed: true} + e := NewExporter(prov, lg, "test") + + // --- export (the REAL pipeline; same producer as a drive export) --- + if err := e.StartExport(ExportRequest{StackName: stack, DestDrive: drive}); err != nil { + t.Fatalf("StartExport: %v", err) + } + job := waitJob(t, e) + if msg := jobErr(job); msg != "" { + t.Fatalf("export failed: %s", msg) + } + entries, _ := os.ReadDir(ExportDir(drive)) + var fabPath string + for _, en := range entries { + if strings.HasSuffix(en.Name(), ".fab") { + fabPath = filepath.Join(ExportDir(drive), en.Name()) + } + } + if fabPath == "" { + t.Fatal("no .fab produced") + } + + // The manifest is readable and names the app (what /api/export/manifest shows pre-import). + man, err := ReadManifestFromFAB(fabPath) + if err != nil { + t.Fatalf("manifest: %v", err) + } + if man.AppName != stack { + t.Fatalf("manifest app = %q", man.AppName) + } + + // --- corrupted copy must be REFUSED (assert the refusal, §10 red-proof of the loop) --- + corrupt := filepath.Join(t.TempDir(), "corrupt.fab") + raw, _ := os.ReadFile(fabPath) + mid := len(raw) / 2 + raw[mid] ^= 0xFF + raw[mid+1] ^= 0xFF + os.WriteFile(corrupt, raw, 0644) + + prov2 := &rtProvider{stackDir: srcStack, stacksDir: t.TempDir(), deployed: false} + e2 := NewExporter(prov2, lg, "test") + if err := e2.StartImport(ImportRequest{FABPath: corrupt}); err != nil { + t.Fatalf("StartImport(corrupt) should start (refusal is async): %v", err) + } + job = waitJob(t, e2) + if msg := jobErr(job); msg == "" { + t.Fatal("a corrupted bundle must FAIL the import (gzip CRC), got success") + } + if prov2.started { + t.Fatal("a refused import must not start the app") + } + if _, err := os.Stat(filepath.Join(prov2.stacksDir, stack, "settings.conf")); !os.IsNotExist(err) { + t.Fatal("a refused import must not restore content") + } + + // --- the clean bundle round-trips: restored content is byte-identical --- + prov3 := &rtProvider{stackDir: srcStack, stacksDir: t.TempDir(), deployed: false} + e3 := NewExporter(prov3, lg, "test") + if err := e3.StartImport(ImportRequest{FABPath: fabPath}); err != nil { + t.Fatalf("StartImport: %v", err) + } + job = waitJob(t, e3) + if msg := jobErr(job); msg != "" { + t.Fatalf("import failed: %s", msg) + } + restoredStack := filepath.Join(prov3.stacksDir, stack) + for name, want := range map[string]string{ + "docker-compose.yml": compose, + "settings.conf": marker, + } { + got, err := os.ReadFile(filepath.Join(restoredStack, name)) + if err != nil { + t.Fatalf("restored %s missing: %v", name, err) + } + if string(got) != want { + t.Errorf("restored %s differs:\n got %q\nwant %q", name, got, want) + } + } + if !prov3.started { + t.Error("import must start the restored app") + } +} diff --git a/controller/internal/web/export_download_test.go b/controller/internal/web/export_download_test.go new file mode 100644 index 0000000..ad29407 --- /dev/null +++ b/controller/internal/web/export_download_test.go @@ -0,0 +1,137 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// .fab download exit (v0.124.0 Part 3) — the containment guard, the post-stream cleanup and +// the TTL sweep. The guard is NON-NEGOTIABLE (§9): only an expected-shaped basename resolved +// strictly inside the staging exports dir is ever opened. + +func downloadTestServer(t *testing.T) (*Server, string) { + t.Helper() + s := testServer(t) + s.cfg.Paths.DataDir = t.TempDir() + dir := s.fabDownloadDir() + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + return s, dir +} + +func fetchDownload(s *Server, file string) *httptest.ResponseRecorder { + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/export/download?file="+file, nil) + s.apiExportDownloadFetch(rr, req) + return rr +} + +// Traversal guard: ../, absolute paths, separators and out-of-dir names must all be refused +// with NO file opened; a legit staged bundle streams byte-exactly and is removed afterwards. +// Red-proof: loosen the guard to raw prefix-matching → the `..` case FAILS (serves the decoy). +func TestFabDownload_TraversalGuardAndCleanup(t *testing.T) { + s, dir := downloadTestServer(t) + + // A decoy OUTSIDE the staging dir that a traversal would reach. + decoy := filepath.Join(filepath.Dir(dir), "decoy.fab") + if err := os.WriteFile(decoy, []byte("DECOY"), 0644); err != nil { + t.Fatal(err) + } + // The legit staged bundle. + want := []byte("FAB-BUNDLE-BYTES") + if err := os.WriteFile(filepath.Join(dir, "actualbudget_20260713-080000.fab"), want, 0644); err != nil { + t.Fatal(err) + } + + refused := []string{ + "..%2Fdecoy.fab", // ../decoy.fab + "..%5Cdecoy.fab", // ..\decoy.fab + "%2Fetc%2Fpasswd", // absolute path + "sub%2Fx.fab", // separator + "..", // bare traversal + ".hidden.fab", // not the exporter's naming (leading dot) + "x.txt", // not a .fab + } + for _, f := range refused { + if rr := fetchDownload(s, f); rr.Code != http.StatusBadRequest { + t.Errorf("file=%q: got %d, want 400", f, rr.Code) + } + } + if body, err := os.ReadFile(decoy); err != nil || string(body) != "DECOY" { + t.Fatal("the decoy outside the staging dir was touched") + } + + // A legit-looking name that does not exist → 404 (never an open of anything else). + if rr := fetchDownload(s, "nosuch_20260713-080000.fab"); rr.Code != http.StatusNotFound { + t.Errorf("missing bundle: got %d, want 404", rr.Code) + } + + // The legit bundle: streamed byte-exactly with attachment headers, then REMOVED. + rr := fetchDownload(s, "actualbudget_20260713-080000.fab") + if rr.Code != http.StatusOK { + t.Fatalf("legit download: got %d (%s)", rr.Code, rr.Body.String()) + } + if rr.Body.String() != string(want) { + t.Fatalf("streamed bytes differ from the staged bundle") + } + if cd := rr.Header().Get("Content-Disposition"); !strings.Contains(cd, `attachment; filename="actualbudget_20260713-080000.fab"`) { + t.Errorf("Content-Disposition = %q", cd) + } + if _, err := os.Stat(filepath.Join(dir, "actualbudget_20260713-080000.fab")); !os.IsNotExist(err) { + t.Error("staged bundle must be removed after a successful stream") + } +} + +// The download API inherits the post-claim auth: an unclaimed box answers 401 before any +// handler runs (asserted per §7C even though it rides the existing RequireAuth). +func TestFabDownload_UnauthenticatedRefused(t *testing.T) { + s, _, _ := claimTestServer(t) + mux := s.fullMux() + rr := httptest.NewRecorder() + mux.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/api/export/download?file=x_1.fab", nil)) + if rr.Code != http.StatusUnauthorized { + t.Fatalf("unauthenticated download: got %d, want 401", rr.Code) + } +} + +// TTL sweep: an aged bundle (and exporter temp) is removed; a fresh bundle and non-bundle +// files are kept. Time is injected — no clock dependency. +func TestFabDownload_TTLSweep(t *testing.T) { + dir := t.TempDir() + now := time.Date(2026, 7, 13, 12, 0, 0, 0, time.UTC) + mk := func(name string, age time.Duration) { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(p, now.Add(-age), now.Add(-age)); err != nil { + t.Fatal(err) + } + } + mk("old_20260713-000000.fab", 2*time.Hour) + mk("stale.fab.tmp", 3*time.Hour) + mk("fresh_20260713-115500.fab", 5*time.Minute) + mk("unrelated.txt", 48*time.Hour) + + if n := sweepFabDownloads(dir, now, time.Hour, nil); n != 2 { + t.Fatalf("sweep removed %d entries, want 2 (the aged bundle + temp)", n) + } + for name, wantGone := range map[string]bool{ + "old_20260713-000000.fab": true, + "stale.fab.tmp": true, + "fresh_20260713-115500.fab": false, + "unrelated.txt": false, + } { + _, err := os.Stat(filepath.Join(dir, name)) + gone := os.IsNotExist(err) + if gone != wantGone { + t.Errorf("%s: gone=%v want %v", name, gone, wantGone) + } + } +} diff --git a/controller/internal/web/handler_export.go b/controller/internal/web/handler_export.go index 5928584..09b4fc9 100644 --- a/controller/internal/web/handler_export.go +++ b/controller/internal/web/handler_export.go @@ -43,6 +43,15 @@ func (s *Server) ServeExportAPI(w http.ResponseWriter, r *http.Request) { case path == "/api/export/status" && r.Method == http.MethodGet: s.apiExportStatus(w, r) + // .fab browser download (v0.124.0): estimate → start (existing pipeline, staging dest) → + // guarded stream. Portability exit only — no scheduling, no status surface. + case path == "/api/export/download/estimate" && r.Method == http.MethodGet: + s.apiExportDownloadEstimate(w, r) + case path == "/api/export/download/start" && r.Method == http.MethodPost: + s.apiExportDownloadStart(w, r) + case path == "/api/export/download" && r.Method == http.MethodGet: + s.apiExportDownloadFetch(w, r) + // GET /api/export/bundles — scan for .fab files on all drives case path == "/api/export/bundles" && r.Method == http.MethodGet: s.apiExportBundles(w, r) diff --git a/controller/internal/web/handler_export_download.go b/controller/internal/web/handler_export_download.go new file mode 100644 index 0000000..fbb1741 --- /dev/null +++ b/controller/internal/web/handler_export_download.go @@ -0,0 +1,189 @@ +package web + +import ( + "encoding/json" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-controller/internal/appexport" +) + +// .fab browser download (v0.124.0, decision 3): PORTABILITY, not a backup tier — no scheduling, +// no status surface. The EXISTING async export pipeline produces the bundle (same producer as a +// drive export → byte-identical format); the only new mechanism is the dest (a dedicated staging +// dir under the controller data dir) and a guarded streaming exit. Bundles are removed after a +// successful stream and swept by a TTL cleanup (fabDownloadTTL) on startup and on each start. + +const fabDownloadTTL = time.Hour + +// fabDownloadRoot is the staging root; the exporter writes bundles into ExportDir(root). +func (s *Server) fabDownloadRoot() string { + return filepath.Join(s.cfg.Paths.DataDir, "fab-downloads") +} + +func (s *Server) fabDownloadDir() string { + return appexport.ExportDir(s.fabDownloadRoot()) +} + +// fabDownloadNamePattern matches the exporter's own bundle naming (_.fab); +// a single safe segment, no separators, cannot start with a dot. +var fabDownloadNamePattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.fab$`) + +// validFabDownloadName is the containment guard's first half: only a bare, expected-shaped +// bundle FILENAME is accepted — no separators, no traversal, no absolute paths. +func validFabDownloadName(name string) bool { + if name == "" || strings.ContainsAny(name, `/\`) || strings.Contains(name, "..") { + return false + } + return fabDownloadNamePattern.MatchString(name) +} + +// apiExportDownloadEstimate returns the size estimate for a download export (shown BEFORE +// starting — the pre-download honesty mechanism). GET /api/export/download/estimate?stack=X +func (s *Server) apiExportDownloadEstimate(w http.ResponseWriter, r *http.Request) { + if s.appExporter == nil { + jsonError(w, "App export not available", http.StatusServiceUnavailable) + return + } + stackName := r.URL.Query().Get("stack") + if stackName == "" || !validStackName(stackName) { + jsonError(w, "Missing or invalid stack parameter", http.StatusBadRequest) + return + } + if err := os.MkdirAll(s.fabDownloadDir(), 0755); err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + est, err := s.appExporter.EstimateExport(stackName, s.fabDownloadRoot()) + if err != nil { + s.logger.Printf("[ERROR] [web] download-export estimate failed for %s: %v", stackName, err) + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + jsonResponse(w, map[string]interface{}{"ok": true, "data": est}) +} + +// apiExportDownloadStart starts the EXISTING async export with the staging dir as dest. +// POST /api/export/download/start {stack_name, password, stop_app} +func (s *Server) apiExportDownloadStart(w http.ResponseWriter, r *http.Request) { + if s.appExporter == nil { + jsonError(w, "App export not available", http.StatusServiceUnavailable) + return + } + var req struct { + StackName string `json:"stack_name"` + Password string `json:"password"` + StopApp bool `json:"stop_app"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + jsonError(w, "Invalid request body", http.StatusBadRequest) + return + } + if req.StackName == "" || !validStackName(req.StackName) { + jsonError(w, "Missing or invalid stack_name", http.StatusBadRequest) + return + } + // Opportunistic TTL sweep so an abandoned bundle never outlives fabDownloadTTL by much. + sweepFabDownloads(s.fabDownloadDir(), time.Now(), fabDownloadTTL, s.logger) + if err := os.MkdirAll(s.fabDownloadDir(), 0755); err != nil { + jsonError(w, err.Error(), http.StatusInternalServerError) + return + } + // A concurrent export/import gets the exporter's own busy answer (single-flight). + if err := s.appExporter.StartExport(appexport.ExportRequest{ + StackName: req.StackName, + DestDrive: s.fabDownloadRoot(), + Password: req.Password, + StopApp: req.StopApp, + }); err != nil { + s.logger.Printf("[ERROR] [web] download-export start failed for %s: %v", req.StackName, err) + jsonError(w, err.Error(), http.StatusConflict) + return + } + s.logger.Printf("[INFO] [web] download-export started for %s (staging dir)", req.StackName) + jsonResponse(w, map[string]interface{}{"ok": true}) +} + +// apiExportDownloadFetch streams a produced bundle to the browser and removes it afterwards. +// GET /api/export/download?file=. Containment is NON-NEGOTIABLE: only an +// expected-shaped basename, resolved strictly inside the staging exports dir, is ever opened. +func (s *Server) apiExportDownloadFetch(w http.ResponseWriter, r *http.Request) { + name := r.URL.Query().Get("file") + if !validFabDownloadName(name) { + s.logger.Printf("[WARN] [web] download-export fetch refused: invalid bundle name") + jsonError(w, "Invalid bundle name", http.StatusBadRequest) + return + } + dir := filepath.Clean(s.fabDownloadDir()) + path := filepath.Join(dir, name) + // Second half of the guard: the resolved path's parent must BE the staging dir. + if filepath.Dir(path) != dir { + s.logger.Printf("[WARN] [web] download-export fetch refused: path escapes the staging dir") + jsonError(w, "Invalid bundle name", http.StatusBadRequest) + return + } + f, err := os.Open(path) + if err != nil { + jsonError(w, "Bundle not found", http.StatusNotFound) + return + } + defer f.Close() + fi, err := f.Stat() + if err != nil || fi.IsDir() { + jsonError(w, "Bundle not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(fi.Size(), 10)) + w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) + n, err := io.Copy(w, f) // streaming — never ReadAll + if err != nil { + // Mid-stream failure (client gone): keep the bundle for a retry; the TTL sweep owns it. + s.logger.Printf("[WARN] [web] download-export stream aborted after %d bytes: %v", n, err) + return + } + f.Close() + if rmErr := os.Remove(path); rmErr != nil { + s.logger.Printf("[WARN] [web] download-export cleanup failed (TTL sweep will retry): %v", rmErr) + } + s.logger.Printf("[INFO] [web] download-export streamed %d bytes and removed the staged bundle", n) +} + +// sweepFabDownloads removes staged bundles (and exporter temp files) older than maxAge. +// Returns how many entries were removed. Pure function of (dir, now) — tests inject both. +func sweepFabDownloads(dir string, now time.Time, maxAge time.Duration, logger *log.Logger) int { + entries, err := os.ReadDir(dir) + if err != nil { + return 0 + } + removed := 0 + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + if !strings.HasSuffix(name, ".fab") && !strings.HasSuffix(name, ".tmp") { + continue + } + info, err := e.Info() + if err != nil { + continue + } + if now.Sub(info.ModTime()) > maxAge { + if err := os.Remove(filepath.Join(dir, name)); err == nil { + removed++ + if logger != nil { + logger.Printf("[INFO] [web] download-export TTL sweep removed a staged bundle (age > %s)", maxAge) + } + } + } + } + return removed +} diff --git a/controller/internal/web/server.go b/controller/internal/web/server.go index 7e9a08a..baf8d48 100644 --- a/controller/internal/web/server.go +++ b/controller/internal/web/server.go @@ -157,6 +157,12 @@ func NewServer(cfg *config.Config, stackMgr *stacks.Manager, cpuCollector *syste s.loadTemplates() go s.cleanupSessions() + // .fab download staging (v0.124.0): sweep aged bundles left by a crash/abandoned download. + if cfg.Paths.DataDir != "" { + if n := sweepFabDownloads(s.fabDownloadDir(), time.Now(), fabDownloadTTL, logger); n > 0 { + logger.Printf("[INFO] [web] download-export startup sweep removed %d staged bundle(s)", n) + } + } // Drive-absent gate reconcile (intermediary-mount model): the periodic absent/return detector that // replaced the retired slice-8C watchdog. Stops+blocks apps whose drive vanished, auto-restarts them // when it returns. No-op when the agent is unreachable. diff --git a/controller/internal/web/templates/backups_restore.html b/controller/internal/web/templates/backups_restore.html index 3367ddc..5acc1c9 100644 --- a/controller/internal/web/templates/backups_restore.html +++ b/controller/internal/web/templates/backups_restore.html @@ -89,10 +89,99 @@ {{end}} + + +
+

Hordozható mentéscsomag (.fab)

+

Hordozható pillanatfelvétel — bárhol tárolhatod, és bármikor visszatöltheted egy meghajtóról. A folyamatos védelmet az 1–3. szintű mentés adja.

+ {{if .OffboxApps}} +
+ +
+
+ {{range .OffboxApps}} +
+
+
{{.DisplayName}}
+
+ +
+
+
+ {{end}} +
+
+ + A csomagok egyenként készülnek és töltődnek le — pillanatfelvétel a mostani állapotról. +
+
+ {{else}} +

Nincs telepített alkalmazás.

+ {{end}} +
+ {{end}}