v0.128.0: chunked browser .fab upload on /import (tunnel-proof)
Cloudflare edge caps request bodies (~100 MB, probed live: 120 MiB -> edge 413,
80 MiB -> origin), so the client slices the file into 64 MiB strictly-sequential
chunks; the server streams each to a .part file in the default drive's exports
dir and finalize renames atomically. Scan/validate/import pipeline untouched.
upload/{init,chunk,finalize,abort} inside ServeExportAPI (inherits auth+CSRF);
single-flight; offset==received or 409+echo; free-space gate; collision ->
lowest-free "name (N).fab"; startup GC of *.part-*; 15-min idle abort.
appexport.DiskFree exported (seam web.uploadDiskFree). Scenarios A-F tested,
red-proofs run (traversal / out-of-order / overwrite).
This commit is contained in:
@@ -0,0 +1,407 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// Chunked browser .fab upload (v0.128.0) — scenarios §7 A–F. Red-proofs recorded in REPORT.md:
|
||||
// C: sanitize bypassed (raw join of the client filename) → the traversal test FAILS
|
||||
// (the file lands outside the exports dir).
|
||||
// B: offset check removed (append regardless) → the replay test FAILS (bytes doubled on disk).
|
||||
// F: naive overwrite (rename straight to the requested name) → the collision test FAILS
|
||||
// (the pre-existing bundle's bytes are clobbered).
|
||||
|
||||
// uploadTestServer builds a Server with a DEFAULT registered drive on a TempDir and returns the
|
||||
// drive's exports dir. The free-space seam reports "plenty" unless a test overrides it.
|
||||
func uploadTestServer(t *testing.T) (*Server, string) {
|
||||
t.Helper()
|
||||
s := testServer(t)
|
||||
drive := t.TempDir()
|
||||
if err := s.settings.AddStoragePath(settings.StoragePath{
|
||||
Path: drive, Label: "Teszt HDD", IsDefault: true, Schedulable: true,
|
||||
AddedAt: "2026-07-13T00:00:00Z",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(appexport.ExportDir(drive), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prevFree := uploadDiskFree
|
||||
uploadDiskFree = func(string) int64 { return 1 << 40 } // 1 TiB — never the constraint
|
||||
t.Cleanup(func() {
|
||||
uploadDiskFree = prevFree
|
||||
s.dropUpload(t)
|
||||
})
|
||||
return s, appexport.ExportDir(drive)
|
||||
}
|
||||
|
||||
// dropUpload force-releases a leftover slot so TempDir cleanup never races the idle timer.
|
||||
func (s *Server) dropUpload(t *testing.T) {
|
||||
t.Helper()
|
||||
s.fabUpload.mu.Lock()
|
||||
job := s.fabUpload.cur
|
||||
s.fabUpload.cur = nil
|
||||
s.fabUpload.mu.Unlock()
|
||||
if job != nil {
|
||||
job.timer.Stop()
|
||||
job.file.Close()
|
||||
os.Remove(job.partPath)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadInit(s *Server, filename string, size int64) *httptest.ResponseRecorder {
|
||||
body, _ := json.Marshal(map[string]interface{}{"filename": filename, "size_bytes": size})
|
||||
rr := httptest.NewRecorder()
|
||||
s.apiUploadInit(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/init", bytes.NewReader(body)))
|
||||
return rr
|
||||
}
|
||||
|
||||
func uploadChunk(s *Server, id string, offset int, data []byte) *httptest.ResponseRecorder {
|
||||
rr := httptest.NewRecorder()
|
||||
url := fmt.Sprintf("/api/export/upload/chunk?id=%s&offset=%d", id, offset)
|
||||
s.apiUploadChunk(rr, httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data)))
|
||||
return rr
|
||||
}
|
||||
|
||||
func uploadFinalize(s *Server, id string) *httptest.ResponseRecorder {
|
||||
body, _ := json.Marshal(map[string]string{"upload_id": id})
|
||||
rr := httptest.NewRecorder()
|
||||
s.apiUploadFinalize(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/finalize", bytes.NewReader(body)))
|
||||
return rr
|
||||
}
|
||||
|
||||
func decodeUpload(t *testing.T, rr *httptest.ResponseRecorder) map[string]interface{} {
|
||||
t.Helper()
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &m); err != nil {
|
||||
t.Fatalf("bad JSON %q: %v", rr.Body.String(), err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// mustInit runs a full init and returns the upload id.
|
||||
func mustInit(t *testing.T, s *Server, filename string, size int64) string {
|
||||
t.Helper()
|
||||
rr := uploadInit(s, filename, size)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("init: got %d (%s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
m := decodeUpload(t, rr)
|
||||
id, _ := m["upload_id"].(string)
|
||||
if id == "" {
|
||||
t.Fatal("init returned no upload_id")
|
||||
}
|
||||
if got := int64(m["chunk_bytes"].(float64)); got != uploadChunkBytes {
|
||||
t.Fatalf("chunk_bytes = %d, want %d", got, int64(uploadChunkBytes))
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// Scenario A — happy path: 3 sequential chunks land byte-exactly in the exports dir under the
|
||||
// requested name, and the EXISTING bundle scan lists it (no second lister). The server streams
|
||||
// via io.Copy — nothing here (or in the handler) buffers the file.
|
||||
func TestFabUpload_HappyPath(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
chunks := [][]byte{
|
||||
bytes.Repeat([]byte{0xA1}, 700),
|
||||
bytes.Repeat([]byte{0xB2}, 700),
|
||||
bytes.Repeat([]byte{0xC3}, 300),
|
||||
}
|
||||
var want []byte
|
||||
for _, c := range chunks {
|
||||
want = append(want, c...)
|
||||
}
|
||||
id := mustInit(t, s, "sajatapp_20260713.fab", int64(len(want)))
|
||||
|
||||
off := 0
|
||||
for i, c := range chunks {
|
||||
rr := uploadChunk(s, id, off, c)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk %d: got %d (%s)", i, rr.Code, rr.Body.String())
|
||||
}
|
||||
off += len(c)
|
||||
if got := int64(decodeUpload(t, rr)["received_bytes"].(float64)); got != int64(off) {
|
||||
t.Fatalf("chunk %d: received_bytes = %d, want %d", i, got, off)
|
||||
}
|
||||
}
|
||||
|
||||
rr := uploadFinalize(s, id)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("finalize: got %d (%s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
if got := decodeUpload(t, rr)["filename"]; got != "sajatapp_20260713.fab" {
|
||||
t.Fatalf("finalize filename = %v", got)
|
||||
}
|
||||
|
||||
final := filepath.Join(exportDir, "sajatapp_20260713.fab")
|
||||
got, err := os.ReadFile(final)
|
||||
if err != nil {
|
||||
t.Fatalf("final file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("final bytes differ: got %d bytes, want %d", len(got), len(want))
|
||||
}
|
||||
// No .part residue.
|
||||
entries, _ := os.ReadDir(exportDir)
|
||||
for _, e := range entries {
|
||||
if strings.Contains(e.Name(), ".part-") {
|
||||
t.Fatalf("leftover part file: %s", e.Name())
|
||||
}
|
||||
}
|
||||
// The EXISTING scan lists the upload (drive label included — the same row the page renders).
|
||||
bundles := appexport.ScanForBundles([]appexport.DrivePathInfo{{Path: filepath.Dir(exportDir), Label: "Teszt HDD"}})
|
||||
found := false
|
||||
for _, b := range bundles {
|
||||
if b.FileName == "sajatapp_20260713.fab" && b.SizeBytes == int64(len(want)) {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("bundle scan does not list the upload: %+v", bundles)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario B — offset mismatch: replaying an already-received chunk answers 409 with the current
|
||||
// received_bytes (the client's re-sync datum) and appends NOTHING (asserted via on-disk size).
|
||||
// Red-proof: drop the offset==received check → this FAILS (size doubles).
|
||||
func TestFabUpload_OffsetMismatchRejected(t *testing.T) {
|
||||
s, _ := uploadTestServer(t)
|
||||
|
||||
chunk := bytes.Repeat([]byte{0x55}, 512)
|
||||
id := mustInit(t, s, "b.fab", 1024)
|
||||
if rr := uploadChunk(s, id, 0, chunk); rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk 0: got %d", rr.Code)
|
||||
}
|
||||
|
||||
// Replay chunk at offset 0 (client lost the ack).
|
||||
rr := uploadChunk(s, id, 0, chunk)
|
||||
if rr.Code != http.StatusConflict {
|
||||
t.Fatalf("replay: got %d, want 409", rr.Code)
|
||||
}
|
||||
if got := int64(decodeUpload(t, rr)["received_bytes"].(float64)); got != 512 {
|
||||
t.Fatalf("replay received_bytes = %d, want 512 (the re-sync datum)", got)
|
||||
}
|
||||
s.fabUpload.mu.Lock()
|
||||
part := s.fabUpload.cur.partPath
|
||||
s.fabUpload.mu.Unlock()
|
||||
if fi, err := os.Stat(part); err != nil || fi.Size() != 512 {
|
||||
t.Fatalf("part size after replay = %v (err=%v), want 512 — replay must not append", fi, err)
|
||||
}
|
||||
|
||||
// Re-synced continuation at the echoed offset succeeds.
|
||||
if rr := uploadChunk(s, id, 512, chunk); rr.Code != http.StatusOK {
|
||||
t.Fatalf("re-synced chunk: got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario C — security gates on init: traversal filenames are neutered to a base name INSIDE the
|
||||
// exports dir (never resolved outside), wrong extension refused, oversize vs the (faked) free
|
||||
// space refused with the Hungarian message, and a second concurrent init answers 409.
|
||||
// Red-proof: bypass sanitizeUploadFilename (raw join) → the traversal leg FAILS.
|
||||
func TestFabUpload_SecurityGates(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
// Traversal name: sanitized to its base — the upload lands INSIDE exports, the target of the
|
||||
// traversal stays untouched.
|
||||
outside := filepath.Join(filepath.Dir(exportDir), "evil.fab") // {drive}/evil.fab — where ../evil.fab would land
|
||||
id := mustInit(t, s, "../evil.fab", 4)
|
||||
if rr := uploadChunk(s, id, 0, []byte("EVIL")); rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk: got %d", rr.Code)
|
||||
}
|
||||
if rr := uploadFinalize(s, id); rr.Code != http.StatusOK {
|
||||
t.Fatalf("finalize: got %d", rr.Code)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(exportDir, "evil.fab")); err != nil {
|
||||
t.Fatal("sanitized upload must land inside the exports dir")
|
||||
}
|
||||
if _, err := os.Stat(outside); !os.IsNotExist(err) {
|
||||
t.Fatal("TRAVERSAL: a file appeared outside the exports dir")
|
||||
}
|
||||
|
||||
// Wrong extension / hidden / empty stem / bad charset → 400, no slot claimed.
|
||||
for _, name := range []string{"x.txt", ".hidden.fab", ".fab", "árvíztűrő.fab", "a/b.txt"} {
|
||||
if rr := uploadInit(s, name, 10); rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("init(%q): got %d, want 400", name, rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Oversize vs free space (fake statfs seam): Hungarian message with both numbers.
|
||||
prevFree := uploadDiskFree
|
||||
uploadDiskFree = func(string) int64 { return 2 << 30 } // 2 GiB free
|
||||
rr := uploadInit(s, "big.fab", 3<<30) // 3 GiB declared
|
||||
uploadDiskFree = prevFree
|
||||
if rr.Code != http.StatusConflict {
|
||||
t.Fatalf("oversize init: got %d, want 409", rr.Code)
|
||||
}
|
||||
if msg, _ := decodeUpload(t, rr)["error"].(string); !strings.Contains(msg, "Nincs elég szabad hely") ||
|
||||
!strings.Contains(msg, "3.0 GB") || !strings.Contains(msg, "2.0 GB") {
|
||||
t.Fatalf("oversize error message = %q", msg)
|
||||
}
|
||||
|
||||
// Second concurrent init → 409 (single-flight).
|
||||
first := mustInit(t, s, "one.fab", 10)
|
||||
if rr := uploadInit(s, "two.fab", 10); rr.Code != http.StatusConflict {
|
||||
t.Fatalf("second init: got %d, want 409", rr.Code)
|
||||
}
|
||||
_ = first
|
||||
}
|
||||
|
||||
// Scenario C (route wrap) — the new upload paths live under the SAME main.go mount
|
||||
// (RequireAuth(CsrfProtect(ServeExportAPI))): an unauthenticated POST answers 401 before any
|
||||
// upload handler runs, proving the wrap applies to the new routes with nothing added at the mux.
|
||||
func TestFabUpload_MountInheritsAuth(t *testing.T) {
|
||||
s, _, _ := claimTestServer(t)
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/api/export/", s.RequireAuth(s.CsrfProtect(http.HandlerFunc(s.ServeExportAPI))))
|
||||
|
||||
for _, p := range []string{"/api/export/upload/init", "/api/export/upload/chunk", "/api/export/upload/finalize", "/api/export/upload/abort"} {
|
||||
rr := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, p, strings.NewReader("{}")))
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Errorf("%s unauthenticated: got %d, want 401", p, rr.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario D — finalize size mismatch: 422, the .part deleted, nothing renamed, slot freed.
|
||||
func TestFabUpload_FinalizeSizeMismatch(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
id := mustInit(t, s, "short.fab", 1000)
|
||||
if rr := uploadChunk(s, id, 0, bytes.Repeat([]byte{1}, 400)); rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk: got %d", rr.Code)
|
||||
}
|
||||
rr := uploadFinalize(s, id)
|
||||
if rr.Code != http.StatusUnprocessableEntity {
|
||||
t.Fatalf("finalize: got %d, want 422", rr.Code)
|
||||
}
|
||||
entries, _ := os.ReadDir(exportDir)
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("exports dir must be empty after a size-mismatch finalize, got %v", entries)
|
||||
}
|
||||
// Slot freed: a fresh init succeeds.
|
||||
mustInit(t, s, "again.fab", 10)
|
||||
}
|
||||
|
||||
// Scenario E — GC: startup sweep removes stray .part files; an idle upload is aborted
|
||||
// server-side (timer seam shortened) and its .part deleted, freeing the slot.
|
||||
func TestFabUpload_GCAndIdleTimeout(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
// Startup GC.
|
||||
if err := os.WriteFile(filepath.Join(exportDir, "stray.fab.part-deadbeef"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keep := filepath.Join(exportDir, "keep.fab")
|
||||
if err := os.WriteFile(keep, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.CleanupStaleUploadParts()
|
||||
if _, err := os.Stat(filepath.Join(exportDir, "stray.fab.part-deadbeef")); !os.IsNotExist(err) {
|
||||
t.Fatal("startup GC must remove stray .part files")
|
||||
}
|
||||
if _, err := os.Stat(keep); err != nil {
|
||||
t.Fatal("startup GC must not touch real bundles")
|
||||
}
|
||||
|
||||
// Idle timeout.
|
||||
prev := uploadIdleTimeout
|
||||
uploadIdleTimeout = 30 * time.Millisecond
|
||||
defer func() { uploadIdleTimeout = prev }()
|
||||
|
||||
id := mustInit(t, s, "idle.fab", 100)
|
||||
s.fabUpload.mu.Lock()
|
||||
part := s.fabUpload.cur.partPath
|
||||
s.fabUpload.mu.Unlock()
|
||||
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
for {
|
||||
s.fabUpload.mu.Lock()
|
||||
gone := s.fabUpload.cur == nil
|
||||
s.fabUpload.mu.Unlock()
|
||||
if gone {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("idle upload was never expired")
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if _, err := os.Stat(part); !os.IsNotExist(err) {
|
||||
t.Fatal("idle-expired .part must be deleted")
|
||||
}
|
||||
if rr := uploadChunk(s, id, 0, []byte("late")); rr.Code != http.StatusNotFound {
|
||||
t.Fatalf("chunk after expiry: got %d, want 404", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario F — collision: an existing "app.fab" is never overwritten; the upload lands as
|
||||
// "app (1).fab", and a re-run finds its own prior (1) and lands as "app (2).fab" — never
|
||||
// "(1)(1)". Red-proof: rename straight to the requested name → this FAILS (bytes clobbered).
|
||||
func TestFabUpload_CollisionLowestFreeSuffix(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
pre := []byte("PRE-EXISTING")
|
||||
if err := os.WriteFile(filepath.Join(exportDir, "app.fab"), pre, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
run := func(payload string) string {
|
||||
id := mustInit(t, s, "app.fab", int64(len(payload)))
|
||||
if rr := uploadChunk(s, id, 0, []byte(payload)); rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk: got %d", rr.Code)
|
||||
}
|
||||
rr := uploadFinalize(s, id)
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("finalize: got %d (%s)", rr.Code, rr.Body.String())
|
||||
}
|
||||
return decodeUpload(t, rr)["filename"].(string)
|
||||
}
|
||||
|
||||
if got := run("FIRST-RERUN"); got != "app (1).fab" {
|
||||
t.Fatalf("first collision landed as %q, want \"app (1).fab\"", got)
|
||||
}
|
||||
if got := run("SECOND-RERUN"); got != "app (2).fab" {
|
||||
t.Fatalf("second collision landed as %q, want \"app (2).fab\" (never (1)(1))", got)
|
||||
}
|
||||
if body, err := os.ReadFile(filepath.Join(exportDir, "app.fab")); err != nil || !bytes.Equal(body, pre) {
|
||||
t.Fatal("the pre-existing bundle was overwritten")
|
||||
}
|
||||
if body, _ := os.ReadFile(filepath.Join(exportDir, "app (1).fab")); string(body) != "FIRST-RERUN" {
|
||||
t.Fatalf("app (1).fab content = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Abort deletes the .part and frees the slot (the Megszakítás button's route).
|
||||
func TestFabUpload_Abort(t *testing.T) {
|
||||
s, exportDir := uploadTestServer(t)
|
||||
|
||||
id := mustInit(t, s, "cancelme.fab", 100)
|
||||
if rr := uploadChunk(s, id, 0, bytes.Repeat([]byte{7}, 50)); rr.Code != http.StatusOK {
|
||||
t.Fatalf("chunk: got %d", rr.Code)
|
||||
}
|
||||
body, _ := json.Marshal(map[string]string{"upload_id": id})
|
||||
rr := httptest.NewRecorder()
|
||||
s.apiUploadAbort(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/abort", bytes.NewReader(body)))
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("abort: got %d", rr.Code)
|
||||
}
|
||||
entries, _ := os.ReadDir(exportDir)
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("exports dir must be empty after abort, got %v", entries)
|
||||
}
|
||||
mustInit(t, s, "next.fab", 10) // slot freed
|
||||
}
|
||||
Reference in New Issue
Block a user