admin
fd40b29119
The fab-upload GC test stat-ed the .part immediately after observing the slot free, but expireIdleUpload unlinks AFTER releasing the mutex. Passed alone, failed in the full package once this release's render tests made web heavier. Not a production defect - a new upload mints a fresh random .part. The test now waits for the outcome it asserts on the same deadline; red-proofed by removing the unlink from production.
422 lines
15 KiB
Go
422 lines
15 KiB
Go
package web
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/appexport"
|
||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||
)
|
||
|
||
// Chunked browser .fab upload (v0.128.0) — scenarios §7 A–F. Red-proofs recorded in REPORT.md:
|
||
// C: sanitize bypassed (raw join of the client filename) → the traversal test FAILS
|
||
// (the file lands outside the exports dir).
|
||
// B: offset check removed (append regardless) → the replay test FAILS (bytes doubled on disk).
|
||
// F: naive overwrite (rename straight to the requested name) → the collision test FAILS
|
||
// (the pre-existing bundle's bytes are clobbered).
|
||
|
||
// uploadTestServer builds a Server with a DEFAULT registered drive on a TempDir and returns the
|
||
// drive's exports dir. The free-space seam reports "plenty" unless a test overrides it.
|
||
func uploadTestServer(t *testing.T) (*Server, string) {
|
||
t.Helper()
|
||
s := testServer(t)
|
||
drive := t.TempDir()
|
||
if err := s.settings.AddStoragePath(settings.StoragePath{
|
||
Path: drive, Label: "Teszt HDD", IsDefault: true, Schedulable: true,
|
||
AddedAt: "2026-07-13T00:00:00Z",
|
||
}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if err := os.MkdirAll(appexport.ExportDir(drive), 0755); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
prevFree := uploadDiskFree
|
||
uploadDiskFree = func(string) int64 { return 1 << 40 } // 1 TiB — never the constraint
|
||
t.Cleanup(func() {
|
||
uploadDiskFree = prevFree
|
||
s.dropUpload(t)
|
||
})
|
||
return s, appexport.ExportDir(drive)
|
||
}
|
||
|
||
// dropUpload force-releases a leftover slot so TempDir cleanup never races the idle timer.
|
||
func (s *Server) dropUpload(t *testing.T) {
|
||
t.Helper()
|
||
s.fabUpload.mu.Lock()
|
||
job := s.fabUpload.cur
|
||
s.fabUpload.cur = nil
|
||
s.fabUpload.mu.Unlock()
|
||
if job != nil {
|
||
job.timer.Stop()
|
||
job.file.Close()
|
||
os.Remove(job.partPath)
|
||
}
|
||
}
|
||
|
||
func uploadInit(s *Server, filename string, size int64) *httptest.ResponseRecorder {
|
||
body, _ := json.Marshal(map[string]interface{}{"filename": filename, "size_bytes": size})
|
||
rr := httptest.NewRecorder()
|
||
s.apiUploadInit(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/init", bytes.NewReader(body)))
|
||
return rr
|
||
}
|
||
|
||
func uploadChunk(s *Server, id string, offset int, data []byte) *httptest.ResponseRecorder {
|
||
rr := httptest.NewRecorder()
|
||
url := fmt.Sprintf("/api/export/upload/chunk?id=%s&offset=%d", id, offset)
|
||
s.apiUploadChunk(rr, httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data)))
|
||
return rr
|
||
}
|
||
|
||
func uploadFinalize(s *Server, id string) *httptest.ResponseRecorder {
|
||
body, _ := json.Marshal(map[string]string{"upload_id": id})
|
||
rr := httptest.NewRecorder()
|
||
s.apiUploadFinalize(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/finalize", bytes.NewReader(body)))
|
||
return rr
|
||
}
|
||
|
||
func decodeUpload(t *testing.T, rr *httptest.ResponseRecorder) map[string]interface{} {
|
||
t.Helper()
|
||
var m map[string]interface{}
|
||
if err := json.Unmarshal(rr.Body.Bytes(), &m); err != nil {
|
||
t.Fatalf("bad JSON %q: %v", rr.Body.String(), err)
|
||
}
|
||
return m
|
||
}
|
||
|
||
// mustInit runs a full init and returns the upload id.
|
||
func mustInit(t *testing.T, s *Server, filename string, size int64) string {
|
||
t.Helper()
|
||
rr := uploadInit(s, filename, size)
|
||
if rr.Code != http.StatusOK {
|
||
t.Fatalf("init: got %d (%s)", rr.Code, rr.Body.String())
|
||
}
|
||
m := decodeUpload(t, rr)
|
||
id, _ := m["upload_id"].(string)
|
||
if id == "" {
|
||
t.Fatal("init returned no upload_id")
|
||
}
|
||
if got := int64(m["chunk_bytes"].(float64)); got != uploadChunkBytes {
|
||
t.Fatalf("chunk_bytes = %d, want %d", got, int64(uploadChunkBytes))
|
||
}
|
||
return id
|
||
}
|
||
|
||
// Scenario A — happy path: 3 sequential chunks land byte-exactly in the exports dir under the
|
||
// requested name, and the EXISTING bundle scan lists it (no second lister). The server streams
|
||
// via io.Copy — nothing here (or in the handler) buffers the file.
|
||
func TestFabUpload_HappyPath(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
chunks := [][]byte{
|
||
bytes.Repeat([]byte{0xA1}, 700),
|
||
bytes.Repeat([]byte{0xB2}, 700),
|
||
bytes.Repeat([]byte{0xC3}, 300),
|
||
}
|
||
var want []byte
|
||
for _, c := range chunks {
|
||
want = append(want, c...)
|
||
}
|
||
id := mustInit(t, s, "sajatapp_20260713.fab", int64(len(want)))
|
||
|
||
off := 0
|
||
for i, c := range chunks {
|
||
rr := uploadChunk(s, id, off, c)
|
||
if rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk %d: got %d (%s)", i, rr.Code, rr.Body.String())
|
||
}
|
||
off += len(c)
|
||
if got := int64(decodeUpload(t, rr)["received_bytes"].(float64)); got != int64(off) {
|
||
t.Fatalf("chunk %d: received_bytes = %d, want %d", i, got, off)
|
||
}
|
||
}
|
||
|
||
rr := uploadFinalize(s, id)
|
||
if rr.Code != http.StatusOK {
|
||
t.Fatalf("finalize: got %d (%s)", rr.Code, rr.Body.String())
|
||
}
|
||
if got := decodeUpload(t, rr)["filename"]; got != "sajatapp_20260713.fab" {
|
||
t.Fatalf("finalize filename = %v", got)
|
||
}
|
||
|
||
final := filepath.Join(exportDir, "sajatapp_20260713.fab")
|
||
got, err := os.ReadFile(final)
|
||
if err != nil {
|
||
t.Fatalf("final file: %v", err)
|
||
}
|
||
if !bytes.Equal(got, want) {
|
||
t.Fatalf("final bytes differ: got %d bytes, want %d", len(got), len(want))
|
||
}
|
||
// No .part residue.
|
||
entries, _ := os.ReadDir(exportDir)
|
||
for _, e := range entries {
|
||
if strings.Contains(e.Name(), ".part-") {
|
||
t.Fatalf("leftover part file: %s", e.Name())
|
||
}
|
||
}
|
||
// The EXISTING scan lists the upload (drive label included — the same row the page renders).
|
||
bundles := appexport.ScanForBundles([]appexport.DrivePathInfo{{Path: filepath.Dir(exportDir), Label: "Teszt HDD"}})
|
||
found := false
|
||
for _, b := range bundles {
|
||
if b.FileName == "sajatapp_20260713.fab" && b.SizeBytes == int64(len(want)) {
|
||
found = true
|
||
}
|
||
}
|
||
if !found {
|
||
t.Fatalf("bundle scan does not list the upload: %+v", bundles)
|
||
}
|
||
}
|
||
|
||
// Scenario B — offset mismatch: replaying an already-received chunk answers 409 with the current
|
||
// received_bytes (the client's re-sync datum) and appends NOTHING (asserted via on-disk size).
|
||
// Red-proof: drop the offset==received check → this FAILS (size doubles).
|
||
func TestFabUpload_OffsetMismatchRejected(t *testing.T) {
|
||
s, _ := uploadTestServer(t)
|
||
|
||
chunk := bytes.Repeat([]byte{0x55}, 512)
|
||
id := mustInit(t, s, "b.fab", 1024)
|
||
if rr := uploadChunk(s, id, 0, chunk); rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk 0: got %d", rr.Code)
|
||
}
|
||
|
||
// Replay chunk at offset 0 (client lost the ack).
|
||
rr := uploadChunk(s, id, 0, chunk)
|
||
if rr.Code != http.StatusConflict {
|
||
t.Fatalf("replay: got %d, want 409", rr.Code)
|
||
}
|
||
if got := int64(decodeUpload(t, rr)["received_bytes"].(float64)); got != 512 {
|
||
t.Fatalf("replay received_bytes = %d, want 512 (the re-sync datum)", got)
|
||
}
|
||
s.fabUpload.mu.Lock()
|
||
part := s.fabUpload.cur.partPath
|
||
s.fabUpload.mu.Unlock()
|
||
if fi, err := os.Stat(part); err != nil || fi.Size() != 512 {
|
||
t.Fatalf("part size after replay = %v (err=%v), want 512 — replay must not append", fi, err)
|
||
}
|
||
|
||
// Re-synced continuation at the echoed offset succeeds.
|
||
if rr := uploadChunk(s, id, 512, chunk); rr.Code != http.StatusOK {
|
||
t.Fatalf("re-synced chunk: got %d", rr.Code)
|
||
}
|
||
}
|
||
|
||
// Scenario C — security gates on init: traversal filenames are neutered to a base name INSIDE the
|
||
// exports dir (never resolved outside), wrong extension refused, oversize vs the (faked) free
|
||
// space refused with the Hungarian message, and a second concurrent init answers 409.
|
||
// Red-proof: bypass sanitizeUploadFilename (raw join) → the traversal leg FAILS.
|
||
func TestFabUpload_SecurityGates(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
// Traversal name: sanitized to its base — the upload lands INSIDE exports, the target of the
|
||
// traversal stays untouched.
|
||
outside := filepath.Join(filepath.Dir(exportDir), "evil.fab") // {drive}/evil.fab — where ../evil.fab would land
|
||
id := mustInit(t, s, "../evil.fab", 4)
|
||
if rr := uploadChunk(s, id, 0, []byte("EVIL")); rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk: got %d", rr.Code)
|
||
}
|
||
if rr := uploadFinalize(s, id); rr.Code != http.StatusOK {
|
||
t.Fatalf("finalize: got %d", rr.Code)
|
||
}
|
||
if _, err := os.Stat(filepath.Join(exportDir, "evil.fab")); err != nil {
|
||
t.Fatal("sanitized upload must land inside the exports dir")
|
||
}
|
||
if _, err := os.Stat(outside); !os.IsNotExist(err) {
|
||
t.Fatal("TRAVERSAL: a file appeared outside the exports dir")
|
||
}
|
||
|
||
// Wrong extension / hidden / empty stem / bad charset → 400, no slot claimed.
|
||
for _, name := range []string{"x.txt", ".hidden.fab", ".fab", "árvíztűrő.fab", "a/b.txt"} {
|
||
if rr := uploadInit(s, name, 10); rr.Code != http.StatusBadRequest {
|
||
t.Errorf("init(%q): got %d, want 400", name, rr.Code)
|
||
}
|
||
}
|
||
|
||
// Oversize vs free space (fake statfs seam): Hungarian message with both numbers.
|
||
prevFree := uploadDiskFree
|
||
uploadDiskFree = func(string) int64 { return 2 << 30 } // 2 GiB free
|
||
rr := uploadInit(s, "big.fab", 3<<30) // 3 GiB declared
|
||
uploadDiskFree = prevFree
|
||
if rr.Code != http.StatusConflict {
|
||
t.Fatalf("oversize init: got %d, want 409", rr.Code)
|
||
}
|
||
if msg, _ := decodeUpload(t, rr)["error"].(string); !strings.Contains(msg, "Nincs elég szabad hely") ||
|
||
!strings.Contains(msg, "3.0 GB") || !strings.Contains(msg, "2.0 GB") {
|
||
t.Fatalf("oversize error message = %q", msg)
|
||
}
|
||
|
||
// Second concurrent init → 409 (single-flight).
|
||
first := mustInit(t, s, "one.fab", 10)
|
||
if rr := uploadInit(s, "two.fab", 10); rr.Code != http.StatusConflict {
|
||
t.Fatalf("second init: got %d, want 409", rr.Code)
|
||
}
|
||
_ = first
|
||
}
|
||
|
||
// Scenario C (route wrap) — the new upload paths live under the SAME main.go mount
|
||
// (RequireAuth(CsrfProtect(ServeExportAPI))): an unauthenticated POST answers 401 before any
|
||
// upload handler runs, proving the wrap applies to the new routes with nothing added at the mux.
|
||
func TestFabUpload_MountInheritsAuth(t *testing.T) {
|
||
s, _, _ := claimTestServer(t)
|
||
mux := http.NewServeMux()
|
||
mux.Handle("/api/export/", s.RequireAuth(s.CsrfProtect(http.HandlerFunc(s.ServeExportAPI))))
|
||
|
||
for _, p := range []string{"/api/export/upload/init", "/api/export/upload/chunk", "/api/export/upload/finalize", "/api/export/upload/abort"} {
|
||
rr := httptest.NewRecorder()
|
||
mux.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, p, strings.NewReader("{}")))
|
||
if rr.Code != http.StatusUnauthorized {
|
||
t.Errorf("%s unauthenticated: got %d, want 401", p, rr.Code)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Scenario D — finalize size mismatch: 422, the .part deleted, nothing renamed, slot freed.
|
||
func TestFabUpload_FinalizeSizeMismatch(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
id := mustInit(t, s, "short.fab", 1000)
|
||
if rr := uploadChunk(s, id, 0, bytes.Repeat([]byte{1}, 400)); rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk: got %d", rr.Code)
|
||
}
|
||
rr := uploadFinalize(s, id)
|
||
if rr.Code != http.StatusUnprocessableEntity {
|
||
t.Fatalf("finalize: got %d, want 422", rr.Code)
|
||
}
|
||
entries, _ := os.ReadDir(exportDir)
|
||
if len(entries) != 0 {
|
||
t.Fatalf("exports dir must be empty after a size-mismatch finalize, got %v", entries)
|
||
}
|
||
// Slot freed: a fresh init succeeds.
|
||
mustInit(t, s, "again.fab", 10)
|
||
}
|
||
|
||
// Scenario E — GC: startup sweep removes stray .part files; an idle upload is aborted
|
||
// server-side (timer seam shortened) and its .part deleted, freeing the slot.
|
||
func TestFabUpload_GCAndIdleTimeout(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
// Startup GC.
|
||
if err := os.WriteFile(filepath.Join(exportDir, "stray.fab.part-deadbeef"), []byte("x"), 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
keep := filepath.Join(exportDir, "keep.fab")
|
||
if err := os.WriteFile(keep, []byte("x"), 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
s.CleanupStaleUploadParts()
|
||
if _, err := os.Stat(filepath.Join(exportDir, "stray.fab.part-deadbeef")); !os.IsNotExist(err) {
|
||
t.Fatal("startup GC must remove stray .part files")
|
||
}
|
||
if _, err := os.Stat(keep); err != nil {
|
||
t.Fatal("startup GC must not touch real bundles")
|
||
}
|
||
|
||
// Idle timeout.
|
||
prev := uploadIdleTimeout
|
||
uploadIdleTimeout = 30 * time.Millisecond
|
||
defer func() { uploadIdleTimeout = prev }()
|
||
|
||
id := mustInit(t, s, "idle.fab", 100)
|
||
s.fabUpload.mu.Lock()
|
||
part := s.fabUpload.cur.partPath
|
||
s.fabUpload.mu.Unlock()
|
||
|
||
deadline := time.Now().Add(3 * time.Second)
|
||
for {
|
||
s.fabUpload.mu.Lock()
|
||
gone := s.fabUpload.cur == nil
|
||
s.fabUpload.mu.Unlock()
|
||
if gone {
|
||
break
|
||
}
|
||
if time.Now().After(deadline) {
|
||
t.Fatal("idle upload was never expired")
|
||
}
|
||
time.Sleep(10 * time.Millisecond)
|
||
}
|
||
// The unlink is ASYNCHRONOUS with respect to the slot: expireIdleUpload nils `cur`, releases
|
||
// the mutex, and only then closes + removes the file. So "the slot is free" does not yet mean
|
||
// "the .part is gone", and stat-ing immediately is a race the test loses under package load
|
||
// (observed 2026-07-20 — passes alone, fails in the full package). Wait for the outcome being
|
||
// asserted; the assertion itself is unchanged, and the deadline still fails a part that is
|
||
// never deleted.
|
||
partGone := false
|
||
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); {
|
||
if _, err := os.Stat(part); os.IsNotExist(err) {
|
||
partGone = true
|
||
break
|
||
}
|
||
time.Sleep(10 * time.Millisecond)
|
||
}
|
||
if !partGone {
|
||
t.Fatal("idle-expired .part must be deleted")
|
||
}
|
||
if rr := uploadChunk(s, id, 0, []byte("late")); rr.Code != http.StatusNotFound {
|
||
t.Fatalf("chunk after expiry: got %d, want 404", rr.Code)
|
||
}
|
||
}
|
||
|
||
// Scenario F — collision: an existing "app.fab" is never overwritten; the upload lands as
|
||
// "app (1).fab", and a re-run finds its own prior (1) and lands as "app (2).fab" — never
|
||
// "(1)(1)". Red-proof: rename straight to the requested name → this FAILS (bytes clobbered).
|
||
func TestFabUpload_CollisionLowestFreeSuffix(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
pre := []byte("PRE-EXISTING")
|
||
if err := os.WriteFile(filepath.Join(exportDir, "app.fab"), pre, 0644); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
run := func(payload string) string {
|
||
id := mustInit(t, s, "app.fab", int64(len(payload)))
|
||
if rr := uploadChunk(s, id, 0, []byte(payload)); rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk: got %d", rr.Code)
|
||
}
|
||
rr := uploadFinalize(s, id)
|
||
if rr.Code != http.StatusOK {
|
||
t.Fatalf("finalize: got %d (%s)", rr.Code, rr.Body.String())
|
||
}
|
||
return decodeUpload(t, rr)["filename"].(string)
|
||
}
|
||
|
||
if got := run("FIRST-RERUN"); got != "app (1).fab" {
|
||
t.Fatalf("first collision landed as %q, want \"app (1).fab\"", got)
|
||
}
|
||
if got := run("SECOND-RERUN"); got != "app (2).fab" {
|
||
t.Fatalf("second collision landed as %q, want \"app (2).fab\" (never (1)(1))", got)
|
||
}
|
||
if body, err := os.ReadFile(filepath.Join(exportDir, "app.fab")); err != nil || !bytes.Equal(body, pre) {
|
||
t.Fatal("the pre-existing bundle was overwritten")
|
||
}
|
||
if body, _ := os.ReadFile(filepath.Join(exportDir, "app (1).fab")); string(body) != "FIRST-RERUN" {
|
||
t.Fatalf("app (1).fab content = %q", body)
|
||
}
|
||
}
|
||
|
||
// Abort deletes the .part and frees the slot (the Megszakítás button's route).
|
||
func TestFabUpload_Abort(t *testing.T) {
|
||
s, exportDir := uploadTestServer(t)
|
||
|
||
id := mustInit(t, s, "cancelme.fab", 100)
|
||
if rr := uploadChunk(s, id, 0, bytes.Repeat([]byte{7}, 50)); rr.Code != http.StatusOK {
|
||
t.Fatalf("chunk: got %d", rr.Code)
|
||
}
|
||
body, _ := json.Marshal(map[string]string{"upload_id": id})
|
||
rr := httptest.NewRecorder()
|
||
s.apiUploadAbort(rr, httptest.NewRequest(http.MethodPost, "/api/export/upload/abort", bytes.NewReader(body)))
|
||
if rr.Code != http.StatusOK {
|
||
t.Fatalf("abort: got %d", rr.Code)
|
||
}
|
||
entries, _ := os.ReadDir(exportDir)
|
||
if len(entries) != 0 {
|
||
t.Fatalf("exports dir must be empty after abort, got %v", entries)
|
||
}
|
||
mustInit(t, s, "next.fab", 10) // slot freed
|
||
}
|