fix: handle oversized lines in DB dump validation
Replace bufio.Scanner with bufio.Reader.ReadLine() which gracefully skips lines exceeding the buffer (isPrefix=true) instead of failing. Fixes validation of Immich's PostgreSQL dump which contains COPY lines with binary-encoded image data exceeding the 256KB scanner limit. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -259,15 +260,34 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
|
|||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
|
|
||||||
scanner := bufio.NewScanner(f)
|
// Use bufio.Reader instead of Scanner: ReadLine gracefully handles lines
|
||||||
// Increase token buffer for very long lines (some SQL lines can be large)
|
// longer than the buffer (isPrefix=true) so we can skip them. Only short
|
||||||
scanner.Buffer(make([]byte, 256*1024), 256*1024)
|
// lines matter (headers, CREATE TABLE). Long COPY/INSERT data lines
|
||||||
|
// (e.g., Immich's binary-encoded image data) are skipped without allocating.
|
||||||
|
reader := bufio.NewReaderSize(f, 256*1024)
|
||||||
|
|
||||||
lineNum := 0
|
lineNum := 0
|
||||||
headerFound := false
|
headerFound := false
|
||||||
tableCount := 0
|
tableCount := 0
|
||||||
for scanner.Scan() {
|
for {
|
||||||
line := scanner.Text()
|
lineBytes, isPrefix, err := reader.ReadLine()
|
||||||
|
if err != nil {
|
||||||
|
if err != io.EOF {
|
||||||
|
v.Error = fmt.Sprintf("hiba az olvasás közben: %v", err)
|
||||||
|
log.Printf("[WARN] ValidateDump FAIL: %s — read error: %v", filePath, err)
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
break // EOF
|
||||||
|
}
|
||||||
|
if isPrefix {
|
||||||
|
// Line exceeds buffer — skip remainder (COPY data, large INSERTs)
|
||||||
|
for isPrefix && err == nil {
|
||||||
|
_, isPrefix, err = reader.ReadLine()
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
line := string(lineBytes)
|
||||||
lineNum++
|
lineNum++
|
||||||
|
|
||||||
// Header check — scan first 10 lines for expected dump header
|
// Header check — scan first 10 lines for expected dump header
|
||||||
@@ -294,12 +314,6 @@ func ValidateDump(filePath string, dbType DBType) DumpValidation {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := scanner.Err(); err != nil {
|
|
||||||
v.Error = fmt.Sprintf("hiba az olvasás közben: %v", err)
|
|
||||||
log.Printf("[WARN] ValidateDump FAIL: %s — scanner error: %v", filePath, err)
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
v.TableCount = tableCount
|
v.TableCount = tableCount
|
||||||
|
|
||||||
if !headerFound {
|
if !headerFound {
|
||||||
|
|||||||
Reference in New Issue
Block a user