@
v0.129.0: CAMPAIGN-4 fixes — rate-limiter key (F-B) + volume-blind estimate (F-A) + no-op claim status (F-C) F-B (MED, security): shared clientIP(r) helper (XFF first-hop, else SplitHostPort host, else raw) replaces requestIP + the duplicated inline derivation in handleLogin, so login AND escrow re-auth key on the port-stripped host IP — distinct direct connections no longer evade the failed-attempt counter. XFF-trust out of scope (commented). F-A (MED, honesty): volumeSizer seam reads volume size from a container view (docker run --rm -v vol:/vol:ro alpine du -sb /vol), replacing the host-path du that returned 0 inside the containerized controller. Failed read -> size_unknown + fits_on_dest forced false (never "fits"). Export pre-flight hard-aborts only on a KNOWN doesn.t-fit. HDD branch unchanged. F-C (LOW-MED): escrowClaimAPIHandler relays agent 404 -> clean 404 and 409 -> 409; 410 and genuine-unreachable 502 unchanged (was: 404 fell through to 502). Tests + red-proofs: ratelimit_ip_test.go (F-B x6), estimate_volsize_test.go (F-A x3), TestEscrowClaim_ProxySemantics +3 (F-C). Alpine busybox du -sb verified prod-valid. Claude-Session: https://claude.ai/code/session_01LbMm4T7Ayzs1unB9pN6Uqd @
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package appexport
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -22,6 +23,10 @@ type ExportEstimate struct {
|
||||
DestFreeBytes int64 `json:"dest_free_bytes"`
|
||||
DestFreeHuman string `json:"dest_free_human"`
|
||||
FitsOnDest bool `json:"fits_on_dest"`
|
||||
// SizeUnknown is set (v0.129.0 F-A) when a volume's size could not be read (docker helper
|
||||
// failed). When true, DataSizeBytes is a partial/understated sum and FitsOnDest is FORCED false
|
||||
// — a failed read must NEVER render as "fits". The UI shows "ismeretlen méret".
|
||||
SizeUnknown bool `json:"size_unknown"`
|
||||
}
|
||||
|
||||
// EstimateExport calculates size estimates for an app export.
|
||||
@@ -52,12 +57,23 @@ func (e *Exporter) EstimateExport(stackName, destDrive string) (*ExportEstimate,
|
||||
volumes := e.provider.GetDockerVolumes(stackName)
|
||||
e.debugf("EstimateExport: Docker volumes: %v", volumes)
|
||||
for _, vol := range volumes {
|
||||
volSize := dockerVolumeSize(vol)
|
||||
volSize, err := volumeSizer(vol)
|
||||
if err != nil {
|
||||
// F-A: the controller runs containerized, so a failed helper read must not
|
||||
// silently become 0-that-reads-as-fits. Mark unknown and keep going.
|
||||
e.logger.Printf("[WARN] appexport: volume size unknown for %s: %v", vol, err)
|
||||
est.SizeUnknown = true
|
||||
continue
|
||||
}
|
||||
e.debugf("EstimateExport: volume %s = %s", vol, humanizeBytes(volSize))
|
||||
est.DataSizeBytes += volSize
|
||||
}
|
||||
}
|
||||
est.DataSizeHuman = humanizeBytes(est.DataSizeBytes)
|
||||
if est.SizeUnknown {
|
||||
est.DataSizeHuman = "ismeretlen méret"
|
||||
} else {
|
||||
est.DataSizeHuman = humanizeBytes(est.DataSizeBytes)
|
||||
}
|
||||
|
||||
est.TotalSizeBytes = est.ConfigSizeBytes + est.DataSizeBytes
|
||||
est.TotalSizeHuman = humanizeBytes(est.TotalSizeBytes)
|
||||
@@ -75,9 +91,10 @@ func (e *Exporter) EstimateExport(stackName, destDrive string) (*ExportEstimate,
|
||||
est.DestFreeBytes = DiskFree(exportDir)
|
||||
est.DestFreeHuman = humanizeBytes(est.DestFreeBytes)
|
||||
|
||||
// Need ~10% overhead for tar.gz metadata + compression margin
|
||||
// Need ~10% overhead for tar.gz metadata + compression margin. F-A: a size we could not read
|
||||
// must never render as "fits" — an unknown-size estimate is conservatively not-fits.
|
||||
needed := est.TotalSizeBytes + est.TotalSizeBytes/10
|
||||
est.FitsOnDest = est.DestFreeBytes >= needed
|
||||
est.FitsOnDest = !est.SizeUnknown && est.DestFreeBytes >= needed
|
||||
|
||||
e.debugf("EstimateExport: total=%s free=%s fits=%v needed=%s minutes=%d",
|
||||
est.TotalSizeHuman, est.DestFreeHuman, est.FitsOnDest, humanizeBytes(needed), est.EstimatedMinutes)
|
||||
@@ -118,21 +135,33 @@ func duBytes(path string) int64 {
|
||||
return size
|
||||
}
|
||||
|
||||
// dockerVolumeSize estimates the size of a Docker named volume.
|
||||
func dockerVolumeSize(volumeName string) int64 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
// volumeSizer returns the byte size of a named Docker volume as seen from a CONTAINER view.
|
||||
// Package var so unit tests inject a fake (returning a known size or an error) without shelling out
|
||||
// to real docker. F-A (v0.129.0): the old dockerVolumeSize `du`d the host mountpoint from
|
||||
// `docker volume inspect`, which is NOT visible inside the containerized controller → always 0.
|
||||
var volumeSizer = realVolumeSize
|
||||
|
||||
// realVolumeSize `du -sb`s the volume mounted read-only into a throwaway helper container — the same
|
||||
// container-view pattern the export path uses (appexport/export.go withVolumeHelper). It mounts the
|
||||
// NAMED VOLUME by name (never a controller-host path — the v0.125.0 strand class). Returns an error
|
||||
// on any failure; callers treat that as "unknown size", never as 0.
|
||||
func realVolumeSize(volumeName string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
// Use docker system df -v and parse, or inspect the volume mount path
|
||||
out, err := exec.CommandContext(ctx, "docker", "volume", "inspect",
|
||||
"--format", "{{.Mountpoint}}", volumeName).Output()
|
||||
var out bytes.Buffer
|
||||
stderr, err := dockerExec(ctx, nil, &out, "run", "--rm", "-v", volumeName+":/vol:ro", "alpine", "du", "-sb", "/vol")
|
||||
if err != nil {
|
||||
return 0
|
||||
return 0, fmt.Errorf("sizing volume %s: %s: %w", volumeName, stderr, err)
|
||||
}
|
||||
mountpoint := strings.TrimSpace(string(out))
|
||||
if mountpoint == "" {
|
||||
return 0
|
||||
fields := strings.Fields(out.String())
|
||||
if len(fields) == 0 {
|
||||
return 0, fmt.Errorf("sizing volume %s: empty du output", volumeName)
|
||||
}
|
||||
return duBytes(mountpoint)
|
||||
var size int64
|
||||
if _, err := fmt.Sscanf(fields[0], "%d", &size); err != nil {
|
||||
return 0, fmt.Errorf("sizing volume %s: parse %q: %w", volumeName, fields[0], err)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// DiskFree returns available bytes on the filesystem containing path (0 on any error).
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package appexport
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// hddProvider is an rtProvider that reports an HDD-backed stack (for the regression scenario H).
|
||||
type hddProvider struct {
|
||||
*rtProvider
|
||||
mounts []string
|
||||
}
|
||||
|
||||
func (p *hddProvider) GetStackNeedsHDD(string) bool { return true }
|
||||
func (p *hddProvider) GetStackHDDMounts(string) []string { return p.mounts }
|
||||
|
||||
func newEstimator(t *testing.T, provider ExportStackProvider) *Exporter {
|
||||
t.Helper()
|
||||
return NewExporter(provider, log.New(io.Discard, "", 0), "test")
|
||||
}
|
||||
|
||||
// Scenario F (the F-A fix): a volume-only app with a >1 GiB volume reports the REAL size via the
|
||||
// container-view sizer — not 0/"3.6 KB". This is the F-A red-proof anchor (revert EstimateExport to
|
||||
// dockerVolumeSize → reads 0).
|
||||
func TestEstimate_VolumeSize_RealNotZero(t *testing.T) {
|
||||
const twoGiB = int64(2) << 30
|
||||
orig := volumeSizer
|
||||
volumeSizer = func(vol string) (int64, error) { return twoGiB, nil }
|
||||
defer func() { volumeSizer = orig }()
|
||||
|
||||
e := newEstimator(t, &rtProvider{stackDir: t.TempDir(), volumes: []string{"app_data"}})
|
||||
est, err := e.EstimateExport("app", t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if est.SizeUnknown {
|
||||
t.Fatalf("size must be known when the sizer succeeds")
|
||||
}
|
||||
if est.DataSizeBytes != twoGiB {
|
||||
t.Fatalf("DataSizeBytes = %d, want %d (WRONG would be 0 — the F-A bug)", est.DataSizeBytes, twoGiB)
|
||||
}
|
||||
if !strings.Contains(est.DataSizeHuman, "GB") {
|
||||
t.Fatalf("DataSizeHuman = %q, want GB-scale (WRONG would be \"3.6 KB\")", est.DataSizeHuman)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario G: a failed volume read must never render as "fits". Size is marked unknown, the human
|
||||
// string says so, and FitsOnDest is forced false.
|
||||
func TestEstimate_VolumeSize_FailureNeverFits(t *testing.T) {
|
||||
orig := volumeSizer
|
||||
volumeSizer = func(vol string) (int64, error) { return 0, errors.New("docker: no such image") }
|
||||
defer func() { volumeSizer = orig }()
|
||||
|
||||
e := newEstimator(t, &rtProvider{stackDir: t.TempDir(), volumes: []string{"app_data"}})
|
||||
est, err := e.EstimateExport("app", t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !est.SizeUnknown {
|
||||
t.Fatalf("a failed volume read must set SizeUnknown")
|
||||
}
|
||||
if est.FitsOnDest {
|
||||
t.Fatalf("an unknown size must NEVER render as fits_on_dest:true")
|
||||
}
|
||||
if est.DataSizeHuman != "ismeretlen méret" {
|
||||
t.Fatalf("DataSizeHuman = %q, want \"ismeretlen méret\"", est.DataSizeHuman)
|
||||
}
|
||||
if est.DataSizeBytes != 0 {
|
||||
t.Fatalf("no successful read → DataSizeBytes should be 0, got %d", est.DataSizeBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario H (regression): an HDD-backed stack must NOT touch the new volume sizer — the HDD branch
|
||||
// (duBytes on the mounted /mnt path) is unchanged. Platform-independent: assert the seam is not
|
||||
// invoked and SizeUnknown stays false.
|
||||
func TestEstimate_HDDPath_DoesNotUseVolumeSizer(t *testing.T) {
|
||||
called := false
|
||||
orig := volumeSizer
|
||||
volumeSizer = func(vol string) (int64, error) { called = true; return 0, nil }
|
||||
defer func() { volumeSizer = orig }()
|
||||
|
||||
p := &hddProvider{rtProvider: &rtProvider{stackDir: t.TempDir()}, mounts: []string{t.TempDir()}}
|
||||
e := newEstimator(t, p)
|
||||
est, err := e.EstimateExport("app", t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if called {
|
||||
t.Fatalf("HDD-backed stack must not call the docker volume sizer")
|
||||
}
|
||||
if est.SizeUnknown {
|
||||
t.Fatalf("HDD branch must not set SizeUnknown")
|
||||
}
|
||||
}
|
||||
@@ -197,9 +197,14 @@ func (e *Exporter) executeExport(req ExportRequest, job *Job) {
|
||||
if err != nil {
|
||||
e.debugf("estimate error (non-fatal): %v", err)
|
||||
} else {
|
||||
e.debugf("estimate: config=%s data=%s total=%s destFree=%s fits=%v",
|
||||
est.ConfigSizeHuman, est.DataSizeHuman, est.TotalSizeHuman, est.DestFreeHuman, est.FitsOnDest)
|
||||
if !est.FitsOnDest {
|
||||
e.debugf("estimate: config=%s data=%s total=%s destFree=%s fits=%v unknown=%v",
|
||||
est.ConfigSizeHuman, est.DataSizeHuman, est.TotalSizeHuman, est.DestFreeHuman, est.FitsOnDest, est.SizeUnknown)
|
||||
// Hard-abort only on a KNOWN doesn't-fit. F-A: est.SizeUnknown forces FitsOnDest=false for
|
||||
// the UI honesty signal, but an unmeasured size must NOT block the export here — the tar
|
||||
// streaming and the destination filesystem surface a real ENOSPC if it genuinely won't fit.
|
||||
if est.SizeUnknown {
|
||||
e.logger.Printf("[WARN] appexport: export space pre-check skipped for %s — volume size unknown", req.StackName)
|
||||
} else if !est.FitsOnDest {
|
||||
e.failJob(job, step, fmt.Sprintf("Nincs elég hely: szükséges ~%s, szabad %s",
|
||||
est.TotalSizeHuman, est.DestFreeHuman))
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user