Placement hardening (F-3a-1..4) + enlarge-blocked delivery chain (Task 3a-fix, v0.134.1)
PlaceOffsiteRestore: live target via raw GetStackHDDPath not AppNamespaceRoot (F-3a-1a: no SSD merge; undeployed refused), placement headroom gate (F-3a-1b), stat pre-pass over all placements before any copy (F-3a-4: no partial writes), scratch removed on success/kept on failure (F-3a-2). mapOffsiteRestorePaths refuses the namespace root itself (F-3a-3). Delivery chain: DefaultEnabledEvents + GetNotificationPrefs append-if-absent migration + settings checkbox + handler slice; paired with hub v0.55.0 allowlist (no customerMessages entry — raw dynamic message survives). +8 tests; all 6 controller §10 red-proofs verified.
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// placeFixture builds a manager + provider with a scratch dir for stack on drive, a snapshot whose
|
||||
// paths anchor on `oldNs`, and (per `full`) the reconstructed scratch srcs on disk. Returns the copier
|
||||
// invocation counter pointer and the scratch dir. Free/size seams default to "plenty of room".
|
||||
func placeFixture(t *testing.T, full bool) (*Manager, *offbox3aProvider, string, *int) {
|
||||
t.Helper()
|
||||
drive := t.TempDir()
|
||||
m, _, prov := classifiedOffboxManager(t, drive)
|
||||
prov.hdd["immich"] = drive
|
||||
|
||||
scratch, liveNs, err := m.offboxRestoreScratchDir("immich")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(scratch, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Snapshot paths anchored on a synthetic POSIX namespace (drive-churn realistic; also avoids the
|
||||
// Windows volume-letter that filepath.Join can't nest — prod paths are Linux, no volume).
|
||||
oldNs := "/felhomdata/ns"
|
||||
unitP := oldNs + "/backups/primary/immich"
|
||||
dataP := oldNs + "/appdata/immich"
|
||||
snapPaths := []string{unitP, dataP}
|
||||
|
||||
// Create the reconstructed scratch srcs the code will stat — computed via the pure mapper so the
|
||||
// fixture matches the code's own path arithmetic (no hand-predicting OS separators).
|
||||
placements, err := mapOffsiteRestorePaths(snapPaths, "immich", scratch, liveNs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, pl := range placements {
|
||||
if !full && !pl.isUnit {
|
||||
continue // unit-only scratch: userdata src deliberately absent (Scenario C)
|
||||
}
|
||||
if err := os.MkdirAll(pl.src, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
|
||||
m.SetOffboxSizer(func(string) int64 { return 1 << 20 })
|
||||
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
if contains(args, "snapshots") {
|
||||
return []byte(`[{"short_id":"a","time":"2026-07-15T00:00:00Z","paths":["` + unitP + `","` + dataP + `"]}]`), nil
|
||||
}
|
||||
return nil, nil
|
||||
})
|
||||
var copies int
|
||||
m.SetOffboxPlaceCopier(func(_, _ string) (int, error) { copies++; return 1, nil })
|
||||
return m, prov, scratch, &copies
|
||||
}
|
||||
|
||||
// A (F-3a-1a): undeployed placement refused with ZERO copies (never merges onto the SSD namespace).
|
||||
func TestPlace_UndeployedRefused(t *testing.T) {
|
||||
m, prov, scratch, copies := placeFixture(t, true)
|
||||
prov.hdd["immich"] = "" // undeployed
|
||||
err := m.PlaceOffsiteRestore(context.Background(), "immich")
|
||||
if err == nil || !strings.Contains(err.Error(), "nincs telepítve") {
|
||||
t.Fatalf("undeployed must refuse with 'nincs telepítve', got %v", err)
|
||||
}
|
||||
if *copies != 0 {
|
||||
t.Errorf("copier must NOT run for an undeployed app, got %d", *copies)
|
||||
}
|
||||
if _, sErr := os.Stat(scratch); sErr != nil {
|
||||
t.Error("scratch must be untouched on refusal")
|
||||
}
|
||||
}
|
||||
|
||||
// B (F-3a-1b): placement headroom gate refuses BEFORE any copy.
|
||||
func TestPlace_HeadroomRefused(t *testing.T) {
|
||||
m, _, _, copies := placeFixture(t, true)
|
||||
m.SetOffboxFreeFn(func(string) int64 { return 1 }) // 1 byte free
|
||||
m.SetOffboxSizer(func(string) int64 { return 1 << 30 })
|
||||
err := m.PlaceOffsiteRestore(context.Background(), "immich")
|
||||
if err == nil || !strings.Contains(err.Error(), "Nincs elég szabad hely") {
|
||||
t.Fatalf("headroom gate must refuse, got %v", err)
|
||||
}
|
||||
if *copies != 0 {
|
||||
t.Errorf("copier must NOT run when headroom fails, got %d", *copies)
|
||||
}
|
||||
}
|
||||
|
||||
// C (F-3a-4): a unit-only scratch (userdata src absent) refuses with ZERO copies (stat pre-pass).
|
||||
func TestPlace_IncompleteScratchRefusedNoCopies(t *testing.T) {
|
||||
m, _, _, copies := placeFixture(t, false) // full=false → userdata src missing
|
||||
err := m.PlaceOffsiteRestore(context.Background(), "immich")
|
||||
if err == nil || !strings.Contains(err.Error(), "hiányos") {
|
||||
t.Fatalf("incomplete scratch must refuse with 'hiányos', got %v", err)
|
||||
}
|
||||
if *copies != 0 {
|
||||
t.Errorf("stat pre-pass must refuse BEFORE any copy, got %d copies", *copies)
|
||||
}
|
||||
}
|
||||
|
||||
// E (F-3a-2): success removes the scratch (ready-gate flips false); failure keeps it.
|
||||
func TestPlace_ScratchLifecycle(t *testing.T) {
|
||||
// success
|
||||
m, _, scratch, copies := placeFixture(t, true)
|
||||
if err := m.PlaceOffsiteRestore(context.Background(), "immich"); err != nil {
|
||||
t.Fatalf("placement: %v", err)
|
||||
}
|
||||
if *copies == 0 {
|
||||
t.Error("expected at least one copy on success")
|
||||
}
|
||||
if _, sErr := os.Stat(scratch); !os.IsNotExist(sErr) {
|
||||
t.Errorf("scratch must be removed after success, stat err=%v", sErr)
|
||||
}
|
||||
if m.OffboxFullScratchReady("immich") {
|
||||
t.Error("OffboxFullScratchReady must be false after cleanup")
|
||||
}
|
||||
|
||||
// failure keeps the scratch
|
||||
m2, _, scratch2, _ := placeFixture(t, true)
|
||||
m2.SetOffboxPlaceCopier(func(_, _ string) (int, error) { return 0, os.ErrPermission })
|
||||
if err := m2.PlaceOffsiteRestore(context.Background(), "immich"); err == nil {
|
||||
t.Fatal("a copier failure must surface as an error")
|
||||
}
|
||||
if _, sErr := os.Stat(scratch2); sErr != nil {
|
||||
t.Errorf("scratch must be KEPT after a failed placement (retry), stat err=%v", sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// D (F-3a-3): mapping refuses the namespace root itself among the snapshot paths.
|
||||
func TestMapOffsiteRestorePaths_RefusesNamespaceRoot(t *testing.T) {
|
||||
oldNs := "/old/ns"
|
||||
snap := []string{oldNs + "/backups/primary/app", oldNs} // oldNs itself must be refused
|
||||
if _, err := mapOffsiteRestorePaths(snap, "app", "/scratch", "/new/ns"); err == nil {
|
||||
t.Error("the namespace root itself among snapshot paths must be refused (F-3a-3)")
|
||||
}
|
||||
}
|
||||
@@ -284,7 +284,10 @@ func mapOffsiteRestorePaths(snapPaths []string, stack, scratch, liveNsRoot strin
|
||||
}
|
||||
out := make([]placement, 0, len(snapPaths))
|
||||
for _, p := range snapPaths {
|
||||
if p != oldNs && !strings.HasPrefix(p, oldNs+"/") {
|
||||
// Every captured path must be a STRICT descendant of oldNs. Requiring the trailing "/" also
|
||||
// catches p == oldNs (the namespace root itself — F-3a-3), which would otherwise map to a junk
|
||||
// placement nesting the whole old namespace under the live root.
|
||||
if !strings.HasPrefix(p, oldNs+"/") {
|
||||
return nil, fmt.Errorf("a pillanatkép egy útvonala a névtéren kívülre mutat: %s", p)
|
||||
}
|
||||
rel := strings.TrimPrefix(p, oldNs+"/")
|
||||
@@ -344,22 +347,36 @@ func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error {
|
||||
return err
|
||||
}
|
||||
_ = id
|
||||
liveNs := m.AppNamespaceRoot(stack)
|
||||
if liveNs == "" {
|
||||
return fmt.Errorf("a(z) %s élő adatmeghajtója nem határozható meg", stack)
|
||||
// F-3a-1a: the live target uses the RAW HDD path (mirrors offboxCaptureSet). NOT AppNamespaceRoot —
|
||||
// its systemDataPath fallback would merge userdata onto the SSD system namespace. Empty HDD ⇒
|
||||
// undeployed ⇒ refuse: the app must be restored first, then its data placed under its live drive.
|
||||
hdd := ""
|
||||
if m.stackProvider != nil {
|
||||
hdd = strings.TrimSpace(m.stackProvider.GetStackHDDPath(stack))
|
||||
}
|
||||
if hdd == "" {
|
||||
return fmt.Errorf("a(z) %s nincs telepítve — előbb állítsd helyre az alkalmazást, utána az adatokat", stack)
|
||||
}
|
||||
liveNs := m.namespaceRoot(hdd)
|
||||
// F-3a-1b: headroom gate — a missing-only merge copies at most the scratch size; refuse before any
|
||||
// copy if the live drive lacks that (conservative — scratch and live often share a drive).
|
||||
if free, need := m.offboxFree()(liveNs), m.offboxSize()(scratch); free < need {
|
||||
return fmt.Errorf("Nincs elég szabad hely a visszaállításhoz (%s szükséges, %s szabad).", humanizeBytes(need), humanizeBytes(free))
|
||||
}
|
||||
placements, err := mapOffsiteRestorePaths(paths, stack, scratch, liveNs)
|
||||
if err != nil {
|
||||
return err // whole-placement refusal (no partial writes)
|
||||
}
|
||||
// F-3a-4: stat pre-pass over EVERY placement BEFORE the first copy — an incomplete scratch (e.g. a
|
||||
// unit-only restore, userdata srcs absent) refuses with ZERO copies, making "no partial writes" true.
|
||||
for _, pl := range placements {
|
||||
if _, sErr := os.Stat(pl.src); sErr != nil {
|
||||
return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src))
|
||||
}
|
||||
}
|
||||
copier := m.placeCopier()
|
||||
var placed int
|
||||
for _, pl := range placements {
|
||||
if _, sErr := os.Stat(pl.src); sErr != nil {
|
||||
// The full scratch is incomplete for this path (e.g. only a unit-only restore ran) — refuse
|
||||
// rather than place a partial set.
|
||||
return fmt.Errorf("a teljes visszaállítás hiányos (%s nincs meg) — futtass előbb egy teljes visszaállítást", filepath.Base(pl.src))
|
||||
}
|
||||
if pl.isUnit {
|
||||
if _, liveErr := os.Stat(pl.dst); liveErr == nil {
|
||||
m.logger.Printf("[INFO] [offbox] place %s: live recovery unit present — not overwriting", stack)
|
||||
@@ -368,10 +385,17 @@ func (m *Manager) PlaceOffsiteRestore(ctx context.Context, stack string) error {
|
||||
}
|
||||
n, cErr := copier(pl.src, pl.dst)
|
||||
if cErr != nil {
|
||||
return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr)
|
||||
return fmt.Errorf("a(z) %s helyreállítása sikertelen: %w", stack, cErr) // scratch KEPT for retry
|
||||
}
|
||||
placed += n
|
||||
}
|
||||
// F-3a-2: on FULL success, remove the scratch best-effort (OffboxFullScratchReady then turns false →
|
||||
// the place button disappears). A failed placement returned above, keeping the scratch for a retry.
|
||||
if rmErr := os.RemoveAll(scratch); rmErr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] place %s: scratch cleanup failed (harmless): %v", stack, rmErr)
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] place %s: scratch removed after successful placement", stack)
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] placed %s from offsite scratch: %d file(s) merged (missing-only)", stack, placed)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package settings
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func contains(list []string, want string) bool {
|
||||
for _, e := range list {
|
||||
if e == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func count(list []string, want string) int {
|
||||
n := 0
|
||||
for _, e := range list {
|
||||
if e == want {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// F2a: the new warning type is on by default for new customers.
|
||||
func TestDefaultEnabledEvents_ContainsEnlargeBlocked(t *testing.T) {
|
||||
if !contains(DefaultEnabledEvents, "offbox_enlarge_blocked") {
|
||||
t.Error("DefaultEnabledEvents must contain offbox_enlarge_blocked (new customers get it)")
|
||||
}
|
||||
}
|
||||
|
||||
// F2b: an EXISTING customer's stored prefs (predating the type) gain it via append-if-absent —
|
||||
// idempotent (two reads → one entry) and their OTHER choices are preserved.
|
||||
func TestGetNotificationPrefs_MigratesExisting(t *testing.T) {
|
||||
s, err := Load(filepath.Join(t.TempDir(), "settings.json"), discardLog())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A customer who kept only two events and never had the new one.
|
||||
if err := s.SetNotificationPrefs(&NotificationPrefs{
|
||||
Email: "c@example.com",
|
||||
EnabledEvents: []string{"backup_failed", "disk_warning"},
|
||||
CooldownHours: 6,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p1 := s.GetNotificationPrefs()
|
||||
if !contains(p1.EnabledEvents, "offbox_enlarge_blocked") {
|
||||
t.Error("existing prefs must gain offbox_enlarge_blocked (append-if-absent migration)")
|
||||
}
|
||||
if !contains(p1.EnabledEvents, "backup_failed") || !contains(p1.EnabledEvents, "disk_warning") {
|
||||
t.Error("the customer's existing choices must be preserved")
|
||||
}
|
||||
|
||||
// idempotent: a second read still has exactly ONE entry.
|
||||
p2 := s.GetNotificationPrefs()
|
||||
if c := count(p2.EnabledEvents, "offbox_enlarge_blocked"); c != 1 {
|
||||
t.Errorf("migration must be idempotent, got %d entries", c)
|
||||
}
|
||||
}
|
||||
|
||||
// A customer who already has the type keeps exactly one (no duplication).
|
||||
func TestGetNotificationPrefs_AlreadyPresentNoDuplicate(t *testing.T) {
|
||||
s, err := Load(filepath.Join(t.TempDir(), "settings.json"), discardLog())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.SetNotificationPrefs(&NotificationPrefs{
|
||||
EnabledEvents: []string{"offbox_enlarge_blocked", "backup_failed"},
|
||||
CooldownHours: 6,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c := count(s.GetNotificationPrefs().EnabledEvents, "offbox_enlarge_blocked"); c != 1 {
|
||||
t.Errorf("already-present type must not duplicate, got %d", c)
|
||||
}
|
||||
}
|
||||
@@ -241,6 +241,7 @@ var DefaultEnabledEvents = []string{
|
||||
"health_critical",
|
||||
"expected_backup_missed",
|
||||
"expected_dbdump_missed",
|
||||
"offbox_enlarge_blocked", // 3a-fix (warning-class): remote enlargement refused by the quota gate
|
||||
}
|
||||
|
||||
// PendingEvent is an event queued for the next Hub push cycle.
|
||||
@@ -522,10 +523,23 @@ func (s *Settings) GetNotificationPrefs() *NotificationPrefs {
|
||||
// Return a copy of the slice
|
||||
events := make([]string, len(prefs.EnabledEvents))
|
||||
copy(events, prefs.EnabledEvents)
|
||||
prefs.EnabledEvents = events
|
||||
// 3a-fix append-if-absent migration: an existing customer's stored prefs predate
|
||||
// offbox_enlarge_blocked, so they cannot have deliberately disabled it — surface it enabled so the
|
||||
// checkbox renders checked and the startup sync (main.go:782) carries it to the hub. Idempotent.
|
||||
prefs.EnabledEvents = appendIfAbsent(events, "offbox_enlarge_blocked")
|
||||
return &prefs
|
||||
}
|
||||
|
||||
// appendIfAbsent appends want to list only if it is not already present (idempotent).
|
||||
func appendIfAbsent(list []string, want string) []string {
|
||||
for _, e := range list {
|
||||
if e == want {
|
||||
return list
|
||||
}
|
||||
}
|
||||
return append(list, want)
|
||||
}
|
||||
|
||||
// SetNotificationPrefs updates notification preferences and saves to disk.
|
||||
// H17: Deep-copies prefs so caller mutations after the call don't affect stored state.
|
||||
func (s *Settings) SetNotificationPrefs(prefs *NotificationPrefs) error {
|
||||
|
||||
@@ -1405,7 +1405,7 @@ func (s *Server) settingsNotificationsHandler(w http.ResponseWriter, r *http.Req
|
||||
// Single-event checkboxes
|
||||
for _, evt := range []string{
|
||||
"backup_failed", "db_dump_failed", "backup_integrity_failed",
|
||||
"crossdrive_failed", "storage_disconnected",
|
||||
"crossdrive_failed", "offbox_enlarge_blocked", "storage_disconnected",
|
||||
"node_down", "health_critical",
|
||||
"storage_reconnected", "health_recovered",
|
||||
} {
|
||||
|
||||
@@ -38,6 +38,10 @@
|
||||
<input type="checkbox" name="event_crossdrive_failed" {{with .NotificationPrefs}}{{range .EnabledEvents}}{{if eq . "crossdrive_failed"}}checked{{end}}{{end}}{{end}}>
|
||||
<span class="toggle-label">Másodlagos mentés sikertelen</span>
|
||||
</label>
|
||||
<label class="toggle">
|
||||
<input type="checkbox" name="event_offbox_enlarge_blocked" {{with .NotificationPrefs}}{{range .EnabledEvents}}{{if eq . "offbox_enlarge_blocked"}}checked{{end}}{{end}}{{end}}>
|
||||
<span class="toggle-label">Távoli mentés — tárhelykeret-figyelmeztetés</span>
|
||||
</label>
|
||||
<label class="toggle">
|
||||
<input type="checkbox" name="event_disk_alerts" {{with .NotificationPrefs}}{{range .EnabledEvents}}{{if eq . "disk_warning"}}checked{{end}}{{end}}{{end}}>
|
||||
<span class="toggle-label">Lemez figyelmeztetés (90%+)</span>
|
||||
|
||||
Reference in New Issue
Block a user