v0.243.0: FileBrowser generated admin password (R-513); per-tier whole-guest backup truth (R-517); skip absent-storage tiers (R-518); OOM-killed worker visible (R-514)
gates / gates (push) Successful in 14s
gates / gates (push) Successful in 14s
MinAgent: 0.131.0 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -112,6 +112,9 @@ type Loop struct {
|
||||
// tierNotify (R-97a) reports a tier's backup outcome to the hub. nil = not wired (pre-provisioning).
|
||||
// Init-only: set once at startup via SetTierNotifier, before Run.
|
||||
tierNotify TierNotifier
|
||||
// R-518: tiers skipped for absent storage, noted once per absence (skipAbsentTiers).
|
||||
skipMu sync.Mutex
|
||||
skipNoted map[string]bool
|
||||
// suppressed (R-97b) is stack name → grace expiry (zero = still quiesced). Read by
|
||||
// SuppressedStacks so an app WE stopped is not reported to the customer as broken.
|
||||
suppressMu sync.Mutex
|
||||
@@ -431,8 +434,9 @@ func (l *Loop) allTiersForManualRun(ctx context.Context) []dueTier {
|
||||
}
|
||||
return []dueTier{{target: ""}}
|
||||
}
|
||||
out := make([]dueTier, 0, len(tiers))
|
||||
for _, t := range tiers {
|
||||
kept := l.skipAbsentTiers(tiers)
|
||||
out := make([]dueTier, 0, len(kept))
|
||||
for _, t := range kept {
|
||||
out = append(out, dueTier{target: t.Target})
|
||||
}
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package quiesce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-518 (cheap half) — a tier whose storage the agent reports ABSENT is not attempted, and no app is
|
||||
// stopped for it. BIGNIGHT: the apps stayed stopped through a felhom-pbs tier whose storage did not
|
||||
// exist, then that tier failed.
|
||||
|
||||
type skipRecorder struct {
|
||||
mu sync.Mutex
|
||||
skipped []string
|
||||
}
|
||||
|
||||
func (r *skipRecorder) BackupFailed(string, string, string) {}
|
||||
func (r *skipRecorder) BackupRecovered(string, string) {}
|
||||
func (r *skipRecorder) BackupTierSkipped(tier, _ string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.skipped = append(r.skipped, tier)
|
||||
}
|
||||
|
||||
// RED-PROOF (run 2026-09-15, recorded in REPORT.md): with skipAbsentTiers returning its input
|
||||
// unchanged, the manual run started [local felhom-pbs] and this failed at "absent tier was attempted".
|
||||
func TestManualRun_AbsentTierSkipped(t *testing.T) {
|
||||
be := newTierBackend()
|
||||
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
|
||||
be.phases["local"] = []string{phaseDone}
|
||||
st := &fakeStacks{running: []string{"immich"}}
|
||||
l := newTierLoop(t, be, st, nil)
|
||||
rec := &skipRecorder{}
|
||||
l.SetTierNotifier(rec)
|
||||
|
||||
if err := l.quiesceAndPollTiers(context.Background(), l.allTiersForManualRun(context.Background())); err != nil {
|
||||
t.Fatalf("manual cycle: %v", err)
|
||||
}
|
||||
if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
|
||||
t.Fatalf("absent tier was attempted: started=%v", got)
|
||||
}
|
||||
if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
|
||||
t.Fatalf("want one stop/start for the local tier, got %d/%d", stops, starts)
|
||||
}
|
||||
// A second resolution in the same absence does not notify again.
|
||||
_ = l.allTiersForManualRun(context.Background())
|
||||
if len(rec.skipped) != 1 || rec.skipped[0] != "felhom-pbs" {
|
||||
t.Fatalf("want exactly one skip notice for felhom-pbs, got %v", rec.skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// Only the absent tier is due → no quiesce at all.
|
||||
func TestScheduled_OnlyAbsentTierDue_NoQuiesce(t *testing.T) {
|
||||
be := newTierBackend()
|
||||
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
|
||||
be.dueSet["felhom-pbs"] = true
|
||||
st := &fakeStacks{running: []string{"immich"}}
|
||||
l := newTierLoop(t, be, st, nil)
|
||||
l.now = func() time.Time { return time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC) }
|
||||
_ = l.runOnce(context.Background())
|
||||
if stops := len(st.stoppedNames()); stops != 0 {
|
||||
t.Fatalf("an app was stopped for a tier whose storage does not exist (stops=%d)", stops)
|
||||
}
|
||||
if got := be.startedTargets(); len(got) != 0 {
|
||||
t.Fatalf("absent tier started: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Unknown/legacy storage (StorageAbsent=false) is never skipped — fail toward backing up.
|
||||
func TestManualRun_UnknownStorageNotSkipped(t *testing.T) {
|
||||
be := newTierBackend()
|
||||
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
|
||||
be.phases["local"] = []string{phaseDone}
|
||||
be.phases["felhom-pbs"] = []string{phaseDone}
|
||||
l := newTierLoop(t, be, &fakeStacks{running: []string{"immich"}}, nil)
|
||||
if got := l.allTiersForManualRun(context.Background()); len(got) != 2 {
|
||||
t.Fatalf("a tier without an absent verdict was dropped: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,43 @@ var ErrTiersUnsupported = errors.New("quiesce: agent does not serve /backup/tier
|
||||
type BackupTier struct {
|
||||
Target string
|
||||
Primary bool
|
||||
// StorageAbsent (R-518, agent >= v0.131.0): the agent POSITIVELY determined the tier's storage
|
||||
// does not exist. Only then is the tier skipped; unknown/legacy stay false (fail toward backing up).
|
||||
StorageAbsent bool
|
||||
}
|
||||
|
||||
// TierSkipNotifier is the OPTIONAL notifier extension for a tier skipped because its storage does not
|
||||
// exist (R-518). Optional so every existing TierNotifier keeps compiling.
|
||||
type TierSkipNotifier interface {
|
||||
BackupTierSkipped(tier, message string)
|
||||
}
|
||||
|
||||
// skipAbsentTiers drops the tiers whose storage the agent reports absent (R-518's cheap half,
|
||||
// BIGNIGHT: the apps stayed stopped through a PBS tier whose storage did not exist, then it failed).
|
||||
// It logs every skip and notifies ONCE per tier per absence (a scheduled loop re-resolves every poll;
|
||||
// the note re-arms when the storage reappears).
|
||||
func (l *Loop) skipAbsentTiers(tiers []BackupTier) []BackupTier {
|
||||
out := make([]BackupTier, 0, len(tiers))
|
||||
l.skipMu.Lock()
|
||||
defer l.skipMu.Unlock()
|
||||
if l.skipNoted == nil {
|
||||
l.skipNoted = map[string]bool{}
|
||||
}
|
||||
for _, t := range tiers {
|
||||
if !t.StorageAbsent {
|
||||
delete(l.skipNoted, t.Target)
|
||||
out = append(out, t)
|
||||
continue
|
||||
}
|
||||
l.logger.Printf("[WARN] [quiesce] tier %s skipped: its storage does not exist on the host — no app is stopped for it", tierLabel(t.Target))
|
||||
if !l.skipNoted[t.Target] {
|
||||
l.skipNoted[t.Target] = true
|
||||
if n, ok := l.tierNotify.(TierSkipNotifier); ok && n != nil {
|
||||
n.BackupTierSkipped(t.Target, "Whole-guest backup tier "+t.Target+" skipped: its storage does not exist on the host (never provisioned or removed). No app was stopped for it.")
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TieredBackend is the OPTIONAL R-82 extension to Backend. A backend that does not implement it
|
||||
@@ -120,6 +157,14 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo
|
||||
if terr != nil {
|
||||
return nil, false, terr
|
||||
}
|
||||
if len(tiers) > 0 {
|
||||
if kept := l.skipAbsentTiers(tiers); len(kept) == 0 {
|
||||
l.logger.Printf("[WARN] [quiesce] every advertised tier's storage is absent — nothing to back up this cycle")
|
||||
return nil, false, nil
|
||||
} else {
|
||||
tiers = kept
|
||||
}
|
||||
}
|
||||
if len(tiers) == 0 {
|
||||
// An agent that advertises no tiers cannot be backed up per-tier, but it can still be
|
||||
// backed up untargeted. Fail toward DOING the backup, never toward skipping it.
|
||||
|
||||
Reference in New Issue
Block a user