v0.243.0: FileBrowser generated admin password (R-513); per-tier whole-guest backup truth (R-517); skip absent-storage tiers (R-518); OOM-killed worker visible (R-514)
gates / gates (push) Successful in 14s

MinAgent: 0.131.0

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-15 10:12:08 +02:00
parent 406755fa8f
commit 843b319f35
23 changed files with 1095 additions and 8 deletions
+6 -2
View File
@@ -112,6 +112,9 @@ type Loop struct {
// tierNotify (R-97a) reports a tier's backup outcome to the hub. nil = not wired (pre-provisioning).
// Init-only: set once at startup via SetTierNotifier, before Run.
tierNotify TierNotifier
// R-518: tiers skipped for absent storage, noted once per absence (skipAbsentTiers).
skipMu sync.Mutex
skipNoted map[string]bool
// suppressed (R-97b) is stack name → grace expiry (zero = still quiesced). Read by
// SuppressedStacks so an app WE stopped is not reported to the customer as broken.
suppressMu sync.Mutex
@@ -431,8 +434,9 @@ func (l *Loop) allTiersForManualRun(ctx context.Context) []dueTier {
}
return []dueTier{{target: ""}}
}
out := make([]dueTier, 0, len(tiers))
for _, t := range tiers {
kept := l.skipAbsentTiers(tiers)
out := make([]dueTier, 0, len(kept))
for _, t := range kept {
out = append(out, dueTier{target: t.Target})
}
return out
@@ -0,0 +1,81 @@
package quiesce
import (
"context"
"sync"
"testing"
"time"
)
// R-518 (cheap half) — a tier whose storage the agent reports ABSENT is not attempted, and no app is
// stopped for it. BIGNIGHT: the apps stayed stopped through a felhom-pbs tier whose storage did not
// exist, then that tier failed.
type skipRecorder struct {
mu sync.Mutex
skipped []string
}
func (r *skipRecorder) BackupFailed(string, string, string) {}
func (r *skipRecorder) BackupRecovered(string, string) {}
func (r *skipRecorder) BackupTierSkipped(tier, _ string) {
r.mu.Lock()
defer r.mu.Unlock()
r.skipped = append(r.skipped, tier)
}
// RED-PROOF (run 2026-09-15, recorded in REPORT.md): with skipAbsentTiers returning its input
// unchanged, the manual run started [local felhom-pbs] and this failed at "absent tier was attempted".
func TestManualRun_AbsentTierSkipped(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
be.phases["local"] = []string{phaseDone}
st := &fakeStacks{running: []string{"immich"}}
l := newTierLoop(t, be, st, nil)
rec := &skipRecorder{}
l.SetTierNotifier(rec)
if err := l.quiesceAndPollTiers(context.Background(), l.allTiersForManualRun(context.Background())); err != nil {
t.Fatalf("manual cycle: %v", err)
}
if got := be.startedTargets(); len(got) != 1 || got[0] != "local" {
t.Fatalf("absent tier was attempted: started=%v", got)
}
if stops, starts := len(st.stoppedNames()), len(st.startedNames()); stops != 1 || starts != 1 {
t.Fatalf("want one stop/start for the local tier, got %d/%d", stops, starts)
}
// A second resolution in the same absence does not notify again.
_ = l.allTiersForManualRun(context.Background())
if len(rec.skipped) != 1 || rec.skipped[0] != "felhom-pbs" {
t.Fatalf("want exactly one skip notice for felhom-pbs, got %v", rec.skipped)
}
}
// Only the absent tier is due → no quiesce at all.
func TestScheduled_OnlyAbsentTierDue_NoQuiesce(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs", StorageAbsent: true}}
be.dueSet["felhom-pbs"] = true
st := &fakeStacks{running: []string{"immich"}}
l := newTierLoop(t, be, st, nil)
l.now = func() time.Time { return time.Date(2026, 9, 15, 1, 0, 0, 0, time.UTC) }
_ = l.runOnce(context.Background())
if stops := len(st.stoppedNames()); stops != 0 {
t.Fatalf("an app was stopped for a tier whose storage does not exist (stops=%d)", stops)
}
if got := be.startedTargets(); len(got) != 0 {
t.Fatalf("absent tier started: %v", got)
}
}
// Unknown/legacy storage (StorageAbsent=false) is never skipped — fail toward backing up.
func TestManualRun_UnknownStorageNotSkipped(t *testing.T) {
be := newTierBackend()
be.tiers = []BackupTier{{Target: "local", Primary: true}, {Target: "felhom-pbs"}}
be.phases["local"] = []string{phaseDone}
be.phases["felhom-pbs"] = []string{phaseDone}
l := newTierLoop(t, be, &fakeStacks{running: []string{"immich"}}, nil)
if got := l.allTiersForManualRun(context.Background()); len(got) != 2 {
t.Fatalf("a tier without an absent verdict was dropped: %v", got)
}
}
+45
View File
@@ -35,6 +35,43 @@ var ErrTiersUnsupported = errors.New("quiesce: agent does not serve /backup/tier
type BackupTier struct {
Target string
Primary bool
// StorageAbsent (R-518, agent >= v0.131.0): the agent POSITIVELY determined the tier's storage
// does not exist. Only then is the tier skipped; unknown/legacy stay false (fail toward backing up).
StorageAbsent bool
}
// TierSkipNotifier is the OPTIONAL notifier extension for a tier skipped because its storage does not
// exist (R-518). Optional so every existing TierNotifier keeps compiling.
type TierSkipNotifier interface {
BackupTierSkipped(tier, message string)
}
// skipAbsentTiers drops the tiers whose storage the agent reports absent (R-518's cheap half,
// BIGNIGHT: the apps stayed stopped through a PBS tier whose storage did not exist, then it failed).
// It logs every skip and notifies ONCE per tier per absence (a scheduled loop re-resolves every poll;
// the note re-arms when the storage reappears).
func (l *Loop) skipAbsentTiers(tiers []BackupTier) []BackupTier {
out := make([]BackupTier, 0, len(tiers))
l.skipMu.Lock()
defer l.skipMu.Unlock()
if l.skipNoted == nil {
l.skipNoted = map[string]bool{}
}
for _, t := range tiers {
if !t.StorageAbsent {
delete(l.skipNoted, t.Target)
out = append(out, t)
continue
}
l.logger.Printf("[WARN] [quiesce] tier %s skipped: its storage does not exist on the host — no app is stopped for it", tierLabel(t.Target))
if !l.skipNoted[t.Target] {
l.skipNoted[t.Target] = true
if n, ok := l.tierNotify.(TierSkipNotifier); ok && n != nil {
n.BackupTierSkipped(t.Target, "Whole-guest backup tier "+t.Target+" skipped: its storage does not exist on the host (never provisioned or removed). No app was stopped for it.")
}
}
}
return out
}
// TieredBackend is the OPTIONAL R-82 extension to Backend. A backend that does not implement it
@@ -120,6 +157,14 @@ func (l *Loop) resolveDueTiers(ctx context.Context) (due []dueTier, degraded boo
if terr != nil {
return nil, false, terr
}
if len(tiers) > 0 {
if kept := l.skipAbsentTiers(tiers); len(kept) == 0 {
l.logger.Printf("[WARN] [quiesce] every advertised tier's storage is absent — nothing to back up this cycle")
return nil, false, nil
} else {
tiers = kept
}
}
if len(tiers) == 0 {
// An agent that advertises no tiers cannot be backed up per-tier, but it can still be
// backed up untargeted. Fail toward DOING the backup, never toward skipping it.