v0.194.0 — one operator email per backup run, and nothing dropped without a trace (R-182)
gates / gates (push) Successful in 8s
gates / gates (push) Successful in 8s
MEASURED, not supposed. On 2026-08-03 nine per-app recovery_unit_capture_failed events reached the hub and TWO operator emails went out. The hub's operator cooldown key is customerID:eventType(+tier) and that event carries `app` but no `tier`, so the key held no app identifier: the first refused app took the hour's slot and every other app's failure was discarded BEFORE anything was written down, leaving no row on any channel. The obvious fix — put `app` in the key — was ruled against: on a full disk it produces one email per app, the volume problem wearing the correctness problem's clothes. internal/backup/runsummary.go: a per-run collector with exactly admissionSet's lifetime, fed by all three write legs, emitting backup_run_failures ONCE at the end and only when something failed. A clean run emits nothing. The per-app event stays and becomes the RECORD — the hub routes it record-only, stored and logged every time, never competing for an email slot. The record and the notification are now different things. Deliberate skips (disconnected, decommissioned) are excluded: they have their own alert, and a nightly email about an unplugged drive is one the operator learns to ignore. A manual run always reports: the digest carries a unique run_id the cooldown cannot collapse. Someone pressing the button is actively trying to get a backup. THE PERIODIC SWEEP GETS A DIGEST TOO. With the per-app event now record-only, a capture failure found between runs would be recorded and never notified — a new silence introduced while closing one. That path emits a digest with NO run_id, so the ordinary 1-hour cooldown caps it exactly as before while the mail now lists every failing app instead of whichever was first. A refusal is recorded ONCE, where the verdict is taken, not at the three legs that consult it — R-181's contract is one verdict per app per run. Noting it per leg listed one refused app three times and produced "2 of 1 apps failed". Found by the digest's own test, not in review. Silence is safe because the hub's deadline check raises expected_backup_missed from report freshness, independently of any mail this box sends (monitor/deadline.go:396,417). Confirmed, not assumed. 7 new tests, 4 red-proofs. The main.go seam walk did NOT fail on its first attempt — the AST test walked the backup package and not main.go; the test was fixed and the mutation re-run rather than the pass recorded.
This commit is contained in:
@@ -15,6 +15,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// R-181 — the reserve guards the write that fills the disk, and its promise is true.
|
||||
@@ -33,6 +35,7 @@ import (
|
||||
type admissionProvider struct {
|
||||
stacks []string
|
||||
volumes map[string][]string
|
||||
hdd map[string]string // per-app drive path, for the drive-state skip tests
|
||||
dir string
|
||||
infoHits []string
|
||||
stopped []string
|
||||
@@ -47,7 +50,7 @@ func (p *admissionProvider) ListDeployedStacks() []StackSummary {
|
||||
return out
|
||||
}
|
||||
func (p *admissionProvider) GetStackHDDMounts(string) []string { return nil }
|
||||
func (p *admissionProvider) GetStackHDDPath(string) string { return "" }
|
||||
func (p *admissionProvider) GetStackHDDPath(n string) string { return p.hdd[n] }
|
||||
func (p *admissionProvider) GetImportRoot() string { return "" }
|
||||
func (p *admissionProvider) GetDockerVolumes(name string) []string {
|
||||
if p.volumes == nil {
|
||||
@@ -88,7 +91,7 @@ func newAdmissionHarness(t *testing.T, stacks ...string) *admissionHarness {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
h := &admissionHarness{
|
||||
prov: &admissionProvider{stacks: stacks, dir: dir},
|
||||
prov: &admissionProvider{stacks: stacks, dir: dir, hdd: map[string]string{}},
|
||||
usage: map[string]*UnitSpace{},
|
||||
dir: dir,
|
||||
logs: &bytes.Buffer{},
|
||||
@@ -116,6 +119,45 @@ func newAdmissionHarness(t *testing.T, stacks ...string) *admissionHarness {
|
||||
return h
|
||||
}
|
||||
|
||||
// markDisconnected / markDecommissioned put a real settings row behind the drive-state skips, so
|
||||
// Scenario F exercises the production guards rather than a stub of them.
|
||||
func (h *admissionHarness) markDisconnected(app string) {
|
||||
h.driveState(app, true, false)
|
||||
}
|
||||
|
||||
func (h *admissionHarness) markDecommissioned(app string) {
|
||||
h.driveState(app, false, true)
|
||||
}
|
||||
|
||||
func (h *admissionHarness) driveState(app string, disconnected, decommissioned bool) {
|
||||
if h.m.settings == nil {
|
||||
sett, err := settings.Load(filepath.Join(h.dir, "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
h.m.settings = sett
|
||||
}
|
||||
// Each such app gets its OWN drive path, or marking one would skip them all.
|
||||
p := filepath.Join(h.dir, "drives", app)
|
||||
if err := os.MkdirAll(p, 0o755); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
h.prov.hdd[app] = p
|
||||
if err := h.m.settings.AddStoragePath(settings.StoragePath{Path: p, Label: app}); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if disconnected {
|
||||
if err := h.m.settings.SetDisconnected(p, true, nil); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
if decommissioned {
|
||||
if err := h.m.settings.SetDecommissioned(p, ""); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *admissionHarness) nsRoot() string { return filepath.Join(h.dir, "felhom-data") }
|
||||
|
||||
// setSpace states the filesystem's occupancy as a test INPUT — the whole point of the unitSpaceFn
|
||||
|
||||
Reference in New Issue
Block a user