Review fix 5: the box sends its end records only to a hub that says it reads them (version skew)
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
A new box with an older hub sent `ended` in the window request; that hub reads 4 KiB of it, decided the window on a wrong count, and the weekly clean-up stalled. - The box sends `ended` only when the hub's last window answer said accepts_ended (kept in settings, re-read from every answer, so a rolled-back hub stops getting them after one answer). Otherwise the request is the one every hub always got, and the ordinary clean-up runs. - One request carries at most 5000 snapshot ids. Tests seen red before the fix; 6 mutations seen red (felhom.eu documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
c7e4ec9b5e
commit
c5acb91a44
10 files changed
+179
-5
No files matched your search
@@ -36,6 +36,9 @@ type OffsiteEndedClaim struct {
|
||||
SnapshotIDs []string `json:"snapshot_ids"`
|
||||
}
|
||||
|
||||
// maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs).
|
||||
const maxEndedClaimIDs = 5000
|
||||
|
||||
// OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged).
|
||||
type OffsiteEndedRefusal struct {
|
||||
App string `json:"app"`
|
||||
@@ -319,3 +322,21 @@ func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings
|
||||
func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone {
|
||||
return m.settings.GetOffsiteEndsDone()
|
||||
}
|
||||
|
||||
// boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked
|
||||
// again in a later window: the record stays while a copy of the app is in the store.
|
||||
func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim {
|
||||
left := limit
|
||||
var out []OffsiteEndedClaim
|
||||
for _, c := range claims {
|
||||
if left <= 0 {
|
||||
break
|
||||
}
|
||||
if len(c.SnapshotIDs) > left {
|
||||
c.SnapshotIDs = c.SnapshotIDs[:left]
|
||||
}
|
||||
left -= len(c.SnapshotIDs)
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Review of 2026-11-15, finding 5 — VERSION SKEW. A new box with an older hub: the window request with `ended` can
|
||||
// pass that hub's 4 KiB read limit (about 55 ids); the hub then reads no count, opens the window with the smallest
|
||||
// cap, and the guard refuses the honest plan every week — the ordinary clean-up stalls. The box now sends its end
|
||||
// records ONLY to a hub that said, in its last window answer, that it reads them.
|
||||
// RED-PROOF: felhom.eu/documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt.
|
||||
func TestD84_ANewBoxWithAnOlderHub_NothingEndedIsSentAndTheCleanUpStillRuns(t *testing.T) {
|
||||
m, sett, er, hub := endedBox(t, "immich")
|
||||
if err := sett.SetOffsiteHubAcceptsEnded(false); err != nil { // a box that has never heard a hub say it
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub.oldHub = true
|
||||
pressed(t, sett, "vikunja", 31)
|
||||
old := time.Now().Add(-60 * 24 * time.Hour)
|
||||
er.plan = []guardSnap{tagged("p1", old, "immich"), tagged("p2", old.Add(time.Minute), "immich")} // the week's honest plan
|
||||
er.snaps = append(er.snaps, er.plan...)
|
||||
for week := 1; week <= 2; week++ {
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 {
|
||||
t.Fatalf("week %d: the box sent %d end record(s) to a hub that never said it reads them — that hub reads 4 KiB and would stall", week, len(hub.claims))
|
||||
}
|
||||
er.plan = nil // the policy's two are gone after the first window (the fake's plan does not follow by itself)
|
||||
}
|
||||
// the ordinary weekly clean-up ran: the policy's two went in the first window, and nothing of the removed app
|
||||
if got := er.removedIDs(); strings.Join(got, ",") != strings.Join(ids("p1", "p2"), ",") {
|
||||
t.Fatalf("removed %v, want the policy's own two and nothing of the removed app", got)
|
||||
}
|
||||
if hub.closed[0].Outcome != "pruned" || len(hub.closed[0].RemovedForEndedApps) != 0 {
|
||||
t.Fatalf("the first window: %+v", hub.closed[0])
|
||||
}
|
||||
if _, ok := sett.GetOffsiteEnd("vikunja"); !ok {
|
||||
t.Fatal("the record was lost while the hub could not be asked")
|
||||
}
|
||||
|
||||
// The hub is updated. An ordinary night's answer (not granted) says accepts_ended; the next window carries the
|
||||
// end records, and the removed app's copies go.
|
||||
hub.oldHub, hub.notGranted = false, true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 || !sett.GetOffsiteHubAcceptsEnded() {
|
||||
t.Fatalf("the night the box LEARNS it: claims=%d learned=%v — it must send none yet and remember the answer", len(hub.claims), sett.GetOffsiteHubAcceptsEnded())
|
||||
}
|
||||
hub.notGranted = false
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 1 || hub.claims[0].App != "vikunja" {
|
||||
t.Fatalf("after the hub said it reads end records the box sent %+v", hub.claims)
|
||||
}
|
||||
if got := er.removedIDs(); len(got) != 5 {
|
||||
t.Fatalf("removed %v, want the policy's two and vikunja's three", got)
|
||||
}
|
||||
// … and if the hub is rolled BACK, the box stops sending at once after the first answer without the word
|
||||
pressed(t, sett, "kimai", 31)
|
||||
hub.oldHub = true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") // this one still carries claims (the box had heard yes)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 || sett.GetOffsiteHubAcceptsEnded() {
|
||||
t.Fatalf("after a rolled-back hub's answer the box still sends end records: claims=%d learned=%v", len(hub.claims), sett.GetOffsiteHubAcceptsEnded())
|
||||
}
|
||||
}
|
||||
|
||||
// One request never carries more ids than the hub reads (5000): the rest go in a later window.
|
||||
func TestD84_OneRequestCarriesABoundedNumberOfIds(t *testing.T) {
|
||||
m, sett, er, hub := endedBox(t, "immich")
|
||||
pressed(t, sett, "bigapp", 31)
|
||||
old := time.Now().Add(-60 * 24 * time.Hour)
|
||||
for i := 0; i < maxEndedClaimIDs+200; i++ {
|
||||
er.snaps = append(er.snaps, tagged(fmt.Sprintf("b%05d", i), old, "bigapp"))
|
||||
}
|
||||
hub.notGranted = true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
n := 0
|
||||
for _, c := range hub.claims {
|
||||
n += len(c.SnapshotIDs)
|
||||
}
|
||||
if n != maxEndedClaimIDs {
|
||||
t.Fatalf("one request carried %d snapshot ids, want the limit %d", n, maxEndedClaimIDs)
|
||||
}
|
||||
}
|
||||
@@ -86,12 +86,18 @@ type agreeingHub struct {
|
||||
allowAll func() []string
|
||||
refuse []OffsiteEndedRefusal
|
||||
maxRm int
|
||||
// oldHub: a hub from before ended apps — it never says accepts_ended. notGranted: the answer of an ordinary night.
|
||||
oldHub bool
|
||||
notGranted bool
|
||||
}
|
||||
|
||||
func (h *agreeingHub) Open(ctx context.Context, n int, ended []OffsiteEndedClaim) (OffsiteWindow, error) {
|
||||
h.opened++
|
||||
h.claims = ended
|
||||
w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse}
|
||||
w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse, AcceptsEnded: !h.oldHub}
|
||||
if h.notGranted {
|
||||
w = OffsiteWindow{Reason: "not due", AcceptsEnded: !h.oldHub}
|
||||
}
|
||||
if h.allowAll != nil {
|
||||
w.EndedAllowed = h.allowAll()
|
||||
}
|
||||
@@ -113,6 +119,9 @@ func endedBox(t *testing.T, living ...string) (*Manager, *settings.Settings, *en
|
||||
tagged("k1", old, "kimai"),
|
||||
}}
|
||||
m.SetOffboxRunner(er.run)
|
||||
if err := sett.SetOffsiteHubAcceptsEnded(true); err != nil { // the box has heard this hub say it knows ended apps
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub := &agreeingHub{maxRm: 5}
|
||||
hub.allowAll = func() []string {
|
||||
var all []string
|
||||
|
||||
@@ -60,6 +60,10 @@ type OffsiteWindow struct {
|
||||
// MaxRemove — and what it refused. Empty from a hub that does not know ended apps: then none is removed.
|
||||
EndedAllowed []string
|
||||
EndedRefused []OffsiteEndedRefusal
|
||||
// AcceptsEnded: this hub said, in this answer, that it reads `ended` in a window request (the hub's
|
||||
// `accepts_ended`; review of 2026-11-15, version skew). An older hub does not say it, and then the box sends it
|
||||
// no end records: that hub reads only 4 KiB of the request and would decide the window on a wrong count.
|
||||
AcceptsEnded bool
|
||||
}
|
||||
|
||||
// OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it).
|
||||
@@ -238,11 +242,23 @@ func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string
|
||||
due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving)
|
||||
m.dropEndsWithNoCopies(snaps, due)
|
||||
claims, candidate := endedCandidates(snaps, due)
|
||||
// VERSION SKEW (review of 2026-11-15): the end records go ONLY to a hub that said, in its last window answer,
|
||||
// that it reads them. An older hub reads 4 KiB of the request; a larger one made it open the window on a wrong
|
||||
// count, and the weekly clean-up stalled. A hub that has not said it gets the request it always got.
|
||||
if len(claims) > 0 && !m.settings.GetOffsiteHubAcceptsEnded() {
|
||||
m.logger.Printf("[INFO] [offbox] %d removed app(s) are due, but the hub has not said that it reads end records (an older hub) — none is sent, nothing of them is removed; the ordinary clean-up goes on", len(claims))
|
||||
claims, candidate = nil, map[string]bool{}
|
||||
}
|
||||
claims = boundEndedClaims(claims, maxEndedClaimIDs)
|
||||
w, err := m.offsiteWindow.Open(ctx, len(snaps), claims)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err)
|
||||
return
|
||||
}
|
||||
// Remember what THIS answer said (granted or not): the next request follows it.
|
||||
if serr := m.settings.SetOffsiteHubAcceptsEnded(w.AcceptsEnded); serr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] the hub's answer about end records could not be saved: %v", serr)
|
||||
}
|
||||
if !w.Granted {
|
||||
m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason)
|
||||
return
|
||||
|
||||
@@ -40,6 +40,9 @@ func TestD84_WindowClient_SendsTheClaimsAndReadsTheAgreement(t *testing.T) {
|
||||
if err := json.Unmarshal(gotBody["ended"], &sent); err != nil || len(sent) != 1 || sent[0].App != "vikunja" || strings.Join(sent[0].SnapshotIDs, ",") != "aa,bb,cc" || sent[0].RemovedAt != "2026-11-15T10:00:00Z" {
|
||||
t.Fatalf("the hub did not receive the claim: %s (%v)", gotBody["ended"], err)
|
||||
}
|
||||
if w.AcceptsEnded {
|
||||
t.Fatal("an answer WITHOUT accepts_ended (an older hub) reads as a hub that knows end records")
|
||||
}
|
||||
if !w.Granted || w.ID != 9 || w.MaxRemove != 10 || strings.Join(w.EndedAllowed, ",") != "aa,bb" || len(w.EndedRefused) != 1 || w.EndedRefused[0].App != "immich" {
|
||||
t.Fatalf("window = %+v, want the agreement (aa,bb) and the refusal of immich", w)
|
||||
}
|
||||
@@ -52,9 +55,13 @@ func TestD84_WindowClient_SendsTheClaimsAndReadsTheAgreement(t *testing.T) {
|
||||
t.Fatalf("a request with no claim carries `ended`: %v", gotBody)
|
||||
}
|
||||
|
||||
// A window that is NOT granted carries no agreement, whatever the answer holds.
|
||||
answer = `{"granted":false,"reason":"not due","ended_allowed":["aa","bb"]}`
|
||||
// A window that is NOT granted carries no agreement, whatever the answer holds — and it does carry the hub's
|
||||
// word that it reads end records (the box learns it on an ordinary night).
|
||||
answer = `{"granted":false,"reason":"not due","ended_allowed":["aa","bb"],"accepts_ended":true}`
|
||||
w2, err := c.Open(context.Background(), 20, claims)
|
||||
if err == nil && !w2.AcceptsEnded {
|
||||
t.Fatal("accepts_ended of a not-granted answer was not read")
|
||||
}
|
||||
if err != nil || w2.Granted || len(w2.EndedAllowed) != 0 {
|
||||
t.Fatalf("not granted: %+v %v — an agreement without a window must not reach the box", w2, err)
|
||||
}
|
||||
|
||||
@@ -146,11 +146,13 @@ func (c HubWindowClient) Open(ctx context.Context, countBefore int, ended []back
|
||||
Reason string `json:"reason"`
|
||||
EndedAllowed []string `json:"ended_allowed"`
|
||||
EndedRefused []backup.OffsiteEndedRefusal `json:"ended_refused"`
|
||||
// AcceptsEnded: this hub reads `ended` (in every answer of a hub that does, granted or not).
|
||||
AcceptsEnded bool `json:"accepts_ended"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return backup.OffsiteWindow{}, fmt.Errorf("hub window-open: malformed response")
|
||||
}
|
||||
w := backup.OffsiteWindow{Granted: r.Granted, ID: r.WindowID, MaxRemove: r.MaxRemove, Reason: r.Reason}
|
||||
w := backup.OffsiteWindow{Granted: r.Granted, ID: r.WindowID, MaxRemove: r.MaxRemove, Reason: r.Reason, AcceptsEnded: r.AcceptsEnded}
|
||||
if r.Granted {
|
||||
t, perr := time.Parse(time.RFC3339, r.NewestAllowed)
|
||||
if perr != nil {
|
||||
|
||||
@@ -169,3 +169,22 @@ func (s *Settings) GetOffsiteEndsDone() []OffsiteEndDone {
|
||||
defer s.mu.RUnlock()
|
||||
return append([]OffsiteEndDone(nil), s.OffsiteEndsDone...)
|
||||
}
|
||||
|
||||
// OffsiteHubAcceptsEnded: did the hub's LAST window answer say that it reads end records (`accepts_ended`)? Kept in
|
||||
// settings so a controller restart does not cost a week: the box sends its end records only to a hub that said so.
|
||||
func (s *Settings) GetOffsiteHubAcceptsEnded() bool {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.OffsiteHubAcceptsEnded
|
||||
}
|
||||
|
||||
// SetOffsiteHubAcceptsEnded records the hub's last answer; it writes only on a change.
|
||||
func (s *Settings) SetOffsiteHubAcceptsEnded(v bool) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.OffsiteHubAcceptsEnded == v {
|
||||
return nil
|
||||
}
|
||||
s.OffsiteHubAcceptsEnded = v
|
||||
return s.save()
|
||||
}
|
||||
@@ -233,6 +233,8 @@ type Settings struct {
|
||||
// OffsiteEndsDone is the short history of ends a clean-up window has carried out. See offsite_ends.go.
|
||||
OffsiteEnds map[string]OffsiteEnd `json:"offsite_ends,omitempty"`
|
||||
OffsiteEndsDone []OffsiteEndDone `json:"offsite_ends_done,omitempty"`
|
||||
// OffsiteHubAcceptsEnded: the hub's last window answer said it reads end records (offsite_ends.go).
|
||||
OffsiteHubAcceptsEnded bool `json:"offsite_hub_accepts_ended,omitempty"`
|
||||
// UnhealthyStops (v0.269.0, decision 28) — when the box last stopped each app for a crash loop or an
|
||||
// out-of-memory storm (RFC3339). Survives the hold being lifted by Start, so a second stop within
|
||||
// 24 h is known to be a second one.
|
||||
|
||||
Reference in new issue
Block a user