Review fix 5: the box sends its end records only to a hub that says it reads them (version skew)
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
A new box with an older hub sent `ended` in the window request; that hub reads 4 KiB of it, decided the window on a wrong count, and the weekly clean-up stalled. - The box sends `ended` only when the hub's last window answer said accepts_ended (kept in settings, re-read from every answer, so a rolled-back hub stops getting them after one answer). Otherwise the request is the one every hub always got, and the ordinary clean-up runs. - One request carries at most 5000 snapshot ids. Tests seen red before the fix; 6 mutations seen red (felhom.eu documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt). Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
1 parent
c7e4ec9b5e
commit
c5acb91a44
10 files changed
+179
-5
No files matched your search
@@ -36,6 +36,9 @@ type OffsiteEndedClaim struct {
|
||||
SnapshotIDs []string `json:"snapshot_ids"`
|
||||
}
|
||||
|
||||
// maxEndedClaimIDs is the most snapshot ids ONE window request carries (the hub reads as many: its maxEndedClaimIDs).
|
||||
const maxEndedClaimIDs = 5000
|
||||
|
||||
// OffsiteEndedRefusal is one refusal the hub sent back, in its own words (logged).
|
||||
type OffsiteEndedRefusal struct {
|
||||
App string `json:"app"`
|
||||
@@ -319,3 +322,21 @@ func (m *Manager) PendingOffsiteEnds() []settings.OffsiteEnd { return m.settings
|
||||
func (m *Manager) OffsiteEndsHistory() []settings.OffsiteEndDone {
|
||||
return m.settings.GetOffsiteEndsDone()
|
||||
}
|
||||
|
||||
// boundEndedClaims cuts a request to at most limit snapshot ids in all (the hub reads no more). What is cut is asked
|
||||
// again in a later window: the record stays while a copy of the app is in the store.
|
||||
func boundEndedClaims(claims []OffsiteEndedClaim, limit int) []OffsiteEndedClaim {
|
||||
left := limit
|
||||
var out []OffsiteEndedClaim
|
||||
for _, c := range claims {
|
||||
if left <= 0 {
|
||||
break
|
||||
}
|
||||
if len(c.SnapshotIDs) > left {
|
||||
c.SnapshotIDs = c.SnapshotIDs[:left]
|
||||
}
|
||||
left -= len(c.SnapshotIDs)
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Review of 2026-11-15, finding 5 — VERSION SKEW. A new box with an older hub: the window request with `ended` can
|
||||
// pass that hub's 4 KiB read limit (about 55 ids); the hub then reads no count, opens the window with the smallest
|
||||
// cap, and the guard refuses the honest plan every week — the ordinary clean-up stalls. The box now sends its end
|
||||
// records ONLY to a hub that said, in its last window answer, that it reads them.
|
||||
// RED-PROOF: felhom.eu/documentation/audits/build-release-4-2026-11-15/A/redproof-review-5.txt.
|
||||
func TestD84_ANewBoxWithAnOlderHub_NothingEndedIsSentAndTheCleanUpStillRuns(t *testing.T) {
|
||||
m, sett, er, hub := endedBox(t, "immich")
|
||||
if err := sett.SetOffsiteHubAcceptsEnded(false); err != nil { // a box that has never heard a hub say it
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub.oldHub = true
|
||||
pressed(t, sett, "vikunja", 31)
|
||||
old := time.Now().Add(-60 * 24 * time.Hour)
|
||||
er.plan = []guardSnap{tagged("p1", old, "immich"), tagged("p2", old.Add(time.Minute), "immich")} // the week's honest plan
|
||||
er.snaps = append(er.snaps, er.plan...)
|
||||
for week := 1; week <= 2; week++ {
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 {
|
||||
t.Fatalf("week %d: the box sent %d end record(s) to a hub that never said it reads them — that hub reads 4 KiB and would stall", week, len(hub.claims))
|
||||
}
|
||||
er.plan = nil // the policy's two are gone after the first window (the fake's plan does not follow by itself)
|
||||
}
|
||||
// the ordinary weekly clean-up ran: the policy's two went in the first window, and nothing of the removed app
|
||||
if got := er.removedIDs(); strings.Join(got, ",") != strings.Join(ids("p1", "p2"), ",") {
|
||||
t.Fatalf("removed %v, want the policy's own two and nothing of the removed app", got)
|
||||
}
|
||||
if hub.closed[0].Outcome != "pruned" || len(hub.closed[0].RemovedForEndedApps) != 0 {
|
||||
t.Fatalf("the first window: %+v", hub.closed[0])
|
||||
}
|
||||
if _, ok := sett.GetOffsiteEnd("vikunja"); !ok {
|
||||
t.Fatal("the record was lost while the hub could not be asked")
|
||||
}
|
||||
|
||||
// The hub is updated. An ordinary night's answer (not granted) says accepts_ended; the next window carries the
|
||||
// end records, and the removed app's copies go.
|
||||
hub.oldHub, hub.notGranted = false, true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 || !sett.GetOffsiteHubAcceptsEnded() {
|
||||
t.Fatalf("the night the box LEARNS it: claims=%d learned=%v — it must send none yet and remember the answer", len(hub.claims), sett.GetOffsiteHubAcceptsEnded())
|
||||
}
|
||||
hub.notGranted = false
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 1 || hub.claims[0].App != "vikunja" {
|
||||
t.Fatalf("after the hub said it reads end records the box sent %+v", hub.claims)
|
||||
}
|
||||
if got := er.removedIDs(); len(got) != 5 {
|
||||
t.Fatalf("removed %v, want the policy's two and vikunja's three", got)
|
||||
}
|
||||
// … and if the hub is rolled BACK, the box stops sending at once after the first answer without the word
|
||||
pressed(t, sett, "kimai", 31)
|
||||
hub.oldHub = true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") // this one still carries claims (the box had heard yes)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(hub.claims) != 0 || sett.GetOffsiteHubAcceptsEnded() {
|
||||
t.Fatalf("after a rolled-back hub's answer the box still sends end records: claims=%d learned=%v", len(hub.claims), sett.GetOffsiteHubAcceptsEnded())
|
||||
}
|
||||
}
|
||||
|
||||
// One request never carries more ids than the hub reads (5000): the rest go in a later window.
|
||||
func TestD84_OneRequestCarriesABoundedNumberOfIds(t *testing.T) {
|
||||
m, sett, er, hub := endedBox(t, "immich")
|
||||
pressed(t, sett, "bigapp", 31)
|
||||
old := time.Now().Add(-60 * 24 * time.Hour)
|
||||
for i := 0; i < maxEndedClaimIDs+200; i++ {
|
||||
er.snaps = append(er.snaps, tagged(fmt.Sprintf("b%05d", i), old, "bigapp"))
|
||||
}
|
||||
hub.notGranted = true
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
n := 0
|
||||
for _, c := range hub.claims {
|
||||
n += len(c.SnapshotIDs)
|
||||
}
|
||||
if n != maxEndedClaimIDs {
|
||||
t.Fatalf("one request carried %d snapshot ids, want the limit %d", n, maxEndedClaimIDs)
|
||||
}
|
||||
}
|
||||
@@ -86,12 +86,18 @@ type agreeingHub struct {
|
||||
allowAll func() []string
|
||||
refuse []OffsiteEndedRefusal
|
||||
maxRm int
|
||||
// oldHub: a hub from before ended apps — it never says accepts_ended. notGranted: the answer of an ordinary night.
|
||||
oldHub bool
|
||||
notGranted bool
|
||||
}
|
||||
|
||||
func (h *agreeingHub) Open(ctx context.Context, n int, ended []OffsiteEndedClaim) (OffsiteWindow, error) {
|
||||
h.opened++
|
||||
h.claims = ended
|
||||
w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse}
|
||||
w := OffsiteWindow{Granted: true, ID: 7, NewestAllowed: time.Now(), MaxRemove: h.maxRm, EndedAllowed: h.allow, EndedRefused: h.refuse, AcceptsEnded: !h.oldHub}
|
||||
if h.notGranted {
|
||||
w = OffsiteWindow{Reason: "not due", AcceptsEnded: !h.oldHub}
|
||||
}
|
||||
if h.allowAll != nil {
|
||||
w.EndedAllowed = h.allowAll()
|
||||
}
|
||||
@@ -113,6 +119,9 @@ func endedBox(t *testing.T, living ...string) (*Manager, *settings.Settings, *en
|
||||
tagged("k1", old, "kimai"),
|
||||
}}
|
||||
m.SetOffboxRunner(er.run)
|
||||
if err := sett.SetOffsiteHubAcceptsEnded(true); err != nil { // the box has heard this hub say it knows ended apps
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub := &agreeingHub{maxRm: 5}
|
||||
hub.allowAll = func() []string {
|
||||
var all []string
|
||||
|
||||
@@ -60,6 +60,10 @@ type OffsiteWindow struct {
|
||||
// MaxRemove — and what it refused. Empty from a hub that does not know ended apps: then none is removed.
|
||||
EndedAllowed []string
|
||||
EndedRefused []OffsiteEndedRefusal
|
||||
// AcceptsEnded: this hub said, in this answer, that it reads `ended` in a window request (the hub's
|
||||
// `accepts_ended`; review of 2026-11-15, version skew). An older hub does not say it, and then the box sends it
|
||||
// no end records: that hub reads only 4 KiB of the request and would decide the window on a wrong count.
|
||||
AcceptsEnded bool
|
||||
}
|
||||
|
||||
// OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it).
|
||||
@@ -238,11 +242,23 @@ func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string
|
||||
due := dueOffsiteEnds(m.settings.GetOffsiteEnds(), m.now(), m.offsiteEndLiving)
|
||||
m.dropEndsWithNoCopies(snaps, due)
|
||||
claims, candidate := endedCandidates(snaps, due)
|
||||
// VERSION SKEW (review of 2026-11-15): the end records go ONLY to a hub that said, in its last window answer,
|
||||
// that it reads them. An older hub reads 4 KiB of the request; a larger one made it open the window on a wrong
|
||||
// count, and the weekly clean-up stalled. A hub that has not said it gets the request it always got.
|
||||
if len(claims) > 0 && !m.settings.GetOffsiteHubAcceptsEnded() {
|
||||
m.logger.Printf("[INFO] [offbox] %d removed app(s) are due, but the hub has not said that it reads end records (an older hub) — none is sent, nothing of them is removed; the ordinary clean-up goes on", len(claims))
|
||||
claims, candidate = nil, map[string]bool{}
|
||||
}
|
||||
claims = boundEndedClaims(claims, maxEndedClaimIDs)
|
||||
w, err := m.offsiteWindow.Open(ctx, len(snaps), claims)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err)
|
||||
return
|
||||
}
|
||||
// Remember what THIS answer said (granted or not): the next request follows it.
|
||||
if serr := m.settings.SetOffsiteHubAcceptsEnded(w.AcceptsEnded); serr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] the hub's answer about end records could not be saved: %v", serr)
|
||||
}
|
||||
if !w.Granted {
|
||||
m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason)
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user