v0.162.0 — R-71(a): the apply-bridge settle-gate (kills the F10 day-0 race)
The day-0 race (DIAG-f10): a fresh box boots below the operator floor, the apply-bridge consumes the single-use offsite password, then ~35s later the managed auto-floor update replaces the container mid-install -> the new process finds no installed key -> consume -> 404 -> offsite dead until an operator Re-issue. Recurs on every onboarding whose ISO floor lags the managed floor. Ordering-only fix (consume/install/persist internals + the 404-no-oracle contract + the Consumer UNTOUCHED; R-71(b) rejected-by-design): - New seam offsiteapply.SettleProvider.SettleState() + SettleFunc adapter over the self-updater's own GetFloor()/IsUpdateRunning() (no second floor path). - Bridge.AwaitSettle polls 10s BEFORE the 3-min Reconcile ctx: defers while an update runs or the box is below the known floor; GOes at/above floor on the first poll with zero added latency (B'). Bounds 90s floor sub-bound / 5min overall, both GO+WARN (hub that can't serve a floor can't serve a consume -> no burn risk; R-71c is the belt). ReconcileWhenSettled = gate then reconcile. - main.go: bridge goroutine moved after the updater is built; wired only when an updater exists (nil Settle = reconcile immediately, old behavior). Finding: the floor is in-memory (report-ACK ~5-10s), NOT persisted -> unknown on any restart until the first ACK; the 90s sub-bound is sized to that. Tests (injectable clock, fake SettleState, recorded Consumer): A-E + nil-provider + cancelled-gate. Four red-proofs all observed FAIL then restored: gate removed / updateRunning branch / floor sub-bound / overall bound. Deferral paths ship unit-proven + red-proofed, NOT live-fired -- their precondition is now structurally prevented by the v1.25.0 build gate. Layering: gate prevents, (a) defers, (c) heals. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01N7Drmtm2RzoqbkJZCNSFNQ
This commit is contained in:
@@ -0,0 +1,300 @@
|
||||
package offsiteapply
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
)
|
||||
|
||||
// --- settle-gate fakes (R-71a) ---
|
||||
|
||||
// fakeSettle is a mutable SettleProvider — a test flips its fields between polls to model a managed
|
||||
// update landing (below-floor → at-floor) or an update finishing (updateRunning true → false).
|
||||
type fakeSettle struct {
|
||||
mu sync.Mutex
|
||||
version string
|
||||
floor string
|
||||
updateRunning bool
|
||||
floorKnown bool
|
||||
polls int
|
||||
// afterPoll runs after each SettleState read (poll number passed) so a test can flip state at a
|
||||
// chosen poll — modelling the update converging.
|
||||
afterPoll func(f *fakeSettle, poll int)
|
||||
}
|
||||
|
||||
func (f *fakeSettle) SettleState() (string, string, bool, bool) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.polls++
|
||||
v, fl, ur, fk := f.version, f.floor, f.updateRunning, f.floorKnown
|
||||
if f.afterPoll != nil {
|
||||
f.afterPoll(f, f.polls)
|
||||
}
|
||||
return v, fl, ur, fk
|
||||
}
|
||||
|
||||
// fakeClock is an injectable clock: it never really sleeps. Each Sleep advances virtual time by the
|
||||
// requested duration and records the call, so a test drives the gate through its bounds instantly and
|
||||
// asserts EXACTLY how much virtual wait was consumed (the B′ zero-wait proof).
|
||||
type fakeClock struct {
|
||||
mu sync.Mutex
|
||||
t time.Time
|
||||
sleeps int
|
||||
totalDur time.Duration
|
||||
}
|
||||
|
||||
func newClock() *fakeClock { return &fakeClock{t: time.Unix(0, 0)} }
|
||||
|
||||
func (c *fakeClock) now() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.t
|
||||
}
|
||||
|
||||
func (c *fakeClock) sleep(_ context.Context, d time.Duration) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.sleeps++
|
||||
c.totalDur += d
|
||||
c.t = c.t.Add(d)
|
||||
}
|
||||
|
||||
// settleBridge builds a bridge wired for a FULL Reconcile (so a released gate consumes exactly once)
|
||||
// plus the injectable settle-gate. Prober is nil → the fresh consume+install path runs on release.
|
||||
func settleBridge(t *testing.T, s *fakeSettle, clk *fakeClock) (*Bridge, *fakeConsumer, *bytes.Buffer) {
|
||||
t.Helper()
|
||||
cfg := &config.Config{}
|
||||
cfg.Offsite = goodOffsite()
|
||||
cons := &fakeConsumer{pw: "the-transient-pw"}
|
||||
var logbuf bytes.Buffer
|
||||
b := &Bridge{
|
||||
Cfg: cfg,
|
||||
Consumer: cons,
|
||||
Scanner: &fakeScanner{fp: "SHA256:goodfp", line: "[h]:23 ssh-ed25519 AAAAKEY"},
|
||||
KeyGen: &fakeKeyGen{priv: "PRIVPEM", pub: "ssh-ed25519 AAAAPUB felhom"},
|
||||
Installer: &fakeInstaller{},
|
||||
Enabler: &fakeEnabler{},
|
||||
MarkerPath: filepath.Join(t.TempDir(), "offbox", "applied_marker"),
|
||||
Logger: log.New(&logbuf, "", 0),
|
||||
Settle: s,
|
||||
Now: clk.now,
|
||||
Sleep: clk.sleep,
|
||||
}
|
||||
return b, cons, &logbuf
|
||||
}
|
||||
|
||||
// Scenario A — the race, killed. While below floor the gate consumes NOTHING; when the update lands
|
||||
// (fake flips to at-floor) exactly one Reconcile proceeds → exactly one Consume.
|
||||
func TestSettle_BelowFloorDefersThenGoes(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true}
|
||||
// Model the managed update converging at the 3rd poll: the container comes back at floor.
|
||||
s.afterPoll = func(f *fakeSettle, poll int) {
|
||||
if poll == 3 {
|
||||
f.version = "0.156.0"
|
||||
}
|
||||
}
|
||||
b, cons, logbuf := settleBridge(t, s, clk)
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile-when-settled: %v", err)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("consume must run exactly once AFTER the update lands, got %d", cons.calls)
|
||||
}
|
||||
// It waited (polled) while below floor and did not consume prematurely.
|
||||
if clk.sleeps < 2 {
|
||||
t.Fatalf("expected the gate to defer while below floor (>=2 sleeps), got %d", clk.sleeps)
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "deferring offsite apply: managed update to floor 0.156.0 pending (we are 0.153.0)") {
|
||||
t.Fatalf("expected the below-floor deferral log, got:\n%s", logbuf.String())
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "settle-gate: GO — at/above floor 0.156.0") {
|
||||
t.Fatalf("expected the at-floor GO log, got:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario A red-proof: WITHOUT the gate (Settle nil), a below-floor box consumes IMMEDIATELY — the
|
||||
// exact F10 burn. This proves the gate is load-bearing (remove it → the failure returns).
|
||||
func TestSettle_RedProof_NoGateConsumesBelowFloor(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true}
|
||||
b, cons, _ := settleBridge(t, s, clk)
|
||||
b.Settle = nil // remove the gate → the pre-R-71a behavior
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("RED-PROOF: without the gate the below-floor box consumes immediately (want 1), got %d", cons.calls)
|
||||
}
|
||||
if clk.sleeps != 0 {
|
||||
t.Fatalf("RED-PROOF: without the gate there is no deferral wait, got %d sleeps", clk.sleeps)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario B — an update in progress defers (zero consume); when it finishes at floor the gate GOes.
|
||||
func TestSettle_UpdateRunningDefersThenGoes(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.156.0", floor: "0.156.0", floorKnown: true, updateRunning: true}
|
||||
s.afterPoll = func(f *fakeSettle, poll int) {
|
||||
if poll == 2 {
|
||||
f.updateRunning = false
|
||||
}
|
||||
}
|
||||
b, cons, logbuf := settleBridge(t, s, clk)
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("consume must run once after the update finishes, got %d", cons.calls)
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "a managed update is in progress") {
|
||||
t.Fatalf("expected the update-running deferral log, got:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario C — floor never becomes known: GO after the sub-bound with the WARN; then Consume runs.
|
||||
func TestSettle_FloorUnknownGoesAfterSubBound(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.156.0", floor: "", floorKnown: false} // floor never learned
|
||||
b, cons, logbuf := settleBridge(t, s, clk)
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("consume must run once after the floor-unknown sub-bound GO, got %d", cons.calls)
|
||||
}
|
||||
// It released at the sub-bound (90s), not the overall bound (5m).
|
||||
if clk.totalDur < settleFloorSubBound || clk.totalDur >= settleOverallBound {
|
||||
t.Fatalf("expected release near the sub-bound %s, waited %s", settleFloorSubBound, clk.totalDur)
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "floor still unknown after 1m30s") {
|
||||
t.Fatalf("expected the floor-unknown GO+WARN log, got:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario C red-proof: drop the sub-bound and floor-unknown would only ever release at the overall
|
||||
// bound (5m) — a 3.3× longer hostage window. We assert the sub-bound is what releases it (waited
|
||||
// well under the overall bound); if the sub-bound branch were removed this assertion fails.
|
||||
func TestSettle_RedProof_FloorUnknownWithoutSubBoundWaitsFarLonger(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.156.0", floor: "", floorKnown: false}
|
||||
b, _, _ := settleBridge(t, s, clk)
|
||||
|
||||
b.AwaitSettle(context.Background())
|
||||
// With the sub-bound present, release happens at ~90s. The red-proof: if a maintainer deletes the
|
||||
// `elapsed >= settleFloorSubBound` branch, the ONLY remaining exit for a perpetually-unknown floor
|
||||
// is the 5-minute overall bound — this bound-check pins that regression.
|
||||
if clk.totalDur >= settleOverallBound {
|
||||
t.Fatalf("RED-PROOF: floor-unknown should release at the sub-bound %s, not drag to the overall bound; waited %s",
|
||||
settleFloorSubBound, clk.totalDur)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario D — perpetually below floor (an update that never lands): GO at the overall bound + WARN.
|
||||
func TestSettle_PerpetuallyBelowFloorGoesAtOverallBound(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} // never converges
|
||||
b, cons, logbuf := settleBridge(t, s, clk)
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("consume must run once after the overall-bound GO, got %d", cons.calls)
|
||||
}
|
||||
if clk.totalDur < settleOverallBound {
|
||||
t.Fatalf("expected release at the overall bound %s, waited %s", settleOverallBound, clk.totalDur)
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "settle-gate bound exhausted") || !strings.Contains(logbuf.String(), "R-71c self-heal is the belt") {
|
||||
t.Fatalf("expected the bound-exhausted GO+WARN log, got:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario D red-proof: drop the overall bound and a perpetually-below-floor box loops forever. We
|
||||
// pin the bound by asserting the gate terminates AND that termination is the overall bound (a
|
||||
// deleted bound would never reach this assertion — the fake clock loops without limit).
|
||||
func TestSettle_RedProof_PerpetualBelowFloorTerminatesAtBound(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true}
|
||||
b, _, _ := settleBridge(t, s, clk)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() { b.AwaitSettle(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RED-PROOF: the gate must terminate at the overall bound — without it a below-floor box loops forever")
|
||||
}
|
||||
if clk.totalDur < settleOverallBound {
|
||||
t.Fatalf("RED-PROOF: termination must be the overall bound %s, waited %s", settleOverallBound, clk.totalDur)
|
||||
}
|
||||
}
|
||||
|
||||
// Scenario E — the B′ invariant: an at/above-floor box GOes on the FIRST evaluation with ZERO wait.
|
||||
func TestSettle_AtFloorGoesFirstPollNoWait(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.162.0", floor: "0.156.0", floorKnown: true} // above floor
|
||||
b, cons, logbuf := settleBridge(t, s, clk)
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if s.polls != 1 {
|
||||
t.Fatalf("B′: the happy path must GO on the FIRST poll, got %d polls", s.polls)
|
||||
}
|
||||
if clk.sleeps != 0 || clk.totalDur != 0 {
|
||||
t.Fatalf("B′: zero new latency — no sleep may be consumed, got %d sleeps / %s", clk.sleeps, clk.totalDur)
|
||||
}
|
||||
if cons.calls != 1 {
|
||||
t.Fatalf("the apply must proceed immediately, consume=%d want 1", cons.calls)
|
||||
}
|
||||
if !strings.Contains(logbuf.String(), "settle-gate: GO — at/above floor 0.156.0 (we are 0.162.0)") {
|
||||
t.Fatalf("expected the first-poll GO log, got:\n%s", logbuf.String())
|
||||
}
|
||||
}
|
||||
|
||||
// No SettleProvider wired (self-update disabled) → no gate: immediate reconcile, zero wait.
|
||||
func TestSettle_NilProviderNoGate(t *testing.T) {
|
||||
clk := newClock()
|
||||
b, cons, _ := settleBridge(t, &fakeSettle{}, clk)
|
||||
b.Settle = nil
|
||||
|
||||
if err := b.ReconcileWhenSettled(context.Background()); err != nil {
|
||||
t.Fatalf("reconcile: %v", err)
|
||||
}
|
||||
if cons.calls != 1 || clk.sleeps != 0 {
|
||||
t.Fatalf("nil Settle must reconcile immediately with no wait: consume=%d sleeps=%d", cons.calls, clk.sleeps)
|
||||
}
|
||||
}
|
||||
|
||||
// A cancelled gate context skips the reconcile (shutdown mid-wait): no consume.
|
||||
func TestSettle_CancelledGateSkipsReconcile(t *testing.T) {
|
||||
clk := newClock()
|
||||
s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} // would defer forever
|
||||
b, cons, _ := settleBridge(t, s, clk)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// Cancel after the first poll so the gate's post-sleep ctx check trips.
|
||||
s.afterPoll = func(_ *fakeSettle, poll int) {
|
||||
if poll == 1 {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
err := b.ReconcileWhenSettled(ctx)
|
||||
if err == nil {
|
||||
t.Fatal("a cancelled gate must return the context error, not reconcile")
|
||||
}
|
||||
if cons.calls != 0 {
|
||||
t.Fatalf("a cancelled gate must NOT consume, got %d", cons.calls)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user