package offsiteapply import ( "bytes" "context" "log" "path/filepath" "strings" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/config" ) // --- settle-gate fakes (R-71a) --- // fakeSettle is a mutable SettleProvider — a test flips its fields between polls to model a managed // update landing (below-floor → at-floor) or an update finishing (updateRunning true → false). type fakeSettle struct { mu sync.Mutex version string floor string updateRunning bool floorKnown bool polls int // afterPoll runs after each SettleState read (poll number passed) so a test can flip state at a // chosen poll — modelling the update converging. afterPoll func(f *fakeSettle, poll int) } func (f *fakeSettle) SettleState() (string, string, bool, bool) { f.mu.Lock() defer f.mu.Unlock() f.polls++ v, fl, ur, fk := f.version, f.floor, f.updateRunning, f.floorKnown if f.afterPoll != nil { f.afterPoll(f, f.polls) } return v, fl, ur, fk } // fakeClock is an injectable clock: it never really sleeps. Each Sleep advances virtual time by the // requested duration and records the call, so a test drives the gate through its bounds instantly and // asserts EXACTLY how much virtual wait was consumed (the B′ zero-wait proof). type fakeClock struct { mu sync.Mutex t time.Time sleeps int totalDur time.Duration } func newClock() *fakeClock { return &fakeClock{t: time.Unix(0, 0)} } func (c *fakeClock) now() time.Time { c.mu.Lock() defer c.mu.Unlock() return c.t } func (c *fakeClock) sleep(_ context.Context, d time.Duration) { c.mu.Lock() defer c.mu.Unlock() c.sleeps++ c.totalDur += d c.t = c.t.Add(d) } // settleBridge builds a bridge wired for a FULL Reconcile (so a released gate consumes exactly once) // plus the injectable settle-gate. Prober is nil → the fresh consume+install path runs on release. func settleBridge(t *testing.T, s *fakeSettle, clk *fakeClock) (*Bridge, *fakeConsumer, *bytes.Buffer) { t.Helper() cfg := &config.Config{} cfg.Offsite = goodOffsite() cons := &fakeConsumer{pw: "the-transient-pw"} var logbuf bytes.Buffer b := &Bridge{ Cfg: cfg, Consumer: cons, Scanner: &fakeScanner{fp: "SHA256:goodfp", line: "[h]:23 ssh-ed25519 AAAAKEY"}, KeyGen: &fakeKeyGen{priv: "PRIVPEM", pub: "ssh-ed25519 AAAAPUB felhom"}, Installer: &fakeInstaller{}, Enabler: &fakeEnabler{}, MarkerPath: filepath.Join(t.TempDir(), "offbox", "applied_marker"), Logger: log.New(&logbuf, "", 0), Settle: s, Now: clk.now, Sleep: clk.sleep, } return b, cons, &logbuf } // Scenario A — the race, killed. While below floor the gate consumes NOTHING; when the update lands // (fake flips to at-floor) exactly one Reconcile proceeds → exactly one Consume. func TestSettle_BelowFloorDefersThenGoes(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} // Model the managed update converging at the 3rd poll: the container comes back at floor. s.afterPoll = func(f *fakeSettle, poll int) { if poll == 3 { f.version = "0.156.0" } } b, cons, logbuf := settleBridge(t, s, clk) if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile-when-settled: %v", err) } if cons.calls != 1 { t.Fatalf("consume must run exactly once AFTER the update lands, got %d", cons.calls) } // It waited (polled) while below floor and did not consume prematurely. if clk.sleeps < 2 { t.Fatalf("expected the gate to defer while below floor (>=2 sleeps), got %d", clk.sleeps) } if !strings.Contains(logbuf.String(), "deferring offsite apply: managed update to floor 0.156.0 pending (we are 0.153.0)") { t.Fatalf("expected the below-floor deferral log, got:\n%s", logbuf.String()) } if !strings.Contains(logbuf.String(), "settle-gate: GO — at/above floor 0.156.0") { t.Fatalf("expected the at-floor GO log, got:\n%s", logbuf.String()) } } // Scenario A red-proof: WITHOUT the gate (Settle nil), a below-floor box consumes IMMEDIATELY — the // exact F10 burn. This proves the gate is load-bearing (remove it → the failure returns). func TestSettle_RedProof_NoGateConsumesBelowFloor(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} b, cons, _ := settleBridge(t, s, clk) b.Settle = nil // remove the gate → the pre-R-71a behavior if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if cons.calls != 1 { t.Fatalf("RED-PROOF: without the gate the below-floor box consumes immediately (want 1), got %d", cons.calls) } if clk.sleeps != 0 { t.Fatalf("RED-PROOF: without the gate there is no deferral wait, got %d sleeps", clk.sleeps) } } // Scenario B — an update in progress defers (zero consume); when it finishes at floor the gate GOes. func TestSettle_UpdateRunningDefersThenGoes(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.156.0", floor: "0.156.0", floorKnown: true, updateRunning: true} s.afterPoll = func(f *fakeSettle, poll int) { if poll == 2 { f.updateRunning = false } } b, cons, logbuf := settleBridge(t, s, clk) if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if cons.calls != 1 { t.Fatalf("consume must run once after the update finishes, got %d", cons.calls) } if !strings.Contains(logbuf.String(), "a managed update is in progress") { t.Fatalf("expected the update-running deferral log, got:\n%s", logbuf.String()) } } // Scenario C — floor never becomes known: GO after the sub-bound with the WARN; then Consume runs. func TestSettle_FloorUnknownGoesAfterSubBound(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.156.0", floor: "", floorKnown: false} // floor never learned b, cons, logbuf := settleBridge(t, s, clk) if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if cons.calls != 1 { t.Fatalf("consume must run once after the floor-unknown sub-bound GO, got %d", cons.calls) } // It released at the sub-bound (90s), not the overall bound (5m). if clk.totalDur < settleFloorSubBound || clk.totalDur >= settleOverallBound { t.Fatalf("expected release near the sub-bound %s, waited %s", settleFloorSubBound, clk.totalDur) } if !strings.Contains(logbuf.String(), "floor still unknown after 1m30s") { t.Fatalf("expected the floor-unknown GO+WARN log, got:\n%s", logbuf.String()) } } // Scenario C red-proof: drop the sub-bound and floor-unknown would only ever release at the overall // bound (5m) — a 3.3× longer hostage window. We assert the sub-bound is what releases it (waited // well under the overall bound); if the sub-bound branch were removed this assertion fails. func TestSettle_RedProof_FloorUnknownWithoutSubBoundWaitsFarLonger(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.156.0", floor: "", floorKnown: false} b, _, _ := settleBridge(t, s, clk) b.AwaitSettle(context.Background()) // With the sub-bound present, release happens at ~90s. The red-proof: if a maintainer deletes the // `elapsed >= settleFloorSubBound` branch, the ONLY remaining exit for a perpetually-unknown floor // is the 5-minute overall bound — this bound-check pins that regression. if clk.totalDur >= settleOverallBound { t.Fatalf("RED-PROOF: floor-unknown should release at the sub-bound %s, not drag to the overall bound; waited %s", settleFloorSubBound, clk.totalDur) } } // Scenario D — perpetually below floor (an update that never lands): GO at the overall bound + WARN. func TestSettle_PerpetuallyBelowFloorGoesAtOverallBound(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} // never converges b, cons, logbuf := settleBridge(t, s, clk) if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if cons.calls != 1 { t.Fatalf("consume must run once after the overall-bound GO, got %d", cons.calls) } if clk.totalDur < settleOverallBound { t.Fatalf("expected release at the overall bound %s, waited %s", settleOverallBound, clk.totalDur) } if !strings.Contains(logbuf.String(), "settle-gate bound exhausted") || !strings.Contains(logbuf.String(), "R-71c self-heal is the belt") { t.Fatalf("expected the bound-exhausted GO+WARN log, got:\n%s", logbuf.String()) } } // Scenario D red-proof: drop the overall bound and a perpetually-below-floor box loops forever. We // pin the bound by asserting the gate terminates AND that termination is the overall bound (a // deleted bound would never reach this assertion — the fake clock loops without limit). func TestSettle_RedProof_PerpetualBelowFloorTerminatesAtBound(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} b, _, _ := settleBridge(t, s, clk) done := make(chan struct{}) go func() { b.AwaitSettle(context.Background()); close(done) }() select { case <-done: case <-time.After(5 * time.Second): t.Fatal("RED-PROOF: the gate must terminate at the overall bound — without it a below-floor box loops forever") } if clk.totalDur < settleOverallBound { t.Fatalf("RED-PROOF: termination must be the overall bound %s, waited %s", settleOverallBound, clk.totalDur) } } // Scenario E — the B′ invariant: an at/above-floor box GOes on the FIRST evaluation with ZERO wait. func TestSettle_AtFloorGoesFirstPollNoWait(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.162.0", floor: "0.156.0", floorKnown: true} // above floor b, cons, logbuf := settleBridge(t, s, clk) if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if s.polls != 1 { t.Fatalf("B′: the happy path must GO on the FIRST poll, got %d polls", s.polls) } if clk.sleeps != 0 || clk.totalDur != 0 { t.Fatalf("B′: zero new latency — no sleep may be consumed, got %d sleeps / %s", clk.sleeps, clk.totalDur) } if cons.calls != 1 { t.Fatalf("the apply must proceed immediately, consume=%d want 1", cons.calls) } if !strings.Contains(logbuf.String(), "settle-gate: GO — at/above floor 0.156.0 (we are 0.162.0)") { t.Fatalf("expected the first-poll GO log, got:\n%s", logbuf.String()) } } // No SettleProvider wired (self-update disabled) → no gate: immediate reconcile, zero wait. func TestSettle_NilProviderNoGate(t *testing.T) { clk := newClock() b, cons, _ := settleBridge(t, &fakeSettle{}, clk) b.Settle = nil if err := b.ReconcileWhenSettled(context.Background()); err != nil { t.Fatalf("reconcile: %v", err) } if cons.calls != 1 || clk.sleeps != 0 { t.Fatalf("nil Settle must reconcile immediately with no wait: consume=%d sleeps=%d", cons.calls, clk.sleeps) } } // A cancelled gate context skips the reconcile (shutdown mid-wait): no consume. func TestSettle_CancelledGateSkipsReconcile(t *testing.T) { clk := newClock() s := &fakeSettle{version: "0.153.0", floor: "0.156.0", floorKnown: true} // would defer forever b, cons, _ := settleBridge(t, s, clk) ctx, cancel := context.WithCancel(context.Background()) // Cancel after the first poll so the gate's post-sleep ctx check trips. s.afterPoll = func(_ *fakeSettle, poll int) { if poll == 1 { cancel() } } err := b.ReconcileWhenSettled(ctx) if err == nil { t.Fatal("a cancelled gate must return the context error, not reconcile") } if cons.calls != 0 { t.Fatalf("a cancelled gate must NOT consume, got %d", cons.calls) } }