5da11c4480
gates / gates (push) Successful in 11s
R-384. aggregateState returned StateUnhealthy the moment unhealthy > 0, and the R-51 mixed-case block that asks "is a supervised member dead?" sat below it. A two-container app whose database exits goes unhealthy BECAUSE it cannot reach that database - so the symptom the dead database causes was what suppressed the alarm for it. unhealthy is not a down state, so classifyRunStates never marked the app down and app_start_failed never fired. Measured live on demo-hp 2026-08-22: bookstack-db stopped at 21:27:01 and the F-OBS heartbeat printed "0 currently down" throughout. R-51's 18-hour immich failure, back through a different door. Two things moved, and either alone leaves the defect standing: the supervised test is hoisted above the unhealthy/starting/restarting returns, and "some members are up" now counts ANY member not in the down bucket. The old guard was running > 0, which made the R-51 block unreachable in exactly the case it was written for. IsDownState is byte-identical - unhealthy stays excluded, because an unhealthy container is running and folding it in reintroduces the flapping that exclusion exists to stop. No new state was minted. Only the ORDER changed. The priority comment was rewritten because it asserted an ordering the code no longer has. Three subtests in TestAggregateState_UnchangedBranches were AMENDED: they asserted an unhealthy/starting/restarting member beat an exited peer on unless-stopped, which pinned the defect as settled behaviour. They keep their intent with the down member given a benign policy. R-383. The double-failure message said the previous state's backup EXISTS, built from the returned path without asking the filesystem - and a missing file is one of the two ways that rollback fails. undoCopyPhrase now describes the copy from disk: present, partial, missing (still naming where it should be), or never written. Zero-length counts as missing. Test count 1494 -> 1504. Four red-proofs planted, four seen failing; the two halves of R-384 convict independently.
168 lines
5.7 KiB
Go
168 lines
5.7 KiB
Go
package backup
|
|
|
|
import (
|
|
"go/ast"
|
|
"go/parser"
|
|
"go/token"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// R-383 — the double-failure message claimed the undo copy EXISTED without ever asking the disk.
|
|
//
|
|
// THE DEFECT. The branch where BOTH the replay and the rollback failed ended with
|
|
// „a korábbi állapot mentése megvan: <file>". The filename came from the path `writeSafetyDump`
|
|
// returned. One of the two ways `rollbackSafetyDump` fails is that the file is NOT THERE — so the
|
|
// sentence was most likely to be false in precisely the case it was printed. Measured twice live, on
|
|
// v0.220.2 and v0.221.1.
|
|
//
|
|
// THE LAYER. The guard sits on the phrase builder, because that is where the claim is MADE. A test
|
|
// at the reconstitute level would need a whole failed off-site restore to reach one sentence, and the
|
|
// AST test below is what pins that the sentence is still wired to this builder.
|
|
//
|
|
// RED-PROOF (observed, see REPORT.md): replace undoCopyPhrase's body with the pre-fix shape
|
|
//
|
|
// return "a korábbi állapot mentése megvan: " + filepath.Base(set.First())
|
|
//
|
|
// and TestR383_AbsentUndoCopyIsNotClaimedToExist fails with the message asserting the file exists.
|
|
func TestR383_AbsentUndoCopyIsNotClaimedToExist(t *testing.T) {
|
|
dir := t.TempDir()
|
|
real := filepath.Join(dir, "pre-restore-20260823T120000Z-app-postgres.sql")
|
|
if err := os.WriteFile(real, []byte("-- a real dump\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gone := filepath.Join(dir, "pre-restore-20260823T120000Z-app-mariadb.sql")
|
|
empty := filepath.Join(dir, "pre-restore-20260823T120000Z-app-empty.sql")
|
|
if err := os.WriteFile(empty, nil, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
set := func(paths ...string) safetyDumpSet {
|
|
s := safetyDumpSet{Stamp: "20260823T120000Z"}
|
|
for _, p := range paths {
|
|
s.Files = append(s.Files, safetyDumpFile{Path: p})
|
|
}
|
|
return s
|
|
}
|
|
|
|
cases := []struct {
|
|
name string
|
|
set safetyDumpSet
|
|
mustContain []string
|
|
mustNotHave []string
|
|
}{
|
|
{
|
|
name: "present — the operator still gets the filename",
|
|
set: set(real),
|
|
mustContain: []string{"megvan", filepath.Base(real)},
|
|
},
|
|
{
|
|
// THE DEFECT ITSELF: the file is gone and the sentence used to say it was there.
|
|
name: "absent — must NOT claim it exists, must still name where it should be",
|
|
set: set(gone),
|
|
mustContain: []string{"NEM találjuk", filepath.Base(gone)},
|
|
mustNotHave: []string{"mentése megvan"},
|
|
},
|
|
{
|
|
// A 0-byte dump restores nothing. Calling it present is the same false reassurance.
|
|
name: "zero-length — counts as missing",
|
|
set: set(empty),
|
|
mustContain: []string{"NEM találjuk", filepath.Base(empty)},
|
|
mustNotHave: []string{"mentése megvan"},
|
|
},
|
|
{
|
|
name: "partial — both halves named, neither hidden",
|
|
set: set(real, gone),
|
|
mustContain: []string{"RÉSZBEN", filepath.Base(real), "HIÁNYZIK", filepath.Base(gone)},
|
|
},
|
|
{
|
|
name: "no undo was ever written — said plainly, not silently",
|
|
set: set(),
|
|
mustContain: []string{"nem készült"},
|
|
mustNotHave: []string{"megvan"},
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := undoCopyPhrase(tc.set)
|
|
for _, want := range tc.mustContain {
|
|
if !strings.Contains(got, want) {
|
|
t.Errorf("phrase %q does not contain %q", got, want)
|
|
}
|
|
}
|
|
for _, bad := range tc.mustNotHave {
|
|
if strings.Contains(got, bad) {
|
|
t.Errorf("phrase %q contains %q — it asserts a file that is not on disk", got, bad)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The seam, through production wiring (§10). A correct phrase builder nobody calls is R-106's defect
|
|
// — "seam built but never wired", four instances in this project. This walks the AST rather than
|
|
// grepping for a substring, so a commented-out call or a similarly-named local cannot satisfy it.
|
|
func TestR383_TheDoubleFailureMessageIsWiredToTheBuilder(t *testing.T) {
|
|
fset := token.NewFileSet()
|
|
f, err := parser.ParseFile(fset, "offbox_reconstitute.go", nil, 0)
|
|
if err != nil {
|
|
t.Fatalf("parse: %v", err)
|
|
}
|
|
|
|
var holdCalled, phraseCalled bool
|
|
ast.Inspect(f, func(n ast.Node) bool {
|
|
call, ok := n.(*ast.CallExpr)
|
|
if !ok {
|
|
return true
|
|
}
|
|
switch fn := call.Fun.(type) {
|
|
case *ast.SelectorExpr:
|
|
if fn.Sel.Name == "holdAppAfterFailedRollback" {
|
|
holdCalled = true
|
|
}
|
|
case *ast.Ident:
|
|
if fn.Name == "undoCopyPhrase" {
|
|
phraseCalled = true
|
|
}
|
|
}
|
|
return true
|
|
})
|
|
if !holdCalled {
|
|
t.Fatal("holdAppAfterFailedRollback is no longer called — the double-failure branch moved; " +
|
|
"re-point this test before trusting it")
|
|
}
|
|
if !phraseCalled {
|
|
t.Fatal("undoCopyPhrase is never called from offbox_reconstitute.go — the message is back to " +
|
|
"asserting a file it did not check (R-383)")
|
|
}
|
|
|
|
// And the claim must not be re-typed OUTSIDE the builder. Inside undoCopyPhrase it is the
|
|
// verified branch and belongs there; anywhere else it is unconditional again, which is R-383.
|
|
// The builder's extent comes from the AST, so this cannot be defeated by moving the function.
|
|
var lo, hi token.Pos
|
|
for _, d := range f.Decls {
|
|
if fd, ok := d.(*ast.FuncDecl); ok && fd.Name.Name == "undoCopyPhrase" {
|
|
lo, hi = fd.Pos(), fd.End()
|
|
}
|
|
}
|
|
if lo == token.NoPos {
|
|
t.Fatal("undoCopyPhrase is not declared in offbox_reconstitute.go")
|
|
}
|
|
ast.Inspect(f, func(n ast.Node) bool {
|
|
lit, ok := n.(*ast.BasicLit)
|
|
if !ok || lit.Kind != token.STRING {
|
|
return true
|
|
}
|
|
if lit.Pos() >= lo && lit.End() <= hi {
|
|
return true // inside the builder — the verified branch
|
|
}
|
|
if strings.Contains(lit.Value, "állapot mentése megvan") {
|
|
t.Errorf("%s: the unconditional claim is back outside undoCopyPhrase: %s",
|
|
fset.Position(lit.Pos()), lit.Value)
|
|
}
|
|
return true
|
|
})
|
|
}
|