admin
8b55de734c
gates / gates (push) Successful in 12s
The system-data fallback resolved a scratch fine and removeProofScratch then refused to delete it: its accepted-roots list is built from REGISTERED drives, and a driveless box has none. Observed on demo-felhom: 'refusing to remove ... it is not inside a proof root', with the copy still on disk. Every nightly proof would have left one behind, growing forever, on exactly the boxes the fallback exists for. My defect, introduced with the fallback in the same session. The unit tests missed it because every one of them registers a drive; the new pair deliberately does not, and the second asserts the guard still REFUSES a path outside every proof root, so the fix is not a widening into uselessness.
244 lines
10 KiB
Go
244 lines
10 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
)
|
|
|
|
// R-411 A2 + R-414 — the collision that can no longer happen, and the box that can now be proved.
|
|
|
|
// TestR411_IntegrityCheckSkipsWhileARestoreHoldsIt — A2, the whole point of Part 1.
|
|
//
|
|
// This is last night's collision, from the other side: with the restore now holding the flag, the
|
|
// integrity check must SKIP rather than run, meet the lock and delete it.
|
|
func TestR411_IntegrityCheckSkipsWhileARestoreHoldsIt(t *testing.T) {
|
|
h := newLockHarness(t, "kimai")
|
|
|
|
// Stand in for a restore in flight: it now takes the flag, so hold it.
|
|
if err := h.m.AcquireRunningForTest(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := h.m.settings.GetOffboxTarget().LastIntegrityCheck
|
|
|
|
res := h.m.CheckOffboxIntegrity(context.Background())
|
|
|
|
if !res.Skipped {
|
|
t.Fatalf("the check must SKIP while a restore holds the flag; got %+v", res)
|
|
}
|
|
if len(h.allArgs()) != 0 {
|
|
t.Fatalf("a skipped check must invoke restic ZERO times — this is the non-effect R-411 is about; got %v", h.allArgs())
|
|
}
|
|
for _, args := range h.allArgs() {
|
|
if containsArg(args, "unlock") || containsArg(args, "--remove-all") {
|
|
t.Fatalf("the unlock escalation fired: %v", args)
|
|
}
|
|
}
|
|
if got := h.m.settings.GetOffboxTarget().LastIntegrityCheck; got != before {
|
|
t.Fatalf("a skip must NOT advance due-ness; %q -> %q", before, got)
|
|
}
|
|
}
|
|
|
|
// ── R-414 ───────────────────────────────────────────────────────────────────────────────────────
|
|
|
|
// drivelessHarness is `demo-felhom`'s real shape: deployed apps, and ZERO registered storage paths.
|
|
func drivelessHarness(t *testing.T, stacks ...string) (*Manager, *settings.Settings, *[][]string) {
|
|
t.Helper()
|
|
m, sett := newOffboxManager(t)
|
|
// deliberately NO AddStoragePath — that is the whole point
|
|
deployed := map[string]bool{}
|
|
for _, s := range stacks {
|
|
deployed[s] = true
|
|
}
|
|
m.SetStackProvider(&offbox3aProvider{
|
|
hdd: map[string]string{}, binds: map[string][]ClassifiedBind{},
|
|
has: map[string]bool{}, deployed: deployed,
|
|
})
|
|
var argv [][]string
|
|
m.SetOffboxLatestSnapshotFn(func(_ context.Context, stack string) (string, []string, error) {
|
|
return "snap-" + stack, []string{"/mnt/sys_drive/felhom-data/backups/primary/" + stack}, nil
|
|
})
|
|
m.SetOffboxRunner(func(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
|
argv = append(argv, append([]string{}, args...))
|
|
if containsArg(args, "restore") {
|
|
target, include := argValue(args, "--target"), argValue(args, "--include")
|
|
if target != "" && include != "" {
|
|
dest := filepath.Join(target, strings.TrimPrefix(include, string(filepath.Separator)))
|
|
materialiseUnit(t, dest, unitFixture{})
|
|
}
|
|
}
|
|
return []byte("{}"), nil
|
|
})
|
|
m.SetOffboxFreeFn(func(string) int64 { return 100 << 30 })
|
|
return m, sett, &argv
|
|
}
|
|
|
|
// TestR414_DrivelessBoxReachesAVerdict — C1.
|
|
//
|
|
// RED-PROOF (run 2026-09-01): reverting the resolver to refuse (dropping step 4) AND restoring the
|
|
// Err path makes this fail with `last_proof_result` absent — which is the exact state `demo-felhom`
|
|
// was in every night.
|
|
func TestR414_DrivelessBoxReachesAVerdict(t *testing.T) {
|
|
m, sett, _ := drivelessHarness(t, "opengist")
|
|
|
|
res := m.ProveOffboxUnit(context.Background())
|
|
if res.Verdict() == "" {
|
|
t.Fatalf("a driveless box must reach a RECORDED verdict, never an empty one — empty is what the hub reads as 'controller too old'; got %+v", res)
|
|
}
|
|
m.RecordProofVerdict(res)
|
|
|
|
got := sett.GetOffboxTarget().LastProofResult
|
|
if got == "" {
|
|
t.Fatal("last_proof_result is ABSENT after a run on a driveless box — this is R-414 exactly: the hub cannot tell 'cannot run here' from 'too old'")
|
|
}
|
|
// With the unit-only fallback in place the box can actually be proved.
|
|
if got != string(UnitProofPass) && got != ProofResultCannotRun {
|
|
t.Fatalf("unexpected verdict %q — expected a real judgement (the fallback worked) or %q", got, ProofResultCannotRun)
|
|
}
|
|
t.Logf("driveless box reached verdict %q", got)
|
|
}
|
|
|
|
// TestR414_UnitOnlyRestoreFallsBackToSystemData — C3.
|
|
func TestR414_UnitOnlyRestoreFallsBackToSystemData(t *testing.T) {
|
|
m, _, _ := drivelessHarness(t, "opengist")
|
|
scratch, nsRoot, err := m.offboxProofScratchDir("opengist")
|
|
if err != nil {
|
|
t.Fatalf("a UNIT-ONLY scratch must resolve on a driveless box (R-414); got %v", err)
|
|
}
|
|
sys := m.cfg.Paths.SystemDataPath
|
|
if !strings.HasPrefix(filepath.Clean(scratch), filepath.Clean(sys)) {
|
|
t.Fatalf("the unit-only scratch must fall back to the system data path %q; got %q", sys, scratch)
|
|
}
|
|
if nsRoot == "" {
|
|
t.Fatal("the namespace root must be returned for the free-space probe")
|
|
}
|
|
if !strings.Contains(scratch, "offsite-proof") {
|
|
t.Fatalf("it must still land in the PROOF root, not the customer's; got %q", scratch)
|
|
}
|
|
}
|
|
|
|
// TestR414_FullRestoreDoesNotFallBack — C4. The state-only tier is protected.
|
|
func TestR414_FullRestoreDoesNotFallBack(t *testing.T) {
|
|
m, _, _ := drivelessHarness(t, "opengist")
|
|
_, _, err := m.offboxRestoreScratchDir("opengist")
|
|
if err == nil {
|
|
t.Fatal("the CUSTOMER's scratch must still refuse on a driveless box — a full restore pulls bulk userdata and the internal SSD is a state-only tier (07 §2.2)")
|
|
}
|
|
// ASCII fragment, with a negative control, because an accented grep has returned 0 for strings
|
|
// that were there (R-364).
|
|
if !strings.Contains(err.Error(), "adatmeghajt") {
|
|
t.Fatalf("the R-252 refusal wording must be preserved — it tells the customer what to do; got %q", err.Error())
|
|
}
|
|
if strings.Contains(err.Error(), "ZZZ-NOT-IN-THE-MESSAGE") {
|
|
t.Fatal("negative control matched — the fragment search is not discriminating")
|
|
}
|
|
}
|
|
|
|
// TestR414_AbsentStillMeansNotRecorded — C2. The two meanings must stay distinct.
|
|
func TestR414_AbsentStillMeansNotRecorded(t *testing.T) {
|
|
m, sett, _ := drivelessHarness(t, "opengist")
|
|
if got := sett.GetOffboxTarget().LastProofResult; got != "" {
|
|
t.Fatalf("a box that has never proved must report ABSENT; got %q", got)
|
|
}
|
|
// A skip must still record nothing — only a reached outcome writes.
|
|
if err := m.AcquireRunningForTest(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
res := m.ProveOffboxUnit(context.Background())
|
|
m.RecordProofVerdict(res)
|
|
if got := sett.GetOffboxTarget().LastProofResult; got != "" {
|
|
t.Fatalf("a SKIP must leave the field absent — it looked at nothing; got %q", got)
|
|
}
|
|
m.ReleaseRunningForTest()
|
|
|
|
// And the wire must not carry the key at all when absent.
|
|
b, err := json.Marshal(&OffboxReportStatus{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(b), "last_proof_result") {
|
|
t.Fatalf("an empty status must not emit last_proof_result; got %s", b)
|
|
}
|
|
}
|
|
|
|
// TestR414_CannotRunIsRecordedButDoesNotAdvanceDueness — the half that keeps the app retryable.
|
|
func TestR414_CannotRunIsRecordedButDoesNotAdvanceDueness(t *testing.T) {
|
|
m, sett, _ := drivelessHarness(t, "opengist")
|
|
res := ProofResult{Stack: "opengist", SnapshotID: "snap-opengist",
|
|
CannotRun: true, CannotWhy: "nincs regisztralt adatmeghajto"}
|
|
if res.Verdict() != ProofResultCannotRun {
|
|
t.Fatalf("cannot-run must render as %q, never as empty; got %q", ProofResultCannotRun, res.Verdict())
|
|
}
|
|
m.RecordProofVerdict(res)
|
|
tt := sett.GetOffboxTarget()
|
|
if tt.LastProofResult != ProofResultCannotRun {
|
|
t.Fatalf("cannot-run must be RECORDED so the hub can see it; got %q", tt.LastProofResult)
|
|
}
|
|
if len(tt.ProvedSnapshots) != 0 {
|
|
t.Fatalf("cannot-run must NOT advance per-snapshot due-ness — nothing was proved, and marking it proved would stop the app ever being retried; got %v", tt.ProvedSnapshots)
|
|
}
|
|
if tt.LastProofSnapshot != "" {
|
|
t.Fatalf("cannot-run must not claim a proved snapshot; got %q", tt.LastProofSnapshot)
|
|
}
|
|
}
|
|
|
|
// TestR414_NoCustomerAlarm — C5. The customer's backups are fine and there is nothing for them to do.
|
|
func TestR414_NoCustomerAlarm(t *testing.T) {
|
|
m, _, _ := drivelessHarness(t, "opengist")
|
|
var pushed int
|
|
m.SetOffboxOrphanEvent(func(string, string) { pushed++ })
|
|
res := m.ProveOffboxUnit(context.Background())
|
|
m.RecordProofVerdict(res)
|
|
if pushed != 0 {
|
|
t.Fatalf("a driveless box must raise no event from the backup layer; got %d", pushed)
|
|
}
|
|
}
|
|
|
|
// TestR414_ProofScratchIsDeletedOnADrivelessBox — the leak live validation caught.
|
|
//
|
|
// The fallback resolved a scratch on the system data path, and `removeProofScratch` then refused to
|
|
// delete it because its accepted-roots list is built from REGISTERED drives, which a driveless box has
|
|
// none of. Observed on demo-felhom 2026-09-01: *"refusing to remove … it is not inside a proof root"*,
|
|
// with the copy still on disk. Every nightly proof would have left one behind.
|
|
//
|
|
// The unit tests did not catch it because they all register a drive. This one deliberately does not.
|
|
func TestR414_ProofScratchIsDeletedOnADrivelessBox(t *testing.T) {
|
|
m, _, _ := drivelessHarness(t, "opengist")
|
|
|
|
scratch, _, err := m.offboxProofScratchDir("opengist")
|
|
if err != nil {
|
|
t.Fatalf("the unit-only scratch must resolve: %v", err)
|
|
}
|
|
if err := os.MkdirAll(filepath.Join(scratch, "marker"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(scratch); err != nil {
|
|
t.Fatalf("fixture: the scratch must exist before the removal is attempted: %v", err)
|
|
}
|
|
|
|
m.removeProofScratch("opengist", scratch)
|
|
|
|
if _, err := os.Stat(scratch); !os.IsNotExist(err) {
|
|
t.Fatalf("the proof copy must be removed on a driveless box too; %s still exists (stat err=%v)", scratch, err)
|
|
}
|
|
}
|
|
|
|
// TestR414_RemovalStillRefusesOutsideAProofRoot — the guard must not be widened into uselessness.
|
|
func TestR414_RemovalStillRefusesOutsideAProofRoot(t *testing.T) {
|
|
m, _, _ := drivelessHarness(t, "opengist")
|
|
outside := t.TempDir()
|
|
keep := filepath.Join(outside, "not-a-proof-root")
|
|
if err := os.MkdirAll(keep, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.removeProofScratch("opengist", keep)
|
|
if _, err := os.Stat(keep); err != nil {
|
|
t.Fatalf("a path outside every proof root must be REFUSED, not deleted: %v", err)
|
|
}
|
|
}
|