C9-F1 + C9-F2: a restore that restored nothing, and a crash loop nobody saw (v0.183.0)
Both are the system reporting healthy while the customer is not, and both live in the same status-derivation code. Neither is fixed by making the system quieter. C9-F1 (HIGH) — Tier-2 writes recovery-unit/ on EVERY run and RestoreTier2Files has never read it (tier2_restore.go:101-104 reads hdd/ + userdata/ only). Phase 0 enumerated all 53 catalog templates against both demo boxes: 43 apps have NO readable subtree, so the button stopped the app, restored 0 files, restarted it and said "Nincs hiányzó fájl — minden fájl megvan a helyén." — at the moment the customer pressed it because files were missing, with 156 MB of BookStack's data unread in the same copy. 9 apps have file legs but never their DB or volumes, so the same sentence was also a clean bill of health over data never opened (immich: 1.3 GB Postgres unit). Honesty half shipped: a pre-flight coverage check refuses UP FRONT without stopping the app and NAMES the action that works; a run that proceeds claims only what it EXAMINED and discloses that the database and volumes are not covered. Completeness is filed as C9-F1b — routing to the Tier-1 unit restore puts a destructive operation behind a non-destructive button, so its confirm copy has to carry that difference. C9-F4 filed: nothing reads the Tier-2 recovery-unit/ mirror, so the second local copy that exists for drive loss is unreachable by any customer action. C9-F2 (HIGH) — a crash loop was counted as working. StateRestarting is deliberately NOT added to IsDownState (that alarms on every deploy fleet-wide, the over-correction F-A1 nearly cost us); a sustained run becomes down after crashLoopAfter = 5m, set above the 120s deploy timeout, Mealie's 60s start_period and R-97b's 180s grace. The dashboard counter uses the same predicate, so it no longer contradicts the alarm on the same screen. README's claim that faults "still surface as restarting" was a wish with no test — corrected in place; it is the seventh such instance. Six red-proofs observed, including the one that matters most: adding StateRestarting to IsDownState fails the brief-restart test with "every deploy and update would page the operator". go test ./... rc=0, 27 packages, run and read separately from this commit.
This commit is contained in:
@@ -0,0 +1,177 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backup"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// C9-F1 at the customer surface. Two messages had to change, and they fail differently:
|
||||
//
|
||||
// - the LOUD lie: for the 43 apps whose data the restore cannot read, the customer got an outage
|
||||
// and „Nincs hiányzó fájl — minden fájl megvan a helyén." — pressed precisely BECAUSE files were
|
||||
// missing;
|
||||
// - the QUIET one: for the 9 apps it does cover, it covers only the file legs, never the database
|
||||
// or named volumes, so the same blanket sentence was a clean bill of health over data the
|
||||
// operation never opened (immich's 1.3 GB Postgres unit is the dangerous case).
|
||||
|
||||
// honestProvider is a minimal provider: the restore must never reach StopStack in the no-coverage
|
||||
// test, and `stops` is the assertion that proves it.
|
||||
type honestProvider struct {
|
||||
hdd string
|
||||
stops int32
|
||||
}
|
||||
|
||||
func (p *honestProvider) GetStackComposePath(string) (string, bool) { return "", false }
|
||||
func (p *honestProvider) ListDeployedStacks() []backup.StackSummary { return nil }
|
||||
func (p *honestProvider) GetStackHDDMounts(string) []string { return nil }
|
||||
func (p *honestProvider) GetStackHDDPath(string) string { return p.hdd }
|
||||
func (p *honestProvider) GetImportRoot() string { return "" }
|
||||
func (p *honestProvider) GetDockerVolumes(string) []string { return nil }
|
||||
func (p *honestProvider) StopStack(string) error { atomic.AddInt32(&p.stops, 1); return nil }
|
||||
func (p *honestProvider) StartStack(string) error { return nil }
|
||||
func (p *honestProvider) RefreshAndIsRunning(string) bool { return true }
|
||||
func (p *honestProvider) GetStackRecoveryInfo(string) (backup.RecoveryInfo, bool) {
|
||||
return backup.RecoveryInfo{}, false
|
||||
}
|
||||
func (p *honestProvider) GetStackClassifiedBinds(string) ([]backup.ClassifiedBind, bool) {
|
||||
return nil, false
|
||||
}
|
||||
func (p *honestProvider) RecoverStackSecrets(string, []string) map[string]string { return nil }
|
||||
func (p *honestProvider) RecreateStackDefinitionFromUnit(string, string, map[string]string) error {
|
||||
return nil
|
||||
}
|
||||
func (p *honestProvider) StartStackServices(string, []string) error { return nil }
|
||||
|
||||
// newHonestServer builds a server whose recorded Tier-2 copy has `legs` (each created as a dir) and,
|
||||
// optionally, a recovery unit — so one harness expresses both the class-A and class-B shapes.
|
||||
func newHonestServer(t *testing.T, legs []string, withUnit bool) (*Server, *honestProvider) {
|
||||
t.Helper()
|
||||
tmp := t.TempDir()
|
||||
live := filepath.Join(tmp, "usb")
|
||||
dest := filepath.Join(tmp, "flash")
|
||||
lg := log.New(io.Discard, "", 0)
|
||||
sett, err := settings.Load(filepath.Join(tmp, "settings.json"), lg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, p := range []string{live, dest} {
|
||||
if err := sett.AddStoragePath(settings.StoragePath{Path: p, Label: filepath.Base(p)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := sett.SetCrossDriveConfig("app", &settings.CrossDriveBackup{
|
||||
Enabled: true, Method: "rsync", DestinationPath: dest,
|
||||
LastRun: "2026-07-28T03:30:00Z", LastStatus: "ok",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
destBase := filepath.Join(dest, "backups", "secondary", "app")
|
||||
for _, leg := range legs {
|
||||
if err := os.MkdirAll(filepath.Join(destBase, leg, "appdata"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if withUnit {
|
||||
if err := os.MkdirAll(filepath.Join(destBase, "recovery-unit", "volume-dumps"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(destBase, ".felhom-tier2-layout"), []byte("2"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &config.Config{}
|
||||
cfg.Paths.DataDir = tmp
|
||||
m := backup.NewManager(cfg, sett, lg)
|
||||
prov := &honestProvider{hdd: live}
|
||||
m.SetStackProvider(prov)
|
||||
return &Server{cfg: cfg, backupMgr: m, logger: lg}, prov
|
||||
}
|
||||
|
||||
// SCENARIO D at the surface — the customer is told plainly, up front, and the app is NOT stopped.
|
||||
// The message must NAME the action that works; a dead end for 81% of the catalog is not honesty.
|
||||
//
|
||||
// RED-PROOF (observed): remove the pre-flight `!cov.CanRestore()` block from the handler →
|
||||
//
|
||||
// tier2_honest_message_test.go:118: no flash_error — the customer was told the restore STARTED
|
||||
// tier2_honest_message_test.go:129: THE APP WAS STOPPED (stops=1) for a restore that can never restore anything
|
||||
func TestTier2RestoreHandler_NoCoverage_RefusesUpFrontAndNamesTheAction(t *testing.T) {
|
||||
s, prov := newHonestServer(t, nil, true) // bookstack shape: unit only, no legs
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/backup/tier2/restore", strings.NewReader("stack_name=app"))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
w := httptest.NewRecorder()
|
||||
s.backupTier2RestoreHandler(w, req)
|
||||
|
||||
loc := w.Header().Get("Location")
|
||||
if !strings.Contains(loc, "flash_error=") {
|
||||
t.Fatalf("no flash_error — the customer was told the restore STARTED: %s", loc)
|
||||
}
|
||||
// The refusal must not have started an operation at all.
|
||||
if st := s.backupMgr.RestoreStatus(); st.Running {
|
||||
t.Error("an async op was begun for a restore that cannot do anything")
|
||||
}
|
||||
if got := atomic.LoadInt32(&prov.stops); got != 0 {
|
||||
t.Errorf("THE APP WAS STOPPED (stops=%d) for a restore that can never restore anything", got)
|
||||
}
|
||||
// The message names the working action rather than dead-ending.
|
||||
for _, want := range []string{"nem ebből a másolatból", "nem állt le", "Visszaállítás indítása"} {
|
||||
if !strings.Contains(tier2NoCoverageMsg, want) {
|
||||
t.Errorf("the refusal message is missing %q:\n%s", want, tier2NoCoverageMsg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SCENARIO F — „nothing missing" must claim only what was EXAMINED, and must disclose what this
|
||||
// restore does not cover at all.
|
||||
//
|
||||
// RED-PROOF (observed): restore the old blanket string
|
||||
// (`msg := "Nincs hiányzó fájl — minden fájl megvan a helyén."` with no disclosure) →
|
||||
//
|
||||
// tier2_honest_message_test.go:154: the success message still claims ALL files: "Nincs hiányzó fájl — minden fájl megvan a helyén."
|
||||
// tier2_honest_message_test.go:161: the message does not disclose that the database and volumes were not covered
|
||||
func TestTier2RestoreHandler_CoveredApp_ClaimsOnlyWhatWasExamined(t *testing.T) {
|
||||
s, _ := newHonestServer(t, []string{"hdd"}, true) // paperless shape: a leg AND a unit
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/backup/tier2/restore", strings.NewReader("stack_name=app"))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
w := httptest.NewRecorder()
|
||||
s.backupTier2RestoreHandler(w, req)
|
||||
|
||||
if !strings.Contains(w.Header().Get("Location"), "flash=") {
|
||||
t.Fatalf("a COVERED app was refused — this is the regression guard on Campaign 9's A1 result: %s", w.Header().Get("Location"))
|
||||
}
|
||||
|
||||
var last backup.RestoreOpResult
|
||||
waitFor(t, func() bool {
|
||||
st := s.backupMgr.RestoreStatus()
|
||||
if st.Running || st.Last == nil {
|
||||
return false
|
||||
}
|
||||
last = *st.Last
|
||||
return true
|
||||
}, "the restore to finish")
|
||||
|
||||
if !last.OK {
|
||||
t.Fatalf("a covered app's restore failed: %s", last.Message)
|
||||
}
|
||||
if strings.Contains(last.Message, "minden fájl megvan a helyén") {
|
||||
t.Errorf("the success message still claims ALL files: %q", last.Message)
|
||||
}
|
||||
if !strings.Contains(last.Message, "vizsgált") {
|
||||
t.Errorf("the message does not limit its claim to what was EXAMINED: %q", last.Message)
|
||||
}
|
||||
if !strings.Contains(last.Message, tier2UnitNotCoveredMsg) {
|
||||
t.Errorf("the message does not disclose that the database and volumes were not covered: %q", last.Message)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user