R-1342: a restore copy under the stable-start watch is not deletable (D62 gap); an app removed during the watch raises no alarm — tests red first; commit only
gates / gates (push) Successful in 31s
gates / gates (push) Successful in 31s
This commit is contained in:
1 parent
6c72e3ff34
commit
652a690e5d
10 files changed
+231
-1
No files matched your search
@@ -340,6 +340,9 @@ type Manager struct {
|
||||
// R-550: persistence of the above (restore_record.go) and the per-app interrupted notices.
|
||||
opRecordPath string
|
||||
opInterrupted map[string]RestoreOpResult
|
||||
// R-1342: the restore copies a stable-start watch still holds (copy dir → true), guarded by mu. While one is here the
|
||||
// copy is the way back if the restored app turns out unstable: „Törlés most" refuses it (restore_copies.go).
|
||||
watching map[string]bool
|
||||
// R-519: the app-data run record (run_record.go) and the start of the last run a stop cut off (zero = none).
|
||||
runRecordPath string
|
||||
runInterrupted time.Time
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// r1342Drive registers the rig's app drive, as every drive is registered on a box, so the list and the delete see it.
|
||||
func r1342Drive(t *testing.T, m *Manager) {
|
||||
t.Helper()
|
||||
if m.settings == nil {
|
||||
sett, err := settings.Load(filepath.Join(t.TempDir(), "settings.json"), log.New(io.Discard, "", 0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.settings = sett
|
||||
}
|
||||
if err := m.settings.AddStoragePath(settings.StoragePath{Path: m.GetAppDrivePath("app"), Label: "rig"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1342 (found live on Tester 1, 2026-10-29 10:39 CET): (1) while the box still watches whether a restored app runs
|
||||
// stably, its restore copy is the way back — „Törlés most" must not take it (D62: never while an operation runs on the
|
||||
// app); (2) an app REMOVED during that watch is not „did not start" — no operator alarm, no „the copy is kept".
|
||||
|
||||
// removableProvider is the stable-watch rig's provider that can answer StackRemoved.
|
||||
type removableProvider struct {
|
||||
*sampledProvider
|
||||
removed atomic.Bool
|
||||
}
|
||||
|
||||
func (p *removableProvider) StackRemoved(string) bool { return p.removed.Load() }
|
||||
|
||||
// RED-PROOF: drop the watching check from restoreCopyNeededFor / DeleteRestoreCopy → the copy is offered and deleted
|
||||
// mid-watch → FAILS.
|
||||
func TestR1342_ACopyUnderTheStableWatchIsNotDeletable(t *testing.T) {
|
||||
withStable(t, 400*time.Millisecond, 3*time.Second)
|
||||
m, rig, alarms := fixARig(t, func(int) bool { return true },
|
||||
func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
|
||||
r1342Drive(t, m)
|
||||
m.BeginRestoreOp("restore", "app")
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.EndRestoreOp(true, "ok")
|
||||
copies := m.RestoreCopies()
|
||||
if len(copies) != 1 || copies[0].Needed != RestoreCopyNeededChecking || !copies[0].DeleteOn.IsZero() {
|
||||
t.Fatalf("mid-watch the copy must read „checking\", got %+v", copies)
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy("app", copies[0].Stamp); !errors.Is(err, ErrRestoreCopyNeeded) {
|
||||
t.Fatalf("a delete mid-watch: %v, want ErrRestoreCopyNeeded", err)
|
||||
}
|
||||
if left := undoCopies(t, rig.undoRoot); len(left) != 1 {
|
||||
t.Fatalf("the copy must stay through the watch, found %v", left)
|
||||
}
|
||||
if left := waitNoCopies(t, rig.undoRoot); len(left) != 0 {
|
||||
t.Fatalf("after the stable window the watch releases it, found %v", left)
|
||||
}
|
||||
if len(*alarms) != 0 {
|
||||
t.Fatalf("no alarm: %v", *alarms)
|
||||
}
|
||||
if m.watchingCopy(filepath.Join(rig.undoRoot, copies[0].Stamp)) {
|
||||
t.Fatal("the watch mark must go when the watch ends")
|
||||
}
|
||||
}
|
||||
|
||||
// RED-PROOF: drop the stableRemoved case → the removed app „did not run stably" → one alarm → FAILS.
|
||||
func TestR1342_AnAppRemovedDuringTheWatchRaisesNoAlarm(t *testing.T) {
|
||||
withStable(t, 2*time.Second, 400*time.Millisecond)
|
||||
var up atomic.Bool
|
||||
up.Store(true)
|
||||
m, rig, alarms := fixARig(t, func(int) bool { return up.Load() },
|
||||
func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
|
||||
rp := &removableProvider{sampledProvider: m.stackProvider.(*sampledProvider)}
|
||||
m.stackProvider = rp
|
||||
r1342Drive(t, m)
|
||||
m.BeginRestoreOp("restore", "app")
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.EndRestoreOp(true, "ok")
|
||||
// the household removes the app inside the watch
|
||||
up.Store(false)
|
||||
rp.removed.Store(true)
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
if len(*alarms) != 0 {
|
||||
t.Fatalf("an app removed during the watch must not alarm „did not start\": %v", *alarms)
|
||||
}
|
||||
copies := m.RestoreCopies()
|
||||
if len(copies) != 1 || copies[0].Needed != "" {
|
||||
t.Fatalf("after the watch ended the copy is an unused one (sweep / Törlés most): %+v", copies)
|
||||
}
|
||||
_ = rig
|
||||
}
|
||||
|
||||
// Control: an app that stays installed and never runs stably still alarms once (the R-1270 rule is unchanged).
|
||||
func TestR1342_AnInstalledUnstableAppStillAlarms(t *testing.T) {
|
||||
withStable(t, 2*time.Second, 400*time.Millisecond)
|
||||
var up atomic.Bool
|
||||
up.Store(true)
|
||||
m, _, alarms := fixARig(t, func(int) bool { return up.Load() },
|
||||
func(int) []ContainerSample { return []ContainerSample{healthy("c1", 0)} })
|
||||
m.stackProvider = &removableProvider{sampledProvider: m.stackProvider.(*sampledProvider)}
|
||||
m.BeginRestoreOp("restore", "app")
|
||||
if _, err := m.RestoreFromRecoveryUnit("app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.EndRestoreOp(true, "ok")
|
||||
up.Store(false)
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
if len(*alarms) != 1 {
|
||||
t.Fatalf("an installed app that never runs stably alarms once, got %v", *alarms)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,9 @@ const (
|
||||
RestoreCopyNeededHold = "hold" // the app is held (a failed put-back keeps its copy, D20) — support decides
|
||||
RestoreCopyNeededPutBack = "putback" // a put-back waits for its drive (R-1251) or a cut restore names it (D22)
|
||||
RestoreCopyNeededUnknown = "unknown" // the app-stop marker cannot be read: nothing is offered (fail closed)
|
||||
// RestoreCopyNeededChecking (R-1342): the box still watches whether the restored app runs stably (up to 30 minutes
|
||||
// after the restore, R-1270) — the copy is the way back if it does not.
|
||||
RestoreCopyNeededChecking = "checking"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -119,7 +122,10 @@ func (m *Manager) RestoreCopies() []RestoreCopy {
|
||||
continue
|
||||
}
|
||||
rc := RestoreCopy{App: app, Stamp: c.Name(), At: at, SizeBytes: dirSize(dir), Path: dir, Needed: needed}
|
||||
if needed == "" {
|
||||
if rc.Needed == "" && m.watchingCopy(dir) {
|
||||
rc.Needed = RestoreCopyNeededChecking
|
||||
}
|
||||
if rc.Needed == "" {
|
||||
rc.DeleteOn = at.Add(restoreCopyUnusedAge)
|
||||
}
|
||||
out = append(out, rc)
|
||||
@@ -174,9 +180,37 @@ func (m *Manager) DeleteRestoreCopy(app, stamp string) (RestoreCopy, error) {
|
||||
if found.Needed = m.restoreCopyNeededFor(app, named, ok); found.Needed != "" {
|
||||
return *found, ErrRestoreCopyNeeded
|
||||
}
|
||||
if m.watching[found.Path] { // R-1342: under m.mu, the same lock the watch marks its copy under
|
||||
found.Needed = RestoreCopyNeededChecking
|
||||
return *found, ErrRestoreCopyNeeded
|
||||
}
|
||||
if err := os.RemoveAll(found.Path); err != nil {
|
||||
return *found, fmt.Errorf("removing the restore copy %s: %w", found.Path, err)
|
||||
}
|
||||
_ = os.Remove(filepath.Dir(found.Path)) // the per-app folder, when it is now empty
|
||||
return *found, nil
|
||||
}
|
||||
|
||||
// setWatching marks (on) or clears a copy a stable-start watch holds (R-1342).
|
||||
func (m *Manager) setWatching(dir string, on bool) {
|
||||
if dir == "" {
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if on {
|
||||
if m.watching == nil {
|
||||
m.watching = map[string]bool{}
|
||||
}
|
||||
m.watching[dir] = true
|
||||
return
|
||||
}
|
||||
delete(m.watching, dir)
|
||||
}
|
||||
|
||||
// watchingCopy reports whether a stable-start watch still holds the copy.
|
||||
func (m *Manager) watchingCopy(dir string) bool {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.watching[dir]
|
||||
}
|
||||
@@ -71,9 +71,14 @@ func orNone(s string) string {
|
||||
// so and the kept copy is removed (the normal rule after a successful restore). A new restore of the app ends the watch
|
||||
// untouched.
|
||||
func (m *Manager) watchLateStart(stack, copyDir string, since time.Time, watch time.Duration) {
|
||||
m.setWatching(copyDir, true) // R-1342: the copy is the way back while the app may still start
|
||||
defer m.setWatching(copyDir, false)
|
||||
switch m.waitStableStart(stack, since, since.Add(watch)) {
|
||||
case stableAborted:
|
||||
return // a new restore of this app owns it now
|
||||
case stableRemoved:
|
||||
m.logger.Printf("[INFO] [backup] %s: the app was removed while the box waited for it to start — the watch ends (R-1342)", stack)
|
||||
return
|
||||
case stableNot:
|
||||
m.logger.Printf("[WARN] [backup] %s: still not running stably %s after its restore — the restore copy stays for support (%s) (R-1270)", stack, lateStartWatch, orNone(copyDir))
|
||||
return
|
||||
@@ -133,12 +138,26 @@ type containerSampler interface {
|
||||
ContainerSamples(name string) ([]ContainerSample, error)
|
||||
}
|
||||
|
||||
// StackRemovalReporter (R-1342) is the provider's optional answer to „was this app removed?" — true only when the app is
|
||||
// known and no longer installed. A watch that sees it ends quietly: an app the household removed did not „fail to
|
||||
// start". The production stackAdapter implements it (a compile-time assertion in cmd/controller/main.go).
|
||||
type StackRemovalReporter interface {
|
||||
StackRemoved(name string) bool
|
||||
}
|
||||
|
||||
// stackRemoved asks the provider; a provider without the answer never reads „removed".
|
||||
func (m *Manager) stackRemoved(stack string) bool {
|
||||
r, ok := m.stackProvider.(StackRemovalReporter)
|
||||
return ok && r.StackRemoved(stack)
|
||||
}
|
||||
|
||||
type stableResult int
|
||||
|
||||
const (
|
||||
stableYes stableResult = iota
|
||||
stableNot
|
||||
stableAborted
|
||||
stableRemoved // R-1342: the app was removed during the watch
|
||||
)
|
||||
|
||||
// stableSample is one look: ok, and the fingerprint (container id → restart count) a stable window must keep.
|
||||
@@ -201,6 +220,9 @@ func (m *Manager) waitStableFor(stack string, watchFrom, deadline time.Time, win
|
||||
if m.restoreBegunAfter(stack, watchFrom) {
|
||||
return stableAborted
|
||||
}
|
||||
if m.stackRemoved(stack) {
|
||||
return stableRemoved
|
||||
}
|
||||
ok, fp := m.stableSample(stack)
|
||||
switch {
|
||||
case !ok:
|
||||
@@ -226,13 +248,17 @@ func (m *Manager) releaseAfterStableStart(stack, kind string, undo *liveVolumeCo
|
||||
dir := undo.dir
|
||||
from := time.Now() // taken HERE, before the goroutine: a restore begun after this moment is a new one
|
||||
until := from.Add(stableReleaseWatch)
|
||||
m.setWatching(dir, true) // R-1342: „Törlés most" refuses the copy while the watch holds it
|
||||
go func() {
|
||||
defer m.setWatching(dir, false)
|
||||
switch m.waitStableStart(stack, from, until) {
|
||||
case stableYes:
|
||||
removeCopyDir(m, stack, dir)
|
||||
m.logger.Printf("[INFO] [backup] %s: ran stably for %s after its %s — the restore copy is released (R-1270)", stack, stableStartWindow, kind)
|
||||
case stableAborted:
|
||||
m.logger.Printf("[INFO] [backup] %s: a new restore began before the %s's start was stable — its restore copy stays for support (%s)", stack, kind, dir)
|
||||
case stableRemoved:
|
||||
m.logger.Printf("[INFO] [backup] %s: the app was removed during the stable-start check after its %s — no alarm; the restore copy, if the remove left it, is an unused one (%s) (R-1342)", stack, kind, dir)
|
||||
default:
|
||||
m.logger.Printf("[ERROR] [backup] %s: did not run stably within %s after its %s — the restore copy stays for support (%s) (R-1270)", stack, stableReleaseWatch, kind, dir)
|
||||
if m.restoreNotStartedNotify != nil {
|
||||
@@ -262,6 +288,8 @@ func (m *Manager) restoredAppStarted(stack string) error {
|
||||
return nil
|
||||
case stableAborted:
|
||||
return fmt.Errorf("a new restore of %s began", stack)
|
||||
case stableRemoved:
|
||||
return fmt.Errorf("%s was removed during the restore's start check", stack)
|
||||
}
|
||||
return fmt.Errorf("stack %s did not run stably for %s (it restarted or turned unhealthy) after restore", stack, verdictStableWindow)
|
||||
}
|
||||
|
||||
@@ -2641,6 +2641,7 @@
|
||||
"restore_copy.needed.hold": "Cannot be deleted: the app is stopped, and this copy is its way back. Support decides about it.",
|
||||
"restore_copy.needed.putback": "Cannot be deleted: an interrupted restore puts the data back from this copy.",
|
||||
"restore_copy.needed.unknown": "Cannot be deleted now: the box cannot tell whether this copy is still needed.",
|
||||
"restore_copy.needed.checking": "Cannot be deleted now: the box is still checking that the app started well after the restore (up to 30 minutes).",
|
||||
"restore_copy.delete": "Delete now",
|
||||
"restore_copy.warning": "This is the copy of %s’s data from before the restore on %s. If you delete it, the restore can no longer be undone, and this copy cannot be brought back.",
|
||||
"restore_copy.app_busy": "Cannot be deleted now: an operation runs on the app. Try again when it has ended.",
|
||||
|
||||
@@ -2629,6 +2629,7 @@
|
||||
"restore_copy.needed.hold": "Nem törölhető: az alkalmazás le van állítva, és ez a másolat az út vissza. Az ügyfélszolgálat dönt róla.",
|
||||
"restore_copy.needed.putback": "Nem törölhető: egy félbeszakadt visszaállítás ebből a másolatból teszi vissza az adatokat.",
|
||||
"restore_copy.needed.unknown": "Most nem törölhető: a doboz nem tudja eldönteni, kell-e még ez a másolat.",
|
||||
"restore_copy.needed.checking": "Most nem törölhető: a doboz még figyeli, hogy az alkalmazás rendben elindult-e a visszaállítás után (legfeljebb 30 percig).",
|
||||
"restore_copy.delete": "Törlés most",
|
||||
"restore_copy.warning": "Ez a(z) %s adatainak másolata a %s-i visszaállítás előttről. Ha törlöd, a visszaállítás többé nem vonható vissza, és ez a másolat nem hozható vissza.",
|
||||
"restore_copy.app_busy": "Most nem törölhető: az alkalmazáson éppen egy művelet fut. Próbáld újra, ha véget ért.",
|
||||
|
||||
@@ -231,3 +231,18 @@ func TestD62_ADeleteRemovesOnlyThatCopyAndStoresTheEvent(t *testing.T) {
|
||||
t.Fatalf("second press code %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
// R-1342: every reason a copy can be needed has its one-line sentence in both languages (the row's key is built from the
|
||||
// reason, so a missing key would show the raw key). RED-PROOF: remove restore_copy.needed.checking from BOTH bundles →
|
||||
// FAILS (seen 2026-10-29; a key missing only in en.json falls back to the Hungarian line — i18n_missing_gate counts that).
|
||||
func TestR1342_EveryNeededReasonHasItsSentence(t *testing.T) {
|
||||
f := newD62Fixture(t)
|
||||
for _, why := range []string{backup.RestoreCopyNeededHold, backup.RestoreCopyNeededPutBack, backup.RestoreCopyNeededUnknown, backup.RestoreCopyNeededChecking} {
|
||||
for _, lang := range []string{"hu", "en"} {
|
||||
key := "restore_copy.needed." + why
|
||||
if got := f.s.msgLang(lang, key); got == "" || got == key || strings.Contains(got, "restore_copy.") {
|
||||
t.Errorf("%s/%s renders %q", lang, key, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user