v0.263.0: a failed update puts the app back by itself (09 decision 15, R-637)

The guarded update gains a folder copy of the app's named volumes, taken after the pull where the app stops anyway (decision 19, chosen by the 2026-09-23 bake-off). On a failed health check the box undoes: every copy validated by its finished-marker first, volumes refilled, definition and pin from the job's own pre-update copies, the old version checked with the OLD .felhom.yml probe. It holds only if the undo fails, and the hold sentence says so and what state the data is in. Bind-mounted folders are never touched. - R-637 built; R-638/R-640/R-641 do not arise with a folder copy; R-639 (pre-update copies incl. .felhom.yml kept until the undo is over). - journal phases copying/undoing with power-cut recovery. - app.yaml last_update_undone + one line on the app page (hu/en). - R-642: start/restart never answer "completed". - Removal deletes kept undo copies. MinAgent unchanged (0.131.0). Nine red-proofs in REPORT.md. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-09-23 11:12:49 +02:00
parent b9deec1907
commit 8fc2b4a1a9
26 changed files with 1326 additions and 69 deletions
@@ -0,0 +1,417 @@
+package stacks
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
+)
+
+// v0.263.0 — the undo (09 §3 decision 15, undo.go). Every test runs the REAL job (runGuardedUpdate /
+// RecoverUpdates / ResumeInterruptedUpdates) with the process boundaries faked, and reads the EFFECT
+// back: the data in the fake volume, the pin in app.yaml, the phase, the hold, the journal.
+
+// fakeCopier is the volume boundary. `vols` is each app volume's CONTENT, so "the data came back" is
+// a string compare, and `complete` is each copy's finished-marker.
+type fakeCopier struct {
+	mu         sync.Mutex
+	vols       map[string]string
+	copies     map[string]string
+	complete   map[string]bool
+	calls      []string
+	copyErr    error
+	cutOff     bool // every copy is made WITHOUT its finished-marker (a copy container killed mid-way)
+	restoreErr error
+	bytes      int64
+}
+
+func newFakeCopier(vols map[string]string) *fakeCopier {
+	return &fakeCopier{vols: vols, copies: map[string]string{}, complete: map[string]bool{}, bytes: 1 << 20}
+}
+
+func (f *fakeCopier) note(c string) { f.calls = append(f.calls, c) }
+func (f *fakeCopier) ProjectVolumes(string) ([]string, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for v := range f.vols {
+		out = append(out, v)
+	}
+	sort.Strings(out)
+	return out, nil
+}
+func (f *fakeCopier) VolumeBytes(string) (int64, error) { return f.bytes, nil }
+func (f *fakeCopier) Copy(src, dst, _ string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.note("copy " + src)
+	if f.copyErr != nil {
+		return f.copyErr
+	}
+	f.copies[dst] = f.vols[src]
+	f.complete[dst] = !f.cutOff
+	return nil
+}
+func (f *fakeCopier) Complete(c string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.complete[c]
+}
+
+// Restore refuses a copy with no marker, exactly as the real helper does (`test -f` in the same shell).
+func (f *fakeCopier) Restore(c, v string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.note("restore " + v)
+	if f.restoreErr != nil {
+		return f.restoreErr
+	}
+	if !f.complete[c] {
+		return fmt.Errorf("no finished-marker in %s", c)
+	}
+	f.vols[v] = f.copies[c]
+	return nil
+}
+func (f *fakeCopier) Remove(v string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.note("remove " + v)
+	delete(f.copies, v)
+	delete(f.complete, v)
+	return nil
+}
+func (f *fakeCopier) CopiesOf(string) []string {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for c := range f.copies {
+		out = append(out, c)
+	}
+	return out
+}
+func (f *fakeCopier) vol(v string) string { f.mu.Lock(); defer f.mu.Unlock(); return f.vols[v] }
+func (f *fakeCopier) setVol(v, s string)  { f.mu.Lock(); f.vols[v] = s; f.mu.Unlock() }
+func (f *fakeCopier) nCopies() int        { f.mu.Lock(); defer f.mu.Unlock(); return len(f.copies) }
+func (f *fakeCopier) callsHave(p string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for _, c := range f.calls {
+		if strings.HasPrefix(c, p) {
+			return true
+		}
+	}
+	return false
+}
+
+const (
+	undoMetaOld = "healthcheck:\n  checks:\n    - type: http\n      port: 3000\n"
+	undoMetaNew = "healthcheck:\n  checks:\n    - type: http\n      port: 3999\n" // the drill's wrong probe
+	undoVol     = "nextcloud_db"
+)
+
+// newUndoManager: slice 4's manager plus a data volume holding "OLD", the OLD .felhom.yml in the stack
+// dir, and a compose fake that plays the two things the real world does in between: the catalog's
+// .felhom.yml flows in with the new probe during the pull (§5.4 — .felhom.yml is never frozen), and
+// the NEW version migrates the data on its first `up`.
+func newUndoManager(t *testing.T) (*Manager, string, *fakeGuards, *composeRec, *fakeCopier) {
+	t.Helper()
+	m, dir, g, c := newSlice4Manager(t)
+	mustWrite(t, filepath.Join(dir, ".felhom.yml"), undoMetaOld)
+	fc := newFakeCopier(map[string]string{undoVol: "OLD"})
+	m.undoCopier = fc
+	pulled, migrated := false, false
+	m.updateComposeFn = func(d string, env []string, args ...string) (string, error) {
+		switch args[0] {
+		case "pull":
+			pulled = true
+			mustWrite(t, filepath.Join(dir, ".felhom.yml"), undoMetaNew)
+		case "up":
+			// Only the NEW version migrates: the first `up` after a pull, with the undo copy taken.
+			// (After a failed copy, or in a resumed undo, the `up` starts the OLD version.)
+			if pulled && !migrated && fc.nCopies() > 0 {
+				migrated = true
+				fc.setVol(undoVol, "MIGRATED")
+			}
+		}
+		return c.fn(d, env, args...)
+	}
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "new version unhealthy" }
+	m.updateUndoHealthFn = func(_ context.Context, _ string, _ time.Duration, meta *Metadata) (bool, string) {
+		if meta != nil && meta.HealthCheck != nil && len(meta.HealthCheck.Checks) > 0 && meta.HealthCheck.Checks[0].Port == 3000 {
+			return true, "old probe answered"
+		}
+		return false, "probed the wrong port"
+	}
+	return m, dir, g, c, fc
+}
+
+// TestUndo_FailedUpdateIsPutBackWithItsData is decision 15 in one test: the new version migrates the
+// data and fails its check; the box puts the old version back WITH THE PRE-UPDATE DATA, and nothing is
+// held.
+//
+// COMPANION RED-PROOF 1 (REPORT.md): at v0.262.1's shape — failAndHold without the tryUndo call — the
+// app ends HELD with the data still "MIGRATED", and this test fails on the first assertion.
+func TestUndo_FailedUpdateIsPutBackWithItsData(t *testing.T) {
+	m, dir, g, _, fc := newUndoManager(t)
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
+		t.Fatalf("a failed update must be UNDONE, not held; held=%v phase=%q err=%q", held, st.UpdatePhase, st.UpdateError)
+	}
+	if got := fc.vol(undoVol); got != "OLD" {
+		t.Errorf("the data must be the PRE-UPDATE data again, got %q", got)
+	}
+	if got := pinOf(t, dir); got != "nextcloud:31.0.14-apache" {
+		t.Errorf("the pin must be the old version again, got %q", got)
+	}
+	if got := fileBody(t, filepath.Join(dir, "docker-compose.yml")); got != pinTplOld {
+		t.Errorf("the live definition must be the old one again:\n%s", got)
+	}
+	if st.UpdateError != "" || st.UpdatePhaseLabel != "Visszaállítva az előző változatra" {
+		t.Errorf("an undone update carries no error and the undone label; err=%q label=%q", st.UpdateError, st.UpdatePhaseLabel)
+	}
+	u := readPin(t, dir).LastUpdateUndone
+	if u == nil || u.To["web"] != "nextcloud:34.0.1-apache" || u.At == "" || !strings.Contains(u.Why, "unhealthy") {
+		t.Errorf("app.yaml must remember the undone step (to, at, why), got %+v", u)
+	}
+	if fc.nCopies() != 0 || journalExists(m) {
+		t.Errorf("after a successful undo the copies and the journal are gone; copies=%d journal=%v", fc.nCopies(), journalExists(m))
+	}
+	for _, f := range []string{preUpdateComposeFile, preUpdateAppliedFile, preUpdateMetaDir} {
+		if _, err := os.Stat(filepath.Join(dir, f)); err == nil {
+			t.Errorf("the pre-update copy %s must be removed once the undo is over", f)
+		}
+	}
+}
+
+// TestUndo_CutOffCopyIsRefusedBeforeAnythingMoves: a copy without its finished-marker is detected
+// BEFORE anything is poured back; the outcome is today's HOLD, saying the data is as the new version
+// left it — never a "success".
+//
+// COMPANION RED-PROOF 2 (REPORT.md): delete the Complete() validation loop in tryUndo. Restore is then
+// called on the cut copy and the undo state reads "half" — this test fails on both assertions.
+func TestUndo_CutOffCopyIsRefusedBeforeAnythingMoves(t *testing.T) {
+	m, _, g, _, fc := newUndoManager(t)
+	fc.cutOff = true
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); !held || st.UpdatePhase != UpdatePhaseFailed || g.undoState != UndoStateUntouched {
+		t.Fatalf("a cut-off copy must end HELD with state %q; held=%v phase=%q state=%q", UndoStateUntouched, held, st.UpdatePhase, g.undoState)
+	}
+	if fc.callsHave("restore ") {
+		t.Error("NOTHING may be poured back from a copy that is not whole")
+	}
+	if got := fc.vol(undoVol); got != "MIGRATED" {
+		t.Errorf("the data must be left as the new version left it, got %q", got)
+	}
+	if fc.nCopies() == 0 {
+		t.Error("a failed undo keeps its copies for the operator")
+	}
+}
+
+// TestUndo_UsesTheOldProbe: the new .felhom.yml names a port the old version never answers (the drill
+// case, and a real one whenever a probe moves with a version). The undo must judge the old version
+// with the OLD probe.
+//
+// COMPANION RED-PROOF 3 (REPORT.md): make tryUndo use LoadMetadata(dir) (the current file) instead of
+// entry.PrevMeta — the undo then probes port 3999 and the app ends HELD; this test fails.
+func TestUndo_UsesTheOldProbe(t *testing.T) {
+	m, _, g, _, _ := newUndoManager(t)
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
+		t.Fatalf("with the old probe the old version is healthy and the undo completes; held=%v phase=%q state=%q", held, st.UpdatePhase, g.undoState)
+	}
+}
+
+// TestUndo_OldVersionThatDoesNotStartIsHeldSayingSo: data and definition put back, the old version
+// still unhealthy → HOLD with the not_started state, and the copies kept.
+func TestUndo_OldVersionThatDoesNotStartIsHeldSayingSo(t *testing.T) {
+	m, _, g, _, fc := newUndoManager(t)
+	m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) {
+		return false, "old version broken too"
+	}
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); !held || g.undoState != UndoStateNotStarted || st.UpdatePhase != UpdatePhaseFailed {
+		t.Fatalf("held=%v state=%q phase=%q", held, g.undoState, st.UpdatePhase)
+	}
+	if got := fc.vol(undoVol); got != "OLD" {
+		t.Errorf("the data was put back before the check, got %q", got)
+	}
+}
+
+// TestUndo_RestoreFailureIsHalf: putting the copy back fails part-way → HOLD, state "half".
+func TestUndo_RestoreFailureIsHalf(t *testing.T) {
+	m, _, g, _, fc := newUndoManager(t)
+	fc.restoreErr = errors.New("disk full")
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); !held || g.undoState != UndoStateHalf {
+		t.Fatalf("held=%v state=%q", held, g.undoState)
+	}
+}
+
+// TestUndo_CopyFailureMovesNothing: the copy is taken after the pull; if it fails the update stops
+// there — the partial copy goes, the pin goes back, the old version starts, and nothing is held.
+func TestUndo_CopyFailureMovesNothing(t *testing.T) {
+	m, dir, g, c, fc := newUndoManager(t)
+	fc.copyErr = errors.New("helper exited 137")
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); held {
+		t.Fatal("a failed copy moved nothing and must not hold")
+	}
+	if st.UpdateError != util.Text("hu", "err.stacks.update_undo_copy_failed") {
+		t.Errorf("err = %q", st.UpdateError)
+	}
+	if got := pinOf(t, dir); got != "nextcloud:31.0.14-apache" {
+		t.Errorf("the pin must go back, got %q", got)
+	}
+	if fc.vol(undoVol) != "OLD" || fc.nCopies() != 0 {
+		t.Errorf("data untouched and no copy left; data=%q copies=%d", fc.vol(undoVol), fc.nCopies())
+	}
+	if got := strings.Join(c.list(), " | "); got != "pull | stop | up -d --remove-orphans" {
+		t.Errorf("the previous version must be started again after the failed copy; compose calls = %q", got)
+	}
+}
+
+// TestUndo_NoRoomForTheCopyRefusesBeforeAnythingMoves: decision 19's disk limit.
+func TestUndo_NoRoomForTheCopyRefusesBeforeAnythingMoves(t *testing.T) {
+	m, dir, _, c, fc := newUndoManager(t)
+	fc.bytes = 49 << 30 // 49 GiB of volumes; 50 GiB free; the 2 GiB floor must hold
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if !strings.Contains(st.UpdateError, "nincs elég szabad hely a frissítés előtti adatmásolathoz") {
+		t.Errorf("err = %q", st.UpdateError)
+	}
+	if pinOf(t, dir) != "nextcloud:31.0.14-apache" || len(c.list()) != 0 || fc.callsHave("copy ") {
+		t.Errorf("nothing may move: pin=%q compose=%v", pinOf(t, dir), c.list())
+	}
+}
+
+// TestUndo_PowerCutDuringTheUndoResumesIt: the journal says `undoing`; after a restart the undo runs
+// again from the copies and ends healthy — never "done", never dropped.
+//
+// COMPANION RED-PROOF 4 (REPORT.md): delete the UpdatePhaseUndoing arm from RecoverUpdates — the entry
+// falls to `default` (dropped), nothing is resumed, and this test fails at the first assertion.
+func TestUndo_PowerCutDuringTheUndoResumesIt(t *testing.T) {
+	m, dir, g, _, fc := newUndoManager(t)
+	e := simulateAdvanced(t, m, dir)
+	md, err := savePreUpdateMeta(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e.PrevMeta, e.Copied, e.Phase = md, true, UpdatePhaseUndoing
+	e.NewPin = map[string]string{"web": "nextcloud:34.0.1-apache"}
+	e.UndoCopies = []undoCopy{{Volume: undoVol, Copy: undoVol + ".pre-update-x"}}
+	fc.copies[undoVol+".pre-update-x"], fc.complete[undoVol+".pre-update-x"] = "OLD", true
+	fc.setVol(undoVol, "MIGRATED")
+	writeTestJournal(t, m, "nextcloud", e)
+	guards := m.updateGuards
+	m.updateGuards = nil
+
+	resumed := m.RecoverUpdates()
+	if len(resumed) != 1 || !m.IsUpdating("nextcloud") {
+		t.Fatalf("an interrupted undo must be resumed and the app marked Updating; resumed=%v", resumed)
+	}
+	if st, _ := m.GetStack("nextcloud"); st.UpdatePhase != UpdatePhaseUndoing {
+		t.Errorf("phase after recovery = %q, want %q", st.UpdatePhase, UpdatePhaseUndoing)
+	}
+	m.updateGuards = guards
+	if n := m.ResumeInterruptedUpdates(context.Background()); n != 1 {
+		t.Fatalf("resumed %d", n)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
+		t.Fatalf("the resumed undo must complete; held=%v phase=%q", held, st.UpdatePhase)
+	}
+	if fc.vol(undoVol) != "OLD" || pinOf(t, dir) != "nextcloud:31.0.14-apache" {
+		t.Errorf("data=%q pin=%q", fc.vol(undoVol), pinOf(t, dir))
+	}
+}
+
+// TestUndo_PowerCutDuringTheCopyPutsTheOldVersionBack: interrupted in `copying` — nothing new ran; the
+// partial copies go, the pin goes back and the old version is started.
+func TestUndo_PowerCutDuringTheCopyPutsTheOldVersionBack(t *testing.T) {
+	m, dir, _, c, fc := newUndoManager(t)
+	e := simulateAdvanced(t, m, dir)
+	e.Phase = UpdatePhaseCopying
+	e.UndoCopies = []undoCopy{{Volume: undoVol, Copy: undoVol + ".pre-update-x"}}
+	fc.copies[undoVol+".pre-update-x"] = "OL" // cut
+	writeTestJournal(t, m, "nextcloud", e)
+	if resumed := m.RecoverUpdates(); len(resumed) != 0 {
+		t.Fatalf("resumed = %v", resumed)
+	}
+	if pinOf(t, dir) != "nextcloud:31.0.14-apache" || fc.nCopies() != 0 || journalExists(m) {
+		t.Errorf("pin=%q copies=%d journal=%v", pinOf(t, dir), fc.nCopies(), journalExists(m))
+	}
+	if got := strings.Join(c.list(), " | "); got != "up -d --remove-orphans" {
+		t.Errorf("the old version must be started again; compose calls = %q", got)
+	}
+}
+
+// TestUndo_ASuccessfulUpdateEndsTheUndoneNote: the next update that works clears last_update_undone.
+//
+// COMPANION RED-PROOF 5 (REPORT.md): drop the recordUpdateUndone(nil) call from verifyAndConclude —
+// the note survives a successful update and this test fails.
+func TestUndo_ASuccessfulUpdateEndsTheUndoneNote(t *testing.T) {
+	m, dir, _, _, _ := newUndoManager(t)
+	m.recordUpdateUndone("nextcloud", dir, &UpdateUndone{To: map[string]string{"web": "x"}, At: "2026-09-23T10:00:00Z"})
+	if readPin(t, dir).LastUpdateUndone == nil {
+		t.Fatal("setup: the note was not written")
+	}
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return true, "fine" }
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseDone {
+		t.Fatalf("phase = %q", st.UpdatePhase)
+	}
+	if u := readPin(t, dir).LastUpdateUndone; u != nil {
+		t.Errorf("a successful update must end the undone note, got %+v", u)
+	}
+}
+
+// TestUndo_PhaseLabelsMatchTheBundle pins the Hungarian labels to the born-as-key bundle text.
+func TestUndo_PhaseLabelsMatchTheBundle(t *testing.T) {
+	for phase, key := range map[string]string{UpdatePhaseCopying: "update.phase.copying", UpdatePhaseUndoing: "update.phase.undoing", UpdatePhaseUndone: "update.phase.undone"} {
+		if got, want := UpdatePhaseLabel(phase), util.Text("hu", key); got != want || got == "" || got == key {
+			t.Errorf("%s: label %q, bundle %q", phase, got, want)
+		}
+	}
+	if util.Text("en", "update.phase.undone") != "Put back to the previous version" {
+		t.Errorf("English label = %q", util.Text("en", "update.phase.undone"))
+	}
+}
+
+// TestUndo_RemovalDeletesKeptCopies: a copy kept by a failed undo goes with the app.
+func TestUndo_RemovalDeletesKeptCopies(t *testing.T) {
+	m, _, _, _, fc := newUndoManager(t)
+	fc.copies["nextcloud_db.pre-update-x"] = "OLD"
+	if n := m.RemoveUndoCopies("nextcloud"); n != 1 || fc.nCopies() != 0 {
+		t.Errorf("removed %d, left %d", n, fc.nCopies())
+	}
+}