felhom-controller/controller/internal/stacks/undo_test.go

package stacks

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)

// v0.263.0 — the undo (09 §3 decision 15, undo.go). Every test runs the REAL job (runGuardedUpdate /
// RecoverUpdates / ResumeInterruptedUpdates) with the process boundaries faked, and reads the EFFECT
// back: the data in the fake volume, the pin in app.yaml, the phase, the hold, the journal.

// fakeCopier is the volume boundary. `vols` is each app volume's CONTENT, so "the data came back" is
// a string compare, and `complete` is each copy's finished-marker.
type fakeCopier struct {
	mu         sync.Mutex
	vols       map[string]string
	copies     map[string]string
	complete   map[string]bool
	calls      []string
	copyErr    error
	cutOff     bool // every copy is made WITHOUT its finished-marker (a copy container killed mid-way)
	restoreErr error
	bytes      int64
}

func newFakeCopier(vols map[string]string) *fakeCopier {
	return &fakeCopier{vols: vols, copies: map[string]string{}, complete: map[string]bool{}, bytes: 1 << 20}
}

func (f *fakeCopier) note(c string) { f.calls = append(f.calls, c) }
func (f *fakeCopier) ProjectVolumes(string) ([]string, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	var out []string
	for v := range f.vols {
		out = append(out, v)
	}
	sort.Strings(out)
	return out, nil
}
func (f *fakeCopier) VolumeBytes(string) (int64, error) { return f.bytes, nil }
func (f *fakeCopier) Copy(src, dst, _ string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.note("copy " + src)
	if f.copyErr != nil {
		return f.copyErr
	}
	f.copies[dst] = f.vols[src]
	f.complete[dst] = !f.cutOff
	return nil
}
func (f *fakeCopier) Complete(c string) bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	return f.complete[c]
}

// Restore refuses a copy with no marker, exactly as the real helper does (`test -f` in the same shell).
func (f *fakeCopier) Restore(c, v string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.note("restore " + v)
	if f.restoreErr != nil {
		return f.restoreErr
	}
	if !f.complete[c] {
		return fmt.Errorf("no finished-marker in %s", c)
	}
	f.vols[v] = f.copies[c]
	return nil
}
func (f *fakeCopier) Remove(v string) error {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.note("remove " + v)
	delete(f.copies, v)
	delete(f.complete, v)
	return nil
}
func (f *fakeCopier) CopiesOf(string) []string {
	f.mu.Lock()
	defer f.mu.Unlock()
	var out []string
	for c := range f.copies {
		out = append(out, c)
	}
	return out
}
func (f *fakeCopier) vol(v string) string { f.mu.Lock(); defer f.mu.Unlock(); return f.vols[v] }
func (f *fakeCopier) setVol(v, s string)  { f.mu.Lock(); f.vols[v] = s; f.mu.Unlock() }
func (f *fakeCopier) nCopies() int        { f.mu.Lock(); defer f.mu.Unlock(); return len(f.copies) }
func (f *fakeCopier) callsHave(p string) bool {
	f.mu.Lock()
	defer f.mu.Unlock()
	for _, c := range f.calls {
		if strings.HasPrefix(c, p) {
			return true
		}
	}
	return false
}

const (
	undoMetaOld = "healthcheck:\n  checks:\n    - type: http\n      port: 3000\n"
	undoMetaNew = "healthcheck:\n  checks:\n    - type: http\n      port: 3999\n" // the drill's wrong probe
	undoVol     = "nextcloud_db"
)

// newUndoManager: slice 4's manager plus a data volume holding "OLD", the OLD .felhom.yml in the stack
// dir, and a compose fake that plays the two things the real world does in between: the catalog's
// .felhom.yml flows in with the new probe during the pull (§5.4 — .felhom.yml is never frozen), and
// the NEW version migrates the data on its first `up`.
func newUndoManager(t *testing.T) (*Manager, string, *fakeGuards, *composeRec, *fakeCopier) {
	t.Helper()
	m, dir, g, c := newSlice4Manager(t)
	mustWrite(t, filepath.Join(dir, ".felhom.yml"), undoMetaOld)
	fc := newFakeCopier(map[string]string{undoVol: "OLD"})
	m.undoCopier = fc
	pulled, migrated := false, false
	m.updateComposeFn = func(d string, env []string, args ...string) (string, error) {
		switch args[0] {
		case "pull":
			pulled = true
			mustWrite(t, filepath.Join(dir, ".felhom.yml"), undoMetaNew)
		case "up":
			// Only the NEW version migrates: the first `up` after a pull, with the undo copy taken.
			// (After a failed copy, or in a resumed undo, the `up` starts the OLD version.)
			if pulled && !migrated && fc.nCopies() > 0 {
				migrated = true
				fc.setVol(undoVol, "MIGRATED")
			}
		}
		return c.fn(d, env, args...)
	}
	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "new version unhealthy" }
	m.updateUndoHealthFn = func(_ context.Context, _ string, _ time.Duration, meta *Metadata) (bool, string) {
		if meta != nil && meta.HealthCheck != nil && len(meta.HealthCheck.Checks) > 0 && meta.HealthCheck.Checks[0].Port == 3000 {
			return true, "old probe answered"
		}
		return false, "probed the wrong port"
	}
	return m, dir, g, c, fc
}

// TestUndo_FailedUpdateIsPutBackWithItsData is decision 15 in one test: the new version migrates the
// data and fails its check; the box puts the old version back WITH THE PRE-UPDATE DATA, and nothing is
// held.
//
// COMPANION RED-PROOF 1 (REPORT.md): at v0.262.1's shape — failAndHold without the tryUndo call — the
// app ends HELD with the data still "MIGRATED", and this test fails on the first assertion.
func TestUndo_FailedUpdateIsPutBackWithItsData(t *testing.T) {
	m, dir, g, _, fc := newUndoManager(t)
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
		t.Fatalf("a failed update must be UNDONE, not held; held=%v phase=%q err=%q", held, st.UpdatePhase, st.UpdateError)
	}
	if got := fc.vol(undoVol); got != "OLD" {
		t.Errorf("the data must be the PRE-UPDATE data again, got %q", got)
	}
	if got := pinOf(t, dir); got != "nextcloud:31.0.14-apache" {
		t.Errorf("the pin must be the old version again, got %q", got)
	}
	if got := fileBody(t, filepath.Join(dir, "docker-compose.yml")); got != pinTplOld {
		t.Errorf("the live definition must be the old one again:\n%s", got)
	}
	if st.UpdateError != "" || st.UpdatePhaseLabel != "Visszaállítva az előző változatra" {
		t.Errorf("an undone update carries no error and the undone label; err=%q label=%q", st.UpdateError, st.UpdatePhaseLabel)
	}
	u := readPin(t, dir).LastUpdateUndone
	if u == nil || u.To["web"] != "nextcloud:34.0.1-apache" || u.At == "" || !strings.Contains(u.Why, "unhealthy") {
		t.Errorf("app.yaml must remember the undone step (to, at, why), got %+v", u)
	}
	if fc.nCopies() != 0 || journalExists(m) {
		t.Errorf("after a successful undo the copies and the journal are gone; copies=%d journal=%v", fc.nCopies(), journalExists(m))
	}
	for _, f := range []string{preUpdateComposeFile, preUpdateAppliedFile, preUpdateMetaDir} {
		if _, err := os.Stat(filepath.Join(dir, f)); err == nil {
			t.Errorf("the pre-update copy %s must be removed once the undo is over", f)
		}
	}
}

// TestUndo_CutOffCopyIsRefusedBeforeAnythingMoves: a copy without its finished-marker is detected
// BEFORE anything is poured back; the outcome is today's HOLD, saying the data is as the new version
// left it — never a "success".
//
// COMPANION RED-PROOF 2 (REPORT.md): delete the Complete() validation loop in tryUndo. Restore is then
// called on the cut copy and the undo state reads "half" — this test fails on both assertions.
func TestUndo_CutOffCopyIsRefusedBeforeAnythingMoves(t *testing.T) {
	m, _, g, _, fc := newUndoManager(t)
	fc.cutOff = true
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); !held || st.UpdatePhase != UpdatePhaseFailed || g.undoState != UndoStateUntouched {
		t.Fatalf("a cut-off copy must end HELD with state %q; held=%v phase=%q state=%q", UndoStateUntouched, held, st.UpdatePhase, g.undoState)
	}
	if fc.callsHave("restore ") {
		t.Error("NOTHING may be poured back from a copy that is not whole")
	}
	if got := fc.vol(undoVol); got != "MIGRATED" {
		t.Errorf("the data must be left as the new version left it, got %q", got)
	}
	if fc.nCopies() == 0 {
		t.Error("a failed undo keeps its copies for the operator")
	}
}

// TestUndo_UsesTheOldProbe: the new .felhom.yml names a port the old version never answers (the drill
// case, and a real one whenever a probe moves with a version). The undo must judge the old version
// with the OLD probe.
//
// COMPANION RED-PROOF 3 (REPORT.md): make tryUndo use LoadMetadata(dir) (the current file) instead of
// entry.PrevMeta — the undo then probes port 3999 and the app ends HELD; this test fails.
func TestUndo_UsesTheOldProbe(t *testing.T) {
	m, _, g, _, _ := newUndoManager(t)
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
		t.Fatalf("with the old probe the old version is healthy and the undo completes; held=%v phase=%q state=%q", held, st.UpdatePhase, g.undoState)
	}
}

// TestUndo_OldVersionThatDoesNotStartIsHeldSayingSo: data and definition put back, the old version
// still unhealthy → HOLD with the not_started state, and the copies kept.
func TestUndo_OldVersionThatDoesNotStartIsHeldSayingSo(t *testing.T) {
	m, _, g, _, fc := newUndoManager(t)
	m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) {
		return false, "old version broken too"
	}
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); !held || g.undoState != UndoStateNotStarted || st.UpdatePhase != UpdatePhaseFailed {
		t.Fatalf("held=%v state=%q phase=%q", held, g.undoState, st.UpdatePhase)
	}
	if got := fc.vol(undoVol); got != "OLD" {
		t.Errorf("the data was put back before the check, got %q", got)
	}
}

// TestUndo_RestoreFailureIsHalf: putting the copy back fails part-way → HOLD, state "half".
func TestUndo_RestoreFailureIsHalf(t *testing.T) {
	m, _, g, _, fc := newUndoManager(t)
	fc.restoreErr = errors.New("disk full")
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); !held || g.undoState != UndoStateHalf {
		t.Fatalf("held=%v state=%q", held, g.undoState)
	}
}

// TestUndo_CopyFailureMovesNothing: the copy is taken after the pull; if it fails the update stops
// there — the partial copy goes, the pin goes back, the old version starts, and nothing is held.
func TestUndo_CopyFailureMovesNothing(t *testing.T) {
	m, dir, g, c, fc := newUndoManager(t)
	fc.copyErr = errors.New("helper exited 137")
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); held {
		t.Fatal("a failed copy moved nothing and must not hold")
	}
	if st.UpdateError != util.Text("hu", "err.stacks.update_undo_copy_failed") {
		t.Errorf("err = %q", st.UpdateError)
	}
	if got := pinOf(t, dir); got != "nextcloud:31.0.14-apache" {
		t.Errorf("the pin must go back, got %q", got)
	}
	if fc.vol(undoVol) != "OLD" || fc.nCopies() != 0 {
		t.Errorf("data untouched and no copy left; data=%q copies=%d", fc.vol(undoVol), fc.nCopies())
	}
	if got := strings.Join(c.list(), " | "); got != "pull | stop | up -d --remove-orphans" {
		t.Errorf("the previous version must be started again after the failed copy; compose calls = %q", got)
	}
}

// TestUndo_NoRoomForTheCopyRefusesBeforeAnythingMoves: decision 19's disk limit.
func TestUndo_NoRoomForTheCopyRefusesBeforeAnythingMoves(t *testing.T) {
	m, dir, _, c, fc := newUndoManager(t)
	fc.bytes = 49 << 30 // 49 GiB of volumes; 50 GiB free; the 2 GiB floor must hold
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if !strings.Contains(st.UpdateError, "nincs elég szabad hely a frissítés előtti adatmásolathoz") {
		t.Errorf("err = %q", st.UpdateError)
	}
	if pinOf(t, dir) != "nextcloud:31.0.14-apache" || len(c.list()) != 0 || fc.callsHave("copy ") {
		t.Errorf("nothing may move: pin=%q compose=%v", pinOf(t, dir), c.list())
	}
}

// TestUndo_PowerCutDuringTheUndoResumesIt: the journal says `undoing`; after a restart the undo runs
// again from the copies and ends healthy — never "done", never dropped.
//
// COMPANION RED-PROOF 4 (REPORT.md): delete the UpdatePhaseUndoing arm from RecoverUpdates — the entry
// falls to `default` (dropped), nothing is resumed, and this test fails at the first assertion.
func TestUndo_PowerCutDuringTheUndoResumesIt(t *testing.T) {
	m, dir, g, _, fc := newUndoManager(t)
	e := simulateAdvanced(t, m, dir)
	md, err := savePreUpdateMeta(dir)
	if err != nil {
		t.Fatal(err)
	}
	e.PrevMeta, e.Copied, e.Phase = md, true, UpdatePhaseUndoing
	e.NewPin = map[string]string{"web": "nextcloud:34.0.1-apache"}
	e.UndoCopies = []undoCopy{{Volume: undoVol, Copy: undoVol + ".pre-update-x"}}
	fc.copies[undoVol+".pre-update-x"], fc.complete[undoVol+".pre-update-x"] = "OLD", true
	fc.setVol(undoVol, "MIGRATED")
	writeTestJournal(t, m, "nextcloud", e)
	guards := m.updateGuards
	m.updateGuards = nil

	resumed := m.RecoverUpdates()
	if len(resumed) != 1 || !m.IsUpdating("nextcloud") {
		t.Fatalf("an interrupted undo must be resumed and the app marked Updating; resumed=%v", resumed)
	}
	if st, _ := m.GetStack("nextcloud"); st.UpdatePhase != UpdatePhaseUndoing {
		t.Errorf("phase after recovery = %q, want %q", st.UpdatePhase, UpdatePhaseUndoing)
	}
	m.updateGuards = guards
	if n := m.ResumeInterruptedUpdates(context.Background()); n != 1 {
		t.Fatalf("resumed %d", n)
	}
	st := waitUpdateDone(t, m, "nextcloud")
	if held, _ := g.HoldFor("nextcloud"); held || st.UpdatePhase != UpdatePhaseUndone {
		t.Fatalf("the resumed undo must complete; held=%v phase=%q", held, st.UpdatePhase)
	}
	if fc.vol(undoVol) != "OLD" || pinOf(t, dir) != "nextcloud:31.0.14-apache" {
		t.Errorf("data=%q pin=%q", fc.vol(undoVol), pinOf(t, dir))
	}
}

// TestUndo_PowerCutDuringTheCopyPutsTheOldVersionBack: interrupted in `copying` — nothing new ran; the
// partial copies go, the pin goes back and the old version is started.
func TestUndo_PowerCutDuringTheCopyPutsTheOldVersionBack(t *testing.T) {
	m, dir, _, c, fc := newUndoManager(t)
	e := simulateAdvanced(t, m, dir)
	e.Phase = UpdatePhaseCopying
	e.UndoCopies = []undoCopy{{Volume: undoVol, Copy: undoVol + ".pre-update-x"}}
	fc.copies[undoVol+".pre-update-x"] = "OL" // cut
	writeTestJournal(t, m, "nextcloud", e)
	if resumed := m.RecoverUpdates(); len(resumed) != 0 {
		t.Fatalf("resumed = %v", resumed)
	}
	if pinOf(t, dir) != "nextcloud:31.0.14-apache" || fc.nCopies() != 0 || journalExists(m) {
		t.Errorf("pin=%q copies=%d journal=%v", pinOf(t, dir), fc.nCopies(), journalExists(m))
	}
	if got := strings.Join(c.list(), " | "); got != "up -d --remove-orphans" {
		t.Errorf("the old version must be started again; compose calls = %q", got)
	}
}

// TestUndo_ASuccessfulUpdateEndsTheUndoneNote: the next update that works clears last_update_undone.
//
// COMPANION RED-PROOF 5 (REPORT.md): drop the recordUpdateUndone(nil) call from verifyAndConclude —
// the note survives a successful update and this test fails.
func TestUndo_ASuccessfulUpdateEndsTheUndoneNote(t *testing.T) {
	m, dir, _, _, _ := newUndoManager(t)
	m.recordUpdateUndone("nextcloud", dir, &UpdateUndone{To: map[string]string{"web": "x"}, At: "2026-09-23T10:00:00Z"})
	if readPin(t, dir).LastUpdateUndone == nil {
		t.Fatal("setup: the note was not written")
	}
	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return true, "fine" }
	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
		t.Fatal(err)
	}
	if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseDone {
		t.Fatalf("phase = %q", st.UpdatePhase)
	}
	if u := readPin(t, dir).LastUpdateUndone; u != nil {
		t.Errorf("a successful update must end the undone note, got %+v", u)
	}
}

// TestUndo_PhaseLabelsMatchTheBundle pins the Hungarian labels to the born-as-key bundle text.
func TestUndo_PhaseLabelsMatchTheBundle(t *testing.T) {
	for phase, key := range map[string]string{UpdatePhaseCopying: "update.phase.copying", UpdatePhaseUndoing: "update.phase.undoing", UpdatePhaseUndone: "update.phase.undone"} {
		if got, want := UpdatePhaseLabel(phase), util.Text("hu", key); got != want || got == "" || got == key {
			t.Errorf("%s: label %q, bundle %q", phase, got, want)
		}
	}
	if util.Text("en", "update.phase.undone") != "Put back to the previous version" {
		t.Errorf("English label = %q", util.Text("en", "update.phase.undone"))
	}
}

// TestUndo_RemovalDeletesKeptCopies: a copy kept by a failed undo goes with the app.
func TestUndo_RemovalDeletesKeptCopies(t *testing.T) {
	m, _, _, _, fc := newUndoManager(t)
	fc.copies["nextcloud_db.pre-update-x"] = "OLD"
	if n := m.RemoveUndoCopies("nextcloud"); n != 1 || fc.nCopies() != 0 {
		t.Errorf("removed %d, left %d", n, fc.nCopies())
	}
}

// TestUndo_OldProbeRunsOnAnAppTheCurrentProbeMarkedUnhealthy drives the REAL wait loop
// (waitUpdateHealthyMeta → RefreshStatus → `docker ps` → the health-probe override → the state gate →
// findProbeContainerMeta) with only the network probe faked. The app's CURRENT .felhom.yml probe has
// failed, so the refresh reads it Unhealthy; the undo's OLD probe answers.
//
// FOUND LIVE, NOT BY A TEST: v0.263.0's first build gated on StateRunning, so the old probe was never
// asked and docmost's undo on 9202 was reported "did not start" after the full 90 s while the old
// version served. TestUndo_UsesTheOldProbe could not see it — it injects updateUndoHealthFn and so
// never reaches this loop (the seam-boundary class: a test proves only what is behind its seam).
//
// COMPANION RED-PROOF (REPORT.md): drop `|| undoProbe` from the state case — the wait times out with
// `last: state unhealthy` and this test fails.
func TestUndo_OldProbeRunsOnAnAppTheCurrentProbeMarkedUnhealthy(t *testing.T) {
	m, dir, _, _, _ := newUndoManager(t)
	mustWrite(t, filepath.Join(dir, ".felhom.yml"), undoMetaNew) // the catalog's probe (3999) is current
	m.mu.Lock()
	m.stacks["nextcloud"].Meta = LoadMetadata(dir)
	m.stacks["nextcloud"].HealthProbe = &HealthProbeResult{Healthy: false} // the periodic probe failed on 3999
	m.mu.Unlock()
	m.execFn = func(name string, args ...string) (string, error) {
		if name == "docker" && len(args) > 0 && args[0] == "ps" {
			return "nextcloud\tnextcloud:31.0.14-apache\trunning\tUp 30 seconds\tnextcloud\n", nil
		}
		return "", nil
	}
	m.inspectRestartPolicyFn = func(string) (string, error) { return "unless-stopped", nil }
	m.updateNowFn = time.Now // the shared setup freezes the clock; this loop's deadline needs it moving
	var probed []int
	m.probeRunFn = func(pt probeTarget) *HealthProbeResult {
		probed = append(probed, pt.checks[0].Port)
		return &HealthProbeResult{Healthy: pt.checks[0].Port == 3000}
	}
	if err := m.RefreshStatus(); err != nil {
		t.Fatal(err)
	}
	if st, _ := m.GetStack("nextcloud"); st.State != StateUnhealthy {
		t.Fatalf("setup: the refresh must read the app Unhealthy (the current probe failed), got %q", st.State)
	}
	old, err := savePreUpdateMeta(dir) // what the job kept — but it holds the NEW file here, so write the OLD one
	if err != nil {
		t.Fatal(err)
	}
	mustWrite(t, filepath.Join(old, ".felhom.yml"), undoMetaOld)
	oldMeta := LoadMetadata(old)

	ok, detail := m.waitUpdateHealthyMeta(context.Background(), "nextcloud", time.Second, &oldMeta)
	if !ok {
		t.Fatalf("the old version answers its own probe and must be judged healthy; got %q, probed ports %v", detail, probed)
	}
	if len(probed) == 0 || probed[0] != 3000 {
		t.Errorf("the undo must probe the OLD port (3000), probed %v", probed)
	}
	// And without an override the same Unhealthy app is NOT probed or settled — the normal update path
	// is unchanged.
	probed = nil
	m.mu.Lock()
	m.stacks["nextcloud"].HealthProbe = &HealthProbeResult{Healthy: false} // the first call stored its healthy result
	m.mu.Unlock()
	if ok, _ := m.waitUpdateHealthyMeta(context.Background(), "nextcloud", time.Second, nil); ok || len(probed) != 0 {
		t.Errorf("without an override an Unhealthy app must stay unhealthy and unprobed; ok=%v probed=%v", ok, probed)
	}
}