controller v0.271.0: automatic app updates — the update leg after the off-site copy, the backup gate waits, the switch (09 6.4 part 7; R-680, R-678, R-643)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/internal/stacks/deploy.go

@@ -158,6 +158,14 @@ type AppConfig struct {
 	// written only by a successful undo, cleared by the next successful update. The page shows one
 	// line from it; the future automatic caller reads it so it never re-presses the same step.
 	LastUpdateUndone *UpdateUndone `yaml:"last_update_undone,omitempty" json:"last_update_undone,omitempty"`
+	// FailedStep (v0.271.0, R-680) is the ladder step whose update was UNDONE or HELD — written by the
+	// undo and by the hold, cleared by the next successful update. The automatic leg never presses it
+	// again while the catalog's ladder for this app is the one it failed on (Ladder = LadderPrint); a
+	// person still can.
+	FailedStep *FailedStep `yaml:"failed_update_step,omitempty" json:"failed_update_step,omitempty"`
+	// LastAutoUpdate (v0.271.0, `09` §6.4 part 7) is the automatic leg's last step on this app — the
+	// line on the app page („Automatikus frissítés %s-kor — sikeres").
+	LastAutoUpdate *AutoUpdateRecord `yaml:"last_auto_update,omitempty" json:"last_auto_update,omitempty"`
 }
 
 // InstalledImage is one compose service's observed image. See AppConfig.InstalledImages.

controller/internal/stacks/ladder.go

@@ -44,6 +44,30 @@ type LadderEntry struct {
 	Verdict string            `yaml:"verdict" json:"verdict"`
 	// TestedAt is when the step was proven (RFC3339) — the badge compares it with the install (v0.269.0).
 	TestedAt string `yaml:"tested_at" json:"tested_at"`
+	// Marks are decision 13's two exceptions the test sets on a step (v0.271.0 reads them): the
+	// automatic leg never takes a step that needs a person, and takes a files-may-change step only when
+	// a fresh WHOLE copy exists. A person's press ignores both — the marks bind the leg only.
+	Marks LadderMarks `yaml:"marks" json:"marks"`
+}
+
+// LadderMarks is the `marks` object of a ladder entry. NeedsPerson is JSON null (nil) or the tester's
+// reason; an entry with no `marks` key reads as no marks.
+type LadderMarks struct {
+	FilesMayChange bool    `yaml:"files_may_change" json:"files_may_change"`
+	NeedsPerson    *string `yaml:"needs_person" json:"needs_person"`
+	MemoryTight    bool    `yaml:"memory_tight" json:"memory_tight"`
+}
+
+// LadderPrint is a short fingerprint of an app's whole ladder as the box parsed it — what R-680's
+// failed-step record is tied to: when the catalog changes the ladder (a new step, a re-test, a mark),
+// the print changes and the automatic leg may try again. "" = no ladder.
+func LadderPrint(ladder []LadderEntry) string {
+	if len(ladder) == 0 {
+		return ""
+	}
+	b, _ := json.Marshal(ladder)
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])[:16]
 }
 
 type ladderDoc struct {

controller/internal/stacks/manager.go

@@ -227,6 +227,8 @@ type Manager struct {
 	updateHealthMetaFn func(ctx context.Context, name string, timeout time.Duration, meta *Metadata) (bool, string)
 	// unhealthy (v0.269.0, decision 28): RestartCount / OOM-kill samples per app for the crash-loop stop.
 	unhealthy unhealthyWatch
+	// leg (v0.271.0, `09` §6.4 part 7): the automatic update leg's state (unattended.go).
+	leg updateLegState
 	// selfUpdating (v0.261.0) reports whether the CONTROLLER is swapping itself. Set by
 	// SetSelfUpdatingCheck; nil means no gate. See update.go.
 	selfUpdating func() bool

controller/internal/stacks/unattended.go

@@ -0,0 +1,510 @@
+package stacks
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
+)
+
+// ── The automatic update leg (`09` §3 decisions 11, 12, 14, 15, 20; §6.4 part 7; controller v0.271.0) ──
+//
+// WHAT IT IS. One more leg of the nightly chain: after the off-site copy, before the full-system backup.
+// It presses the SAME public guarded Update a person presses (StartGuardedUpdate) — no second update
+// path — one app at a time, ONE tested step per app per night, and never:
+//   - an app whose installed version has no ladder entry (older than the ladder, or no test record);
+//   - a step whose verdict is not `proven`, or that carries the `needs_person` mark;
+//   - a `files_may_change` step unless a fresh WHOLE copy of the app exists on the box (decision 25/26);
+//   - the step that was undone or held last time, while the catalog's ladder is unchanged (R-680);
+//   - a held app, an app that is current, ahead, or unorderable.
+// It starts no step at or after W+5h (decision 20); a step already running finishes.
+//
+// WHERE IT IS CALLED FROM. The `offbox-backup` scheduler job, after the off-site copy, on EVERY path —
+// configured, not configured, failed, skipped, panicked (cmd/controller chainUpdateLeg, pinned by
+// TestChainUpdateLeg_*). The full-system backup's gate asks UpdateLegState and waits while the leg runs,
+// until W+5h (quiesce.Options.UpdateLegFn).
+//
+// WHAT THE HOUSEHOLD HEARS. Nothing new: an undone or held step already sends app_update_undone /
+// app_update_held from the update itself. A successful automatic step sends NO mail; the app page shows
+// one line from app.yaml's last_auto_update. The operator gets one summary line per night in the log and
+// in the hub report (LastUpdateLegSummary).
+
+// Leg skip reasons — stable keys for the log, the summary and tests.
+const (
+	LegSkipHeld            = "held"
+	LegSkipUnpinned        = "unpinned"
+	LegSkipUnknownOrder    = "order_unknown"
+	LegSkipAhead           = "ahead"
+	LegSkipNoTestRecord    = "no_test_record"
+	LegSkipOlderThanLadder = "older_than_ladder"
+	LegSkipNotProven       = "not_proven"
+	LegSkipNeedsPerson     = "needs_person"
+	LegSkipFilesNoCopy     = "files_may_change_no_whole_copy"
+	LegSkipFailedBefore    = "failed_before"
+	LegSkipWindowEnd       = "window_end"
+	LegSkipSwitchedOff     = "switched_off"
+	LegSkipCancelled       = "cancelled"
+	legCurrent             = "current" // not a skip: nothing to do
+
+	LegOutcomeDone    = "done"
+	LegOutcomeUndone  = "undone"
+	LegOutcomeHeld    = "held"
+	LegOutcomeFailed  = "failed"
+	LegOutcomeSkipped = "skipped"
+)
+
+// legTransient are the refusals a later press may not meet (R-609's split): retried ONCE later in the leg.
+var legTransient = map[string]bool{"busy": true, "updating": true, "deploying": true, "migrating": true, "self_updating": true}
+
+// UpdateLegOptions wires the leg (main.go). INIT-ONLY via SetUpdateLeg.
+type UpdateLegOptions struct {
+	// Enabled is the per-box switch `app_update.unattended` (decision 12, default ON). Read at the start
+	// and before every press, so switching it off stops the leg before its next step.
+	Enabled func() bool
+	// WindowStart is the effective backup window start W "HH:MM" — the leg starts no step at or after W+5h.
+	WindowStart func() string
+	// FreshWholeCopy answers decision 13's `files_may_change` mark: is there a fresh copy on this box that
+	// brings the app back WHOLE (decision 25/26's truth table)? nil = no → such a step is always skipped.
+	FreshWholeCopy func(ctx context.Context, name string) (bool, string)
+	// Location is the wall clock W is read in (Europe/Budapest); nil = time.Local.
+	Location *time.Location
+	// Poll is how often the leg looks whether its step has ended (default 5 s).
+	Poll time.Duration
+	// RetryWait is how long the leg waits before its one retry of transient refusals (default 2 min).
+	RetryWait time.Duration
+	// Now is the clock (tests); nil = time.Now.
+	Now func() time.Time
+}
+
+// LegStep is one app's line in the night's summary.
+type LegStep struct {
+	App     string            `json:"app"`
+	Outcome string            `json:"outcome"` // done | undone | held | failed | skipped
+	Reason  string            `json:"reason,omitempty"`
+	From    map[string]string `json:"from,omitempty"`
+	To      map[string]string `json:"to,omitempty"`
+	Seconds float64           `json:"seconds,omitempty"`
+}
+
+// UpdateLegSummary is one night's leg — the operator's summary line and the hub report's `update_leg`.
+type UpdateLegSummary struct {
+	Trigger   string    `json:"trigger"`
+	StartedAt time.Time `json:"started_at"`
+	EndedAt   time.Time `json:"ended_at"`
+	Deadline  time.Time `json:"deadline"`
+	Enabled   bool      `json:"enabled"`
+	Done      int       `json:"done"`
+	Undone    int       `json:"undone"`
+	Held      int       `json:"held"`
+	Failed    int       `json:"failed"`
+	Skipped   int       `json:"skipped"`
+	// Stopped says why the leg ended early: window_end | switched_off | cancelled | "" (it ran through).
+	Stopped string    `json:"stopped,omitempty"`
+	Steps   []LegStep `json:"steps"`
+}
+
+// Line is the one operator-English summary line.
+func (s *UpdateLegSummary) Line() string {
+	var skips []string
+	for _, st := range s.Steps {
+		if st.Outcome == LegOutcomeSkipped {
+			skips = append(skips, st.App+"="+st.Reason)
+		}
+	}
+	stop := ""
+	if s.Stopped != "" {
+		stop = " stopped=" + s.Stopped
+	}
+	return fmt.Sprintf("update leg (%s): done=%d undone=%d held=%d failed=%d skipped=%d%s in %s [skipped: %s]",
+		s.Trigger, s.Done, s.Undone, s.Held, s.Failed, s.Skipped, stop,
+		s.EndedAt.Sub(s.StartedAt).Round(time.Second), strings.Join(skips, ", "))
+}
+
+func (s *UpdateLegSummary) add(st LegStep) {
+	s.Steps = append(s.Steps, st)
+	switch st.Outcome {
+	case LegOutcomeDone:
+		s.Done++
+	case LegOutcomeUndone:
+		s.Undone++
+	case LegOutcomeHeld:
+		s.Held++
+	case LegOutcomeFailed:
+		s.Failed++
+	default:
+		s.Skipped++
+	}
+}
+
+type updateLegState struct {
+	run         sync.Mutex // single-flight: one leg at a time
+	mu          sync.Mutex // guards the fields below
+	opts        *UpdateLegOptions
+	active      bool
+	stepRunning bool
+	last        *UpdateLegSummary
+}
+
+// SetUpdateLeg wires the leg. INIT-ONLY (main.go). Without it RunUpdateLeg does nothing and says so.
+func (m *Manager) SetUpdateLeg(o UpdateLegOptions) {
+	if o.Poll <= 0 {
+		o.Poll = 5 * time.Second
+	}
+	if o.RetryWait <= 0 {
+		o.RetryWait = 2 * time.Minute
+	}
+	if o.Location == nil {
+		o.Location = time.Local
+	}
+	if o.Now == nil {
+		o.Now = time.Now
+	}
+	m.leg.mu.Lock()
+	m.leg.opts = &o
+	m.leg.mu.Unlock()
+}
+
+// UpdateLegState is what the full-system backup's gate asks (decision 20): is the leg running, and is one
+// of its steps in flight right now.
+func (m *Manager) UpdateLegState() (active, stepRunning bool) {
+	m.leg.mu.Lock()
+	defer m.leg.mu.Unlock()
+	return m.leg.active, m.leg.stepRunning
+}
+
+// LastUpdateLegSummary is the last leg's summary, or nil when none ran since the controller started.
+func (m *Manager) LastUpdateLegSummary() *UpdateLegSummary {
+	m.leg.mu.Lock()
+	defer m.leg.mu.Unlock()
+	if m.leg.last == nil {
+		return nil
+	}
+	c := *m.leg.last
+	c.Steps = append([]LegStep(nil), m.leg.last.Steps...)
+	return &c
+}
+
+func (m *Manager) setLegFlags(active, step bool) {
+	m.leg.mu.Lock()
+	m.leg.active, m.leg.stepRunning = active, step
+	m.leg.mu.Unlock()
+}
+
+// LegDeadline is the instant the leg starts no more steps: W+5h of the night that contains `now` (the
+// most recent W at or before now). An unparseable W falls back to the default window.
+func LegDeadline(now time.Time, window string, loc *time.Location) time.Time {
+	if loc == nil {
+		loc = time.Local
+	}
+	startMin, err := backupwindow.ParseHHMM(window)
+	if err != nil {
+		startMin, _ = backupwindow.ParseHHMM(backupwindow.DefaultWindow)
+	}
+	n := now.In(loc)
+	w := time.Date(n.Year(), n.Month(), n.Day(), startMin/60, startMin%60, 0, 0, loc)
+	if w.After(n) {
+		w = w.AddDate(0, 0, -1)
+	}
+	return w.Add(time.Duration(backupwindow.UpdateLegStopOffsetMin) * time.Minute)
+}
+
+// RunUpdateLeg runs one night's leg and returns its summary (nil when another leg is running or the leg
+// is not wired). trigger names the caller for the log ("after-offsite").
+func (m *Manager) RunUpdateLeg(ctx context.Context, trigger string) *UpdateLegSummary {
+	if !m.leg.run.TryLock() {
+		m.logger.Printf("[WARN] [update-leg] a leg is already running — this call (%s) does nothing", trigger)
+		return nil
+	}
+	defer m.leg.run.Unlock()
+	m.leg.mu.Lock()
+	o := m.leg.opts
+	m.leg.mu.Unlock()
+	if o == nil {
+		m.logger.Printf("[WARN] [update-leg] not wired (SetUpdateLeg) — no automatic app updates")
+		return nil
+	}
+	sum := &UpdateLegSummary{Trigger: trigger, StartedAt: o.Now(), Steps: []LegStep{}}
+	finish := func() *UpdateLegSummary {
+		sum.EndedAt = o.Now()
+		m.logger.Printf("[INFO] [update-leg] %s", sum.Line())
+		m.leg.mu.Lock()
+		m.leg.last = sum
+		m.leg.mu.Unlock()
+		return sum
+	}
+	sum.Enabled = o.Enabled == nil || o.Enabled()
+	if !sum.Enabled {
+		sum.Stopped = LegSkipSwitchedOff
+		m.logger.Printf("[INFO] [update-leg] automatic app updates are switched OFF on this box (app_update.unattended=false) — no app is pressed tonight")
+		return finish()
+	}
+	window := ""
+	if o.WindowStart != nil {
+		window = o.WindowStart()
+	}
+	sum.Deadline = LegDeadline(sum.StartedAt, window, o.Location)
+	m.setLegFlags(true, false)
+	defer m.setLegFlags(false, false)
+	m.logger.Printf("[INFO] [update-leg] started (%s): window %s, no step starts at or after %s", trigger, window, sum.Deadline.In(o.Location).Format("15:04"))
+
+	// The freshest view of the catalog and the pins — a sync may have landed since the last scan.
+	if err := m.ScanStacks(); err != nil {
+		m.logger.Printf("[WARN] [update-leg] rescan before the leg failed: %v — using the last scan", err)
+	}
+	var names []string
+	for _, st := range m.GetStacks() {
+		if st.Deployed && !st.Protected && !m.cfg.IsProtectedStack(st.Name) {
+			names = append(names, st.Name)
+		}
+	}
+	sort.Strings(names)
+
+	var retry []string
+	for pass := 0; pass < 2 && sum.Stopped == ""; pass++ {
+		list := names
+		if pass == 1 {
+			list, retry = retry, nil
+			if len(list) == 0 {
+				break
+			}
+			m.logger.Printf("[INFO] [update-leg] %d app(s) were refused for a passing reason — one retry in %s: %v", len(list), o.RetryWait, list)
+			select {
+			case <-ctx.Done():
+			case <-time.After(o.RetryWait):
+			}
+		}
+		for i, name := range list {
+			if stop := m.legStop(ctx, o, sum); stop != "" {
+				sum.Stopped = stop
+				for _, rest := range list[i:] {
+					sum.add(LegStep{App: rest, Outcome: LegOutcomeSkipped, Reason: stop})
+				}
+				if pass == 0 {
+					for _, rest := range retry {
+						sum.add(LegStep{App: rest, Outcome: LegOutcomeSkipped, Reason: stop})
+					}
+				}
+				break
+			}
+			entry, reason := m.legCandidate(ctx, name, o)
+			if reason == legCurrent {
+				continue
+			}
+			if reason != "" {
+				m.logger.Printf("[INFO] [update-leg] %s: skipped — %s", name, reason)
+				sum.add(LegStep{App: name, Outcome: LegOutcomeSkipped, Reason: reason})
+				continue
+			}
+			t0 := o.Now()
+			if err := m.StartGuardedUpdate(name); err != nil {
+				why := "refused"
+				var ref *UpdateRefusal
+				if errors.As(err, &ref) {
+					why = ref.Reason
+				}
+				if legTransient[why] && pass == 0 {
+					m.logger.Printf("[INFO] [update-leg] %s: refused (%s) — a passing reason, retried once later tonight", name, why)
+					retry = append(retry, name)
+					continue
+				}
+				m.logger.Printf("[INFO] [update-leg] %s: skipped — the update refused (%s)", name, why)
+				sum.add(LegStep{App: name, Outcome: LegOutcomeSkipped, Reason: "refused:" + why})
+				continue
+			}
+			m.logger.Printf("[INFO] [update-leg] %s: step pressed %s → %s", name, summarisePin(entry.From), summarisePin(entry.To))
+			m.setLegFlags(true, true)
+			outcome := m.legWait(ctx, o, name)
+			m.setLegFlags(true, false)
+			step := LegStep{App: name, Outcome: outcome, From: entry.From, To: entry.To, Seconds: o.Now().Sub(t0).Round(100 * time.Millisecond).Seconds()}
+			sum.add(step)
+			m.recordAutoUpdate(name, &AutoUpdateRecord{At: t0.UTC().Format(time.RFC3339), Outcome: outcome, From: entry.From, To: entry.To})
+			m.logger.Printf("[INFO] [update-leg] %s: step ended %s after %.1f s", name, outcome, step.Seconds)
+		}
+	}
+	return finish()
+}
+
+// legStop is the leg's own stop check before every press: cancelled, switched off, or W+5h reached.
+func (m *Manager) legStop(ctx context.Context, o *UpdateLegOptions, sum *UpdateLegSummary) string {
+	switch {
+	case ctx.Err() != nil:
+		m.logger.Printf("[WARN] [update-leg] cancelled (the controller is stopping) — the remaining apps wait for the next night")
+		return LegSkipCancelled
+	case o.Enabled != nil && !o.Enabled():
+		m.logger.Printf("[INFO] [update-leg] the switch was turned OFF during the leg — no further step tonight")
+		return LegSkipSwitchedOff
+	case !o.Now().Before(sum.Deadline):
+		m.logger.Printf("[INFO] [update-leg] W+5h reached (%s) — no new step starts; the full-system backup keeps its hour, the rest waits for the next night", sum.Deadline.In(o.Location).Format("15:04"))
+		return LegSkipWindowEnd
+	}
+	return ""
+}
+
+// legWait waits until the pressed step has ended and returns its outcome. A cancelled context stops the
+// WAIT only — the update job itself runs on, and its journal finishes it after a restart.
+func (m *Manager) legWait(ctx context.Context, o *UpdateLegOptions, name string) string {
+	for m.IsUpdating(name) {
+		select {
+		case <-ctx.Done():
+			return LegOutcomeFailed
+		case <-time.After(o.Poll):
+		}
+	}
+	st, ok := m.GetStack(name)
+	if !ok {
+		return LegOutcomeFailed
+	}
+	switch {
+	case st.UpdatePhase == UpdatePhaseDone:
+		return LegOutcomeDone
+	case st.UpdatePhase == UpdatePhaseUndone:
+		return LegOutcomeUndone
+	case st.updateHeld || st.UpdateErrorKey == UpdateErrorKeyHeld || st.HoldReason != "":
+		return LegOutcomeHeld
+	}
+	return LegOutcomeFailed
+}
+
+// legCandidate decides whether the leg may press this app tonight, and which step it would take. It
+// returns ("", entry) to press, legCurrent when there is nothing to do, or a skip reason.
+func (m *Manager) legCandidate(ctx context.Context, name string, o *UpdateLegOptions) (LadderEntry, string) {
+	st, ok := m.GetStack(name)
+	if !ok || !st.Deployed {
+		return LadderEntry{}, legCurrent
+	}
+	if st.HoldReason != "" || st.updateHeld {
+		return LadderEntry{}, LegSkipHeld
+	}
+	if g := m.guards(); g != nil {
+		if held, _ := g.HoldFor(name); held {
+			return LadderEntry{}, LegSkipHeld
+		}
+	}
+	switch CatalogOrder(*st) {
+	case UpdateOrderCurrent:
+		return LadderEntry{}, legCurrent
+	case UpdateOrderAhead:
+		return LadderEntry{}, LegSkipAhead
+	case UpdateOrderBehind:
+	default:
+		return LadderEntry{}, LegSkipUnknownOrder
+	}
+	if st.AppConfig == nil || len(st.AppConfig.PinnedImages) == 0 {
+		return LadderEntry{}, LegSkipUnpinned
+	}
+	pinned := st.AppConfig.PinnedImages
+	tplDir := filepath.Dir(m.CatalogTemplatePath(name, "docker-compose.yml"))
+	ladder, err := LoadLadder(filepath.Join(tplDir, ".felhom.yml"))
+	if err != nil || len(ladder) == 0 {
+		return LadderEntry{}, LegSkipNoTestRecord
+	}
+	idx := -1
+	for i := len(ladder) - 1; i >= 0; i-- { // the SAME choice nextLadderStep makes
+		if sameRefs(ladder[i].From, pinned) {
+			idx = i
+			break
+		}
+	}
+	if idx < 0 {
+		if sameRefs(ladder[len(ladder)-1].To, pinned) {
+			return LadderEntry{}, LegSkipNoTestRecord // at the head, behind only by something no step records
+		}
+		m.logger.Printf("[INFO] [update-leg] %s: the installed version %s matches no update_ladder entry — an app older than the ladder is never pressed by the leg (a person can)", name, summarisePin(pinned))
+		return LadderEntry{}, LegSkipOlderThanLadder
+	}
+	e := ladder[idx]
+	if e.Verdict != "proven" {
+		return e, LegSkipNotProven
+	}
+	if e.Marks.NeedsPerson != nil && strings.TrimSpace(*e.Marks.NeedsPerson) != "" {
+		return e, LegSkipNeedsPerson
+	}
+	if fs := st.AppConfig.FailedStep; fs != nil && sameRefs(fs.To, e.To) && fs.Ladder == LadderPrint(ladder) {
+		return e, LegSkipFailedBefore
+	}
+	if e.Marks.FilesMayChange {
+		whole, why := false, "no whole-copy check wired"
+		if o.FreshWholeCopy != nil {
+			whole, why = o.FreshWholeCopy(ctx, name)
+		}
+		if !whole {
+			m.logger.Printf("[INFO] [update-leg] %s: the step may change the app's files and no fresh whole copy exists (%s)", name, why)
+			return e, LegSkipFilesNoCopy
+		}
+	}
+	return e, ""
+}
+
+// ── The records the leg and the update write into app.yaml ─────────────────────────────────────────
+
+// FailedStep is R-680's record: the step whose update was undone or held.
+type FailedStep struct {
+	To      map[string]string `yaml:"to" json:"to"`
+	Ladder  string            `yaml:"ladder" json:"ladder"` // LadderPrint of the ladder it failed on
+	At      string            `yaml:"at" json:"at"`
+	Outcome string            `yaml:"outcome" json:"outcome"` // undone | held
+}
+
+// AutoUpdateRecord is the leg's last step on an app — the page's line.
+type AutoUpdateRecord struct {
+	At      string            `yaml:"at" json:"at"`
+	Outcome string            `yaml:"outcome" json:"outcome"`
+	From    map[string]string `yaml:"from,omitempty" json:"from,omitempty"`
+	To      map[string]string `yaml:"to,omitempty" json:"to,omitempty"`
+}
+
+// mutateAppConfig loads app.yaml, applies fn (false = nothing to write), saves it and mirrors the
+// in-memory copy. A failed write is logged and never fails the caller — these are records.
+func (m *Manager) mutateAppConfig(name, dir, what string, fn func(cfg *AppConfig) bool) {
+	cfg := LoadAppConfig(dir)
+	if cfg == nil || !fn(cfg) {
+		return
+	}
+	meta := LoadMetadata(dir)
+	if err := SaveAppConfig(dir, cfg, m.encKey, SensitiveEnvVars(&meta)); err != nil {
+		m.logger.Printf("[ERROR] [stacks] %s: recording %s failed: %v", name, what, err)
+		return
+	}
+	m.mu.Lock()
+	if st, ok := m.stacks[name]; ok && st.AppConfig != nil {
+		fn(st.AppConfig)
+	}
+	m.mu.Unlock()
+}
+
+// recordFailedStep writes R-680's record for the step `to` (undone or held), tied to the ladder the
+// catalog carries now.
+func (m *Manager) recordFailedStep(name, dir string, to map[string]string, outcome string) {
+	if len(to) == 0 {
+		return
+	}
+	ladder, _ := LoadLadder(m.CatalogTemplatePath(name, ".felhom.yml"))
+	rec := &FailedStep{To: to, Ladder: LadderPrint(ladder), At: m.now().UTC().Format(time.RFC3339), Outcome: outcome}
+	m.mutateAppConfig(name, dir, "failed_update_step", func(cfg *AppConfig) bool { cfg.FailedStep = rec; return true })
+	m.logger.Printf("[INFO] [stacks] update %s: step %s recorded as %s — the automatic leg will not press it again until the catalog's ladder changes (ladder %s)", name, summarisePin(to), outcome, rec.Ladder)
+}
+
+// clearFailedStep drops R-680's record after a successful update.
+func (m *Manager) clearFailedStep(name, dir string) {
+	m.mutateAppConfig(name, dir, "failed_update_step", func(cfg *AppConfig) bool {
+		if cfg.FailedStep == nil {
+			return false
+		}
+		cfg.FailedStep = nil
+		return true
+	})
+}
+
+func (m *Manager) recordAutoUpdate(name string, rec *AutoUpdateRecord) {
+	st, ok := m.GetStack(name)
+	if !ok {
+		return
+	}
+	m.mutateAppConfig(name, filepath.Dir(st.ComposePath), "last_auto_update", func(cfg *AppConfig) bool { cfg.LastAutoUpdate = rec; return true })
+}

controller/internal/stacks/unattended_test.go

@@ -0,0 +1,454 @@
+package stacks
+
+import (
+	"bytes"
+	"context"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// v0.271.0 — the automatic update leg (`09` §3 decisions 11–15, 20; §6.4 part 7). Every test runs the
+// REAL leg over the REAL guarded update job with the process boundaries faked (ladderManager: nextcloud
+// pinned at A, the catalog at C, a two-step ladder A→B→C, B's own definition in steps/), and reads the
+// EFFECT back: the pin in app.yaml, the definitions `up` ran on, app.yaml's records, the summary.
+
+// legNow is inside the night of window 02:30 (the leg starts at W+105m = 04:15).
+var legNow = time.Date(2026, 9, 25, 4, 15, 0, 0, time.UTC)
+
+func legOpts(m *Manager, mut func(o *UpdateLegOptions)) {
+	o := UpdateLegOptions{
+		Enabled:     func() bool { return true },
+		WindowStart: func() string { return "02:30" },
+		Location:    time.UTC,
+		Poll:        2 * time.Millisecond,
+		RetryWait:   time.Millisecond,
+		Now:         func() time.Time { return legNow },
+	}
+	if mut != nil {
+		mut(&o)
+	}
+	m.SetUpdateLeg(o)
+}
+
+// writeLadder replaces the catalog's .felhom.yml with the given ladder lines.
+func writeLadder(t *testing.T, m *Manager, lines ...string) {
+	t.Helper()
+	catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
+	mustWrite(t, filepath.Join(catDir, ".felhom.yml"), "display_name: Nextcloud\nupdate_ladder:\n"+strings.Join(lines, ""))
+}
+
+// ladderLineMarks is ladderLine with a verdict and a marks object.
+func ladderLineMarks(from, to, verdict, marks string) string {
+	return `  - {"from": {"web": "` + from + `"}, "to": {"web": "` + to + `"}, "digest": {"web": "sha256:` +
+		strings.Repeat("a", 64) + `"}, "verdict": "` + verdict + `", "marks": ` + marks + `}` + "\n"
+}
+
+func legStepFor(s *UpdateLegSummary, app string) LegStep {
+	for _, st := range s.Steps {
+		if st.App == app {
+			return st
+		}
+	}
+	return LegStep{}
+}
+
+// legManager is ladderManager plus what a real box has and the fakes do not produce: an installed-image
+// record (CatalogOrder answers Unknown without one, and the leg never presses Unknown). Every `up` records
+// the image it brought up, as recordInstalledImages does on a box from the running container.
+func legManager(t *testing.T) (*Manager, string, *fakeGuards, *[]string, *bytes.Buffer) {
+	t.Helper()
+	m, dir, g, ups, logBuf := ladderManager(t, true)
+	setInstalled := func(img string) {
+		cfg := LoadAppConfig(dir)
+		cfg.InstalledImages = map[string]InstalledImage{"web": {Ref: img}}
+		if err := SaveAppConfig(dir, cfg, m.encKey, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+	setInstalled(ladderA)
+	inner := m.updateComposeFn
+	m.updateComposeFn = func(d string, env []string, args ...string) (string, error) {
+		out, err := inner(d, env, args...)
+		if err == nil && args[0] == "up" {
+			if imgs, perr := ParseComposeImages(ComposePathIn(d)); perr == nil {
+				setInstalled(strings.SplitN(imgs["web"], "@", 2)[0])
+			}
+		}
+		return out, err
+	}
+	return m, dir, g, ups, logBuf
+}
+
+func mustLeg(t *testing.T, m *Manager) *UpdateLegSummary {
+	t.Helper()
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	s := m.RunUpdateLeg(context.Background(), "test")
+	if s == nil {
+		t.Fatal("the leg did not run")
+	}
+	return s
+}
+
+// TestLeg_OneStepPerAppPerNight — a box two steps behind takes ONE step a night (decision 14 + the
+// brief: "one tested step per app"), with the step's own definition, and the summary says done=1.
+//
+// COMPANION RED-PROOF (REPORT.md): make the leg loop while the app stays behind — this test fails at
+// "the leg took 2 steps in one night".
+func TestLeg_OneStepPerAppPerNight(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 1 {
+		t.Fatalf("the leg took %d steps in one night (ups=%v), want exactly one", len(*ups), *ups)
+	}
+	if pinOf(t, dir) != ladderB {
+		t.Fatalf("pinned %s, want the first tested step B", pinOf(t, dir))
+	}
+	if s.Done != 1 || legStepFor(s, "nextcloud").Outcome != LegOutcomeDone {
+		t.Fatalf("summary %+v, want done=1 for nextcloud", s)
+	}
+	cfg := LoadAppConfig(dir)
+	if cfg.LastAutoUpdate == nil || cfg.LastAutoUpdate.Outcome != LegOutcomeDone || cfg.LastAutoUpdate.To["web"] != ladderB {
+		t.Fatalf("app.yaml must carry the page's record of the automatic step, got %+v", cfg.LastAutoUpdate)
+	}
+	// the next night climbs the next step
+	s2 := mustLeg(t, m)
+	if pinOf(t, dir) != ladderC || s2.Done != 1 {
+		t.Fatalf("night 2 ended on %s (done=%d), want C", pinOf(t, dir), s2.Done)
+	}
+	// and the third night finds nothing to do — no press, not even counted as a skip
+	s3 := mustLeg(t, m)
+	if len(*ups) != 2 || s3.Done+s3.Skipped != 0 {
+		t.Fatalf("night 3 at the head pressed or counted something: ups=%v summary=%+v", *ups, s3)
+	}
+}
+
+// TestR678_StepsLeftFreshAtDone — the page's steps-left count and the pin are current the moment the
+// update says `done`, with no scan in between. MEASURED 2026-09-24: ~50 s stale, six re-presses.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the ScanStacks call in verifyAndConclude — this test fails at
+// "steps left read 2 right after the step ended done".
+func TestR678_StepsLeftFreshAtDone(t *testing.T) {
+	m, _, _, _, _ := ladderManager(t, true)
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	if st, _ := m.GetStack("nextcloud"); st.LadderStepsLeft != 2 {
+		t.Fatalf("before: steps left %d, want 2", st.LadderStepsLeft)
+	}
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatal(err)
+	}
+	st := waitUpdateDone(t, m, "nextcloud")
+	if st.UpdatePhase != UpdatePhaseDone {
+		t.Fatalf("ended %q", st.UpdatePhase)
+	}
+	if st.LadderStepsLeft != 1 {
+		t.Fatalf("steps left read %d right after the step ended done, want 1", st.LadderStepsLeft)
+	}
+	if st.AppConfig == nil || st.AppConfig.PinnedImages["web"] != ladderB {
+		t.Fatalf("the in-memory pin is stale after done: %+v", st.AppConfig)
+	}
+}
+
+// TestR680_FailedStepIsNotPressedAgain — B fails and is undone: the box records the failed step, the
+// next night's leg skips it (failed_before) and nothing is brought up; a PERSON can still press it; and
+// when the catalog's ladder changes, the leg may try again.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the FailedStep check in legCandidate — this test fails at
+// "night 2 pressed the step that was undone on night 1".
+func TestR680_FailedStepIsNotPressedAgain(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	m.undoCopier = newFakeCopier(map[string]string{undoVol: "OLD"})
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) { return false, "B unhealthy" }
+	m.updateUndoHealthFn = func(context.Context, string, time.Duration, *Metadata) (bool, string) { return true, "A answers" }
+	legOpts(m, nil)
+
+	s1 := mustLeg(t, m)
+	if s1.Undone != 1 || pinOf(t, dir) != ladderA {
+		t.Fatalf("night 1: %+v on %s, want undone back on A", s1, pinOf(t, dir))
+	}
+	cfg := LoadAppConfig(dir)
+	if cfg.FailedStep == nil || cfg.FailedStep.To["web"] != ladderB || cfg.FailedStep.Outcome != "undone" || cfg.FailedStep.Ladder == "" {
+		t.Fatalf("the failed step must be recorded on the box, got %+v", cfg.FailedStep)
+	}
+	upsAfter1 := len(*ups)
+
+	s2 := mustLeg(t, m)
+	if len(*ups) != upsAfter1 {
+		t.Fatalf("night 2 pressed the step that was undone on night 1 (ups=%v)", *ups)
+	}
+	if st := legStepFor(s2, "nextcloud"); st.Outcome != LegOutcomeSkipped || st.Reason != LegSkipFailedBefore {
+		t.Fatalf("night 2 must skip with %q, got %+v", LegSkipFailedBefore, st)
+	}
+
+	// a person still can — the record binds the leg only
+	if err := m.StartGuardedUpdate("nextcloud"); err != nil {
+		t.Fatalf("a person's press must not be refused by the failed-step record: %v", err)
+	}
+	waitUpdateDone(t, m, "nextcloud")
+
+	// the catalog re-tests the step (a new tested_at): the ladder's print changes → the leg tries again
+	writeLadder(t, m,
+		`  - {"from": {"web": "`+ladderA+`"}, "to": {"web": "`+ladderB+`"}, "digest": {"web": "sha256:`+strings.Repeat("a", 64)+`"}, "verdict": "proven", "tested_at": "2026-09-26T01:00:00Z"}`+"\n",
+		ladderLine(ladderB, ladderC))
+	before := len(*ups)
+	mustLeg(t, m)
+	if len(*ups) <= before || (*ups)[before] != ladderB {
+		t.Fatalf("after the catalog changed the ladder the leg must try the step again (ups=%v)", *ups)
+	}
+}
+
+// TestLeg_NeedsPersonIsNeverTaken — decision 13's mark: the leg never takes a needs_person step.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the NeedsPerson check — this test fails at "a needs_person step
+// was pressed by the leg".
+func TestLeg_NeedsPersonIsNeverTaken(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": false, "needs_person": "the admin must re-login after this step", "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || pinOf(t, dir) != ladderA {
+		t.Fatalf("a needs_person step was pressed by the leg (ups=%v)", *ups)
+	}
+	if st := legStepFor(s, "nextcloud"); st.Reason != LegSkipNeedsPerson {
+		t.Fatalf("skip reason %+v, want %q", st, LegSkipNeedsPerson)
+	}
+}
+
+// TestLeg_FilesMayChangeNeedsAWholeCopy — decision 13's other mark: taken only when a fresh WHOLE copy
+// exists (the backup side's truth table, asked through FreshWholeCopy).
+func TestLeg_FilesMayChangeNeedsAWholeCopy(t *testing.T) {
+	m, dir, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	whole := false
+	legOpts(m, func(o *UpdateLegOptions) {
+		o.FreshWholeCopy = func(context.Context, string) (bool, string) { return whole, "fake" }
+	})
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipFilesNoCopy {
+		t.Fatalf("without a whole copy the step must be skipped: ups=%v summary=%+v", *ups, s)
+	}
+	whole = true
+	s = mustLeg(t, m)
+	if len(*ups) != 1 || pinOf(t, dir) != ladderB || s.Done != 1 {
+		t.Fatalf("with a fresh whole copy the step must be taken: ups=%v pin=%s", *ups, pinOf(t, dir))
+	}
+	// unwired check = no whole copy (fail closed)
+	m2, _, _, ups2, _ := legManager(t)
+	writeLadder(t, m2,
+		ladderLineMarks(ladderA, ladderB, "proven", `{"files_may_change": true, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m2, nil)
+	mustLeg(t, m2)
+	if len(*ups2) != 0 {
+		t.Fatal("with no whole-copy check wired a files_may_change step must never be taken")
+	}
+}
+
+// TestLeg_OlderThanLadderIsNotTouched — an installed version matching no ladder entry: the leg leaves it
+// alone and says so by name (a person's press still jumps, TestLadder_UnknownInstalledJumpsAndSaysSo).
+func TestLeg_OlderThanLadderIsNotTouched(t *testing.T) {
+	m, dir, _, ups, logBuf := legManager(t)
+	old := "services:\n  web:\n    image: nextcloud:30.0.0-apache\nvolumes:\n  db:\n"
+	mustWrite(t, ComposePathIn(dir), old)
+	mustWrite(t, AppliedComposePath(dir), old)
+	mustWrite(t, filepath.Join(dir, "app.yaml"), "deployed: true\nenv: {}\npinned_images:\n    web: nextcloud:30.0.0-apache\ninstalled_images:\n    web:\n        ref: nextcloud:30.0.0-apache\n")
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 {
+		t.Fatalf("an app older than the ladder was pressed: %v", *ups)
+	}
+	if legStepFor(s, "nextcloud").Reason != LegSkipOlderThanLadder || !strings.Contains(logBuf.String(), "nextcloud:30.0.0-apache") {
+		t.Fatalf("skip must be older_than_ladder and logged by name; summary %+v", s)
+	}
+}
+
+// TestLeg_OnlyProvenStepsAreTaken — an `unrecorded` (backfilled, never tested) entry is not a test.
+func TestLeg_OnlyProvenStepsAreTaken(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	writeLadder(t, m,
+		ladderLineMarks(ladderA, ladderB, "unrecorded", `{"files_may_change": false, "needs_person": null, "memory_tight": false}`),
+		ladderLine(ladderB, ladderC))
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipNotProven {
+		t.Fatalf("an unproven step was taken or mis-reasoned: ups=%v %+v", *ups, s)
+	}
+}
+
+// TestLeg_NoStepAtOrAfterW5h — decision 20: the leg starts nothing at W+5h.
+//
+// COMPANION RED-PROOF (REPORT.md): drop the deadline arm of legStop — this test fails at "a step was
+// started at W+5h".
+func TestLeg_NoStepAtOrAfterW5h(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	legOpts(m, func(o *UpdateLegOptions) {
+		o.Now = func() time.Time { return time.Date(2026, 9, 25, 7, 30, 0, 0, time.UTC) } // W 02:30 + 5h
+	})
+	s := mustLeg(t, m)
+	if len(*ups) != 0 {
+		t.Fatalf("a step was started at W+5h: %v", *ups)
+	}
+	if s.Stopped != LegSkipWindowEnd || legStepFor(s, "nextcloud").Reason != LegSkipWindowEnd {
+		t.Fatalf("summary must say window_end: %+v", s)
+	}
+}
+
+// TestLeg_SwitchOffPressesNothing — decision 12: the switch off means no press at all.
+func TestLeg_SwitchOffPressesNothing(t *testing.T) {
+	m, _, _, ups, _ := legManager(t)
+	legOpts(m, func(o *UpdateLegOptions) { o.Enabled = func() bool { return false } })
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || s.Enabled || s.Stopped != LegSkipSwitchedOff {
+		t.Fatalf("switch off: ups=%v summary=%+v", *ups, s)
+	}
+}
+
+// TestLeg_TransientRefusalIsRetriedOnce — `busy` is a passing reason: one retry, then a named skip.
+func TestLeg_TransientRefusalIsRetriedOnce(t *testing.T) {
+	m, _, g, ups, logBuf := legManager(t)
+	g.busy = true
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != "refused:busy" {
+		t.Fatalf("busy: ups=%v summary=%+v", *ups, s)
+	}
+	if n := strings.Count(logBuf.String(), "REFUSED (busy)"); n != 2 {
+		t.Fatalf("a busy refusal must be pressed exactly twice (once + one retry), saw %d", n)
+	}
+}
+
+// TestLeg_HeldAppIsNotPressed — a held app is terminal for the leg.
+func TestLeg_HeldAppIsNotPressed(t *testing.T) {
+	m, _, g, ups, _ := legManager(t)
+	g.held, g.holdWhy = true, "HELD"
+	legOpts(m, nil)
+	s := mustLeg(t, m)
+	if len(*ups) != 0 || legStepFor(s, "nextcloud").Reason != LegSkipHeld {
+		t.Fatalf("held: ups=%v summary=%+v", *ups, s)
+	}
+}
+
+// TestLeg_GateSeesTheLegAndItsStep — UpdateLegState answers active while the leg runs and stepRunning
+// while its step is in flight (the full-system backup's gate reads exactly this), and both go false after.
+func TestLeg_GateSeesTheLegAndItsStep(t *testing.T) {
+	m, _, _, _, _ := legManager(t)
+	release := make(chan struct{})
+	var mu sync.Mutex
+	var seen [][2]bool
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
+		a, s := m.UpdateLegState()
+		mu.Lock()
+		seen = append(seen, [2]bool{a, s})
+		mu.Unlock()
+		<-release
+		return true, "ok"
+	}
+	legOpts(m, nil)
+	done := make(chan struct{})
+	go func() { mustLeg(t, m); close(done) }()
+	deadline := time.Now().Add(3 * time.Second)
+	for {
+		mu.Lock()
+		n := len(seen)
+		mu.Unlock()
+		if n > 0 || time.Now().After(deadline) {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+	close(release)
+	<-done
+	if len(seen) == 0 || !seen[0][0] || !seen[0][1] {
+		t.Fatalf("during the step's verify the gate must see (active, stepRunning) = (true, true), saw %v", seen)
+	}
+	if a, s := m.UpdateLegState(); a || s {
+		t.Fatalf("after the leg the gate must see (false, false), got (%v, %v)", a, s)
+	}
+}
+
+// TestD28_NoCrashLoopStopDuringAnAutomaticStep — decision 28's stop must not fire while the leg's step
+// restarts the app (pull, up, verify, undo): an app that is Updating is not sampled and its history is
+// dropped, so restarts climbing during the step never make a verdict.
+//
+// COMPANION RED-PROOF (REPORT.md): drop `st.Updating` from ObserveUnhealthy's skip — this test fails at
+// "a crash-loop verdict fired during an automatic step".
+func TestD28_NoCrashLoopStopDuringAnAutomaticStep(t *testing.T) {
+	m, _, _, _, _ := legManager(t)
+	if err := m.ScanStacks(); err != nil {
+		t.Fatal(err)
+	}
+	var cmu sync.Mutex
+	restarts := 0
+	m.execFn = func(name string, args ...string) (string, error) {
+		if name == "docker" && len(args) > 0 && args[0] == "inspect" {
+			cmu.Lock()
+			defer cmu.Unlock()
+			return "/nextcloud-web-1|" + strconv.Itoa(restarts) + "\n", nil
+		}
+		return "", nil
+	}
+	release := make(chan struct{})
+	inVerify := make(chan struct{}, 1)
+	m.updateHealthFn = func(context.Context, string, time.Duration) (bool, string) {
+		inVerify <- struct{}{}
+		<-release
+		return true, "ok"
+	}
+	legOpts(m, nil)
+	done := make(chan struct{})
+	go func() { m.RunUpdateLeg(context.Background(), "test"); close(done) }()
+	select {
+	case <-inVerify:
+	case <-time.After(3 * time.Second):
+		t.Fatal("the step never reached its verify")
+	}
+	// the container the samples read (set now: the leg's rescan rebuilt the stack before the step)
+	m.mu.Lock()
+	m.stacks["nextcloud"].Containers = []ContainerInfo{{Name: "nextcloud-web-1"}}
+	m.mu.Unlock()
+	t0 := time.Date(2026, 9, 25, 4, 20, 0, 0, time.UTC)
+	for i := 0; i < 10; i++ { // ten samples a minute apart, +3 restarts each: 27 restarts in 9 minutes
+		cmu.Lock()
+		restarts += 3
+		cmu.Unlock()
+		if v := m.ObserveUnhealthy(t0.Add(time.Duration(i)*time.Minute), nil); len(v) > 0 {
+			close(release)
+			<-done
+			t.Fatalf("a crash-loop verdict fired during an automatic step: %+v", v)
+		}
+	}
+	close(release)
+	<-done
+}
+
+// TestLegDeadline — W+5h of the night that contains now, across midnight too.
+func TestLegDeadline(t *testing.T) {
+	loc := time.UTC
+	cases := []struct {
+		now    time.Time
+		window string
+		want   time.Time
+	}{
+		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "02:30", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)},
+		{time.Date(2026, 9, 25, 1, 0, 0, 0, loc), "23:00", time.Date(2026, 9, 25, 4, 0, 0, 0, loc)},
+		{time.Date(2026, 9, 25, 2, 0, 0, 0, loc), "02:30", time.Date(2026, 9, 24, 7, 30, 0, 0, loc)}, // before W: last night's
+		{time.Date(2026, 9, 25, 4, 15, 0, 0, loc), "junk", time.Date(2026, 9, 25, 7, 30, 0, 0, loc)}, // default 02:30
+	}
+	for _, c := range cases {
+		if got := LegDeadline(c.now, c.window, loc); !got.Equal(c.want) {
+			t.Errorf("LegDeadline(%s, %q) = %s, want %s", c.now.Format("15:04"), c.window, got, c.want)
+		}
+	}
+}

controller/internal/stacks/undo.go

@@ -553,9 +553,13 @@ func (m *Manager) tryUndo(ctx context.Context, name, dir, why string, entry *upd
 	m.recordInstalledImages(name, dir, env)
 	m.removeUndoCopies(name, entry.UndoCopies)
 	m.recordUpdateUndone(name, dir, &UpdateUndone{To: entry.NewPin, At: start.UTC().Format(time.RFC3339), Why: why})
+	m.recordFailedStep(name, dir, entry.NewPin, "undone") // R-680
 	m.removePreUpdateCopies(dir)
 	_ = m.RefreshStatus()
 	m.clearJournal(name)
+	if err := m.ScanStacks(); err != nil { // R-678: the page reads the pin the undo put back
+		m.logger.Printf("[WARN] [stacks] update %s: rescan after the undo failed: %v", name, err)
+	}
 	m.finishUpdate(name, UpdatePhaseUndone, "")
 	m.emitUpdateEvent(UpdateEventUndone, name, entry, UpdateRestorePoint{}, true)
 	m.logger.Printf("[INFO] [stacks] update %s: UNDONE in %s — the previous version is running on the data from before the update (%s)", name, m.now().Sub(start).Round(time.Second), detail)

controller/internal/stacks/update.go

@@ -944,8 +944,16 @@ func (m *Manager) verifyAndConclude(ctx context.Context, name, dir string, env [
 	_ = m.RefreshStatus()
 	m.removeUndoCopies(name, entry.UndoCopies)
 	m.recordUpdateUndone(name, dir, nil) // a successful update ends the "undone" note
+	m.clearFailedStep(name, dir)         // R-680: and the failed-step record
 	m.clearJournal(name)
 	m.removePreUpdateCopies(dir)
+	// R-678 (v0.271.0): the app's catalog fields — ladder_steps_left, the badge's inputs, the pin — are
+	// re-read NOW, before Updating goes false, so neither a person nor the automatic leg ever reads the
+	// pre-update values after `done`. MEASURED 2026-09-24: ~50 s stale, six re-presses by the caller.
+	// COMPANION RED-PROOF (REPORT.md): drop this scan — TestR678_StepsLeftFreshAtDone fails.
+	if err := m.ScanStacks(); err != nil {
+		m.logger.Printf("[WARN] [stacks] update %s: rescan after the update failed: %v — the page catches up at the next scan", name, err)
+	}
 	m.finishUpdate(name, UpdatePhaseDone, "")
 	m.logger.Printf("[INFO] [stacks] update %s: DONE in %s", name, m.now().Sub(start).Round(time.Second))
 }
@@ -1008,6 +1016,9 @@ func (m *Manager) failAndHold(ctx context.Context, name, dir string, env []strin
 		holdWhy = w
 		m.markUpdateHeld(name)
 	}
+	if entry != nil { // R-680: the automatic leg never presses this step again on this ladder
+		m.recordFailedStep(name, dir, entry.NewPin, "held")
+	}
 	_ = m.RefreshStatus()
 	m.clearJournal(name)
 	m.removePreUpdateCopies(dir)