controller v0.271.0: automatic app updates — the update leg after the off-site copy, the backup gate waits, the switch (09 6.4 part 7; R-680, R-678, R-643)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

controller/cmd/controller/main.go

@@ -19,6 +19,7 @@ import (
 
 	"crypto/subtle"
 	"strings"
+	"sync"
 
 	"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
 	"gitea.dooplex.hu/admin/felhom-controller/internal/api"
@@ -567,6 +568,23 @@ func main() {
 	// An UNWIRED manager also refuses (fail closed); TestSlice4_UpdateGuardsAreWiredAtStartup walks
 	// this file for the call, because a seam built and never wired has shipped here seven times.
 	stackMgr.SetUpdateGuards(&updateGuardsAdapter{b: backupMgr, q: quiesceLoop})
+	// v0.271.0 (`09` §6.4 part 7): the automatic update leg. The switch is read at the leg's start and
+	// before every press; W is the SAME effective window every nightly leg reads. The files-may-change
+	// mark asks the backup side's truth table (decisions 25/26), never a second definition of "whole".
+	// Pinned by TestUpdateLegIsWiredAtStartup (an AST walk of this file).
+	stackMgr.SetUpdateLeg(stacks.UpdateLegOptions{
+		Enabled: sett.GetAppUpdateUnattended,
+		WindowStart: func() string {
+			return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
+		},
+		FreshWholeCopy: func(ctx context.Context, name string) (bool, string) {
+			if backupMgr == nil {
+				return false, "backup is disabled on this box"
+			}
+			return backupMgr.FreshWholeCopy(ctx, name, cfg.Update.BackupMaxAgeDuration(), time.Now())
+		},
+		Location: budapestLoc(),
+	})
 	// v0.238.1: the nightly legs (capture, Tier 2, volume dump) leave an app alone WHILE it is being
 	// updated, not only once it is held — found live in Scenario F, see backup.Manager.isHeld.
 	if backupMgr != nil {
@@ -718,7 +736,13 @@ func main() {
 		// `pulling`, `starting` and `verifying` do not — and the last two are where the new version
 		// may already have touched the customer's data.
 		updater.SetAppUpdatingCheck(func() bool {
-			return stackMgr != nil && stackMgr.AnyUpdating()
+			if stackMgr == nil {
+				return false
+			}
+			// v0.271.0: the whole automatic update leg counts, not only a step in flight — a controller
+			// swap between two steps would cancel the rest of the night's leg.
+			legActive, _ := stackMgr.UpdateLegState()
+			return stackMgr.AnyUpdating() || legActive
 		})
 		if stackMgr != nil {
 			stackMgr.SetSelfUpdatingCheck(updater.IsUpdateRunning)
@@ -1264,12 +1288,19 @@ func main() {
 					"A korábbi távoli mentések a türelmi idő lejártával törlésre kerültek, az ügyfél döntése alapján. A hozzájuk tartozó lezárt helyreállítási csomag eltávolítását is kértük.", map[string]string{"deleted_path": renamedTo})
 			}
 		})
+		// v0.271.0 (`09` §3 decision 11, §6.4.2 point 2): the automatic update leg is CHAINED to the
+		// off-site job — it runs when this function's off-site half has ended, on every path: configured
+		// or not, ok, failed, any of RunOffboxBackup's early returns, even a panic. A box with no off-site
+		// target runs it at W+105m. One call site, no signal to go stale.
 		sched.Daily("offbox-backup", offboxLeg, func(ctx context.Context) error {
-			t := sett.GetOffboxTarget()
-			if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
-				return nil // not configured / not scheduled
-			}
-			return backupMgr.RunOffboxBackup(ctx)
+			return chainUpdateLeg(ctx, func(ctx context.Context) error {
+				t := sett.GetOffboxTarget()
+				if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
+					logger.Printf("[INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now")
+					return nil // not configured / not scheduled
+				}
+				return backupMgr.RunOffboxBackup(ctx)
+			}, func(ctx context.Context) { stackMgr.RunUpdateLeg(ctx, "after-offsite") })
 		})
 		// R-241 — the abandonment terminal step. DAILY and not on the backup leg, deliberately: it must
 		// run on a box whose off-site tier is NOT configured for runs (an abandoning box may be sitting
@@ -3237,6 +3268,8 @@ func startQuiesceLoop(ctx context.Context, cfg *config.Config, sett *settings.Se
 		WindowStartFn: func() string {
 			return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
 		},
+		// v0.271.0 (`09` decision 20): the full-system backup waits for the automatic update leg.
+		UpdateLegFn: stackMgr.UpdateLegState,
 	})
 	loop.Recover() // crash-safety: restart any stacks stranded-down by a mid-quiesce crash
 	go loop.Run(ctx)
@@ -3689,3 +3722,34 @@ func stopUnhealthyApps(logger *log.Logger, d unhealthyDeps, ooms []stacks.OOMCon
 	}
 	return stopped
 }
+
+// chainUpdateLeg runs the off-site half, then the update leg — ALWAYS, whatever the off-site half did:
+// returned nil (ran, or had nothing to do), returned an error, or panicked. The off-site half's error is
+// returned (the scheduler logs it); a panic becomes an error. `09` §6.4.2 point 2; pinned by
+// TestChainUpdateLeg_EveryPath.
+func chainUpdateLeg(ctx context.Context, offsite func(context.Context) error, leg func(context.Context)) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("the off-site leg panicked: %v", r)
+		}
+		leg(ctx)
+	}()
+	return offsite(ctx)
+}
+
+var (
+	budapestLocOnce sync.Once
+	budapestLocVal  *time.Location
+)
+
+// budapestLoc is the wall clock the backup window is read in (the scheduler's and the quiesce gate's).
+func budapestLoc() *time.Location {
+	budapestLocOnce.Do(func() {
+		loc, err := time.LoadLocation("Europe/Budapest")
+		if err != nil {
+			loc = time.Local
+		}
+		budapestLocVal = loc
+	})
+	return budapestLocVal
+}

controller/cmd/controller/update_leg_wiring_test.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"os"
+	"strings"
+	"testing"
+)
+
+// v0.271.0 — the automatic update leg (`09` §6.4 part 7).
+
+// TestChainUpdateLeg_EveryPath — the leg runs after the off-site half on EVERY path it can take:
+// nothing to do (no target), ran ok, returned an error (incl. every RunOffboxBackup early return, which
+// returns to here), and panicked. The off-site error reaches the scheduler unchanged; a panic becomes one.
+//
+// COMPANION RED-PROOF (REPORT.md): call the leg after `return offsite(ctx)` instead of in the defer —
+// the error and panic cases fail at "the leg did not run after the off-site half".
+func TestChainUpdateLeg_EveryPath(t *testing.T) {
+	boom := errors.New("sftp: connection refused")
+	cases := []struct {
+		name    string
+		offsite func(context.Context) error
+		wantErr string
+	}{
+		{"no off-site target (nothing to do)", func(context.Context) error { return nil }, ""},
+		{"off-site ran ok", func(context.Context) error { return nil }, ""},
+		{"off-site failed / an early return", func(context.Context) error { return boom }, "connection refused"},
+		{"off-site panicked", func(context.Context) error { panic("nil map") }, "panicked"},
+	}
+	for _, c := range cases {
+		ran := 0
+		err := chainUpdateLeg(context.Background(), c.offsite, func(context.Context) { ran++ })
+		if ran != 1 {
+			t.Errorf("%s: the leg did not run after the off-site half (ran %d times)", c.name, ran)
+		}
+		if c.wantErr == "" && err != nil {
+			t.Errorf("%s: unexpected error %v", c.name, err)
+		}
+		if c.wantErr != "" && (err == nil || !strings.Contains(err.Error(), c.wantErr)) {
+			t.Errorf("%s: error %v, want one containing %q", c.name, err, c.wantErr)
+		}
+	}
+}
+
+// TestUpdateLegIsWiredAtStartup — the leg, its chain and the gate's interlock are CALLED from main.go (an
+// AST walk; a comment naming them would not count). A seam built and never wired is this project's
+// commonest defect.
+//
+// COMPANION RED-PROOF (REPORT.md): comment out the SetUpdateLeg call — this fails.
+func TestUpdateLegIsWiredAtStartup(t *testing.T) {
+	lines, _, _ := slice4CallLines(t)
+	for _, callee := range []string{"SetUpdateLeg", "chainUpdateLeg", "RunUpdateLeg", "FreshWholeCopy"} {
+		if len(lines[callee]) == 0 {
+			t.Errorf("main.go never calls %s — the automatic update leg is not wired", callee)
+		}
+	}
+	src, err := os.ReadFile("main.go")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(src), "Enabled: sett.GetAppUpdateUnattended,") {
+		t.Error("the leg's switch must be the household's app_update.unattended (sett.GetAppUpdateUnattended)")
+	}
+	if !strings.Contains(string(src), "UpdateLegFn: stackMgr.UpdateLegState") {
+		t.Error("the quiesce loop must be given stackMgr.UpdateLegState (decision 20's interlock)")
+	}
+	// the chain must sit INSIDE the offbox-backup job, not beside it
+	i := strings.Index(string(src), `sched.Daily("offbox-backup"`)
+	if i < 0 || !strings.Contains(string(src)[i:i+600], "chainUpdateLeg(") {
+		t.Error("the offbox-backup job must run through chainUpdateLeg")
+	}
+}