controller v0.271.0: automatic app updates — the update leg after the off-site copy, the backup gate waits, the switch (09 6.4 part 7; R-680, R-678, R-643)
gates / gates (push) Successful in 24s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 22:07:24 +02:00
parent 1495ca43fb
commit 9cf13a3add
33 changed files with 1672 additions and 14 deletions
+70 -6
View File
@@ -19,6 +19,7 @@ import (
"crypto/subtle"
"strings"
"sync"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
"gitea.dooplex.hu/admin/felhom-controller/internal/api"
@@ -567,6 +568,23 @@ func main() {
// An UNWIRED manager also refuses (fail closed); TestSlice4_UpdateGuardsAreWiredAtStartup walks
// this file for the call, because a seam built and never wired has shipped here seven times.
stackMgr.SetUpdateGuards(&updateGuardsAdapter{b: backupMgr, q: quiesceLoop})
// v0.271.0 (`09` §6.4 part 7): the automatic update leg. The switch is read at the leg's start and
// before every press; W is the SAME effective window every nightly leg reads. The files-may-change
// mark asks the backup side's truth table (decisions 25/26), never a second definition of "whole".
// Pinned by TestUpdateLegIsWiredAtStartup (an AST walk of this file).
stackMgr.SetUpdateLeg(stacks.UpdateLegOptions{
Enabled: sett.GetAppUpdateUnattended,
WindowStart: func() string {
return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
},
FreshWholeCopy: func(ctx context.Context, name string) (bool, string) {
if backupMgr == nil {
return false, "backup is disabled on this box"
}
return backupMgr.FreshWholeCopy(ctx, name, cfg.Update.BackupMaxAgeDuration(), time.Now())
},
Location: budapestLoc(),
})
// v0.238.1: the nightly legs (capture, Tier 2, volume dump) leave an app alone WHILE it is being
// updated, not only once it is held — found live in Scenario F, see backup.Manager.isHeld.
if backupMgr != nil {
@@ -718,7 +736,13 @@ func main() {
// `pulling`, `starting` and `verifying` do not — and the last two are where the new version
// may already have touched the customer's data.
updater.SetAppUpdatingCheck(func() bool {
return stackMgr != nil && stackMgr.AnyUpdating()
if stackMgr == nil {
return false
}
// v0.271.0: the whole automatic update leg counts, not only a step in flight — a controller
// swap between two steps would cancel the rest of the night's leg.
legActive, _ := stackMgr.UpdateLegState()
return stackMgr.AnyUpdating() || legActive
})
if stackMgr != nil {
stackMgr.SetSelfUpdatingCheck(updater.IsUpdateRunning)
@@ -1264,12 +1288,19 @@ func main() {
"A korábbi távoli mentések a türelmi idő lejártával törlésre kerültek, az ügyfél döntése alapján. A hozzájuk tartozó lezárt helyreállítási csomag eltávolítását is kértük.", map[string]string{"deleted_path": renamedTo})
}
})
// v0.271.0 (`09` §3 decision 11, §6.4.2 point 2): the automatic update leg is CHAINED to the
// off-site job — it runs when this function's off-site half has ended, on every path: configured
// or not, ok, failed, any of RunOffboxBackup's early returns, even a panic. A box with no off-site
// target runs it at W+105m. One call site, no signal to go stale.
sched.Daily("offbox-backup", offboxLeg, func(ctx context.Context) error {
t := sett.GetOffboxTarget()
if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
return nil // not configured / not scheduled
}
return backupMgr.RunOffboxBackup(ctx)
return chainUpdateLeg(ctx, func(ctx context.Context) error {
t := sett.GetOffboxTarget()
if t == nil || !t.Enabled || t.Schedule != "daily" || !backupMgr.OffboxConfigured() {
logger.Printf("[INFO] [offbox] no scheduled off-site target on this box — the off-site leg does nothing; the update leg runs now")
return nil // not configured / not scheduled
}
return backupMgr.RunOffboxBackup(ctx)
}, func(ctx context.Context) { stackMgr.RunUpdateLeg(ctx, "after-offsite") })
})
// R-241 — the abandonment terminal step. DAILY and not on the backup leg, deliberately: it must
// run on a box whose off-site tier is NOT configured for runs (an abandoning box may be sitting
@@ -3237,6 +3268,8 @@ func startQuiesceLoop(ctx context.Context, cfg *config.Config, sett *settings.Se
WindowStartFn: func() string {
return backupwindow.EffectiveWindow(sett.GetBackupWindowStart(), cfg.Backup.DBDumpSchedule)
},
// v0.271.0 (`09` decision 20): the full-system backup waits for the automatic update leg.
UpdateLegFn: stackMgr.UpdateLegState,
})
loop.Recover() // crash-safety: restart any stacks stranded-down by a mid-quiesce crash
go loop.Run(ctx)
@@ -3689,3 +3722,34 @@ func stopUnhealthyApps(logger *log.Logger, d unhealthyDeps, ooms []stacks.OOMCon
}
return stopped
}
// chainUpdateLeg runs the off-site half, then the update leg — ALWAYS, whatever the off-site half did:
// returned nil (ran, or had nothing to do), returned an error, or panicked. The off-site half's error is
// returned (the scheduler logs it); a panic becomes an error. `09` §6.4.2 point 2; pinned by
// TestChainUpdateLeg_EveryPath.
func chainUpdateLeg(ctx context.Context, offsite func(context.Context) error, leg func(context.Context)) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("the off-site leg panicked: %v", r)
}
leg(ctx)
}()
return offsite(ctx)
}
var (
budapestLocOnce sync.Once
budapestLocVal *time.Location
)
// budapestLoc is the wall clock the backup window is read in (the scheduler's and the quiesce gate's).
func budapestLoc() *time.Location {
budapestLocOnce.Do(func() {
loc, err := time.LoadLocation("Europe/Budapest")
if err != nil {
loc = time.Local
}
budapestLocVal = loc
})
return budapestLocVal
}