controller v0.264.0: the household is told when an update is undone or held, in its language
gates / gates (push) Successful in 25s
gates / gates (push) Successful in 25s
app_update_undone / app_update_held events (09 decision 15), on by default and seeded once on existing boxes; R-606 update sentences as key+args rendered per reader; R-646 startup applied-meta backfill for apps current with the catalog; R-620 a disabled notifier WARNs once per event type. Needs hub v0.120.0. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -154,6 +154,12 @@ type Stack struct {
|
||||
UpdatePhase string `json:"update_phase,omitempty"`
|
||||
UpdatePhaseLabel string `json:"update_phase_label,omitempty"`
|
||||
UpdateError string `json:"update_error,omitempty"`
|
||||
// UpdateErrorKey / UpdateErrorArgs (v0.264.0, R-606) are UpdateError as a bundle key + arguments,
|
||||
// so a page renders it in the READER's language (UpdateErrorIn). UpdateError stays the Hungarian —
|
||||
// what the API, the logs and older readers have always had. Empty key = an old or composed sentence,
|
||||
// rendered as stored, never as an empty line.
|
||||
UpdateErrorKey string `json:"-"`
|
||||
UpdateErrorArgs []interface{} `json:"-"`
|
||||
// updateHeld (R-480, v0.240.0) — the last update ended HELD, so UpdateError is the hold's own
|
||||
// sentence („… leállítva marad"). It is shown only while that hold is in force; fillHoldReason.
|
||||
updateHeld bool
|
||||
@@ -243,6 +249,7 @@ type Manager struct {
|
||||
undoCopier volumeCopier
|
||||
updateUndoHealthFn func(ctx context.Context, name string, timeout time.Duration, meta *Metadata) (bool, string)
|
||||
probeRunFn func(t probeTarget) *HealthProbeResult // the health wait's network probe; nil ⇒ runChecks
|
||||
updateEventSink func(UpdateEvent) // v0.264.0: the notifier; nil ⇒ no events
|
||||
updateMemoryFn func(newReqMB, newLimitMB, releasedReqMB, releasedLimitMB int) (refusal error, warning string)
|
||||
updateDiskFreeFn func() (freeGiB float64, ok bool)
|
||||
updateNowFn func() time.Time
|
||||
|
||||
@@ -401,6 +401,7 @@ func (m *Manager) tryUndo(ctx context.Context, name, dir, why string, entry *upd
|
||||
_ = m.RefreshStatus()
|
||||
m.clearJournal(name)
|
||||
m.finishUpdate(name, UpdatePhaseUndone, "")
|
||||
m.emitUpdateEvent(UpdateEventUndone, name, entry, UpdateRestorePoint{}, true)
|
||||
m.logger.Printf("[INFO] [stacks] update %s: UNDONE in %s — the previous version is running on the data from before the update (%s)", name, m.now().Sub(start).Round(time.Second), detail)
|
||||
return ""
|
||||
}
|
||||
@@ -424,3 +425,86 @@ func (m *Manager) recordUpdateUndone(name, dir string, u *UpdateUndone) {
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// ── The household and the operator are TOLD (v0.264.0, `09` §3 decision 15, §6.4 part 2) ─────────
|
||||
|
||||
// Update event kinds — the hub event types, one register for both repos (hub allowedEventTypes).
|
||||
const (
|
||||
UpdateEventUndone = "app_update_undone" // the box put the previous version back — warning
|
||||
UpdateEventHeld = "app_update_held" // the update (or its undo) failed and the app is HELD — error
|
||||
)
|
||||
|
||||
// UpdateEvent is what the update job hands to the notifier. The stacks package composes no sentence
|
||||
// for it: the notifier renders the undone line from a bundle key, and the held line is the hold's own
|
||||
// sentence, rendered by the backup side in each language (main.go wires both).
|
||||
type UpdateEvent struct {
|
||||
Kind string
|
||||
App string
|
||||
From, To map[string]string
|
||||
At time.Time
|
||||
CopyTier int
|
||||
CopyDate time.Time
|
||||
// HoldRecorded is false when the hold could not be persisted — the event still goes (the operator
|
||||
// must hear about exactly that), and the household sentence is then update.error.hold_unsaved.
|
||||
HoldRecorded bool
|
||||
}
|
||||
|
||||
// SetUpdateEventSink wires the notifier (main.go). INIT-ONLY. Nil = no events (tests, setup mode).
|
||||
// Pinned by TestUpdateEventSinkIsWiredAtStartup (an AST walk of main.go).
|
||||
func (m *Manager) SetUpdateEventSink(fn func(UpdateEvent)) {
|
||||
m.mu.Lock()
|
||||
m.updateEventSink = fn
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// emitUpdateEvent sends ONE event for an undone or held update. Called exactly once per outcome by
|
||||
// tryUndo (undone) and failAndHold (held) — the job never retries (decision 15), so one outcome is one
|
||||
// event; the hub's per-app cooldown is the second belt against a storm (R-629).
|
||||
func (m *Manager) emitUpdateEvent(kind, name string, entry *updateJournalEntry, rp UpdateRestorePoint, holdRecorded bool) {
|
||||
m.mu.RLock()
|
||||
sink := m.updateEventSink
|
||||
m.mu.RUnlock()
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
ev := UpdateEvent{Kind: kind, App: name, At: m.now(), CopyTier: rp.Tier, CopyDate: rp.ProvenAt, HoldRecorded: holdRecorded}
|
||||
if entry != nil {
|
||||
ev.From, ev.To = entry.PrevPin, entry.NewPin
|
||||
}
|
||||
m.logger.Printf("[INFO] [stacks] update %s: event %s (hold recorded: %v)", name, kind, holdRecorded)
|
||||
sink(ev)
|
||||
}
|
||||
|
||||
// BackfillAppliedMeta is R-646's startup pass (v0.264.0), beside AdoptPins. An app pinned before
|
||||
// v0.263.2 has no applied-meta record, so its FIRST undo would judge the old version with whatever
|
||||
// .felhom.yml the catalog sync last put in place. For an app that is CURRENT with the catalog, that file
|
||||
// IS the pinned version's own — so it is recorded now. An app that is Behind or Unknown is skipped and
|
||||
// NAMED: its pinned version's file is already gone, and guessing it would be worse than saying so.
|
||||
// Idempotent; never overwrites an existing record. Returns (recorded, skipped) app names.
|
||||
func (m *Manager) BackfillAppliedMeta() (recorded, skipped []string) {
|
||||
for _, st := range m.GetStacks() {
|
||||
if !st.Deployed || st.AppConfig == nil || len(st.AppConfig.PinnedImages) == 0 {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(st.ComposePath)
|
||||
if _, err := os.Stat(filepath.Join(dir, appliedMetaDir, ".felhom.yml")); err == nil {
|
||||
continue // already recorded — never overwritten
|
||||
}
|
||||
if CatalogOrder(st) != UpdateOrderCurrent {
|
||||
skipped = append(skipped, st.Name)
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, ".felhom.yml"))
|
||||
if err == nil {
|
||||
err = storeAppliedMeta(dir, b)
|
||||
}
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [stacks] applied-meta backfill: %s: %v", st.Name, err)
|
||||
skipped = append(skipped, st.Name)
|
||||
continue
|
||||
}
|
||||
recorded = append(recorded, st.Name)
|
||||
}
|
||||
m.logger.Printf("[INFO] [stacks] applied-meta backfill (R-646): recorded %d %v; skipped %d %v — not current with the catalog, their pinned version's .felhom.yml is no longer on the box", len(recorded), recorded, len(skipped), skipped)
|
||||
return recorded, skipped
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/i18n"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/system"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
@@ -352,26 +353,26 @@ func (m *Manager) UpdatePreflight(name string) *UpdateRefusal {
|
||||
return m.refuseUpdate(name, "not_found", fmt.Sprintf("stack %q not found", name), "no such stack")
|
||||
}
|
||||
if !st.Deployed {
|
||||
return m.refuseUpdate(name, "not_deployed", MsgUpdateNotDeployed, "not deployed")
|
||||
return m.refuseUpdateErr(name, "not_deployed", util.MsgError("update.refusal.not_deployed"), "not deployed")
|
||||
}
|
||||
g := m.guards()
|
||||
if g == nil {
|
||||
return m.refuseUpdate(name, "guards_unwired", MsgUpdateNoGuards, "no UpdateGuards wired — fail closed")
|
||||
return m.refuseUpdateErr(name, "guards_unwired", util.MsgError("update.error.no_guards"), "no UpdateGuards wired — fail closed")
|
||||
}
|
||||
if st.Deploying {
|
||||
return m.refuseUpdate(name, "deploying", fmt.Sprintf(MsgUpdateDeployingFmt, name), "a deploy is in progress")
|
||||
return m.refuseUpdateErr(name, "deploying", util.MsgError("update.refusal.deploying", name), "a deploy is in progress")
|
||||
}
|
||||
if st.Updating {
|
||||
return m.refuseUpdate(name, "updating", fmt.Sprintf(MsgUpdateAlreadyFmt, name), "an update is already in progress")
|
||||
return m.refuseUpdateErr(name, "updating", util.MsgError("update.refusal.already", name), "an update is already in progress")
|
||||
}
|
||||
if held, why := g.HoldFor(name); held {
|
||||
return m.refuseUpdate(name, "held", why, "the app is held")
|
||||
}
|
||||
if busy, why := g.Busy(name); busy {
|
||||
return m.refuseUpdate(name, "busy", MsgUpdateBusy, why)
|
||||
return m.refuseUpdateErr(name, "busy", util.MsgError("update.refusal.busy"), why)
|
||||
}
|
||||
if m.IsMigrating() {
|
||||
return m.refuseUpdate(name, "migrating", MsgUpdateMigrating, "a data migration is running")
|
||||
return m.refuseUpdateErr(name, "migrating", util.MsgError("update.refusal.migrating"), "a data migration is running")
|
||||
}
|
||||
// v0.261.0 — the other half of the self-update lock. The controller's swap restarts this process;
|
||||
// starting an app update into that is how an update loses its own supervisor mid-flight. TRANSIENT:
|
||||
@@ -408,7 +409,7 @@ func (m *Manager) UpdatePreflight(name string) *UpdateRefusal {
|
||||
// with a copy but no way to back up is refused too.
|
||||
if canBackUp, why := g.CanBackUp(name); !canBackUp {
|
||||
if _, found, seen := g.RestorePoints(context.Background(), name, nil); !found {
|
||||
return m.refuseUpdate(name, "no_backup", fmt.Sprintf(MsgUpdateNoBackupFmt, name),
|
||||
return m.refuseUpdateErr(name, "no_backup", util.MsgError("update.refusal.no_backup", name),
|
||||
fmt.Sprintf("no copy on any tier (found: %s) and no backup can be taken now: %s", describeRestorePoints(m.now(), seen), why))
|
||||
}
|
||||
m.logger.Printf("[WARN] [stacks] update %s: no backup can be taken now (%s) — an existing copy must carry the update", name, why)
|
||||
@@ -421,7 +422,7 @@ func (m *Manager) UpdatePreflight(name string) *UpdateRefusal {
|
||||
case !known:
|
||||
m.logger.Printf("[WARN] [stacks] update %s: free space on the Docker data root is unreadable — proceeding without the %.0f GB floor", name, updateDiskFloorGiB)
|
||||
case free < updateDiskFloorGiB:
|
||||
return m.refuseUpdate(name, "disk", fmt.Sprintf(MsgUpdateDiskFmt, free, updateDiskFloorGiB),
|
||||
return m.refuseUpdateErr(name, "disk", util.MsgError("update.refusal.disk", free, updateDiskFloorGiB),
|
||||
fmt.Sprintf("%.2f GiB free on the Docker data root, floor %.0f GiB (fixed floor — image size unknown)", free, updateDiskFloorGiB))
|
||||
}
|
||||
return nil
|
||||
@@ -478,7 +479,7 @@ func (m *Manager) StartGuardedUpdate(name string) error {
|
||||
// A second press between the preflight and here is the race this lock closes.
|
||||
if s.Updating || s.Deploying {
|
||||
m.mu.Unlock()
|
||||
return m.refuseUpdate(name, "updating", fmt.Sprintf(MsgUpdateAlreadyFmt, name), "lost the race for the Updating flag")
|
||||
return m.refuseUpdateErr(name, "updating", util.MsgError("update.refusal.already", name), "lost the race for the Updating flag")
|
||||
}
|
||||
s.Updating, s.UpdateError, s.updateHeld = true, "", false
|
||||
s.UpdatePhase, s.UpdatePhaseLabel = UpdatePhaseChecking, UpdatePhaseLabel(UpdatePhaseChecking)
|
||||
@@ -552,17 +553,49 @@ func (m *Manager) setUpdatePhase(name, phase string) {
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// finishUpdate is the ONE place Updating goes false. msg is the customer sentence on failure.
|
||||
// finishUpdate is the ONE place Updating goes false. msg is the customer sentence on failure — a
|
||||
// finished one (the hold's own sentence, or none). A sentence the job owns goes through finishUpdateKey.
|
||||
func (m *Manager) finishUpdate(name, phase, msg string) {
|
||||
m.finishUpdateKey(name, phase, "", msg)
|
||||
}
|
||||
|
||||
// finishUpdateKey is finishUpdate with the sentence as a bundle KEY (v0.264.0, R-606): UpdateError
|
||||
// keeps the Hungarian (byte-identical to the MsgUpdate* literal it replaced), and the key + args ride
|
||||
// beside it for the page. key "" = `plain` is a finished sentence and is stored as it is.
|
||||
func (m *Manager) finishUpdateKey(name, phase, key, plain string, args ...interface{}) {
|
||||
msg := plain
|
||||
if key != "" {
|
||||
msg = util.Text(i18n.Default, key, args...)
|
||||
}
|
||||
m.mu.Lock()
|
||||
if s, ok := m.stacks[name]; ok {
|
||||
s.Updating = false
|
||||
s.UpdatePhase, s.UpdatePhaseLabel = phase, UpdatePhaseLabel(phase)
|
||||
s.UpdateError = msg
|
||||
s.UpdateError, s.UpdateErrorKey, s.UpdateErrorArgs = msg, key, args
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// UpdatePhaseLabelIn is a phase's label in lang (v0.264.0, R-606). Hungarian is the updatePhaseLabels
|
||||
// map itself; another language reads `update.phase.<phase>` and falls back to the Hungarian.
|
||||
func UpdatePhaseLabelIn(lang, phase string) string {
|
||||
if lang == i18n.Default || phase == "" {
|
||||
return UpdatePhaseLabel(phase)
|
||||
}
|
||||
if b, err := i18n.Shared(); err == nil && b.Has(lang, "update.phase."+phase) {
|
||||
return b.Msg(lang, "update.phase."+phase)
|
||||
}
|
||||
return UpdatePhaseLabel(phase)
|
||||
}
|
||||
|
||||
// UpdateErrorIn is the stack's update sentence in lang (v0.264.0, R-606).
|
||||
func (s Stack) UpdateErrorIn(lang string) string {
|
||||
if s.UpdateErrorKey == "" || lang == i18n.Default {
|
||||
return s.UpdateError
|
||||
}
|
||||
return util.Text(lang, s.UpdateErrorKey, s.UpdateErrorArgs...)
|
||||
}
|
||||
|
||||
func (m *Manager) updateCompose(dir string, env []string, args ...string) (string, error) {
|
||||
if m.updateComposeFn != nil {
|
||||
return m.updateComposeFn(dir, env, args...)
|
||||
@@ -608,18 +641,18 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
dir := filepath.Dir(st.ComposePath)
|
||||
g := m.guards()
|
||||
entry := updateJournalEntry{StartedAt: start}
|
||||
fail := func(msg, detail string) {
|
||||
fail := func(key, detail string, args ...interface{}) {
|
||||
m.logger.Printf("[ERROR] [stacks] update %s FAILED in phase %s after %s — nothing was moved: %s", name, entry.Phase, m.now().Sub(start).Round(time.Millisecond), detail)
|
||||
m.clearJournal(name)
|
||||
m.finishUpdate(name, UpdatePhaseFailed, msg)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, key, "", args...)
|
||||
}
|
||||
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhaseChecking) {
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateJournalFailed)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.journal_failed", "")
|
||||
return
|
||||
}
|
||||
if g == nil {
|
||||
fail(MsgUpdateNoGuards, "no UpdateGuards wired")
|
||||
fail("update.error.no_guards", "no UpdateGuards wired")
|
||||
return
|
||||
}
|
||||
// R-475: the precondition is a copy on ANY tier, chosen in the order 2, 1, 3, and the age rule
|
||||
@@ -633,17 +666,17 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [stacks] update %s: no usable copy on any tier — younger than %s and not older than this install's deploy (%s) (found: %s) — backing up first", name, maxAge, fmtDeployTime(deployedAt), describeRestorePoints(start, seen))
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhaseBackingUp) {
|
||||
fail(MsgUpdateJournalFailed, "journal write failed")
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
if err := g.BackupNow(ctx, name); err != nil {
|
||||
fail(fmt.Sprintf(MsgUpdateBackupFailFmt, err), "pre-update backup: "+err.Error())
|
||||
fail("update.error.backup_failed", "pre-update backup: "+err.Error(), err.Error())
|
||||
return
|
||||
}
|
||||
now := m.now()
|
||||
rp, ok, seen = g.RestorePoints(ctx, name, usableRestorePoint(now, maxAge, deployedAt))
|
||||
if !ok {
|
||||
fail(MsgUpdateBackupNoUnit, fmt.Sprintf("after the backup there is still no copy younger than %s on any tier (found: %s)", maxAge, describeRestorePoints(now, seen)))
|
||||
fail("update.error.backup_no_unit", fmt.Sprintf("after the backup there is still no copy younger than %s on any tier (found: %s)", maxAge, describeRestorePoints(now, seen)))
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [stacks] update %s: precondition met after the backup — %s copy from %s", name, updateTierName(rp.Tier), rp.ProvenAt.UTC().Format(time.RFC3339))
|
||||
@@ -653,12 +686,12 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
|
||||
// SAFETY DUMP BEFORE THE PIN MOVES — "a minute ago", before any migration can have run.
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhaseSafetyDump) {
|
||||
fail(MsgUpdateJournalFailed, "journal write failed")
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
paths, err := g.SafetyDump(ctx, name)
|
||||
if err != nil {
|
||||
fail(fmt.Sprintf(MsgUpdateDumpFailFmt, err), "safety dump: "+err.Error())
|
||||
fail("update.error.dump_failed", "safety dump: "+err.Error(), err.Error())
|
||||
return
|
||||
}
|
||||
m.logger.Printf("[INFO] [stacks] update %s: safety dump done (%d file(s)) %v", name, len(paths), paths)
|
||||
@@ -668,9 +701,9 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
undoVols, perr := m.planUndoCopies(name)
|
||||
if perr != nil {
|
||||
if se, ok := perr.(*undoSpaceError); ok {
|
||||
fail(undoMsg("err.stacks.update_undo_space", se.need, se.free, updateDiskFloorGiB), "undo copy: "+perr.Error())
|
||||
fail("err.stacks.update_undo_space", "undo copy: "+perr.Error(), se.need, se.free, updateDiskFloorGiB)
|
||||
} else {
|
||||
fail(undoMsg("err.stacks.update_undo_copy_failed"), "undo copy plan: "+perr.Error())
|
||||
fail("err.stacks.update_undo_copy_failed", "undo copy plan: "+perr.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -679,11 +712,11 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
// at any later instant can put it back (Scenario G).
|
||||
prevLive, err := os.ReadFile(st.ComposePath)
|
||||
if err != nil {
|
||||
fail(MsgUpdatePinFailed, "reading the live compose file: "+err.Error())
|
||||
fail("update.error.pin_failed", "reading the live compose file: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, preUpdateComposeFile), prevLive, 0o644); err != nil {
|
||||
fail(MsgUpdateJournalFailed, "saving the pre-update compose copy: "+err.Error())
|
||||
fail("update.error.journal_failed", "saving the pre-update compose copy: "+err.Error())
|
||||
return
|
||||
}
|
||||
entry.PrevCompose = filepath.Join(dir, preUpdateComposeFile)
|
||||
@@ -711,12 +744,12 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
}
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhasePinning) {
|
||||
m.removePreUpdateCopies(dir)
|
||||
fail(MsgUpdateJournalFailed, "journal write failed")
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
if err := m.advancePinToCatalog(name, dir); err != nil {
|
||||
m.pinBack(name, dir, entry)
|
||||
fail(MsgUpdatePinFailed, "advancing the pin: "+err.Error())
|
||||
fail("update.error.pin_failed", "advancing the pin: "+err.Error())
|
||||
return
|
||||
}
|
||||
if cfg := LoadAppConfig(dir); cfg != nil {
|
||||
@@ -726,7 +759,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
env := m.stackEnv(dir)
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhasePulling) {
|
||||
m.pinBack(name, dir, entry)
|
||||
fail(MsgUpdateJournalFailed, "journal write failed")
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
if _, err := m.updateCompose(dir, env, "pull"); err != nil {
|
||||
@@ -734,7 +767,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
// state is the old pin and the old file — put both back.
|
||||
m.pinBack(name, dir, entry)
|
||||
m.logger.Printf("[ERROR] [stacks] update %s: pull failed — pin and definition PUT BACK; the app was not touched. Docker said: %v", name, err)
|
||||
fail(MsgUpdatePullFailed, "pull failed: "+err.Error())
|
||||
fail("update.error.pull_failed", "pull failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -743,7 +776,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
// start again.
|
||||
if !m.enterUpdatePhase(name, &entry, UpdatePhaseCopying) {
|
||||
m.pinBack(name, dir, entry)
|
||||
fail(MsgUpdateJournalFailed, "journal write failed")
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
if err := m.makeUndoCopies(name, dir, env, undoVols, &entry); err != nil {
|
||||
@@ -753,7 +786,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
if _, uerr := m.updateCompose(dir, m.stackEnv(dir), "up", "-d", "--remove-orphans"); uerr != nil {
|
||||
m.logger.Printf("[ERROR] [stacks] update %s: restarting the previous version after the failed copy also failed: %v", name, uerr)
|
||||
}
|
||||
fail(undoMsg("err.stacks.update_undo_copy_failed"), "undo copy: "+err.Error())
|
||||
fail("err.stacks.update_undo_copy_failed", "undo copy: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
@@ -842,19 +875,25 @@ func (m *Manager) failAndHold(ctx context.Context, name, dir string, env []strin
|
||||
m.logger.Printf("[ERROR] [stacks] update %s: stopping the failed app also failed: %v", name, err)
|
||||
}
|
||||
}
|
||||
msg := MsgUpdateHoldUnsaved
|
||||
holdWhy := ""
|
||||
if g := m.guards(); g == nil {
|
||||
m.logger.Printf("[ERROR] [stacks] update %s: no UpdateGuards — the hold CANNOT be recorded", name)
|
||||
} else if err := g.HoldAfterFailedUpdate(name, m.now(), rp, undoState); err != nil {
|
||||
m.logger.Printf("[ERROR] [stacks] update %s: %v", name, err)
|
||||
} else if _, why := g.HoldFor(name); why != "" {
|
||||
msg = why
|
||||
} else if _, w := g.HoldFor(name); w != "" {
|
||||
holdWhy = w
|
||||
m.markUpdateHeld(name)
|
||||
}
|
||||
_ = m.RefreshStatus()
|
||||
m.clearJournal(name)
|
||||
m.removePreUpdateCopies(dir)
|
||||
m.finishUpdate(name, UpdatePhaseFailed, msg)
|
||||
if holdWhy == "" {
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.hold_unsaved", "")
|
||||
} else {
|
||||
// The hold's own sentence (the page renders it per reader through RestoreHoldForLang).
|
||||
m.finishUpdate(name, UpdatePhaseFailed, holdWhy)
|
||||
}
|
||||
m.emitUpdateEvent(UpdateEventHeld, name, entry, rp, holdWhy != "")
|
||||
}
|
||||
|
||||
// pinBack restores the pin, the stored definition and the live file from the journaled copies, and
|
||||
@@ -1147,12 +1186,12 @@ func (m *Manager) RecoverUpdates() []string {
|
||||
case UpdatePhaseChecking, UpdatePhaseBackingUp, UpdatePhaseSafetyDump:
|
||||
m.logger.Printf("[WARN] [stacks] update recovery: %s was interrupted in %s (started %s) — nothing had moved; dropping it", name, e.Phase, e.StartedAt.Format(time.RFC3339))
|
||||
m.clearJournal(name)
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateInterrupted)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.interrupted", "")
|
||||
case UpdatePhasePinning, UpdatePhasePulling:
|
||||
m.logger.Printf("[WARN] [stacks] update recovery: %s was interrupted in %s (started %s) — nothing had run; putting the pin back", name, e.Phase, e.StartedAt.Format(time.RFC3339))
|
||||
m.pinBack(name, dir, e)
|
||||
m.clearJournal(name)
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateInterrupted)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.interrupted", "")
|
||||
case UpdatePhaseCopying:
|
||||
// v0.263.0: the app was STOPPED for the copy and nothing new ran. The partial copies go, the
|
||||
// pin goes back, and the previous version is started again.
|
||||
@@ -1163,7 +1202,7 @@ func (m *Manager) RecoverUpdates() []string {
|
||||
m.logger.Printf("[ERROR] [stacks] update recovery: %s: starting the previous version failed: %v", name, err)
|
||||
}
|
||||
m.clearJournal(name)
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateInterrupted)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.interrupted", "")
|
||||
case UpdatePhaseUndoing:
|
||||
// v0.263.0: a power cut DURING the undo. Resumed like `starting` — the undo runs again from
|
||||
// the copies (still there: they are removed only after the undo succeeded) and then probes.
|
||||
@@ -1205,14 +1244,14 @@ func (m *Manager) ResumeInterruptedUpdates(ctx context.Context) int {
|
||||
for _, name := range names {
|
||||
st, ok := m.GetStack(name)
|
||||
if !ok {
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateInterrupted)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.interrupted", "")
|
||||
continue
|
||||
}
|
||||
m.updateJournalMu.Lock()
|
||||
e, ok := m.readUpdateJournal().Updates[name]
|
||||
m.updateJournalMu.Unlock()
|
||||
if !ok {
|
||||
m.finishUpdate(name, UpdatePhaseFailed, MsgUpdateInterrupted)
|
||||
m.finishUpdateKey(name, UpdatePhaseFailed, "update.error.interrupted", "")
|
||||
continue
|
||||
}
|
||||
provenAt, _ := time.Parse(time.RFC3339, e.ProvenCopyAt)
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/i18n"
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
|
||||
// v0.264.0 — the update's outcome is TOLD (`09` §3 decision 15): one event per undone update, one per
|
||||
// held update, handed to the sink main.go wires to the notifier. And R-646 / R-606 on the stacks side.
|
||||
|
||||
type eventRec struct {
|
||||
mu sync.Mutex
|
||||
evs []UpdateEvent
|
||||
}
|
||||
|
||||
func (r *eventRec) sink(ev UpdateEvent) { r.mu.Lock(); r.evs = append(r.evs, ev); r.mu.Unlock() }
|
||||
func (r *eventRec) list() []UpdateEvent {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]UpdateEvent(nil), r.evs...)
|
||||
}
|
||||
|
||||
// COMPANION RED-PROOF (REPORT.md): delete the emitUpdateEvent call from tryUndo — no event is sent,
|
||||
// and this test fails on the count.
|
||||
func TestUpdateEvents_AnUndoneUpdateIsToldOnce(t *testing.T) {
|
||||
m, _, _, _, _ := newUndoManager(t)
|
||||
rec := &eventRec{}
|
||||
m.SetUpdateEventSink(rec.sink)
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseUndone {
|
||||
t.Fatalf("setup: phase %q", st.UpdatePhase)
|
||||
}
|
||||
evs := rec.list()
|
||||
if len(evs) != 1 || evs[0].Kind != UpdateEventUndone || evs[0].App != "nextcloud" {
|
||||
t.Fatalf("an undone update must produce exactly ONE %s event, got %+v", UpdateEventUndone, evs)
|
||||
}
|
||||
if evs[0].From["web"] != "nextcloud:31.0.14-apache" || evs[0].To["web"] != "nextcloud:34.0.1-apache" {
|
||||
t.Errorf("the event must name the step (from → to), got %+v → %+v", evs[0].From, evs[0].To)
|
||||
}
|
||||
}
|
||||
|
||||
// COMPANION RED-PROOF (REPORT.md): delete the emitUpdateEvent call from failAndHold — a held update
|
||||
// sends nothing, and this test fails on the count.
|
||||
func TestUpdateEvents_AHeldUpdateIsToldOnce(t *testing.T) {
|
||||
m, _, _, _, fc := newUndoManager(t)
|
||||
fc.cutOff = true // the undo fails → HOLD
|
||||
rec := &eventRec{}
|
||||
m.SetUpdateEventSink(rec.sink)
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitUpdateDone(t, m, "nextcloud")
|
||||
evs := rec.list()
|
||||
if len(evs) != 1 || evs[0].Kind != UpdateEventHeld || !evs[0].HoldRecorded {
|
||||
t.Fatalf("a held update must produce exactly ONE recorded %s event, got %+v", UpdateEventHeld, evs)
|
||||
}
|
||||
if evs[0].CopyTier != UpdateTierSecondDrive || evs[0].CopyDate.IsZero() {
|
||||
t.Errorf("the held event must name the copy the hold points at, got tier %d date %v", evs[0].CopyTier, evs[0].CopyDate)
|
||||
}
|
||||
}
|
||||
|
||||
// A hold that could not be SAVED still tells the operator — that is exactly what they must hear.
|
||||
func TestUpdateEvents_AnUnsavedHoldIsStillTold(t *testing.T) {
|
||||
m, _, g, _, fc := newUndoManager(t)
|
||||
fc.cutOff = true
|
||||
g.holdErr = os.ErrPermission
|
||||
rec := &eventRec{}
|
||||
m.SetUpdateEventSink(rec.sink)
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitUpdateDone(t, m, "nextcloud")
|
||||
evs := rec.list()
|
||||
if len(evs) != 1 || evs[0].Kind != UpdateEventHeld || evs[0].HoldRecorded {
|
||||
t.Fatalf("an unsaved hold must still send ONE held event, marked unrecorded; got %+v", evs)
|
||||
}
|
||||
if st.UpdateError != MsgUpdateHoldUnsaved || st.UpdateErrorKey != "update.error.hold_unsaved" {
|
||||
t.Errorf("err=%q key=%q", st.UpdateError, st.UpdateErrorKey)
|
||||
}
|
||||
}
|
||||
|
||||
// R-646. COMPANION RED-PROOF (REPORT.md): make BackfillAppliedMeta skip the storeAppliedMeta call —
|
||||
// the current app then has no record and this test fails.
|
||||
func TestR646_BackfillRecordsOnlyAppsCurrentWithTheCatalog(t *testing.T) {
|
||||
m, dir, _, _, _ := newUndoManager(t) // nextcloud: pinned 31, catalog 34 → Behind
|
||||
if err := os.RemoveAll(filepath.Join(dir, appliedMetaDir)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second app, CURRENT: installed == catalog.
|
||||
cur := filepath.Join(m.cfg.Paths.StacksDir, "vikunja")
|
||||
if err := os.MkdirAll(cur, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mustWrite(t, filepath.Join(cur, "docker-compose.yml"), "services:\n web:\n image: vikunja/vikunja:2.3.0\n")
|
||||
mustWrite(t, filepath.Join(cur, ".felhom.yml"), undoMetaOld)
|
||||
mustWrite(t, filepath.Join(cur, "app.yaml"), "deployed: true\nenv: {}\npinned_images:\n web: vikunja/vikunja:2.3.0\n")
|
||||
m.mu.Lock()
|
||||
m.stacks["vikunja"] = &Stack{Name: "vikunja", Deployed: true, ComposePath: filepath.Join(cur, "docker-compose.yml"),
|
||||
AppConfig: LoadAppConfig(cur),
|
||||
CatalogImages: map[string]string{"web": "vikunja/vikunja:2.3.0"}}
|
||||
m.stacks["vikunja"].AppConfig.InstalledImages = map[string]InstalledImage{"web": {Ref: "vikunja/vikunja:2.3.0"}}
|
||||
m.stacks["nextcloud"].CatalogImages = map[string]string{"web": "nextcloud:34.0.1-apache"}
|
||||
m.stacks["nextcloud"].AppConfig.InstalledImages = map[string]InstalledImage{"web": {Ref: "nextcloud:31.0.14-apache"}}
|
||||
m.mu.Unlock()
|
||||
|
||||
recorded, skipped := m.BackfillAppliedMeta()
|
||||
if len(recorded) != 1 || recorded[0] != "vikunja" || len(skipped) != 1 || skipped[0] != "nextcloud" {
|
||||
t.Fatalf("recorded=%v skipped=%v — want the current app recorded and the behind app skipped by name", recorded, skipped)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(cur, appliedMetaDir, ".felhom.yml")); err != nil {
|
||||
t.Errorf("the current app must have its record: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, appliedMetaDir, ".felhom.yml")); err == nil {
|
||||
t.Error("a BEHIND app must get no record — its pinned version's file is gone, and guessing it is worse")
|
||||
}
|
||||
// Idempotent, and never overwrites.
|
||||
mustWrite(t, filepath.Join(cur, appliedMetaDir, ".felhom.yml"), "marker: kept\n")
|
||||
if rec2, _ := m.BackfillAppliedMeta(); len(rec2) != 0 {
|
||||
t.Errorf("a second pass must record nothing, recorded %v", rec2)
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(cur, appliedMetaDir, ".felhom.yml")); string(b) != "marker: kept\n" {
|
||||
t.Errorf("an existing record must never be overwritten, got %q", b)
|
||||
}
|
||||
}
|
||||
|
||||
// R-606 — the Hungarian is byte-identical to the literals the keys replaced, and the other language is
|
||||
// really the other language.
|
||||
func TestR606_UpdateSentencesHungarianUnchangedAndTranslated(t *testing.T) {
|
||||
for key, lit := range map[string]string{
|
||||
"update.error.interrupted": MsgUpdateInterrupted,
|
||||
"update.error.pull_failed": MsgUpdatePullFailed,
|
||||
"update.error.pin_failed": MsgUpdatePinFailed,
|
||||
"update.error.journal_failed": MsgUpdateJournalFailed,
|
||||
"update.error.backup_no_unit": MsgUpdateBackupNoUnit,
|
||||
"update.error.hold_unsaved": MsgUpdateHoldUnsaved,
|
||||
"update.error.no_guards": MsgUpdateNoGuards,
|
||||
"update.error.backup_failed": MsgUpdateBackupFailFmt,
|
||||
"update.error.dump_failed": MsgUpdateDumpFailFmt,
|
||||
"update.refusal.busy": MsgUpdateBusy,
|
||||
"update.refusal.disk": MsgUpdateDiskFmt,
|
||||
"update.refusal.no_backup": MsgUpdateNoBackupFmt,
|
||||
} {
|
||||
b, _ := i18n.Shared()
|
||||
if got := b.Msg(i18n.Default, key); got != lit {
|
||||
t.Errorf("%s: Hungarian changed:\n got %q\nwant %q", key, got, lit)
|
||||
}
|
||||
if en := b.Msg("en", key); en == "" || en == lit {
|
||||
t.Errorf("%s: no English", key)
|
||||
}
|
||||
}
|
||||
for ph, lbl := range updatePhaseLabels {
|
||||
if got := UpdatePhaseLabelIn(i18n.Default, ph); got != lbl {
|
||||
t.Errorf("phase %s: Hungarian label %q, want %q", ph, got, lbl)
|
||||
}
|
||||
if en := UpdatePhaseLabelIn("en", ph); en == lbl {
|
||||
t.Errorf("phase %s: no English label", ph)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// R-606 — a stored update sentence renders in the reader's language; an old stored one (no key) renders
|
||||
// as stored, never empty.
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT.md): make UpdateErrorIn return s.UpdateError always — the English
|
||||
// assertion fails.
|
||||
func TestR606_UpdateErrorRendersInTheReadersLanguage(t *testing.T) {
|
||||
m, _, _, _, _ := newUndoManager(t)
|
||||
m.finishUpdateKey("nextcloud", UpdatePhaseFailed, "update.error.backup_failed", "", "disk full")
|
||||
st, _ := m.GetStack("nextcloud")
|
||||
if st.UpdateError != util.Text(i18n.Default, "update.error.backup_failed", "disk full") {
|
||||
t.Errorf("UpdateError must stay the Hungarian, got %q", st.UpdateError)
|
||||
}
|
||||
en := st.UpdateErrorIn("en")
|
||||
if en != "The update did not start, because the backup before the update did not succeed: disk full. The app keeps running unchanged." {
|
||||
t.Errorf("English = %q", en)
|
||||
}
|
||||
old := Stack{UpdateError: "egy régi, kulcs nélküli mondat"}
|
||||
if old.UpdateErrorIn("en") != "egy régi, kulcs nélküli mondat" {
|
||||
t.Error("a sentence stored by an older version must render as stored, never as an empty line")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user