v0.270.0: no update for a current app (R-679); an interrupted install is reported (R-681); a restore brings back the pinned version's health check (R-669); R-674
gates / gates (push) Successful in 24s

Five red-proofs. MinAgent 0.131.0 unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 16:49:43 +02:00
parent 7c6140d95b
commit 504eae018b
23 changed files with 1121 additions and 16 deletions
+14
View File
@@ -397,6 +397,16 @@ func (m *Manager) DeployStack(req DeployRequest) (string, error) {
}
m.mu.Unlock()
// R-681: the install's own journal — removed when it ends either way; found at start = interrupted.
if err := markInstallPending(stackDir); err != nil {
m.logger.Printf("[WARN] [stacks] Stack %s: cannot write the install marker (%v) — a restart mid-install would go unreported", req.StackName, err)
}
m.mu.Lock()
if s, ok := m.stacks[req.StackName]; ok {
s.InstallInterrupted = false
}
m.mu.Unlock()
// Run docker compose up -d asynchronously
go m.runComposeDeploy(req.StackName, stackDir, env, appCfg)
@@ -453,6 +463,7 @@ func (m *Manager) runComposeDeploy(name, stackDir string, env map[string]string,
// The app.yaml is deliberately NOT deleted here: it is the crash-safe record written with
// Deployed:false, it carries the settings the customer typed, and a redeploy reuses them. The
// state the surfaces read is `not_deployed`, which is the fact that matters.
clearInstallPending(stackDir) // R-681: the install ended (badly) and said so
if m.deployDoneHook != nil {
m.deployDoneHook(name, false, composeErr.Error())
}
@@ -477,9 +488,12 @@ func (m *Manager) runComposeDeploy(name, stackDir string, env map[string]string,
s.AppConfig = nil
}
m.mu.Unlock()
clearInstallPending(stackDir)
return
}
clearInstallPending(stackDir) // R-681: the durable record says deployed — the install is over
// Clear deploying flag
m.mu.Lock()
if s, ok := m.stacks[name]; ok {
@@ -0,0 +1,100 @@
package stacks
import (
"os"
"path/filepath"
"time"
)
// ── An install cut off by a controller restart is finished, not forgotten (R-681, v0.270.0) ─────────
//
// MEASURED 2026-09-24 (night, chaos round 2): `systemctl restart docker` 20 s into n8n's install took the
// controller down with it. After the restart the box logged NOTHING about n8n, the app read not_deployed
// and the household's page showed it as never installed — the install had simply vanished, while its
// half-written files stayed in the stack dir. An update journals itself and resumes after the same
// accident; an install did not.
//
// THE MECHANISM: DeployStack writes installPendingFile before the compose-up goroutine starts;
// runComposeDeploy removes it when the install ENDS either way (the failure path already reports itself,
// R-536/R-649). A pending marker found at start therefore means the process died mid-install, and
// RecoverInterruptedInstalls finishes it through the SAME failure path a failed install takes:
// - the durable record says Deployed:true → the install had finished; only the marker goes;
// - otherwise → `compose down` (volumes kept, R-649), the stale pin records cleared, the app reads
// not-installed with installInterruptedFile set (the page's sentence, surviving further restarts
// until the next install), and the deploy-done hook fires `app_deploy_failed` with the reason.
// Pinned by r681_install_interrupted_test.go.
const (
installPendingFile = ".felhom-install-pending"
installInterruptedFile = ".felhom-install-interrupted"
)
// installInterruptedReason is the detail the event and the deploy page carry.
const installInterruptedReason = "interrupted by a controller restart before it finished — install it again"
func markInstallPending(stackDir string) error {
_ = os.Remove(filepath.Join(stackDir, installInterruptedFile)) // a new install supersedes the old sentence
return os.WriteFile(filepath.Join(stackDir, installPendingFile), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644)
}
func clearInstallPending(stackDir string) {
_ = os.Remove(filepath.Join(stackDir, installPendingFile))
}
func installInterrupted(stackDir string) bool {
_, err := os.Stat(filepath.Join(stackDir, installInterruptedFile))
return err == nil
}
// RecoverInterruptedInstalls runs once at start, after the deploy-done hook is wired. It returns the
// names it resolved as interrupted.
func (m *Manager) RecoverInterruptedInstalls() []string {
m.mu.RLock()
type cand struct{ name, dir string }
var cands []cand
for name, st := range m.stacks {
dir := filepath.Dir(st.ComposePath)
if _, err := os.Stat(filepath.Join(dir, installPendingFile)); err == nil {
cands = append(cands, cand{name, dir})
}
}
m.mu.RUnlock()
var out []string
for _, c := range cands {
if cfg := LoadAppConfig(c.dir); cfg != nil && cfg.Deployed {
clearInstallPending(c.dir)
m.logger.Printf("[INFO] [stacks] install %s: its record says deployed — the install had finished before the restart; marker cleared (R-681)", c.name)
continue
}
m.logger.Printf("[WARN] [stacks] install %s was INTERRUPTED by a controller restart — removing what it started (volumes kept) and reporting it (R-681)", c.name)
down := m.composeDownFn
if down == nil {
down = func(dir string) error { _, err := m.composeExecCustomEnv(dir, m.stackEnv(dir), "down"); return err }
}
if err := down(c.dir); err != nil {
m.logger.Printf("[ERROR] [stacks] install %s: removing what the interrupted install started failed: %v — Remove clears it", c.name, err)
}
for _, p := range []string{AppliedComposePath(c.dir), filepath.Join(c.dir, appliedMetaDir)} {
_ = os.RemoveAll(p) // the pin of a version that never became real
}
if err := os.WriteFile(filepath.Join(c.dir, installInterruptedFile), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
m.logger.Printf("[WARN] [stacks] install %s: cannot record the interruption for the page: %v", c.name, err)
}
clearInstallPending(c.dir)
m.mu.Lock()
if s, ok := m.stacks[c.name]; ok {
s.Deployed = false
s.Deploying = false
s.AppConfig = nil
s.DeployError = installInterruptedReason
s.InstallInterrupted = true
}
m.mu.Unlock()
if m.deployDoneHook != nil {
m.deployDoneHook(c.name, false, installInterruptedReason)
}
out = append(out, c.name)
}
return out
}
+7
View File
@@ -136,6 +136,13 @@ func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, e
}
}
if idx < 0 {
// R-674 (v0.270.0): a pin equal to the newest entry's `to` is AT THE HEAD, not "older than the
// ladder" — the old sentence sent an operator looking for a missing record. Pinned by
// TestR674_HeadIsNotCalledOlder.
if sameRefs(ladder[len(ladder)-1].To, pinned) {
return LadderStep{Index: -1, Source: current, Meta: currentMeta,
Why: fmt.Sprintf("the installed version %s is AT THE HEAD of the update_ladder (%d entries) — the catalog's current definition", summarisePin(pinned), len(ladder))}, nil
}
return LadderStep{Index: -1, Source: current, Meta: currentMeta,
Why: fmt.Sprintf("the installed version %s matches no update_ladder entry (%d entries) — an app older than the ladder has no record to climb; the catalog's current definition", summarisePin(pinned), len(ladder))}, nil
}
+21 -13
View File
@@ -148,6 +148,9 @@ type Stack struct {
AppConfig *AppConfig `json:"app_config,omitempty"`
Deploying bool `json:"deploying"` // compose up in progress
DeployError string `json:"deploy_error,omitempty"` // last async deploy error
// InstallInterrupted (R-681, v0.270.0): the last install was cut off by a controller restart; the
// page says so until the next install (read from the stack dir's marker at every scan).
InstallInterrupted bool `json:"install_interrupted,omitempty"`
// Updating / UpdatePhase / UpdatePhaseLabel / UpdateError (update arc slice 4, v0.237.0) are the
// guarded update's in-memory progress, the same shape as Deploying/DeployError: the API answers
// 202 at once and the page polls GET /api/stacks/{name}. See update.go.
@@ -267,7 +270,10 @@ type Manager struct {
// --- guarded update (slice 4, update.go) ---
updateGuards UpdateGuards // init-only, SetUpdateGuards; nil ⇒ every update is REFUSED
updateComposeFn func(dir string, env []string, args ...string) (string, error)
updateHealthFn func(ctx context.Context, name string, timeout time.Duration) (bool, string)
// composeDownFn is RecoverInterruptedInstalls' seam (R-681): nil → the real `compose down`. Tests set a
// fake so they never reach real Docker (R-650).
composeDownFn func(dir string) error
updateHealthFn func(ctx context.Context, name string, timeout time.Duration) (bool, string)
// v0.263.0 undo seams (undo.go): the volume copier (nil ⇒ docker) and the undo's health wait,
// which receives the probe it must use (nil ⇒ waitUpdateHealthyMeta).
undoCopier volumeCopier
@@ -654,21 +660,23 @@ func (m *Manager) ScanStacks() error {
existing.CatalogImages = catImages
existing.LadderStepsLeft = stepsLeft
existing.CatalogDigests, existing.CatalogTestedAt = catDigests, catTestedAt
existing.InstallInterrupted = !deployed && installInterrupted(stackDir) // R-681
}
} else {
m.stacks[name] = &Stack{
Name: name,
Meta: meta,
ComposePath: composePath,
State: StateNotDeployed,
Deployed: deployed,
Protected: m.cfg.IsProtectedStack(name),
AppConfig: appCfg,
TemplateImages: tplImages,
CatalogImages: catImages,
LadderStepsLeft: stepsLeft,
CatalogDigests: catDigests,
CatalogTestedAt: catTestedAt,
Name: name,
Meta: meta,
ComposePath: composePath,
State: StateNotDeployed,
Deployed: deployed,
Protected: m.cfg.IsProtectedStack(name),
AppConfig: appCfg,
TemplateImages: tplImages,
CatalogImages: catImages,
LadderStepsLeft: stepsLeft,
InstallInterrupted: !deployed && installInterrupted(stackDir), // R-681
CatalogDigests: catDigests,
CatalogTestedAt: catTestedAt,
}
}
}
@@ -0,0 +1,33 @@
package stacks
import (
"os"
"path/filepath"
"strings"
"testing"
)
// R-669 (v0.270.0): after a restore writes the app's .felhom.yml, that file IS the pinned version's
// record — so the NEXT update's undo judges the old version with the restored probe, not the failed
// step's. Asserted on what the undo actually reads: savePreUpdateMeta's copy.
//
// COMPANION RED-PROOF (REPORT.md): make RecordRestoredAppliedMeta a no-op (v0.269.1: no restore path
// rewrote applied-meta) → "the next undo would judge with the failed step's probe".
func TestR669_RestoreResetsTheAppliedRecord(t *testing.T) {
m, _, _, _ := newSlice4Manager(t)
dir := filepath.Dir(m.stacks["nextcloud"].ComposePath)
if err := os.MkdirAll(filepath.Dir(AppliedMetaFile(dir)), 0o755); err != nil {
t.Fatal(err)
}
mustWrite(t, AppliedMetaFile(dir), "healthcheck:\n checks:\n - type: api\n port: 8999\n") // left by the failed step
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "healthcheck:\n checks:\n - type: api\n port: 80\n") // written by the restore
m.RecordRestoredAppliedMeta("nextcloud", dir)
md, err := savePreUpdateMeta(dir)
if err != nil {
t.Fatal(err)
}
b, _ := os.ReadFile(filepath.Join(md, ".felhom.yml"))
if strings.Contains(string(b), "8999") || !strings.Contains(string(b), "port: 80") {
t.Fatalf("the next undo would judge with the failed step's probe:\n%s", b)
}
}
@@ -0,0 +1,30 @@
package stacks
import (
"path/filepath"
"strings"
"testing"
)
// R-674 (v0.270.0): a pin equal to the ladder's newest `to` is AT THE HEAD — the log must not call it
// "older than the ladder".
//
// COMPANION RED-PROOF (REPORT.md): drop the head branch in nextLadderStep → "the head was called older
// than the ladder".
func TestR674_HeadIsNotCalledOlder(t *testing.T) {
dir := t.TempDir()
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "display_name: X\nupdate_ladder:\n"+ladderLine(ladderA, ladderB)+ladderLine(ladderB, ladderC))
mustWrite(t, filepath.Join(dir, "docker-compose.yml"), "services:\n web:\n image: "+ladderC+"\n")
step, err := nextLadderStep(dir, map[string]string{"web": ladderC})
if err != nil {
t.Fatal(err)
}
if strings.Contains(step.Why, "older than the ladder") || !strings.Contains(step.Why, "AT THE HEAD") {
t.Fatalf("the head was called older than the ladder: %q", step.Why)
}
// Control: a pin the ladder does not know is still said so.
step, _ = nextLadderStep(dir, map[string]string{"web": "nextcloud:30.0.0-apache"})
if !strings.Contains(step.Why, "matches no update_ladder entry") {
t.Fatalf("the unknown-pin sentence changed: %q", step.Why)
}
}
@@ -0,0 +1,48 @@
package stacks
import (
"strings"
"testing"
"time"
)
// R-679 (v0.270.0): an Update pressed on an app already at the catalog head is refused BEFORE anything
// moves — no backup, no pull, no restart — with its own reason and a sentence in the household's
// language. A re-tested digest for the same tag is NOT current and still updates.
//
// COMPANION RED-PROOF (REPORT.md): drop the UpdateOrderCurrent check from UpdatePreflight → "an app at
// the head was allowed to update".
func TestR679_CurrentAppIsRefused(t *testing.T) {
m, _, _, _ := newSlice4Manager(t)
st := m.stacks["nextcloud"]
st.AppConfig.InstalledImages = map[string]InstalledImage{"web": oi("nextcloud:31.0.14-apache")}
st.CatalogImages = map[string]string{"web": "nextcloud:31.0.14-apache"}
ref := m.UpdatePreflight("nextcloud")
if ref == nil {
t.Fatal("an app at the head was allowed to update")
}
if ref.Reason != "already_current" || ref.Cause == nil || !strings.Contains(ref.Message, "legfrissebb") {
t.Fatalf("refusal = %q %q (cause %v) — want already_current, a key-bearing Cause, the Hungarian sentence", ref.Reason, ref.Message, ref.Cause)
}
}
func TestR679_BehindAndRetestedDigestStillUpdate(t *testing.T) {
m, _, _, _ := newSlice4Manager(t)
st := m.stacks["nextcloud"]
st.AppConfig.InstalledImages = map[string]InstalledImage{"web": oi("nextcloud:31.0.13-apache")}
st.CatalogImages = map[string]string{"web": "nextcloud:31.0.14-apache"}
if ref := m.UpdatePreflight("nextcloud"); ref != nil {
t.Fatalf("a behind app was refused: %q", ref.Reason)
}
// same tag, a newer TESTED digest than the install → behind, not current
inst := oi("redis:7-alpine")
inst.Digest = "sha256:" + strings.Repeat("a", 64)
inst.At = time.Now().Add(-48 * time.Hour).UTC().Format(time.RFC3339)
st.AppConfig.InstalledImages = map[string]InstalledImage{"web": inst}
st.CatalogImages = map[string]string{"web": "redis:7-alpine"}
st.CatalogDigests = map[string]string{"web": "sha256:" + strings.Repeat("b", 64)}
st.CatalogTestedAt = time.Now()
if ref := m.UpdatePreflight("nextcloud"); ref != nil {
t.Fatalf("a re-tested digest was refused as current: %q", ref.Reason)
}
}
@@ -0,0 +1,69 @@
package stacks
import (
"os"
"path/filepath"
"testing"
)
// R-681 (v0.270.0): an install a controller restart cut off is FINISHED through the failure path and
// REPORTED — never silent. The consequences asserted: `compose down` ran (never real Docker — a fake), the
// household is told (the deploy-done hook fires as a failure with the reason), the stale pin records are
// gone, and the page's flag survives a rescan.
//
// COMPANION RED-PROOF (REPORT.md): make RecoverInterruptedInstalls return without acting (v0.269.1: no
// install marker, nothing looked) → "an interrupted install was not reported".
func TestR681_InterruptedInstallIsFinishedAndReported(t *testing.T) {
m, dir := newPinManager(t, pinTplOld, pinTplOld, "deployed: false\nenv: {}\n")
m.stacks["nextcloud"].Deployed = false
mustWrite(t, filepath.Join(dir, installPendingFile), "2026-09-24T11:48:42Z\n")
mustWrite(t, AppliedComposePath(dir), pinTplOld) // stale, from an earlier install
var downs []string
m.composeDownFn = func(d string) error { downs = append(downs, d); return nil }
type call struct {
name string
ok bool
detail string
}
var hooks []call
m.SetDeployDoneHook(func(n string, ok bool, d string) { hooks = append(hooks, call{n, ok, d}) })
got := m.RecoverInterruptedInstalls()
if len(got) != 1 || len(hooks) != 1 || hooks[0].ok || hooks[0].detail != installInterruptedReason {
t.Fatalf("an interrupted install was not reported: resolved=%v hooks=%+v", got, hooks)
}
if len(downs) != 1 || downs[0] != dir {
t.Fatalf("what the interrupted install started was not removed: downs=%v", downs)
}
if _, err := os.Stat(AppliedComposePath(dir)); err == nil {
t.Fatal("the stale pin record of a version that never ran is still there")
}
if _, err := os.Stat(filepath.Join(dir, installPendingFile)); err == nil {
t.Fatal("the pending marker survived — the next start would report it again")
}
if err := m.ScanStacks(); err != nil {
t.Fatal(err)
}
if st, _ := m.GetStack("nextcloud"); !st.InstallInterrupted || st.Deployed {
t.Fatalf("after a rescan: interrupted=%v deployed=%v — the page must still say it", st.InstallInterrupted, st.Deployed)
}
// A second start finds nothing to report.
if again := m.RecoverInterruptedInstalls(); len(again) != 0 || len(hooks) != 1 {
t.Fatalf("reported twice: %v", again)
}
}
// An install that FINISHED before the restart (its record says deployed) is left alone: no down, no event.
func TestR681_FinishedInstallIsLeftAlone(t *testing.T) {
m, dir := newPinManager(t, pinTplOld, pinTplOld, "deployed: true\nenv: {}\n")
mustWrite(t, filepath.Join(dir, installPendingFile), "x\n")
downs, hooks := 0, 0
m.composeDownFn = func(string) error { downs++; return nil }
m.SetDeployDoneHook(func(string, bool, string) { hooks++ })
if got := m.RecoverInterruptedInstalls(); len(got) != 0 || downs != 0 || hooks != 0 {
t.Fatalf("a finished install was treated as interrupted: resolved=%v downs=%d hooks=%d", got, downs, hooks)
}
if _, err := os.Stat(filepath.Join(dir, installPendingFile)); err == nil {
t.Fatal("the marker of a finished install was not cleared")
}
}
+12
View File
@@ -86,6 +86,18 @@ const preUpdateMetaDir = "pre-update-meta"
// time, got the new probe (romm :8999), and judged the serving old version "did not start".
const appliedMetaDir = "applied-meta"
// AppliedMetaFile is the pinned version's .felhom.yml record (R-669, v0.270.0: the recovery unit captures
// it, so a restore brings back the PINNED version's health check, not the sync's newer one).
func AppliedMetaFile(stackDir string) string { return filepath.Join(stackDir, appliedMetaDir, ".felhom.yml") }
// RecordRestoredAppliedMeta makes the .felhom.yml a restore just wrote the pinned version's record
// (R-669, v0.270.0). MEASURED 2026-09-24 (A2): after a failed step ended in a restore, applied-meta still
// held the FAILED step's probe, and the next failed update's undo judged the correct old version with it
// and held the app. Pinned by TestR669_RestoreResetsTheAppliedRecord.
func (m *Manager) RecordRestoredAppliedMeta(name, stackDir string) {
m.storeAppliedMetaFrom(name, stackDir, filepath.Join(stackDir, ".felhom.yml"))
}
// storeAppliedMeta records the .felhom.yml of the version just pinned. A failure is logged by the
// caller and never fails the act — without it the undo falls back to the current file, loudly.
func storeAppliedMeta(stackDir string, src []byte) error {
+14
View File
@@ -416,6 +416,20 @@ func (m *Manager) UpdatePreflight(name string) *UpdateRefusal {
fmt.Sprintf("installed is provably NEWER than the catalog on every differing service (installed=%v catalog=%v)",
st.AppConfig.InstalledImages, st.CatalogImages))
}
// R-679 (v0.270.0) — AN APP ALREADY AT THE HEAD IS NOT UPDATED. MEASURED 2026-09-24 (night, Part C):
// navidrome at the catalog head was pressed four times; each press made a safety dump, pulled and
// restarted the app (8.5 s of downtime) and changed nothing. "Current" is CatalogOrder's own verdict:
// the installed images equal the catalog's AND no newer tested digest exists for a floating tag — so a
// re-tested digest still updates. Unknown / unorderable / behind fall through as before. Before this, a
// same-version Update was called "the repair path" in a test comment; Restart is the repair path now.
//
// COMPANION RED-PROOF (REPORT.md): drop this check → TestR679_CurrentAppIsRefused fails at "an app at
// the head was allowed to update".
if CatalogOrder(*st) == UpdateOrderCurrent {
return m.refuseUpdateErr(name, "already_current", util.MsgError("err.stacks.update_already_current"),
fmt.Sprintf("installed equals the catalog head on every service and no newer tested digest (installed=%v catalog=%v)",
st.AppConfig.InstalledImages, st.CatalogImages))
}
// R-475: any tier counts, and an app with no copy at all is backed up first by the job. So the only
// refusal left here is Scenario L — no copy on any tier AND no way to make one now. (With a copy
// but no way to back up, the job still applies the age rule and refuses then if the copy is stale.)
@@ -202,7 +202,6 @@ func TestR524_PreflightRefusesDowngrade(t *testing.T) {
}{
{"ahead — the downgrade is refused", "nextcloud:31.0.15-apache", "nextcloud:31.0.14-apache", "downgrade"},
{"behind — the ordinary update is allowed", "nextcloud:31.0.13-apache", "nextcloud:31.0.14-apache", ""},
{"level — allowed (a same-version update is the repair path)", "nextcloud:31.0.14-apache", "nextcloud:31.0.14-apache", ""},
{"unorderable — allowed, exactly as before v0.260.0", "nextcloud:31-apache", "nextcloud:30-apache", ""},
}
for _, c := range cases {