controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)
gates / gates (push) Successful in 27s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-09-24 12:18:39 +02:00
parent 7c3b3a9694
commit 3c6b49b31c
141 changed files with 3401 additions and 237 deletions
+15
View File
@@ -1,6 +1,8 @@
package stacks
import (
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
"bufio"
"context"
"fmt"
@@ -171,6 +173,19 @@ type HDDDataResponse struct {
Stack string `json:"stack"`
HDDPaths []HDDPath `json:"hdd_paths"`
HasHDDData bool `json:"has_hdd_data"`
// KeepDataOnly (v0.269.0, `09` §3 decision 27, R-666): the app is held and its page says support is
// informed — the remove dialog offers only "remove the app, keep my data", and the API refuses the rest.
KeepDataOnly bool `json:"keep_data_only,omitempty"`
}
// ErrRemoveKeepDataWhileSupport is the refusal of a data-deleting remove while support is informed.
var ErrRemoveKeepDataWhileSupport = util.MsgError("err.stacks.remove_keep_data_while_support")
// RemoveKeepsDataOnly reports decision 27's condition for one app: held, and the hold names no whole
// copy (the page says support is informed).
func (m *Manager) RemoveKeepsDataOnly(name string) bool {
st, ok := m.GetStack(name)
return ok && st.HoldReason != "" && st.HoldNoWholeCopy
}
// HDDPath represents a single HDD bind mount path and its status.
+115
View File
@@ -0,0 +1,115 @@
package stacks
import (
"path/filepath"
"regexp"
"strings"
"time"
)
// ── Exact image fingerprints on the box (`09` §6.4 part 6, box half; §3 decision 17; v0.269.0) ────
//
// The catalog records, per ladder entry, the registry digest of every `to` ref at the moment the step was
// tested (`scripts/image_digest.py`). The box:
//
// 1. RENDERS `name:tag@sha256:…` into the compose file it runs whenever the entry for exactly those refs
// carries a digest — so a pull fetches the TESTED image, not whatever the tag points at today.
// Docker and Compose accept the form and refuse a digest that does not exist (measured on 9202,
// 2026-09-23, `audits/update-rulings-2026-09-23/70-…`).
// 2. Keeps every PIN and every RECORD digest-free: ParseComposeImages strips `@…`, and the installed
// record's Ref is stripped too (its Digest is a field of its own). One strip at each door, so the
// pin, the ladder, the badge and the syncer all compare plain `name:tag`.
// 3. Reads the badge from the TESTED digest only — never a registry query (`09` §8.1): same tag, a
// different installed digest, and a catalog test NEWER than the install → „Frissítés elérhető".
var digestRe = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
// StripDigest removes a `@sha256:…` suffix from an image reference.
func StripDigest(ref string) string {
if at := strings.LastIndex(ref, "@"); at >= 0 {
return ref[:at]
}
return ref
}
var serviceLineRe = regexp.MustCompile(`^ ([A-Za-z0-9_-]+):\s*$`)
var imageLineRe = regexp.MustCompile(`^(\s+image:\s*)["']?([^\s"'#]+)["']?(.*)$`)
// renderDigests rewrites each service's own `image:` line to `ref@digest` when digests carries a valid
// one for that service. Line-based (the catalog's own reading, `ladder.images_in`), so comments and
// every other byte are kept. A service with no valid digest keeps its line.
func renderDigests(compose []byte, digests map[string]string) []byte {
if len(digests) == 0 {
return compose
}
lines := strings.Split(string(compose), "\n")
svc := ""
done := map[string]bool{}
inServices := false
for i, l := range lines {
if strings.HasPrefix(l, "services:") {
inServices = true
continue
}
if l != "" && !strings.HasPrefix(l, " ") && !strings.HasPrefix(l, "#") {
inServices = false
}
if !inServices {
continue
}
if m := serviceLineRe.FindStringSubmatch(l); m != nil {
svc = m[1]
continue
}
m := imageLineRe.FindStringSubmatch(l)
if m == nil || svc == "" || done[svc] {
continue
}
d := digests[svc]
if !digestRe.MatchString(d) {
continue
}
lines[i] = m[1] + StripDigest(m[2]) + "@" + d + m[3]
done[svc] = true
}
return []byte(strings.Join(lines, "\n"))
}
// ladderEntryFor returns the NEWEST ladder entry whose `to` is exactly these (digest-free) refs.
func ladderEntryFor(templateDir string, refs map[string]string) (LadderEntry, bool) {
ladder, err := LoadLadder(filepath.Join(templateDir, ".felhom.yml"))
if err != nil {
return LadderEntry{}, false
}
for i := len(ladder) - 1; i >= 0; i-- {
if sameRefs(ladder[i].To, refs) {
return ladder[i], true
}
}
return LadderEntry{}, false
}
// RenderWithLadderDigests is what the syncer and the update write: the compose bytes with the tested
// digests of the ladder entry for exactly its refs. No entry, or no digest → the bytes unchanged.
func RenderWithLadderDigests(templateDir string, compose []byte) []byte {
refs, err := parseComposeImagesBytes(compose)
if err != nil || len(refs) == 0 {
return compose
}
e, ok := ladderEntryFor(templateDir, refs)
if !ok {
return compose
}
return renderDigests(compose, e.Digest)
}
// catalogTestedDigests is the badge's input: the tested digest per service of the catalog's current
// refs, and when that test ran. Empty when the ladder has no entry for them.
func catalogTestedDigests(templateDir string, catalogRefs map[string]string) (map[string]string, time.Time) {
e, ok := ladderEntryFor(templateDir, catalogRefs)
if !ok {
return nil, time.Time{}
}
t, _ := time.Parse(time.RFC3339, e.TestedAt)
return e.Digest, t
}
+80
View File
@@ -0,0 +1,80 @@
package stacks
import (
"path/filepath"
"strings"
"testing"
"time"
)
var dA = "sha256:" + strings.Repeat("a", 64)
var dB = "sha256:" + strings.Repeat("b", 64)
func TestDigest_RenderAndStrip(t *testing.T) {
in := "# header\nservices:\n web:\n image: redis:7-alpine # pinned\n restart: unless-stopped\n db:\n image: \"mariadb:11.8\"\nvolumes:\n x:\n"
out := string(renderDigests([]byte(in), map[string]string{"web": dA, "db": "not-a-digest"}))
if !strings.Contains(out, " image: redis:7-alpine@"+dA+" # pinned") {
t.Fatalf("web not rendered:\n%s", out)
}
if !strings.Contains(out, " image: \"mariadb:11.8\"") {
t.Fatalf("an invalid digest must leave the line alone:\n%s", out)
}
again := string(renderDigests([]byte(out), map[string]string{"web": dB}))
if strings.Count(again, "@sha256:") != 1 || !strings.Contains(again, "redis:7-alpine@"+dB) {
t.Fatalf("re-render must replace, not stack, the digest:\n%s", again)
}
imgs, err := parseComposeImagesBytes([]byte(again))
if err != nil || imgs["web"] != "redis:7-alpine" {
t.Fatalf("the parsed ref must be digest-free: %v %v", imgs, err)
}
}
// TestDigest_FloatingTagBehindOnlyForANewerTestedDigest — the badge rule, every arm.
//
// COMPANION RED-PROOF (REPORT): make digestBehind return false — the "newer test" case then reads
// Current and this test fails.
func TestDigest_FloatingTagBehindOnlyForANewerTestedDigest(t *testing.T) {
installAt := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
mk := func(instDigest, instAt string, cat map[string]string, testedAt time.Time) Stack {
return Stack{Deployed: true, CatalogImages: map[string]string{"web": "redis:7-alpine"},
CatalogDigests: cat, CatalogTestedAt: testedAt,
AppConfig: &AppConfig{InstalledImages: map[string]InstalledImage{"web": {Ref: "redis:7-alpine", Digest: instDigest, At: instAt}}}}
}
cases := []struct {
name string
s Stack
want UpdateOrder
}{
{"same digest", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dA}, installAt.Add(time.Hour)), UpdateOrderCurrent},
{"newer tested digest", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderBehind},
{"test older than the install", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(-time.Hour)), UpdateOrderCurrent},
{"install time unknown", mk(dA, "", map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderCurrent},
{"installed digest unknown", mk("", installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderCurrent},
{"no tested digest", mk(dA, installAt.Format(time.RFC3339), nil, time.Time{}), UpdateOrderCurrent},
}
for _, c := range cases {
if got := CatalogOrder(c.s); got != c.want {
t.Errorf("%s: %v, want %v", c.name, got, c.want)
}
}
}
// The update runs EXACTLY the tested image: the pinned step's compose carries the entry's digest, while
// the pin itself stays a plain ref.
func TestDigest_TheUpdateRendersTheStepsTestedDigest(t *testing.T) {
m, dir, _, _, _ := ladderManager(t, true)
catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
// the ladder fixture's digests are sha256:aaaa…
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
t.Fatal(err)
}
waitUpdateDone(t, m, "nextcloud")
body := fileBody(t, ComposePathIn(dir))
if !strings.Contains(body, "image: "+ladderB+"@"+dA) {
t.Fatalf("the live compose does not pin the tested digest:\n%s", body)
}
if pinOf(t, dir) != ladderB {
t.Fatalf("the pin must stay a plain ref, got %q", pinOf(t, dir))
}
_ = catDir
}
+14 -3
View File
@@ -92,16 +92,27 @@ func ParseComposeImages(composePath string) (map[string]string, error) {
if err != nil {
return nil, fmt.Errorf("reading compose file: %w", err)
}
out, err := parseComposeImagesBytes(data)
if err != nil {
return nil, fmt.Errorf("parsing compose file %s: %w", composePath, err)
}
return out, nil
}
// parseComposeImagesBytes is ParseComposeImages over bytes. v0.269.0 (`09` §6.4 part 6): a rendered
// `name:tag@sha256:…` is returned WITHOUT its digest — pins, the ladder, the badge and the syncer compare
// plain refs; the digest lives only in the compose file that runs (digest.go).
func parseComposeImagesBytes(data []byte) (map[string]string, error) {
var doc composeImagesDoc
if err := yaml.Unmarshal(data, &doc); err != nil {
return nil, fmt.Errorf("parsing compose file %s: %w", composePath, err)
return nil, err
}
out := make(map[string]string, len(doc.Services))
for svc, def := range doc.Services {
if strings.TrimSpace(def.Image) == "" {
continue
}
out[svc] = strings.TrimSpace(def.Image)
out[svc] = StripDigest(strings.TrimSpace(def.Image))
}
return out, nil
}
@@ -212,7 +223,7 @@ func (m *Manager) observeInstalledImages(stackDir string, env []string) (map[str
continue
}
out[svc] = InstalledImage{
Ref: f.ref,
Ref: StripDigest(f.ref), // v0.269.0: a digest-pinned run records the plain ref; the digest is Digest
Digest: pickDigest(f.ref, digests[f.imageID]),
At: now,
}
+40 -5
View File
@@ -42,6 +42,8 @@ type LadderEntry struct {
To map[string]string `yaml:"to" json:"to"`
Digest map[string]string `yaml:"digest" json:"digest"`
Verdict string `yaml:"verdict" json:"verdict"`
// TestedAt is when the step was proven (RFC3339) — the badge compares it with the install (v0.269.0).
TestedAt string `yaml:"tested_at" json:"tested_at"`
}
type ladderDoc struct {
@@ -80,6 +82,11 @@ func StepFile(templateDir string, to map[string]string) string {
return filepath.Join(templateDir, "steps", StepKey(to)+".yml")
}
// StepMetaFile is the step's own `.felhom.yml` (R-664, v0.269.0): its probe, memory and applied record.
func StepMetaFile(templateDir string, to map[string]string) string {
return filepath.Join(templateDir, "steps", StepKey(to)+".felhom.yml")
}
func sameRefs(a, b map[string]string) bool {
if len(a) != len(b) {
return false
@@ -100,6 +107,10 @@ type LadderStep struct {
Left int
// Source is the compose file the step pins: a steps/ file, or the template's docker-compose.yml.
Source string
// Meta is the `.felhom.yml` that belongs to Source (R-664): steps/<key>.felhom.yml when the catalog
// carries it, else the template's own — never the stack dir's, which a restore may have rewritten
// with an older one (R-665).
Meta string
// Why is one operator-English sentence for the log.
Why string
}
@@ -109,12 +120,13 @@ type LadderStep struct {
// update refuses before anything moves (a jump past a tested step is the thing this exists to stop).
func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, error) {
current := filepath.Join(templateDir, "docker-compose.yml")
currentMeta := filepath.Join(templateDir, ".felhom.yml")
ladder, err := LoadLadder(filepath.Join(templateDir, ".felhom.yml"))
if err != nil && !os.IsNotExist(err) {
return LadderStep{}, err
}
if len(ladder) == 0 {
return LadderStep{Index: -1, Source: current, Why: "the template carries no update_ladder — the catalog's current definition"}, nil
return LadderStep{Index: -1, Source: current, Meta: currentMeta, Why: "the template carries no update_ladder — the catalog's current definition"}, nil
}
idx := -1
for i := len(ladder) - 1; i >= 0; i-- { // the NEWEST entry whose `from` is what runs
@@ -124,12 +136,12 @@ func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, e
}
}
if idx < 0 {
return LadderStep{Index: -1, Source: current,
return LadderStep{Index: -1, Source: current, Meta: currentMeta,
Why: fmt.Sprintf("the installed version %s matches no update_ladder entry (%d entries) — an app older than the ladder has no record to climb; the catalog's current definition", summarisePin(pinned), len(ladder))}, nil
}
left := len(ladder) - idx
if idx == len(ladder)-1 {
return LadderStep{Index: idx, Left: left, Source: current,
return LadderStep{Index: idx, Left: left, Source: current, Meta: currentMeta,
Why: fmt.Sprintf("the last step (%d of %d) — the catalog's current definition", idx+1, len(ladder))}, nil
}
src := StepFile(templateDir, ladder[idx].To)
@@ -140,8 +152,12 @@ func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, e
if !sameRefs(imgs, ladder[idx].To) {
return LadderStep{}, fmt.Errorf("step %d of %d: %s names %s, the ladder says %s", idx+1, len(ladder), src, summarisePin(imgs), summarisePin(ladder[idx].To))
}
return LadderStep{Index: idx, Left: left, Source: src,
Why: fmt.Sprintf("step %d of %d: %s → %s, from %s", idx+1, len(ladder), summarisePin(ladder[idx].From), summarisePin(ladder[idx].To), filepath.Base(src))}, nil
meta := StepMetaFile(templateDir, ladder[idx].To)
if _, err := os.Stat(meta); err != nil {
meta = currentMeta // a step written before R-664: the template's own, said in the log
}
return LadderStep{Index: idx, Left: left, Source: src, Meta: meta,
Why: fmt.Sprintf("step %d of %d: %s → %s, from %s (probe from %s)", idx+1, len(ladder), summarisePin(ladder[idx].From), summarisePin(ladder[idx].To), filepath.Base(src), filepath.Base(meta))}, nil
}
// ladderStepsLeft is the page's count: how many tested steps separate this pin from the catalog's
@@ -161,3 +177,22 @@ func ladderStepsLeft(templateDir string, pinned map[string]string) int {
}
return 0
}
// loadMetadataFile reads a `.felhom.yml` that is not named `.felhom.yml` (a step's
// `steps/<key>.felhom.yml`) through LoadMetadata — the ONE validating reader — by giving it a scratch
// directory. Only the health check and the resources are read from the result.
func loadMetadataFile(path string) (Metadata, error) {
data, err := os.ReadFile(path)
if err != nil {
return Metadata{}, err
}
tmp, err := os.MkdirTemp("", "felhom-step-meta-")
if err != nil {
return Metadata{}, err
}
defer os.RemoveAll(tmp)
if err := os.WriteFile(filepath.Join(tmp, ".felhom.yml"), data, 0o600); err != nil {
return Metadata{}, err
}
return LoadMetadata(tmp), nil
}
+6 -4
View File
@@ -82,10 +82,12 @@ func TestLadder_TwoPressesTwoSteps(t *testing.T) {
if got := pinOf(t, dir); got != ladderB {
t.Fatalf("press 1 pinned %s, want the tested step B %s — one press must be one step", got, ladderB)
}
if body := fileBody(t, ComposePathIn(dir)); body != ladderBDef {
t.Fatalf("press 1 rendered\n%s\nwant step B's OWN definition (with STEP_B_OWN_DEFINITION)", body)
// v0.269.0: the step's own definition, run with its TESTED digest (`09` §6.4 part 6)
wantB := string(renderDigests([]byte(ladderBDef), map[string]string{"web": "sha256:" + strings.Repeat("a", 64)}))
if body := fileBody(t, ComposePathIn(dir)); body != wantB {
t.Fatalf("press 1 rendered\n%s\nwant step B's OWN definition (with STEP_B_OWN_DEFINITION) and its digest", body)
}
if body := fileBody(t, AppliedComposePath(dir)); body != ladderBDef {
if body := fileBody(t, AppliedComposePath(dir)); body != wantB {
t.Fatal("the stored applied definition is not step B's — the sync would render the wrong file while the app sits at B")
}
if n := ladderStepsLeft(catDir, map[string]string{"web": ladderB}); n != 1 {
@@ -99,7 +101,7 @@ func TestLadder_TwoPressesTwoSteps(t *testing.T) {
if st.UpdatePhase != UpdatePhaseDone || pinOf(t, dir) != ladderC {
t.Fatalf("press 2 ended %q on %s, want done on C", st.UpdatePhase, pinOf(t, dir))
}
if body := fileBody(t, ComposePathIn(dir)); body != pinTplNew {
if body := fileBody(t, ComposePathIn(dir)); body != string(renderDigests([]byte(pinTplNew), map[string]string{"web": "sha256:" + strings.Repeat("a", 64)})) {
t.Fatalf("press 2 must render the catalog's current definition; got\n%s", body)
}
if strings.Join(*ups, ",") != ladderB+","+ladderC {
+24 -3
View File
@@ -170,9 +170,12 @@ type Stack struct {
HoldReason string `json:"hold_reason,omitempty"`
// HoldNoWholeCopy (v0.268.0, R-659): the hold names NO copy — none on this box brings the app back
// whole — so the page offers no restore button beside the sentence (the restore would refuse).
HoldNoWholeCopy bool `json:"hold_no_whole_copy,omitempty"`
HealthProbe *HealthProbeResult `json:"health_probe,omitempty"` // controller-side probe result
LastUpdated time.Time `json:"last_updated"`
HoldNoWholeCopy bool `json:"hold_no_whole_copy,omitempty"`
// HoldKind (v0.269.0): which hold — "update_failed", "unhealthy_stop" (decision 28: the page offers
// Start, not a restore), "restore"; "" when none.
HoldKind string `json:"hold_kind,omitempty"`
HealthProbe *HealthProbeResult `json:"health_probe,omitempty"` // controller-side probe result
LastUpdated time.Time `json:"last_updated"`
// RestartingSince (C9-F2) is when this stack was FIRST observed in StateRestarting during the
// current restarting run; zero whenever the stack is in any other state. It is what turns a brief
// restart (normal: deploy, update, quiesce restart) into a distinguishable crash loop — see
@@ -205,10 +208,22 @@ type Stack struct {
// pin and the catalog's head; one press climbs one. 0 = unknown or none. The page shows it while
// the badge says „Frissítés elérhető".
LadderStepsLeft int `json:"ladder_steps_left,omitempty"`
// CatalogDigests / CatalogTestedAt (v0.269.0, `09` §6.4 part 6): the TESTED digest per service of the
// catalog's current refs and when that test ran — the badge's input for a floating tag (never a
// registry query). Empty when the ladder has no entry for them.
CatalogDigests map[string]string `json:"catalog_digests,omitempty"`
CatalogTestedAt time.Time `json:"catalog_tested_at,omitempty"`
}
// Manager handles all docker compose stack operations.
type Manager struct {
// lastVerifyMeta (v0.269.0) is the .felhom.yml the last update verify judged by — read by a test.
lastVerifyMeta string
// updateHealthMetaFn (test seam, v0.269.0): the verify with the chosen .felhom.yml; nil → the real wait.
updateHealthMetaFn func(ctx context.Context, name string, timeout time.Duration, meta *Metadata) (bool, string)
// unhealthy (v0.269.0, decision 28): RestartCount / OOM-kill samples per app for the crash-loop stop.
unhealthy unhealthyWatch
// selfUpdating (v0.261.0) reports whether the CONTROLLER is swapping itself. Set by
// SetSelfUpdatingCheck; nil means no gate. See update.go.
selfUpdating func() bool
@@ -611,6 +626,8 @@ func (m *Manager) ScanStacks() error {
// catalog template by definition, and warning once per app per scan would be noise.
var catImages map[string]string
stepsLeft := 0
var catDigests map[string]string
var catTestedAt time.Time
if deployed && !m.cfg.IsProtectedStack(name) {
catPath := m.CatalogTemplatePath(name, "docker-compose.yml")
if appCfg != nil {
@@ -618,6 +635,7 @@ func (m *Manager) ScanStacks() error {
}
if imgs, cerr := ParseComposeImages(catPath); cerr == nil {
catImages = imgs
catDigests, catTestedAt = catalogTestedDigests(filepath.Dir(catPath), imgs)
} else if m.isDebug() {
m.logger.Printf("[DEBUG] [stacks] ScanStacks: no readable catalog template for %s (%v) — the update badge will render nothing", name, cerr)
}
@@ -635,6 +653,7 @@ func (m *Manager) ScanStacks() error {
existing.TemplateImages = tplImages
existing.CatalogImages = catImages
existing.LadderStepsLeft = stepsLeft
existing.CatalogDigests, existing.CatalogTestedAt = catDigests, catTestedAt
}
} else {
m.stacks[name] = &Stack{
@@ -648,6 +667,8 @@ func (m *Manager) ScanStacks() error {
TemplateImages: tplImages,
CatalogImages: catImages,
LadderStepsLeft: stepsLeft,
CatalogDigests: catDigests,
CatalogTestedAt: catTestedAt,
}
}
}
+8 -5
View File
@@ -328,12 +328,12 @@ func (m *Manager) CatalogTemplatePath(appName, filename string) string {
// the syncer has already copied the catalog verbatim into its stack dir, so `pull` + `up -d` do
// today's job with no help from here.
func (m *Manager) advancePinToCatalog(name, stackDir string) error {
return m.advancePinTo(name, stackDir, m.CatalogTemplatePath(name, "docker-compose.yml"))
return m.advancePinTo(name, stackDir, m.CatalogTemplatePath(name, "docker-compose.yml"), m.CatalogTemplatePath(name, ".felhom.yml"))
}
// advancePinTo is advancePinToCatalog with the definition named: the catalog's current compose file,
// or — on a ladder (v0.268.0, `09` §3 decision 14) — one step's own definition from `steps/`.
func (m *Manager) advancePinTo(name, stackDir, src string) error {
func (m *Manager) advancePinTo(name, stackDir, src, metaSrc string) error {
cfg := LoadAppConfig(stackDir)
if cfg == nil || len(cfg.PinnedImages) == 0 {
return nil // unpinned — today's behaviour, unchanged
@@ -346,6 +346,8 @@ func (m *Manager) advancePinTo(name, stackDir, src string) error {
return fmt.Errorf("cannot read the catalog's current definition for %s (%s): %w", name, src, err)
}
// v0.269.0 (`09` §6.4 part 6): run EXACTLY the tested images — the ladder entry's digests for these refs.
data = RenderWithLadderDigests(filepath.Dir(m.CatalogTemplatePath(name, "docker-compose.yml")), data)
live := ComposePathIn(stackDir)
if err := StoreAppliedDefinition(stackDir, data); err != nil {
return fmt.Errorf("storing the new applied definition for %s: %w", name, err)
@@ -365,8 +367,9 @@ func (m *Manager) advancePinTo(name, stackDir, src string) error {
}
m.mu.Unlock()
// v0.263.2: the new version's own .felhom.yml becomes the pinned version's record.
m.storeAppliedMetaFrom(name, stackDir, m.CatalogTemplatePath(name, ".felhom.yml"))
m.logger.Printf("[INFO] [stacks] update %s: pin advanced to the catalog's current definition (%s)", name, summarisePin(pin))
// v0.263.2: the new version's own .felhom.yml becomes the pinned version's record — the STEP's own
// when the ladder carries one (R-664, v0.269.0).
m.storeAppliedMetaFrom(name, stackDir, metaSrc)
m.logger.Printf("[INFO] [stacks] update %s: pin advanced to %s (%s)", name, src, summarisePin(pin))
return nil
}
@@ -0,0 +1,59 @@
package stacks
import (
"context"
"path/filepath"
"testing"
"time"
)
// R-665 + R-664 (v0.269.0) — the new version is judged by ITS OWN .felhom.yml: the catalog's, or the
// ladder step's; never the stack dir's, which a restore rewrites with the unit's older file.
//
// COMPANION RED-PROOF (REPORT): make updateHealthFor ignore metaFile (the v0.268.0 behaviour) — the probe
// port read is then the RESTORED 1111 and both cases fail.
func probePortSeen(t *testing.T, m *Manager) *int {
t.Helper()
port := new(int)
m.updateHealthMetaFn = func(_ context.Context, _ string, _ time.Duration, meta *Metadata) (bool, string) {
if meta != nil && meta.HealthCheck != nil && len(meta.HealthCheck.Checks) > 0 {
*port = meta.HealthCheck.Checks[0].Port
}
return true, "fake"
}
return port
}
func TestR665_TheVerifyUsesTheNewVersionsOwnFile(t *testing.T) {
m, dir, _, _ := newSlice4Manager(t)
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 1111\n") // what a restore wrote
mustWrite(t, m.CatalogTemplatePath("nextcloud", ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 2222\n")
port := probePortSeen(t, m)
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
t.Fatal(err)
}
if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseDone {
t.Fatalf("ended %q", st.UpdatePhase)
}
if *port != 2222 {
t.Fatalf("verified with probe port %d, want the catalog's 2222 (1111 = the restored file, R-665)", *port)
}
}
func TestR664_AStepIsVerifiedWithItsOwnFile(t *testing.T) {
m, dir, _, _, _ := ladderManager(t, true)
catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 1111\n")
mustWriteMk(t, StepMetaFile(catDir, map[string]string{"web": ladderB}), "healthcheck:\n checks:\n - type: http\n port: 3333\n")
port := probePortSeen(t, m)
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
t.Fatal(err)
}
waitUpdateDone(t, m, "nextcloud")
if *port != 3333 {
t.Fatalf("step B verified with probe port %d, want its own 3333", *port)
}
if got := fileBody(t, filepath.Join(dir, appliedMetaDir, ".felhom.yml")); got != "healthcheck:\n checks:\n - type: http\n port: 3333\n" {
t.Fatalf("applied-meta after step B is not the step's own file: %q", got)
}
}
+190
View File
@@ -0,0 +1,190 @@
package stacks
import (
"sort"
"strconv"
"strings"
"sync"
"time"
)
// ── A crash loop and an out-of-memory storm are stopped by the box (`09` §3 decision 28, R-667) ────
//
// WHY THE OLD DETECTOR MISSED IT. Stack.CrashLooping asked for crashLoopAfter (5 min) of UNINTERRUPTED
// `restarting`, measured by RestartingSince — a clock the status pass resets whenever it catches the
// container `running` between two crashes. Measured on 9202 2026-09-24: gokapi at 385 → 546 restarts,
// `restarting_since` a minute old at every look, „0 currently down".
//
// SO THIS COUNTS WHAT DOCKER COUNTS: each container's RestartCount, summed per app, sampled every scan.
// RestartCount only grows for one container run and resets when compose RECREATES the container — a
// drop is a reset and starts the window again, never a negative count.
//
// THE THRESHOLDS, from evidence (`audits/night-2026-09-24/A3/`):
// - crash loop: >= CrashLoopRestarts (6) restarts within CrashLoopWindow (10 min). Docker's restart
// back-off caps a steady crash loop at about ONE restart a minute (gokapi: 539 → 546 in 7 min), so
// the brief's "10 in 10 minutes" sits on the edge and can miss a steady loop. Across all 40
// containers of both demo boxes and 9202 no healthy container had restarted more than ONCE.
// - out-of-memory storm: the existing app_oom_storm rule — >= 20 kernel OOM kills within 30 min.
const (
CrashLoopRestarts = 6
CrashLoopWindow = 10 * time.Minute
OOMStormKills = 20
OOMStormWindow = 30 * time.Minute
UnhealthyCrashLoop = "crash_loop"
UnhealthyOOMStorm = "oom_storm"
)
type unhealthySample struct {
at time.Time
restarts int64
kills int64 // -1: unknown
}
// UnhealthyVerdict is one app that crossed a threshold in the latest observation.
type UnhealthyVerdict struct {
Stack string
Kind string // UnhealthyCrashLoop / UnhealthyOOMStorm
Count int64 // restarts or kills inside the window
Window time.Duration
}
type unhealthyWatch struct {
mu sync.Mutex
samples map[string][]unhealthySample
}
// judgeUnhealthy is the pure verdict over one app's samples (oldest first, the last one = now).
func judgeUnhealthy(samples []unhealthySample) (kind string, count int64, window time.Duration) {
if len(samples) < 2 {
return "", 0, 0
}
last := samples[len(samples)-1]
// A drop in either counter is a reset (the container was recreated): only samples after it count.
start := 0
for i := 1; i < len(samples); i++ {
if samples[i].restarts < samples[i-1].restarts {
start = i
}
}
for i := start; i < len(samples); i++ {
if last.at.Sub(samples[i].at) <= CrashLoopWindow {
if d := last.restarts - samples[i].restarts; d >= CrashLoopRestarts {
return UnhealthyCrashLoop, d, CrashLoopWindow
}
break
}
}
if last.kills >= 0 {
kstart := 0
for i := 1; i < len(samples); i++ {
if samples[i].kills >= 0 && samples[i-1].kills >= 0 && samples[i].kills < samples[i-1].kills {
kstart = i
}
}
for i := kstart; i < len(samples); i++ {
if samples[i].kills < 0 {
continue
}
if last.at.Sub(samples[i].at) <= OOMStormWindow {
if d := last.kills - samples[i].kills; d >= OOMStormKills {
return UnhealthyOOMStorm, d, OOMStormWindow
}
break
}
}
}
return "", 0, 0
}
// ObserveUnhealthy takes one sample per deployed app — RestartCount summed over its containers (one
// `docker inspect` for all of them) and the kernel OOM kills from the latest ScanOOMKilled — and returns
// the apps whose window crossed a threshold. An app that is deploying, updating or already held is not
// sampled and its history is dropped (a deploy or an update recreates containers on purpose).
func (m *Manager) ObserveUnhealthy(now time.Time, ooms []OOMContainer) []UnhealthyVerdict {
type appC struct{ names []string }
m.mu.RLock()
apps := map[string]*appC{}
owner := map[string]string{}
var all []string
skip := map[string]bool{}
for name, st := range m.stacks {
if !st.Deployed || st.Protected {
continue
}
if st.Deploying || st.Updating || st.HoldReason != "" || st.updateHeld {
skip[name] = true
continue
}
a := &appC{}
for _, c := range st.Containers {
a.names = append(a.names, c.Name)
owner[c.Name] = name
all = append(all, c.Name)
}
apps[name] = a
}
m.mu.RUnlock()
restarts := map[string]int64{}
if len(all) > 0 {
sort.Strings(all)
args := append([]string{"inspect", "-f", "{{.Name}}|{{.RestartCount}}"}, all...)
out, _ := m.execCommand("docker", args...) // a vanished container fails its own line only
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.SplitN(strings.TrimSpace(line), "|", 2)
if len(f) != 2 {
continue
}
n, err := strconv.ParseInt(strings.TrimSpace(f[1]), 10, 64)
if err != nil {
continue
}
if app, ok := owner[strings.TrimPrefix(f[0], "/")]; ok {
restarts[app] += n
}
}
}
kills := map[string]int64{}
for _, o := range ooms {
if o.Kills >= 0 {
kills[o.Stack] += o.Kills
}
}
m.unhealthy.mu.Lock()
defer m.unhealthy.mu.Unlock()
if m.unhealthy.samples == nil {
m.unhealthy.samples = map[string][]unhealthySample{}
}
for name := range m.unhealthy.samples {
if _, live := apps[name]; !live {
delete(m.unhealthy.samples, name) // removed, deploying, updating or held: history dropped
}
}
var out []UnhealthyVerdict
for name := range apps {
if skip[name] {
continue
}
k := int64(-1)
if v, ok := kills[name]; ok {
k = v
} else {
k = 0
}
ss := append(m.unhealthy.samples[name], unhealthySample{at: now, restarts: restarts[name], kills: k})
// keep 35 minutes of history — the longer window plus a margin
for len(ss) > 1 && now.Sub(ss[0].at) > OOMStormWindow+5*time.Minute {
ss = ss[1:]
}
m.unhealthy.samples[name] = ss
if kind, n, win := judgeUnhealthy(ss); kind != "" {
out = append(out, UnhealthyVerdict{Stack: name, Kind: kind, Count: n, Window: win})
delete(m.unhealthy.samples, name) // one verdict per episode
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Stack < out[j].Stack })
return out
}
@@ -0,0 +1,102 @@
package stacks
import (
"fmt"
"io"
"log"
"strings"
"testing"
"time"
)
var uT0 = time.Date(2026, 9, 24, 9, 22, 0, 0, time.UTC)
// gokapi's measured shape: Docker's back-off caps a steady crash loop at ~1 restart a minute (539 → 546
// in 7 min on 9202). Sampled every 30 s, as the dead-app loop does.
func gokapiSamples(minutes int) []unhealthySample {
var ss []unhealthySample
for i := 0; i <= minutes*2; i++ {
ss = append(ss, unhealthySample{at: uT0.Add(time.Duration(i) * 30 * time.Second), restarts: 539 + int64(i/2), kills: 0})
}
return ss
}
// TestR667_TheMeasuredCrashLoopTrips — the shape the old detector never saw.
//
// COMPANION RED-PROOF (REPORT): make judgeUnhealthy compare only the LAST TWO samples (the "clock that
// resets at every look" shape of RestartingSince) — this test then fails at "gokapi's loop did not trip".
func TestR667_TheMeasuredCrashLoopTrips(t *testing.T) {
if k, _, _ := judgeUnhealthy(gokapiSamples(5)); k != "" {
t.Fatalf("5 minutes of 1/min restarts tripped already (%s) — the threshold is 6 in 10 min", k)
}
k, n, w := judgeUnhealthy(gokapiSamples(7))
if k != UnhealthyCrashLoop || n < CrashLoopRestarts || w != CrashLoopWindow {
t.Fatalf("gokapi's loop did not trip: kind=%q n=%d", k, n)
}
}
func TestR667_SlowStartsAndResetsDoNotTrip(t *testing.T) {
// the worst healthy container measured on the demo boxes: one restart at first start
one := []unhealthySample{{at: uT0, restarts: 0}, {at: uT0.Add(30 * time.Second), restarts: 1}, {at: uT0.Add(9 * time.Minute), restarts: 1}}
if k, _, _ := judgeUnhealthy(one); k != "" {
t.Fatalf("a single first-start restart tripped: %s", k)
}
// compose recreated the container: RestartCount 40 → 0 → 3 is a reset, not -37 or +43
reset := []unhealthySample{{at: uT0, restarts: 40}, {at: uT0.Add(time.Minute), restarts: 0}, {at: uT0.Add(2 * time.Minute), restarts: 3}}
if k, _, _ := judgeUnhealthy(reset); k != "" {
t.Fatalf("a recreate read as a crash loop: %s", k)
}
// restarts spread over more than the window: 6 restarts in 30 min is not a loop
var spread []unhealthySample
for i := 0; i <= 6; i++ {
spread = append(spread, unhealthySample{at: uT0.Add(time.Duration(i) * 5 * time.Minute), restarts: int64(i)})
}
if k, _, _ := judgeUnhealthy(spread); k != "" {
t.Fatalf("6 restarts in 30 min tripped: %s", k)
}
}
func TestR667_OOMStormTrips(t *testing.T) {
ss := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(10 * time.Minute), kills: 12}, {at: uT0.Add(25 * time.Minute), kills: 23}}
if k, n, _ := judgeUnhealthy(ss); k != UnhealthyOOMStorm || n != 20 {
t.Fatalf("kind=%q n=%d, want an OOM storm of 20", k, n)
}
calm := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(25 * time.Minute), kills: 10}}
if k, _, _ := judgeUnhealthy(calm); k != "" {
t.Fatalf("7 kills tripped: %s", k)
}
}
// ObserveUnhealthy through its docker seam: RestartCount summed per app, one verdict per episode, and an
// app that is updating is never judged.
func TestR667_ObserveSumsPerAppAndSkipsUpdating(t *testing.T) {
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
"gokapi": {Name: "gokapi", Deployed: true, Containers: []ContainerInfo{{Name: "gokapi"}}},
"romm": {Name: "romm", Deployed: true, Updating: true, Containers: []ContainerInfo{{Name: "romm"}}},
}}
count := int64(539)
m.execFn = func(name string, args ...string) (string, error) {
a := strings.Join(args, " ")
out := ""
if strings.Contains(a, "gokapi") {
out += fmt.Sprintf("/gokapi|%d\n", count)
}
if strings.Contains(a, "romm") {
out += "/romm|999\n"
}
return out, nil
}
var got []UnhealthyVerdict
for i := 0; i <= 16 && len(got) == 0; i++ {
got = m.ObserveUnhealthy(uT0.Add(time.Duration(i)*30*time.Second), nil)
if i%2 == 1 {
count++
}
}
if len(got) != 1 || got[0].Stack != "gokapi" || got[0].Kind != UnhealthyCrashLoop {
t.Fatalf("verdicts = %+v, want exactly gokapi's crash loop", got)
}
if again := m.ObserveUnhealthy(uT0.Add(9*time.Minute), nil); len(again) != 0 {
t.Fatalf("the same episode tripped twice: %+v", again)
}
}
+51 -4
View File
@@ -295,6 +295,9 @@ func fillHoldReason(g UpdateGuards, st *Stack) {
if nw, ok := g.(holdWholeCopy); ok {
st.HoldNoWholeCopy = nw.HoldNoWholeCopy(st.Name)
}
if hk, ok := g.(holdKinder); ok {
st.HoldKind = hk.HoldKind(st.Name)
}
}
}
// R-480: an update that ended HELD carries the hold's sentence as its UpdateError. Once that hold
@@ -307,6 +310,11 @@ func fillHoldReason(g UpdateGuards, st *Stack) {
}
}
// holdKinder is the OPTIONAL half of UpdateGuards that names the hold's kind (v0.269.0).
type holdKinder interface {
HoldKind(name string) string
}
// holdWholeCopy is the OPTIONAL half of UpdateGuards that says a hold names no copy (R-659).
type holdWholeCopy interface {
HoldNoWholeCopy(name string) bool
@@ -445,6 +453,14 @@ func (m *Manager) updateMemoryRefusal(name string, st *Stack) *UpdateRefusal {
return nil
}
newMeta := LoadMetadata(filepath.Dir(catPath))
// R-664 (v0.269.0): on a ladder the NEXT STEP's own memory request is what this press installs.
if st.AppConfig != nil && len(st.AppConfig.PinnedImages) > 0 {
if step, err := nextLadderStep(filepath.Dir(catPath), st.AppConfig.PinnedImages); err == nil && step.Meta != catPath {
if mm, merr := loadMetadataFile(step.Meta); merr == nil {
newMeta = mm
}
}
}
newReq, newLim := ParseMemoryMB(newMeta.Resources.MemRequest), ParseMemoryMB(newMeta.Resources.MemLimit)
if newReq == 0 {
m.logger.Printf("[WARN] [stacks] update %s: the new template declares no memory request — proceeding without the memory check", name)
@@ -657,9 +673,32 @@ func (m *Manager) updateCompose(dir string, env []string, args ...string) (strin
}
func (m *Manager) updateHealth(ctx context.Context, name string, timeout time.Duration) (bool, string) {
if m.updateHealthFn != nil {
return m.updateHealthFor(ctx, name, timeout, "")
}
// updateHealthFor is the verify with the NEW version's own .felhom.yml (R-665/R-664): metaFile is the
// catalog's (or the step's) file journaled at the start of the job; "" = the stack dir's (an update
// journaled by an older controller).
func (m *Manager) updateHealthFor(ctx context.Context, name string, timeout time.Duration, metaFile string) (bool, string) {
if m.updateHealthFn != nil && m.updateHealthMetaFn == nil {
return m.updateHealthFn(ctx, name, timeout)
}
if metaFile != "" {
if _, err := os.Stat(metaFile); err == nil {
meta := LoadMetadata(filepath.Dir(metaFile))
if filepath.Base(metaFile) != ".felhom.yml" {
if mm, err := loadMetadataFile(metaFile); err == nil {
meta = mm
}
}
m.lastVerifyMeta = metaFile
if m.updateHealthMetaFn != nil {
return m.updateHealthMetaFn(ctx, name, timeout, &meta)
}
return m.waitUpdateHealthyMeta(ctx, name, timeout, &meta)
}
m.logger.Printf("[WARN] [stacks] update %s: the new version's .felhom.yml %s is gone — judging with the stack dir's", name, metaFile)
}
return m.waitUpdateHealthy(ctx, name, timeout)
}
@@ -712,15 +751,20 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
// first, before anything moves, so a step the catalog promises and does not carry refuses here
// rather than jumping past it. An unpinned app is left to today's behaviour (advancePinTo no-ops).
stepSrc := m.CatalogTemplatePath(name, "docker-compose.yml")
stepMeta := m.CatalogTemplatePath(name, ".felhom.yml")
if cfg := LoadAppConfig(dir); cfg != nil && len(cfg.PinnedImages) > 0 {
step, serr := nextLadderStep(filepath.Dir(stepSrc), cfg.PinnedImages)
if serr != nil {
fail("update.error.pin_failed", "update ladder: "+serr.Error())
return
}
stepSrc = step.Source
stepSrc, stepMeta = step.Source, step.Meta
m.logger.Printf("[INFO] [stacks] update %s: ladder — %s", name, step.Why)
}
// R-665 (v0.269.0): the new version is judged by ITS OWN .felhom.yml, journaled so a resumed verify
// uses it too — never by the stack dir's, which a restore rewrites with the unit's older file until
// the next catalog sync (measured on 9202 2026-09-24: a failing edge passed on the restored probe).
entry.NewMeta = stepMeta
// R-475: the precondition is a copy on ANY tier, chosen in the order 2, 1, 3, and the age rule
// applies to whichever tier is chosen. The first FRESH copy wins — not merely the first copy — so a
// stale second-drive mirror never forces a backup while the app's own unit is minutes old.
@@ -813,7 +857,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
fail("update.error.journal_failed", "journal write failed")
return
}
if err := m.advancePinTo(name, dir, stepSrc); err != nil {
if err := m.advancePinTo(name, dir, stepSrc, stepMeta); err != nil {
m.pinBack(name, dir, entry)
fail("update.error.pin_failed", "advancing the pin: "+err.Error())
return
@@ -876,7 +920,7 @@ func (m *Manager) verifyAndConclude(ctx context.Context, name, dir string, env [
}
timeout := m.healthTimeout()
waitStart := m.now()
healthy, detail := m.updateHealth(ctx, name, timeout)
healthy, detail := m.updateHealthFor(ctx, name, timeout, entry.NewMeta)
if !healthy {
m.failAndHold(ctx, name, dir, env, rp, "not healthy: "+detail, entry)
return
@@ -1131,6 +1175,9 @@ type updateJournalEntry struct {
Copied bool `json:"copied,omitempty"`
PrevMeta string `json:"prev_meta,omitempty"`
NewPin map[string]string `json:"new_pin,omitempty"`
// NewMeta (v0.269.0, R-665/R-664) is the NEW version's own .felhom.yml — the catalog's, or the
// ladder step's — used for the verify, so a resumed verify judges by the same file.
NewMeta string `json:"new_meta,omitempty"`
}
type updateJournal struct {
+28
View File
@@ -2,6 +2,7 @@ package stacks
import (
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
)
@@ -79,11 +80,38 @@ func CatalogOrder(s Stack) UpdateOrder {
}
}
if differing == 0 {
// v0.269.0 (`09` §6.4 part 6): the same TAG can be a different image. Behind only when the catalog
// holds a TESTED digest for the service, the installed digest is known and differs, and the test
// is NEWER than this install — never from a registry query, never on an unknown.
if digestBehind(s) {
return UpdateOrderBehind
}
return UpdateOrderCurrent
}
return UpdateOrderAhead
}
// digestBehind: see CatalogOrder. Pinned by TestDigest_FloatingTagBehindOnlyForANewerTestedDigest.
func digestBehind(s Stack) bool {
if len(s.CatalogDigests) == 0 || s.CatalogTestedAt.IsZero() {
return false
}
for svc, want := range s.CatalogDigests {
got, ok := s.AppConfig.InstalledImages[svc]
if !ok || got.Digest == "" || !digestRe.MatchString(want) || got.Digest == want {
continue
}
at, err := time.Parse(time.RFC3339, got.At)
if err != nil {
continue // when the install happened is unknown → never "behind" on it
}
if s.CatalogTestedAt.After(at) {
return true
}
}
return false
}
// CompareImageRefs orders two image references the way a human reads them: -1 when a is older than
// b, 0 when they are the same version, 1 when a is newer. The second return is the whole point —
// FALSE means "these two cannot be ordered", and every caller must treat that as "do not know"