controller v0.269.0: whole restore from the second drive; crash loops stopped; exact image digests; steps judged by their own .felhom.yml (decisions 26-28, R-661 R-666 R-667 R-668 R-664 R-665 R-662, 09 6.4 part 6)
gates / gates (push) Successful in 27s
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
@@ -171,6 +173,19 @@ type HDDDataResponse struct {
|
||||
Stack string `json:"stack"`
|
||||
HDDPaths []HDDPath `json:"hdd_paths"`
|
||||
HasHDDData bool `json:"has_hdd_data"`
|
||||
// KeepDataOnly (v0.269.0, `09` §3 decision 27, R-666): the app is held and its page says support is
|
||||
// informed — the remove dialog offers only "remove the app, keep my data", and the API refuses the rest.
|
||||
KeepDataOnly bool `json:"keep_data_only,omitempty"`
|
||||
}
|
||||
|
||||
// ErrRemoveKeepDataWhileSupport is the refusal of a data-deleting remove while support is informed.
|
||||
var ErrRemoveKeepDataWhileSupport = util.MsgError("err.stacks.remove_keep_data_while_support")
|
||||
|
||||
// RemoveKeepsDataOnly reports decision 27's condition for one app: held, and the hold names no whole
|
||||
// copy (the page says support is informed).
|
||||
func (m *Manager) RemoveKeepsDataOnly(name string) bool {
|
||||
st, ok := m.GetStack(name)
|
||||
return ok && st.HoldReason != "" && st.HoldNoWholeCopy
|
||||
}
|
||||
|
||||
// HDDPath represents a single HDD bind mount path and its status.
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── Exact image fingerprints on the box (`09` §6.4 part 6, box half; §3 decision 17; v0.269.0) ────
|
||||
//
|
||||
// The catalog records, per ladder entry, the registry digest of every `to` ref at the moment the step was
|
||||
// tested (`scripts/image_digest.py`). The box:
|
||||
//
|
||||
// 1. RENDERS `name:tag@sha256:…` into the compose file it runs whenever the entry for exactly those refs
|
||||
// carries a digest — so a pull fetches the TESTED image, not whatever the tag points at today.
|
||||
// Docker and Compose accept the form and refuse a digest that does not exist (measured on 9202,
|
||||
// 2026-09-23, `audits/update-rulings-2026-09-23/70-…`).
|
||||
// 2. Keeps every PIN and every RECORD digest-free: ParseComposeImages strips `@…`, and the installed
|
||||
// record's Ref is stripped too (its Digest is a field of its own). One strip at each door, so the
|
||||
// pin, the ladder, the badge and the syncer all compare plain `name:tag`.
|
||||
// 3. Reads the badge from the TESTED digest only — never a registry query (`09` §8.1): same tag, a
|
||||
// different installed digest, and a catalog test NEWER than the install → „Frissítés elérhető".
|
||||
|
||||
var digestRe = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// StripDigest removes a `@sha256:…` suffix from an image reference.
|
||||
func StripDigest(ref string) string {
|
||||
if at := strings.LastIndex(ref, "@"); at >= 0 {
|
||||
return ref[:at]
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
var serviceLineRe = regexp.MustCompile(`^ ([A-Za-z0-9_-]+):\s*$`)
|
||||
var imageLineRe = regexp.MustCompile(`^(\s+image:\s*)["']?([^\s"'#]+)["']?(.*)$`)
|
||||
|
||||
// renderDigests rewrites each service's own `image:` line to `ref@digest` when digests carries a valid
|
||||
// one for that service. Line-based (the catalog's own reading, `ladder.images_in`), so comments and
|
||||
// every other byte are kept. A service with no valid digest keeps its line.
|
||||
func renderDigests(compose []byte, digests map[string]string) []byte {
|
||||
if len(digests) == 0 {
|
||||
return compose
|
||||
}
|
||||
lines := strings.Split(string(compose), "\n")
|
||||
svc := ""
|
||||
done := map[string]bool{}
|
||||
inServices := false
|
||||
for i, l := range lines {
|
||||
if strings.HasPrefix(l, "services:") {
|
||||
inServices = true
|
||||
continue
|
||||
}
|
||||
if l != "" && !strings.HasPrefix(l, " ") && !strings.HasPrefix(l, "#") {
|
||||
inServices = false
|
||||
}
|
||||
if !inServices {
|
||||
continue
|
||||
}
|
||||
if m := serviceLineRe.FindStringSubmatch(l); m != nil {
|
||||
svc = m[1]
|
||||
continue
|
||||
}
|
||||
m := imageLineRe.FindStringSubmatch(l)
|
||||
if m == nil || svc == "" || done[svc] {
|
||||
continue
|
||||
}
|
||||
d := digests[svc]
|
||||
if !digestRe.MatchString(d) {
|
||||
continue
|
||||
}
|
||||
lines[i] = m[1] + StripDigest(m[2]) + "@" + d + m[3]
|
||||
done[svc] = true
|
||||
}
|
||||
return []byte(strings.Join(lines, "\n"))
|
||||
}
|
||||
|
||||
// ladderEntryFor returns the NEWEST ladder entry whose `to` is exactly these (digest-free) refs.
|
||||
func ladderEntryFor(templateDir string, refs map[string]string) (LadderEntry, bool) {
|
||||
ladder, err := LoadLadder(filepath.Join(templateDir, ".felhom.yml"))
|
||||
if err != nil {
|
||||
return LadderEntry{}, false
|
||||
}
|
||||
for i := len(ladder) - 1; i >= 0; i-- {
|
||||
if sameRefs(ladder[i].To, refs) {
|
||||
return ladder[i], true
|
||||
}
|
||||
}
|
||||
return LadderEntry{}, false
|
||||
}
|
||||
|
||||
// RenderWithLadderDigests is what the syncer and the update write: the compose bytes with the tested
|
||||
// digests of the ladder entry for exactly its refs. No entry, or no digest → the bytes unchanged.
|
||||
func RenderWithLadderDigests(templateDir string, compose []byte) []byte {
|
||||
refs, err := parseComposeImagesBytes(compose)
|
||||
if err != nil || len(refs) == 0 {
|
||||
return compose
|
||||
}
|
||||
e, ok := ladderEntryFor(templateDir, refs)
|
||||
if !ok {
|
||||
return compose
|
||||
}
|
||||
return renderDigests(compose, e.Digest)
|
||||
}
|
||||
|
||||
// catalogTestedDigests is the badge's input: the tested digest per service of the catalog's current
|
||||
// refs, and when that test ran. Empty when the ladder has no entry for them.
|
||||
func catalogTestedDigests(templateDir string, catalogRefs map[string]string) (map[string]string, time.Time) {
|
||||
e, ok := ladderEntryFor(templateDir, catalogRefs)
|
||||
if !ok {
|
||||
return nil, time.Time{}
|
||||
}
|
||||
t, _ := time.Parse(time.RFC3339, e.TestedAt)
|
||||
return e.Digest, t
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var dA = "sha256:" + strings.Repeat("a", 64)
|
||||
var dB = "sha256:" + strings.Repeat("b", 64)
|
||||
|
||||
func TestDigest_RenderAndStrip(t *testing.T) {
|
||||
in := "# header\nservices:\n web:\n image: redis:7-alpine # pinned\n restart: unless-stopped\n db:\n image: \"mariadb:11.8\"\nvolumes:\n x:\n"
|
||||
out := string(renderDigests([]byte(in), map[string]string{"web": dA, "db": "not-a-digest"}))
|
||||
if !strings.Contains(out, " image: redis:7-alpine@"+dA+" # pinned") {
|
||||
t.Fatalf("web not rendered:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, " image: \"mariadb:11.8\"") {
|
||||
t.Fatalf("an invalid digest must leave the line alone:\n%s", out)
|
||||
}
|
||||
again := string(renderDigests([]byte(out), map[string]string{"web": dB}))
|
||||
if strings.Count(again, "@sha256:") != 1 || !strings.Contains(again, "redis:7-alpine@"+dB) {
|
||||
t.Fatalf("re-render must replace, not stack, the digest:\n%s", again)
|
||||
}
|
||||
imgs, err := parseComposeImagesBytes([]byte(again))
|
||||
if err != nil || imgs["web"] != "redis:7-alpine" {
|
||||
t.Fatalf("the parsed ref must be digest-free: %v %v", imgs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDigest_FloatingTagBehindOnlyForANewerTestedDigest — the badge rule, every arm.
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT): make digestBehind return false — the "newer test" case then reads
|
||||
// Current and this test fails.
|
||||
func TestDigest_FloatingTagBehindOnlyForANewerTestedDigest(t *testing.T) {
|
||||
installAt := time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)
|
||||
mk := func(instDigest, instAt string, cat map[string]string, testedAt time.Time) Stack {
|
||||
return Stack{Deployed: true, CatalogImages: map[string]string{"web": "redis:7-alpine"},
|
||||
CatalogDigests: cat, CatalogTestedAt: testedAt,
|
||||
AppConfig: &AppConfig{InstalledImages: map[string]InstalledImage{"web": {Ref: "redis:7-alpine", Digest: instDigest, At: instAt}}}}
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
s Stack
|
||||
want UpdateOrder
|
||||
}{
|
||||
{"same digest", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dA}, installAt.Add(time.Hour)), UpdateOrderCurrent},
|
||||
{"newer tested digest", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderBehind},
|
||||
{"test older than the install", mk(dA, installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(-time.Hour)), UpdateOrderCurrent},
|
||||
{"install time unknown", mk(dA, "", map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderCurrent},
|
||||
{"installed digest unknown", mk("", installAt.Format(time.RFC3339), map[string]string{"web": dB}, installAt.Add(time.Hour)), UpdateOrderCurrent},
|
||||
{"no tested digest", mk(dA, installAt.Format(time.RFC3339), nil, time.Time{}), UpdateOrderCurrent},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := CatalogOrder(c.s); got != c.want {
|
||||
t.Errorf("%s: %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The update runs EXACTLY the tested image: the pinned step's compose carries the entry's digest, while
|
||||
// the pin itself stays a plain ref.
|
||||
func TestDigest_TheUpdateRendersTheStepsTestedDigest(t *testing.T) {
|
||||
m, dir, _, _, _ := ladderManager(t, true)
|
||||
catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
|
||||
// the ladder fixture's digests are sha256:aaaa…
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitUpdateDone(t, m, "nextcloud")
|
||||
body := fileBody(t, ComposePathIn(dir))
|
||||
if !strings.Contains(body, "image: "+ladderB+"@"+dA) {
|
||||
t.Fatalf("the live compose does not pin the tested digest:\n%s", body)
|
||||
}
|
||||
if pinOf(t, dir) != ladderB {
|
||||
t.Fatalf("the pin must stay a plain ref, got %q", pinOf(t, dir))
|
||||
}
|
||||
_ = catDir
|
||||
}
|
||||
@@ -92,16 +92,27 @@ func ParseComposeImages(composePath string) (map[string]string, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading compose file: %w", err)
|
||||
}
|
||||
out, err := parseComposeImagesBytes(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing compose file %s: %w", composePath, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// parseComposeImagesBytes is ParseComposeImages over bytes. v0.269.0 (`09` §6.4 part 6): a rendered
|
||||
// `name:tag@sha256:…` is returned WITHOUT its digest — pins, the ladder, the badge and the syncer compare
|
||||
// plain refs; the digest lives only in the compose file that runs (digest.go).
|
||||
func parseComposeImagesBytes(data []byte) (map[string]string, error) {
|
||||
var doc composeImagesDoc
|
||||
if err := yaml.Unmarshal(data, &doc); err != nil {
|
||||
return nil, fmt.Errorf("parsing compose file %s: %w", composePath, err)
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string]string, len(doc.Services))
|
||||
for svc, def := range doc.Services {
|
||||
if strings.TrimSpace(def.Image) == "" {
|
||||
continue
|
||||
}
|
||||
out[svc] = strings.TrimSpace(def.Image)
|
||||
out[svc] = StripDigest(strings.TrimSpace(def.Image))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -212,7 +223,7 @@ func (m *Manager) observeInstalledImages(stackDir string, env []string) (map[str
|
||||
continue
|
||||
}
|
||||
out[svc] = InstalledImage{
|
||||
Ref: f.ref,
|
||||
Ref: StripDigest(f.ref), // v0.269.0: a digest-pinned run records the plain ref; the digest is Digest
|
||||
Digest: pickDigest(f.ref, digests[f.imageID]),
|
||||
At: now,
|
||||
}
|
||||
|
||||
@@ -42,6 +42,8 @@ type LadderEntry struct {
|
||||
To map[string]string `yaml:"to" json:"to"`
|
||||
Digest map[string]string `yaml:"digest" json:"digest"`
|
||||
Verdict string `yaml:"verdict" json:"verdict"`
|
||||
// TestedAt is when the step was proven (RFC3339) — the badge compares it with the install (v0.269.0).
|
||||
TestedAt string `yaml:"tested_at" json:"tested_at"`
|
||||
}
|
||||
|
||||
type ladderDoc struct {
|
||||
@@ -80,6 +82,11 @@ func StepFile(templateDir string, to map[string]string) string {
|
||||
return filepath.Join(templateDir, "steps", StepKey(to)+".yml")
|
||||
}
|
||||
|
||||
// StepMetaFile is the step's own `.felhom.yml` (R-664, v0.269.0): its probe, memory and applied record.
|
||||
func StepMetaFile(templateDir string, to map[string]string) string {
|
||||
return filepath.Join(templateDir, "steps", StepKey(to)+".felhom.yml")
|
||||
}
|
||||
|
||||
func sameRefs(a, b map[string]string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
@@ -100,6 +107,10 @@ type LadderStep struct {
|
||||
Left int
|
||||
// Source is the compose file the step pins: a steps/ file, or the template's docker-compose.yml.
|
||||
Source string
|
||||
// Meta is the `.felhom.yml` that belongs to Source (R-664): steps/<key>.felhom.yml when the catalog
|
||||
// carries it, else the template's own — never the stack dir's, which a restore may have rewritten
|
||||
// with an older one (R-665).
|
||||
Meta string
|
||||
// Why is one operator-English sentence for the log.
|
||||
Why string
|
||||
}
|
||||
@@ -109,12 +120,13 @@ type LadderStep struct {
|
||||
// update refuses before anything moves (a jump past a tested step is the thing this exists to stop).
|
||||
func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, error) {
|
||||
current := filepath.Join(templateDir, "docker-compose.yml")
|
||||
currentMeta := filepath.Join(templateDir, ".felhom.yml")
|
||||
ladder, err := LoadLadder(filepath.Join(templateDir, ".felhom.yml"))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return LadderStep{}, err
|
||||
}
|
||||
if len(ladder) == 0 {
|
||||
return LadderStep{Index: -1, Source: current, Why: "the template carries no update_ladder — the catalog's current definition"}, nil
|
||||
return LadderStep{Index: -1, Source: current, Meta: currentMeta, Why: "the template carries no update_ladder — the catalog's current definition"}, nil
|
||||
}
|
||||
idx := -1
|
||||
for i := len(ladder) - 1; i >= 0; i-- { // the NEWEST entry whose `from` is what runs
|
||||
@@ -124,12 +136,12 @@ func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, e
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return LadderStep{Index: -1, Source: current,
|
||||
return LadderStep{Index: -1, Source: current, Meta: currentMeta,
|
||||
Why: fmt.Sprintf("the installed version %s matches no update_ladder entry (%d entries) — an app older than the ladder has no record to climb; the catalog's current definition", summarisePin(pinned), len(ladder))}, nil
|
||||
}
|
||||
left := len(ladder) - idx
|
||||
if idx == len(ladder)-1 {
|
||||
return LadderStep{Index: idx, Left: left, Source: current,
|
||||
return LadderStep{Index: idx, Left: left, Source: current, Meta: currentMeta,
|
||||
Why: fmt.Sprintf("the last step (%d of %d) — the catalog's current definition", idx+1, len(ladder))}, nil
|
||||
}
|
||||
src := StepFile(templateDir, ladder[idx].To)
|
||||
@@ -140,8 +152,12 @@ func nextLadderStep(templateDir string, pinned map[string]string) (LadderStep, e
|
||||
if !sameRefs(imgs, ladder[idx].To) {
|
||||
return LadderStep{}, fmt.Errorf("step %d of %d: %s names %s, the ladder says %s", idx+1, len(ladder), src, summarisePin(imgs), summarisePin(ladder[idx].To))
|
||||
}
|
||||
return LadderStep{Index: idx, Left: left, Source: src,
|
||||
Why: fmt.Sprintf("step %d of %d: %s → %s, from %s", idx+1, len(ladder), summarisePin(ladder[idx].From), summarisePin(ladder[idx].To), filepath.Base(src))}, nil
|
||||
meta := StepMetaFile(templateDir, ladder[idx].To)
|
||||
if _, err := os.Stat(meta); err != nil {
|
||||
meta = currentMeta // a step written before R-664: the template's own, said in the log
|
||||
}
|
||||
return LadderStep{Index: idx, Left: left, Source: src, Meta: meta,
|
||||
Why: fmt.Sprintf("step %d of %d: %s → %s, from %s (probe from %s)", idx+1, len(ladder), summarisePin(ladder[idx].From), summarisePin(ladder[idx].To), filepath.Base(src), filepath.Base(meta))}, nil
|
||||
}
|
||||
|
||||
// ladderStepsLeft is the page's count: how many tested steps separate this pin from the catalog's
|
||||
@@ -161,3 +177,22 @@ func ladderStepsLeft(templateDir string, pinned map[string]string) int {
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// loadMetadataFile reads a `.felhom.yml` that is not named `.felhom.yml` (a step's
|
||||
// `steps/<key>.felhom.yml`) through LoadMetadata — the ONE validating reader — by giving it a scratch
|
||||
// directory. Only the health check and the resources are read from the result.
|
||||
func loadMetadataFile(path string) (Metadata, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
tmp, err := os.MkdirTemp("", "felhom-step-meta-")
|
||||
if err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
if err := os.WriteFile(filepath.Join(tmp, ".felhom.yml"), data, 0o600); err != nil {
|
||||
return Metadata{}, err
|
||||
}
|
||||
return LoadMetadata(tmp), nil
|
||||
}
|
||||
|
||||
@@ -82,10 +82,12 @@ func TestLadder_TwoPressesTwoSteps(t *testing.T) {
|
||||
if got := pinOf(t, dir); got != ladderB {
|
||||
t.Fatalf("press 1 pinned %s, want the tested step B %s — one press must be one step", got, ladderB)
|
||||
}
|
||||
if body := fileBody(t, ComposePathIn(dir)); body != ladderBDef {
|
||||
t.Fatalf("press 1 rendered\n%s\nwant step B's OWN definition (with STEP_B_OWN_DEFINITION)", body)
|
||||
// v0.269.0: the step's own definition, run with its TESTED digest (`09` §6.4 part 6)
|
||||
wantB := string(renderDigests([]byte(ladderBDef), map[string]string{"web": "sha256:" + strings.Repeat("a", 64)}))
|
||||
if body := fileBody(t, ComposePathIn(dir)); body != wantB {
|
||||
t.Fatalf("press 1 rendered\n%s\nwant step B's OWN definition (with STEP_B_OWN_DEFINITION) and its digest", body)
|
||||
}
|
||||
if body := fileBody(t, AppliedComposePath(dir)); body != ladderBDef {
|
||||
if body := fileBody(t, AppliedComposePath(dir)); body != wantB {
|
||||
t.Fatal("the stored applied definition is not step B's — the sync would render the wrong file while the app sits at B")
|
||||
}
|
||||
if n := ladderStepsLeft(catDir, map[string]string{"web": ladderB}); n != 1 {
|
||||
@@ -99,7 +101,7 @@ func TestLadder_TwoPressesTwoSteps(t *testing.T) {
|
||||
if st.UpdatePhase != UpdatePhaseDone || pinOf(t, dir) != ladderC {
|
||||
t.Fatalf("press 2 ended %q on %s, want done on C", st.UpdatePhase, pinOf(t, dir))
|
||||
}
|
||||
if body := fileBody(t, ComposePathIn(dir)); body != pinTplNew {
|
||||
if body := fileBody(t, ComposePathIn(dir)); body != string(renderDigests([]byte(pinTplNew), map[string]string{"web": "sha256:" + strings.Repeat("a", 64)})) {
|
||||
t.Fatalf("press 2 must render the catalog's current definition; got\n%s", body)
|
||||
}
|
||||
if strings.Join(*ups, ",") != ladderB+","+ladderC {
|
||||
|
||||
@@ -170,9 +170,12 @@ type Stack struct {
|
||||
HoldReason string `json:"hold_reason,omitempty"`
|
||||
// HoldNoWholeCopy (v0.268.0, R-659): the hold names NO copy — none on this box brings the app back
|
||||
// whole — so the page offers no restore button beside the sentence (the restore would refuse).
|
||||
HoldNoWholeCopy bool `json:"hold_no_whole_copy,omitempty"`
|
||||
HealthProbe *HealthProbeResult `json:"health_probe,omitempty"` // controller-side probe result
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
HoldNoWholeCopy bool `json:"hold_no_whole_copy,omitempty"`
|
||||
// HoldKind (v0.269.0): which hold — "update_failed", "unhealthy_stop" (decision 28: the page offers
|
||||
// Start, not a restore), "restore"; "" when none.
|
||||
HoldKind string `json:"hold_kind,omitempty"`
|
||||
HealthProbe *HealthProbeResult `json:"health_probe,omitempty"` // controller-side probe result
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
// RestartingSince (C9-F2) is when this stack was FIRST observed in StateRestarting during the
|
||||
// current restarting run; zero whenever the stack is in any other state. It is what turns a brief
|
||||
// restart (normal: deploy, update, quiesce restart) into a distinguishable crash loop — see
|
||||
@@ -205,10 +208,22 @@ type Stack struct {
|
||||
// pin and the catalog's head; one press climbs one. 0 = unknown or none. The page shows it while
|
||||
// the badge says „Frissítés elérhető".
|
||||
LadderStepsLeft int `json:"ladder_steps_left,omitempty"`
|
||||
|
||||
// CatalogDigests / CatalogTestedAt (v0.269.0, `09` §6.4 part 6): the TESTED digest per service of the
|
||||
// catalog's current refs and when that test ran — the badge's input for a floating tag (never a
|
||||
// registry query). Empty when the ladder has no entry for them.
|
||||
CatalogDigests map[string]string `json:"catalog_digests,omitempty"`
|
||||
CatalogTestedAt time.Time `json:"catalog_tested_at,omitempty"`
|
||||
}
|
||||
|
||||
// Manager handles all docker compose stack operations.
|
||||
type Manager struct {
|
||||
// lastVerifyMeta (v0.269.0) is the .felhom.yml the last update verify judged by — read by a test.
|
||||
lastVerifyMeta string
|
||||
// updateHealthMetaFn (test seam, v0.269.0): the verify with the chosen .felhom.yml; nil → the real wait.
|
||||
updateHealthMetaFn func(ctx context.Context, name string, timeout time.Duration, meta *Metadata) (bool, string)
|
||||
// unhealthy (v0.269.0, decision 28): RestartCount / OOM-kill samples per app for the crash-loop stop.
|
||||
unhealthy unhealthyWatch
|
||||
// selfUpdating (v0.261.0) reports whether the CONTROLLER is swapping itself. Set by
|
||||
// SetSelfUpdatingCheck; nil means no gate. See update.go.
|
||||
selfUpdating func() bool
|
||||
@@ -611,6 +626,8 @@ func (m *Manager) ScanStacks() error {
|
||||
// catalog template by definition, and warning once per app per scan would be noise.
|
||||
var catImages map[string]string
|
||||
stepsLeft := 0
|
||||
var catDigests map[string]string
|
||||
var catTestedAt time.Time
|
||||
if deployed && !m.cfg.IsProtectedStack(name) {
|
||||
catPath := m.CatalogTemplatePath(name, "docker-compose.yml")
|
||||
if appCfg != nil {
|
||||
@@ -618,6 +635,7 @@ func (m *Manager) ScanStacks() error {
|
||||
}
|
||||
if imgs, cerr := ParseComposeImages(catPath); cerr == nil {
|
||||
catImages = imgs
|
||||
catDigests, catTestedAt = catalogTestedDigests(filepath.Dir(catPath), imgs)
|
||||
} else if m.isDebug() {
|
||||
m.logger.Printf("[DEBUG] [stacks] ScanStacks: no readable catalog template for %s (%v) — the update badge will render nothing", name, cerr)
|
||||
}
|
||||
@@ -635,6 +653,7 @@ func (m *Manager) ScanStacks() error {
|
||||
existing.TemplateImages = tplImages
|
||||
existing.CatalogImages = catImages
|
||||
existing.LadderStepsLeft = stepsLeft
|
||||
existing.CatalogDigests, existing.CatalogTestedAt = catDigests, catTestedAt
|
||||
}
|
||||
} else {
|
||||
m.stacks[name] = &Stack{
|
||||
@@ -648,6 +667,8 @@ func (m *Manager) ScanStacks() error {
|
||||
TemplateImages: tplImages,
|
||||
CatalogImages: catImages,
|
||||
LadderStepsLeft: stepsLeft,
|
||||
CatalogDigests: catDigests,
|
||||
CatalogTestedAt: catTestedAt,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -328,12 +328,12 @@ func (m *Manager) CatalogTemplatePath(appName, filename string) string {
|
||||
// the syncer has already copied the catalog verbatim into its stack dir, so `pull` + `up -d` do
|
||||
// today's job with no help from here.
|
||||
func (m *Manager) advancePinToCatalog(name, stackDir string) error {
|
||||
return m.advancePinTo(name, stackDir, m.CatalogTemplatePath(name, "docker-compose.yml"))
|
||||
return m.advancePinTo(name, stackDir, m.CatalogTemplatePath(name, "docker-compose.yml"), m.CatalogTemplatePath(name, ".felhom.yml"))
|
||||
}
|
||||
|
||||
// advancePinTo is advancePinToCatalog with the definition named: the catalog's current compose file,
|
||||
// or — on a ladder (v0.268.0, `09` §3 decision 14) — one step's own definition from `steps/`.
|
||||
func (m *Manager) advancePinTo(name, stackDir, src string) error {
|
||||
func (m *Manager) advancePinTo(name, stackDir, src, metaSrc string) error {
|
||||
cfg := LoadAppConfig(stackDir)
|
||||
if cfg == nil || len(cfg.PinnedImages) == 0 {
|
||||
return nil // unpinned — today's behaviour, unchanged
|
||||
@@ -346,6 +346,8 @@ func (m *Manager) advancePinTo(name, stackDir, src string) error {
|
||||
return fmt.Errorf("cannot read the catalog's current definition for %s (%s): %w", name, src, err)
|
||||
}
|
||||
|
||||
// v0.269.0 (`09` §6.4 part 6): run EXACTLY the tested images — the ladder entry's digests for these refs.
|
||||
data = RenderWithLadderDigests(filepath.Dir(m.CatalogTemplatePath(name, "docker-compose.yml")), data)
|
||||
live := ComposePathIn(stackDir)
|
||||
if err := StoreAppliedDefinition(stackDir, data); err != nil {
|
||||
return fmt.Errorf("storing the new applied definition for %s: %w", name, err)
|
||||
@@ -365,8 +367,9 @@ func (m *Manager) advancePinTo(name, stackDir, src string) error {
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
// v0.263.2: the new version's own .felhom.yml becomes the pinned version's record.
|
||||
m.storeAppliedMetaFrom(name, stackDir, m.CatalogTemplatePath(name, ".felhom.yml"))
|
||||
m.logger.Printf("[INFO] [stacks] update %s: pin advanced to the catalog's current definition (%s)", name, summarisePin(pin))
|
||||
// v0.263.2: the new version's own .felhom.yml becomes the pinned version's record — the STEP's own
|
||||
// when the ladder carries one (R-664, v0.269.0).
|
||||
m.storeAppliedMetaFrom(name, stackDir, metaSrc)
|
||||
m.logger.Printf("[INFO] [stacks] update %s: pin advanced to %s (%s)", name, src, summarisePin(pin))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// R-665 + R-664 (v0.269.0) — the new version is judged by ITS OWN .felhom.yml: the catalog's, or the
|
||||
// ladder step's; never the stack dir's, which a restore rewrites with the unit's older file.
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT): make updateHealthFor ignore metaFile (the v0.268.0 behaviour) — the probe
|
||||
// port read is then the RESTORED 1111 and both cases fail.
|
||||
func probePortSeen(t *testing.T, m *Manager) *int {
|
||||
t.Helper()
|
||||
port := new(int)
|
||||
m.updateHealthMetaFn = func(_ context.Context, _ string, _ time.Duration, meta *Metadata) (bool, string) {
|
||||
if meta != nil && meta.HealthCheck != nil && len(meta.HealthCheck.Checks) > 0 {
|
||||
*port = meta.HealthCheck.Checks[0].Port
|
||||
}
|
||||
return true, "fake"
|
||||
}
|
||||
return port
|
||||
}
|
||||
|
||||
func TestR665_TheVerifyUsesTheNewVersionsOwnFile(t *testing.T) {
|
||||
m, dir, _, _ := newSlice4Manager(t)
|
||||
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 1111\n") // what a restore wrote
|
||||
mustWrite(t, m.CatalogTemplatePath("nextcloud", ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 2222\n")
|
||||
port := probePortSeen(t, m)
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := waitUpdateDone(t, m, "nextcloud"); st.UpdatePhase != UpdatePhaseDone {
|
||||
t.Fatalf("ended %q", st.UpdatePhase)
|
||||
}
|
||||
if *port != 2222 {
|
||||
t.Fatalf("verified with probe port %d, want the catalog's 2222 (1111 = the restored file, R-665)", *port)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR664_AStepIsVerifiedWithItsOwnFile(t *testing.T) {
|
||||
m, dir, _, _, _ := ladderManager(t, true)
|
||||
catDir := filepath.Dir(m.CatalogTemplatePath("nextcloud", "docker-compose.yml"))
|
||||
mustWrite(t, filepath.Join(dir, ".felhom.yml"), "healthcheck:\n checks:\n - type: http\n port: 1111\n")
|
||||
mustWriteMk(t, StepMetaFile(catDir, map[string]string{"web": ladderB}), "healthcheck:\n checks:\n - type: http\n port: 3333\n")
|
||||
port := probePortSeen(t, m)
|
||||
if err := m.StartGuardedUpdate("nextcloud"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitUpdateDone(t, m, "nextcloud")
|
||||
if *port != 3333 {
|
||||
t.Fatalf("step B verified with probe port %d, want its own 3333", *port)
|
||||
}
|
||||
if got := fileBody(t, filepath.Join(dir, appliedMetaDir, ".felhom.yml")); got != "healthcheck:\n checks:\n - type: http\n port: 3333\n" {
|
||||
t.Fatalf("applied-meta after step B is not the step's own file: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── A crash loop and an out-of-memory storm are stopped by the box (`09` §3 decision 28, R-667) ────
|
||||
//
|
||||
// WHY THE OLD DETECTOR MISSED IT. Stack.CrashLooping asked for crashLoopAfter (5 min) of UNINTERRUPTED
|
||||
// `restarting`, measured by RestartingSince — a clock the status pass resets whenever it catches the
|
||||
// container `running` between two crashes. Measured on 9202 2026-09-24: gokapi at 385 → 546 restarts,
|
||||
// `restarting_since` a minute old at every look, „0 currently down".
|
||||
//
|
||||
// SO THIS COUNTS WHAT DOCKER COUNTS: each container's RestartCount, summed per app, sampled every scan.
|
||||
// RestartCount only grows for one container run and resets when compose RECREATES the container — a
|
||||
// drop is a reset and starts the window again, never a negative count.
|
||||
//
|
||||
// THE THRESHOLDS, from evidence (`audits/night-2026-09-24/A3/`):
|
||||
// - crash loop: >= CrashLoopRestarts (6) restarts within CrashLoopWindow (10 min). Docker's restart
|
||||
// back-off caps a steady crash loop at about ONE restart a minute (gokapi: 539 → 546 in 7 min), so
|
||||
// the brief's "10 in 10 minutes" sits on the edge and can miss a steady loop. Across all 40
|
||||
// containers of both demo boxes and 9202 no healthy container had restarted more than ONCE.
|
||||
// - out-of-memory storm: the existing app_oom_storm rule — >= 20 kernel OOM kills within 30 min.
|
||||
|
||||
const (
|
||||
CrashLoopRestarts = 6
|
||||
CrashLoopWindow = 10 * time.Minute
|
||||
OOMStormKills = 20
|
||||
OOMStormWindow = 30 * time.Minute
|
||||
|
||||
UnhealthyCrashLoop = "crash_loop"
|
||||
UnhealthyOOMStorm = "oom_storm"
|
||||
)
|
||||
|
||||
type unhealthySample struct {
|
||||
at time.Time
|
||||
restarts int64
|
||||
kills int64 // -1: unknown
|
||||
}
|
||||
|
||||
// UnhealthyVerdict is one app that crossed a threshold in the latest observation.
|
||||
type UnhealthyVerdict struct {
|
||||
Stack string
|
||||
Kind string // UnhealthyCrashLoop / UnhealthyOOMStorm
|
||||
Count int64 // restarts or kills inside the window
|
||||
Window time.Duration
|
||||
}
|
||||
|
||||
type unhealthyWatch struct {
|
||||
mu sync.Mutex
|
||||
samples map[string][]unhealthySample
|
||||
}
|
||||
|
||||
// judgeUnhealthy is the pure verdict over one app's samples (oldest first, the last one = now).
|
||||
func judgeUnhealthy(samples []unhealthySample) (kind string, count int64, window time.Duration) {
|
||||
if len(samples) < 2 {
|
||||
return "", 0, 0
|
||||
}
|
||||
last := samples[len(samples)-1]
|
||||
// A drop in either counter is a reset (the container was recreated): only samples after it count.
|
||||
start := 0
|
||||
for i := 1; i < len(samples); i++ {
|
||||
if samples[i].restarts < samples[i-1].restarts {
|
||||
start = i
|
||||
}
|
||||
}
|
||||
for i := start; i < len(samples); i++ {
|
||||
if last.at.Sub(samples[i].at) <= CrashLoopWindow {
|
||||
if d := last.restarts - samples[i].restarts; d >= CrashLoopRestarts {
|
||||
return UnhealthyCrashLoop, d, CrashLoopWindow
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.kills >= 0 {
|
||||
kstart := 0
|
||||
for i := 1; i < len(samples); i++ {
|
||||
if samples[i].kills >= 0 && samples[i-1].kills >= 0 && samples[i].kills < samples[i-1].kills {
|
||||
kstart = i
|
||||
}
|
||||
}
|
||||
for i := kstart; i < len(samples); i++ {
|
||||
if samples[i].kills < 0 {
|
||||
continue
|
||||
}
|
||||
if last.at.Sub(samples[i].at) <= OOMStormWindow {
|
||||
if d := last.kills - samples[i].kills; d >= OOMStormKills {
|
||||
return UnhealthyOOMStorm, d, OOMStormWindow
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", 0, 0
|
||||
}
|
||||
|
||||
// ObserveUnhealthy takes one sample per deployed app — RestartCount summed over its containers (one
|
||||
// `docker inspect` for all of them) and the kernel OOM kills from the latest ScanOOMKilled — and returns
|
||||
// the apps whose window crossed a threshold. An app that is deploying, updating or already held is not
|
||||
// sampled and its history is dropped (a deploy or an update recreates containers on purpose).
|
||||
func (m *Manager) ObserveUnhealthy(now time.Time, ooms []OOMContainer) []UnhealthyVerdict {
|
||||
type appC struct{ names []string }
|
||||
m.mu.RLock()
|
||||
apps := map[string]*appC{}
|
||||
owner := map[string]string{}
|
||||
var all []string
|
||||
skip := map[string]bool{}
|
||||
for name, st := range m.stacks {
|
||||
if !st.Deployed || st.Protected {
|
||||
continue
|
||||
}
|
||||
if st.Deploying || st.Updating || st.HoldReason != "" || st.updateHeld {
|
||||
skip[name] = true
|
||||
continue
|
||||
}
|
||||
a := &appC{}
|
||||
for _, c := range st.Containers {
|
||||
a.names = append(a.names, c.Name)
|
||||
owner[c.Name] = name
|
||||
all = append(all, c.Name)
|
||||
}
|
||||
apps[name] = a
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
restarts := map[string]int64{}
|
||||
if len(all) > 0 {
|
||||
sort.Strings(all)
|
||||
args := append([]string{"inspect", "-f", "{{.Name}}|{{.RestartCount}}"}, all...)
|
||||
out, _ := m.execCommand("docker", args...) // a vanished container fails its own line only
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
f := strings.SplitN(strings.TrimSpace(line), "|", 2)
|
||||
if len(f) != 2 {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(f[1]), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if app, ok := owner[strings.TrimPrefix(f[0], "/")]; ok {
|
||||
restarts[app] += n
|
||||
}
|
||||
}
|
||||
}
|
||||
kills := map[string]int64{}
|
||||
for _, o := range ooms {
|
||||
if o.Kills >= 0 {
|
||||
kills[o.Stack] += o.Kills
|
||||
}
|
||||
}
|
||||
|
||||
m.unhealthy.mu.Lock()
|
||||
defer m.unhealthy.mu.Unlock()
|
||||
if m.unhealthy.samples == nil {
|
||||
m.unhealthy.samples = map[string][]unhealthySample{}
|
||||
}
|
||||
for name := range m.unhealthy.samples {
|
||||
if _, live := apps[name]; !live {
|
||||
delete(m.unhealthy.samples, name) // removed, deploying, updating or held: history dropped
|
||||
}
|
||||
}
|
||||
var out []UnhealthyVerdict
|
||||
for name := range apps {
|
||||
if skip[name] {
|
||||
continue
|
||||
}
|
||||
k := int64(-1)
|
||||
if v, ok := kills[name]; ok {
|
||||
k = v
|
||||
} else {
|
||||
k = 0
|
||||
}
|
||||
ss := append(m.unhealthy.samples[name], unhealthySample{at: now, restarts: restarts[name], kills: k})
|
||||
// keep 35 minutes of history — the longer window plus a margin
|
||||
for len(ss) > 1 && now.Sub(ss[0].at) > OOMStormWindow+5*time.Minute {
|
||||
ss = ss[1:]
|
||||
}
|
||||
m.unhealthy.samples[name] = ss
|
||||
if kind, n, win := judgeUnhealthy(ss); kind != "" {
|
||||
out = append(out, UnhealthyVerdict{Stack: name, Kind: kind, Count: n, Window: win})
|
||||
delete(m.unhealthy.samples, name) // one verdict per episode
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Stack < out[j].Stack })
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var uT0 = time.Date(2026, 9, 24, 9, 22, 0, 0, time.UTC)
|
||||
|
||||
// gokapi's measured shape: Docker's back-off caps a steady crash loop at ~1 restart a minute (539 → 546
|
||||
// in 7 min on 9202). Sampled every 30 s, as the dead-app loop does.
|
||||
func gokapiSamples(minutes int) []unhealthySample {
|
||||
var ss []unhealthySample
|
||||
for i := 0; i <= minutes*2; i++ {
|
||||
ss = append(ss, unhealthySample{at: uT0.Add(time.Duration(i) * 30 * time.Second), restarts: 539 + int64(i/2), kills: 0})
|
||||
}
|
||||
return ss
|
||||
}
|
||||
|
||||
// TestR667_TheMeasuredCrashLoopTrips — the shape the old detector never saw.
|
||||
//
|
||||
// COMPANION RED-PROOF (REPORT): make judgeUnhealthy compare only the LAST TWO samples (the "clock that
|
||||
// resets at every look" shape of RestartingSince) — this test then fails at "gokapi's loop did not trip".
|
||||
func TestR667_TheMeasuredCrashLoopTrips(t *testing.T) {
|
||||
if k, _, _ := judgeUnhealthy(gokapiSamples(5)); k != "" {
|
||||
t.Fatalf("5 minutes of 1/min restarts tripped already (%s) — the threshold is 6 in 10 min", k)
|
||||
}
|
||||
k, n, w := judgeUnhealthy(gokapiSamples(7))
|
||||
if k != UnhealthyCrashLoop || n < CrashLoopRestarts || w != CrashLoopWindow {
|
||||
t.Fatalf("gokapi's loop did not trip: kind=%q n=%d", k, n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR667_SlowStartsAndResetsDoNotTrip(t *testing.T) {
|
||||
// the worst healthy container measured on the demo boxes: one restart at first start
|
||||
one := []unhealthySample{{at: uT0, restarts: 0}, {at: uT0.Add(30 * time.Second), restarts: 1}, {at: uT0.Add(9 * time.Minute), restarts: 1}}
|
||||
if k, _, _ := judgeUnhealthy(one); k != "" {
|
||||
t.Fatalf("a single first-start restart tripped: %s", k)
|
||||
}
|
||||
// compose recreated the container: RestartCount 40 → 0 → 3 is a reset, not -37 or +43
|
||||
reset := []unhealthySample{{at: uT0, restarts: 40}, {at: uT0.Add(time.Minute), restarts: 0}, {at: uT0.Add(2 * time.Minute), restarts: 3}}
|
||||
if k, _, _ := judgeUnhealthy(reset); k != "" {
|
||||
t.Fatalf("a recreate read as a crash loop: %s", k)
|
||||
}
|
||||
// restarts spread over more than the window: 6 restarts in 30 min is not a loop
|
||||
var spread []unhealthySample
|
||||
for i := 0; i <= 6; i++ {
|
||||
spread = append(spread, unhealthySample{at: uT0.Add(time.Duration(i) * 5 * time.Minute), restarts: int64(i)})
|
||||
}
|
||||
if k, _, _ := judgeUnhealthy(spread); k != "" {
|
||||
t.Fatalf("6 restarts in 30 min tripped: %s", k)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR667_OOMStormTrips(t *testing.T) {
|
||||
ss := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(10 * time.Minute), kills: 12}, {at: uT0.Add(25 * time.Minute), kills: 23}}
|
||||
if k, n, _ := judgeUnhealthy(ss); k != UnhealthyOOMStorm || n != 20 {
|
||||
t.Fatalf("kind=%q n=%d, want an OOM storm of 20", k, n)
|
||||
}
|
||||
calm := []unhealthySample{{at: uT0, kills: 3}, {at: uT0.Add(25 * time.Minute), kills: 10}}
|
||||
if k, _, _ := judgeUnhealthy(calm); k != "" {
|
||||
t.Fatalf("7 kills tripped: %s", k)
|
||||
}
|
||||
}
|
||||
|
||||
// ObserveUnhealthy through its docker seam: RestartCount summed per app, one verdict per episode, and an
|
||||
// app that is updating is never judged.
|
||||
func TestR667_ObserveSumsPerAppAndSkipsUpdating(t *testing.T) {
|
||||
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
|
||||
"gokapi": {Name: "gokapi", Deployed: true, Containers: []ContainerInfo{{Name: "gokapi"}}},
|
||||
"romm": {Name: "romm", Deployed: true, Updating: true, Containers: []ContainerInfo{{Name: "romm"}}},
|
||||
}}
|
||||
count := int64(539)
|
||||
m.execFn = func(name string, args ...string) (string, error) {
|
||||
a := strings.Join(args, " ")
|
||||
out := ""
|
||||
if strings.Contains(a, "gokapi") {
|
||||
out += fmt.Sprintf("/gokapi|%d\n", count)
|
||||
}
|
||||
if strings.Contains(a, "romm") {
|
||||
out += "/romm|999\n"
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var got []UnhealthyVerdict
|
||||
for i := 0; i <= 16 && len(got) == 0; i++ {
|
||||
got = m.ObserveUnhealthy(uT0.Add(time.Duration(i)*30*time.Second), nil)
|
||||
if i%2 == 1 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if len(got) != 1 || got[0].Stack != "gokapi" || got[0].Kind != UnhealthyCrashLoop {
|
||||
t.Fatalf("verdicts = %+v, want exactly gokapi's crash loop", got)
|
||||
}
|
||||
if again := m.ObserveUnhealthy(uT0.Add(9*time.Minute), nil); len(again) != 0 {
|
||||
t.Fatalf("the same episode tripped twice: %+v", again)
|
||||
}
|
||||
}
|
||||
@@ -295,6 +295,9 @@ func fillHoldReason(g UpdateGuards, st *Stack) {
|
||||
if nw, ok := g.(holdWholeCopy); ok {
|
||||
st.HoldNoWholeCopy = nw.HoldNoWholeCopy(st.Name)
|
||||
}
|
||||
if hk, ok := g.(holdKinder); ok {
|
||||
st.HoldKind = hk.HoldKind(st.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
// R-480: an update that ended HELD carries the hold's sentence as its UpdateError. Once that hold
|
||||
@@ -307,6 +310,11 @@ func fillHoldReason(g UpdateGuards, st *Stack) {
|
||||
}
|
||||
}
|
||||
|
||||
// holdKinder is the OPTIONAL half of UpdateGuards that names the hold's kind (v0.269.0).
|
||||
type holdKinder interface {
|
||||
HoldKind(name string) string
|
||||
}
|
||||
|
||||
// holdWholeCopy is the OPTIONAL half of UpdateGuards that says a hold names no copy (R-659).
|
||||
type holdWholeCopy interface {
|
||||
HoldNoWholeCopy(name string) bool
|
||||
@@ -445,6 +453,14 @@ func (m *Manager) updateMemoryRefusal(name string, st *Stack) *UpdateRefusal {
|
||||
return nil
|
||||
}
|
||||
newMeta := LoadMetadata(filepath.Dir(catPath))
|
||||
// R-664 (v0.269.0): on a ladder the NEXT STEP's own memory request is what this press installs.
|
||||
if st.AppConfig != nil && len(st.AppConfig.PinnedImages) > 0 {
|
||||
if step, err := nextLadderStep(filepath.Dir(catPath), st.AppConfig.PinnedImages); err == nil && step.Meta != catPath {
|
||||
if mm, merr := loadMetadataFile(step.Meta); merr == nil {
|
||||
newMeta = mm
|
||||
}
|
||||
}
|
||||
}
|
||||
newReq, newLim := ParseMemoryMB(newMeta.Resources.MemRequest), ParseMemoryMB(newMeta.Resources.MemLimit)
|
||||
if newReq == 0 {
|
||||
m.logger.Printf("[WARN] [stacks] update %s: the new template declares no memory request — proceeding without the memory check", name)
|
||||
@@ -657,9 +673,32 @@ func (m *Manager) updateCompose(dir string, env []string, args ...string) (strin
|
||||
}
|
||||
|
||||
func (m *Manager) updateHealth(ctx context.Context, name string, timeout time.Duration) (bool, string) {
|
||||
if m.updateHealthFn != nil {
|
||||
return m.updateHealthFor(ctx, name, timeout, "")
|
||||
}
|
||||
|
||||
// updateHealthFor is the verify with the NEW version's own .felhom.yml (R-665/R-664): metaFile is the
|
||||
// catalog's (or the step's) file journaled at the start of the job; "" = the stack dir's (an update
|
||||
// journaled by an older controller).
|
||||
func (m *Manager) updateHealthFor(ctx context.Context, name string, timeout time.Duration, metaFile string) (bool, string) {
|
||||
if m.updateHealthFn != nil && m.updateHealthMetaFn == nil {
|
||||
return m.updateHealthFn(ctx, name, timeout)
|
||||
}
|
||||
if metaFile != "" {
|
||||
if _, err := os.Stat(metaFile); err == nil {
|
||||
meta := LoadMetadata(filepath.Dir(metaFile))
|
||||
if filepath.Base(metaFile) != ".felhom.yml" {
|
||||
if mm, err := loadMetadataFile(metaFile); err == nil {
|
||||
meta = mm
|
||||
}
|
||||
}
|
||||
m.lastVerifyMeta = metaFile
|
||||
if m.updateHealthMetaFn != nil {
|
||||
return m.updateHealthMetaFn(ctx, name, timeout, &meta)
|
||||
}
|
||||
return m.waitUpdateHealthyMeta(ctx, name, timeout, &meta)
|
||||
}
|
||||
m.logger.Printf("[WARN] [stacks] update %s: the new version's .felhom.yml %s is gone — judging with the stack dir's", name, metaFile)
|
||||
}
|
||||
return m.waitUpdateHealthy(ctx, name, timeout)
|
||||
}
|
||||
|
||||
@@ -712,15 +751,20 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
// first, before anything moves, so a step the catalog promises and does not carry refuses here
|
||||
// rather than jumping past it. An unpinned app is left to today's behaviour (advancePinTo no-ops).
|
||||
stepSrc := m.CatalogTemplatePath(name, "docker-compose.yml")
|
||||
stepMeta := m.CatalogTemplatePath(name, ".felhom.yml")
|
||||
if cfg := LoadAppConfig(dir); cfg != nil && len(cfg.PinnedImages) > 0 {
|
||||
step, serr := nextLadderStep(filepath.Dir(stepSrc), cfg.PinnedImages)
|
||||
if serr != nil {
|
||||
fail("update.error.pin_failed", "update ladder: "+serr.Error())
|
||||
return
|
||||
}
|
||||
stepSrc = step.Source
|
||||
stepSrc, stepMeta = step.Source, step.Meta
|
||||
m.logger.Printf("[INFO] [stacks] update %s: ladder — %s", name, step.Why)
|
||||
}
|
||||
// R-665 (v0.269.0): the new version is judged by ITS OWN .felhom.yml, journaled so a resumed verify
|
||||
// uses it too — never by the stack dir's, which a restore rewrites with the unit's older file until
|
||||
// the next catalog sync (measured on 9202 2026-09-24: a failing edge passed on the restored probe).
|
||||
entry.NewMeta = stepMeta
|
||||
// R-475: the precondition is a copy on ANY tier, chosen in the order 2, 1, 3, and the age rule
|
||||
// applies to whichever tier is chosen. The first FRESH copy wins — not merely the first copy — so a
|
||||
// stale second-drive mirror never forces a backup while the app's own unit is minutes old.
|
||||
@@ -813,7 +857,7 @@ func (m *Manager) runGuardedUpdate(ctx context.Context, name string) {
|
||||
fail("update.error.journal_failed", "journal write failed")
|
||||
return
|
||||
}
|
||||
if err := m.advancePinTo(name, dir, stepSrc); err != nil {
|
||||
if err := m.advancePinTo(name, dir, stepSrc, stepMeta); err != nil {
|
||||
m.pinBack(name, dir, entry)
|
||||
fail("update.error.pin_failed", "advancing the pin: "+err.Error())
|
||||
return
|
||||
@@ -876,7 +920,7 @@ func (m *Manager) verifyAndConclude(ctx context.Context, name, dir string, env [
|
||||
}
|
||||
timeout := m.healthTimeout()
|
||||
waitStart := m.now()
|
||||
healthy, detail := m.updateHealth(ctx, name, timeout)
|
||||
healthy, detail := m.updateHealthFor(ctx, name, timeout, entry.NewMeta)
|
||||
if !healthy {
|
||||
m.failAndHold(ctx, name, dir, env, rp, "not healthy: "+detail, entry)
|
||||
return
|
||||
@@ -1131,6 +1175,9 @@ type updateJournalEntry struct {
|
||||
Copied bool `json:"copied,omitempty"`
|
||||
PrevMeta string `json:"prev_meta,omitempty"`
|
||||
NewPin map[string]string `json:"new_pin,omitempty"`
|
||||
// NewMeta (v0.269.0, R-665/R-664) is the NEW version's own .felhom.yml — the catalog's, or the
|
||||
// ladder step's — used for the verify, so a resumed verify judges by the same file.
|
||||
NewMeta string `json:"new_meta,omitempty"`
|
||||
}
|
||||
|
||||
type updateJournal struct {
|
||||
|
||||
@@ -2,6 +2,7 @@ package stacks
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/util"
|
||||
)
|
||||
@@ -79,11 +80,38 @@ func CatalogOrder(s Stack) UpdateOrder {
|
||||
}
|
||||
}
|
||||
if differing == 0 {
|
||||
// v0.269.0 (`09` §6.4 part 6): the same TAG can be a different image. Behind only when the catalog
|
||||
// holds a TESTED digest for the service, the installed digest is known and differs, and the test
|
||||
// is NEWER than this install — never from a registry query, never on an unknown.
|
||||
if digestBehind(s) {
|
||||
return UpdateOrderBehind
|
||||
}
|
||||
return UpdateOrderCurrent
|
||||
}
|
||||
return UpdateOrderAhead
|
||||
}
|
||||
|
||||
// digestBehind: see CatalogOrder. Pinned by TestDigest_FloatingTagBehindOnlyForANewerTestedDigest.
|
||||
func digestBehind(s Stack) bool {
|
||||
if len(s.CatalogDigests) == 0 || s.CatalogTestedAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
for svc, want := range s.CatalogDigests {
|
||||
got, ok := s.AppConfig.InstalledImages[svc]
|
||||
if !ok || got.Digest == "" || !digestRe.MatchString(want) || got.Digest == want {
|
||||
continue
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339, got.At)
|
||||
if err != nil {
|
||||
continue // when the install happened is unknown → never "behind" on it
|
||||
}
|
||||
if s.CatalogTestedAt.After(at) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CompareImageRefs orders two image references the way a human reads them: -1 when a is older than
|
||||
// b, 0 when they are the same version, 1 when a is newer. The second return is the whole point —
|
||||
// FALSE means "these two cannot be ordered", and every caller must treat that as "do not know"
|
||||
|
||||
Reference in New Issue
Block a user