v0.262.0: six defects two drill nights found in the update, remove and hold paths
gates / gates (push) Successful in 26s
gates / gates (push) Successful in 26s
R-630 (P1): waitUpdateHealthy kept the probe inside `if hc != nil && len(hc.Checks) > 0`, and when findProbeContainer returned "" its else set last="no probe container" and LOOPED - the settle path sat in the outer else, unreachable. So verifying could only time out and failAndHold then stopped a working app. Measured on paperless-ngx: three containers healthy, failed at +313.0s, front door 404 after. It now falls through to the same settle path with a WARN naming the candidates. The probe target is decidable now: HealthCheckConfig.Container plus findProbeContainerMeta resolve by exact stack name -> explicit container -> a UNIQUE prefix -> nothing with the candidates returned. The old rule took the FIRST prefix match. A skipped stack records why instead of silence. R-634 (half): RemoveStack refused on the !Deployed FLAG while the machine had containers, a compose file and an app.yaml. It now asks whether anything EXISTS. The mechanism producing the bad record is still not diagnosed and R-634 stays open for it. R-633/R-626: RemoveStack consults UpdateGuards.Busy and IsUpdating and refuses with the app's own sentence - the product already refused this clash for update and for restore. And because `down` returning 0 is a request not a result, the project is watched for 25s afterwards, anything carrying its label is removed by name with its labels logged, and the answer carries `verified`. R-621: failAndHold writes compose logs --tail 400 into <stackdir>/hold-logs/<ts>/ BEFORE the down that destroys them. Two existing tests pin the compose sequence and correctly caught the new step; their expectations are updated with the reason that the ORDER is the assertion. R-614: RemoveStack calls ClearUpdateState. NOT in this release: R-625 (a held app still renders an Update button). Named, not half-done. Three new sentences, each born as a key in both bundles. Four red-proofs seen failing. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -22,6 +22,7 @@ type probeTarget struct {
|
||||
// Called by the scheduler every minute.
|
||||
func (m *Manager) RunHealthProbes() error {
|
||||
// Phase 1: collect targets (under lock)
|
||||
var noProbe []noProbeStack
|
||||
m.mu.RLock()
|
||||
var targets []probeTarget
|
||||
skippedNotDue := 0
|
||||
@@ -57,10 +58,14 @@ func (m *Manager) RunHealthProbes() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Find the main container to probe (matching stack name)
|
||||
containerName := findProbeContainer(name, stack.Containers)
|
||||
// Find the main container to probe. R-630: a stack whose check resolves to no container
|
||||
// used to be skipped SILENTLY — paperless-ngx's probe had therefore never run on any box,
|
||||
// and nothing on any screen could say so. It now gets a RESULT that says why, so the app
|
||||
// page can tell "checked and healthy" from "never checked".
|
||||
containerName, candidates := findProbeContainerMeta(name, &stack.Meta, stack.Containers)
|
||||
if containerName == "" {
|
||||
skippedNoContainer++
|
||||
noProbe = append(noProbe, noProbeStack{name: name, candidates: candidates})
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -72,6 +77,29 @@ func (m *Manager) RunHealthProbes() error {
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
// Record the "no probe container" stacks OUTSIDE the read lock, then write their results under
|
||||
// the write lock — the same shape the probe results themselves use.
|
||||
for _, np := range noProbe {
|
||||
m.mu.Lock()
|
||||
if s, ok := m.stacks[np.name]; ok {
|
||||
s.HealthProbe = &HealthProbeResult{
|
||||
Healthy: true, // never RED on our own inability to look — R-630's whole point
|
||||
LastCheck: m.now(),
|
||||
Details: []HealthCheckDetail{{
|
||||
Type: "none",
|
||||
Target: np.name,
|
||||
Healthy: true,
|
||||
Error: MsgHealthNoProbeContainer,
|
||||
MessageKey: KeyHealthNoProbeContainer,
|
||||
}},
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if m.isDebug() {
|
||||
m.logger.Printf("[DEBUG] [stacks] RunHealthProbes: %s has a health check but no container to probe — candidates: %v", np.name, np.candidates)
|
||||
}
|
||||
}
|
||||
|
||||
if m.isDebug() {
|
||||
m.logger.Printf("[DEBUG] [stacks] RunHealthProbes: collected %d targets (%d skipped not due, %d skipped no container)",
|
||||
len(targets), skippedNotDue, skippedNoContainer)
|
||||
@@ -294,21 +322,71 @@ func (m *Manager) probeHTTP(containerName string, check HealthCheckItem, target
|
||||
return detail
|
||||
}
|
||||
|
||||
// findProbeContainer returns the container name to probe for a stack.
|
||||
// Prefers exact match with stack name, then prefix match (stack-service-N).
|
||||
// The one sentence R-630 adds. Hungarian bytes by default, the key beside it — util.MsgError's
|
||||
// contract, because HealthProbe is serialised raw to the API and has no localisation seam of its own.
|
||||
const (
|
||||
MsgHealthNoProbeContainer = "Nem futott egészségellenőrzés: nincs hozzá tartozó konténer."
|
||||
KeyHealthNoProbeContainer = "health.no_probe_container"
|
||||
)
|
||||
|
||||
// noProbeStack is a stack that declares a health check which resolves to no container.
|
||||
type noProbeStack struct {
|
||||
name string
|
||||
candidates []string
|
||||
}
|
||||
|
||||
// findProbeContainer returns the container to probe for a stack, and the candidates it rejected.
|
||||
//
|
||||
// Four rules, in order (R-630):
|
||||
//
|
||||
// 1. the container whose name EQUALS the stack name;
|
||||
// 2. the container named by `healthcheck.container` in `.felhom.yml`;
|
||||
// 3. a prefix match — but ONLY when exactly one running container matches. The old code took the
|
||||
// FIRST prefix match, which for `immich` (four `immich-*` containers and no exact match) meant
|
||||
// whichever the container list happened to yield, and which was seen live on `outline` probing
|
||||
// `outline-postgres:3000` during startup because the exactly-named container was not up yet;
|
||||
// 4. nothing — and the CANDIDATES come back with it, because "no container" with no list is the
|
||||
// silence this whole row is about.
|
||||
//
|
||||
// `meta` may be nil; the caller is not required to have one.
|
||||
func findProbeContainerMeta(stackName string, meta *Metadata, containers []ContainerInfo) (string, []string) {
|
||||
probeable := func(c ContainerInfo) bool {
|
||||
return c.State == StateRunning || c.State == StateUnhealthy
|
||||
}
|
||||
for _, c := range containers {
|
||||
if c.Name == stackName && probeable(c) {
|
||||
return c.Name, nil
|
||||
}
|
||||
}
|
||||
if meta != nil && meta.HealthCheck != nil && meta.HealthCheck.Container != "" {
|
||||
want := meta.HealthCheck.Container
|
||||
for _, c := range containers {
|
||||
if c.Name == want && probeable(c) {
|
||||
return c.Name, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
var prefix []string
|
||||
for _, c := range containers {
|
||||
if strings.HasPrefix(c.Name, stackName) && probeable(c) {
|
||||
prefix = append(prefix, c.Name)
|
||||
}
|
||||
}
|
||||
if len(prefix) == 1 {
|
||||
return prefix[0], nil
|
||||
}
|
||||
// Ambiguous or empty: name what was seen so the log and the page can say why.
|
||||
all := make([]string, 0, len(containers))
|
||||
for _, c := range containers {
|
||||
all = append(all, c.Name+"("+string(c.State)+")")
|
||||
}
|
||||
return "", all
|
||||
}
|
||||
|
||||
// findProbeContainer is the one-value form kept for callers that only want the name.
|
||||
func findProbeContainer(stackName string, containers []ContainerInfo) string {
|
||||
for _, c := range containers {
|
||||
if c.Name == stackName && (c.State == StateRunning || c.State == StateUnhealthy) {
|
||||
return c.Name
|
||||
}
|
||||
}
|
||||
// Fallback: first running container with matching prefix
|
||||
for _, c := range containers {
|
||||
if strings.HasPrefix(c.Name, stackName) && (c.State == StateRunning || c.State == StateUnhealthy) {
|
||||
return c.Name
|
||||
}
|
||||
}
|
||||
return ""
|
||||
n, _ := findProbeContainerMeta(stackName, nil, containers)
|
||||
return n
|
||||
}
|
||||
|
||||
// parseInterval parses a duration string like "5m", "30s", "1h".
|
||||
|
||||
Reference in New Issue
Block a user