felhom-controller/controller/internal/stacks/unhealthy.go

package stacks

import (
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

// ── A crash loop and an out-of-memory storm are stopped by the box (`09` §3 decision 28, R-667) ────
//
// WHY THE OLD DETECTOR MISSED IT. Stack.CrashLooping asked for crashLoopAfter (5 min) of UNINTERRUPTED
// `restarting`, measured by RestartingSince — a clock the status pass resets whenever it catches the
// container `running` between two crashes. Measured on 9202 2026-09-24: gokapi at 385 → 546 restarts,
// `restarting_since` a minute old at every look, „0 currently down".
//
// SO THIS COUNTS WHAT DOCKER COUNTS: each container's RestartCount, summed per app, sampled every scan.
// RestartCount only grows for one container run and resets when compose RECREATES the container — a
// drop is a reset and starts the window again, never a negative count.
//
// THE THRESHOLDS, from evidence (`audits/night-2026-09-24/A3/`):
//   - crash loop: >= CrashLoopRestarts (6) restarts within CrashLoopWindow (10 min). Docker's restart
//     back-off caps a steady crash loop at about ONE restart a minute (gokapi: 539 → 546 in 7 min), so
//     the brief's "10 in 10 minutes" sits on the edge and can miss a steady loop. Across all 40
//     containers of both demo boxes and 9202 no healthy container had restarted more than ONCE.
//   - out-of-memory storm: the existing app_oom_storm rule — >= 20 kernel OOM kills within 30 min.

const (
	CrashLoopRestarts = 6
	CrashLoopWindow   = 10 * time.Minute
	OOMStormKills     = 20
	OOMStormWindow    = 30 * time.Minute

	UnhealthyCrashLoop = "crash_loop"
	UnhealthyOOMStorm  = "oom_storm"
)

type unhealthySample struct {
	at       time.Time
	restarts int64
	kills    int64 // -1: unknown
}

// UnhealthyVerdict is one app that crossed a threshold in the latest observation.
type UnhealthyVerdict struct {
	Stack  string
	Kind   string // UnhealthyCrashLoop / UnhealthyOOMStorm
	Count  int64  // restarts or kills inside the window
	Window time.Duration
}

type unhealthyWatch struct {
	mu      sync.Mutex
	samples map[string][]unhealthySample
}

// judgeUnhealthy is the pure verdict over one app's samples (oldest first, the last one = now).
func judgeUnhealthy(samples []unhealthySample) (kind string, count int64, window time.Duration) {
	if len(samples) < 2 {
		return "", 0, 0
	}
	last := samples[len(samples)-1]
	// A drop in either counter is a reset (the container was recreated): only samples after it count.
	start := 0
	for i := 1; i < len(samples); i++ {
		if samples[i].restarts < samples[i-1].restarts {
			start = i
		}
	}
	for i := start; i < len(samples); i++ {
		if last.at.Sub(samples[i].at) <= CrashLoopWindow {
			if d := last.restarts - samples[i].restarts; d >= CrashLoopRestarts {
				return UnhealthyCrashLoop, d, CrashLoopWindow
			}
			break
		}
	}
	if last.kills >= 0 {
		kstart := 0
		for i := 1; i < len(samples); i++ {
			if samples[i].kills >= 0 && samples[i-1].kills >= 0 && samples[i].kills < samples[i-1].kills {
				kstart = i
			}
		}
		for i := kstart; i < len(samples); i++ {
			if samples[i].kills < 0 {
				continue
			}
			if last.at.Sub(samples[i].at) <= OOMStormWindow {
				if d := last.kills - samples[i].kills; d >= OOMStormKills {
					return UnhealthyOOMStorm, d, OOMStormWindow
				}
				break
			}
		}
	}
	return "", 0, 0
}

// ObserveUnhealthy takes one sample per deployed app — RestartCount summed over its containers (one
// `docker inspect` for all of them) and the kernel OOM kills from the latest ScanOOMKilled — and returns
// the apps whose window crossed a threshold. An app that is deploying, updating or already held is not
// sampled and its history is dropped (a deploy or an update recreates containers on purpose).
func (m *Manager) ObserveUnhealthy(now time.Time, ooms []OOMContainer) []UnhealthyVerdict {
	type appC struct{ names []string }
	m.mu.RLock()
	apps := map[string]*appC{}
	owner := map[string]string{}
	var all []string
	skip := map[string]bool{}
	for name, st := range m.stacks {
		if !st.Deployed || st.Protected {
			continue
		}
		if st.Deploying || st.Updating || st.HoldReason != "" || st.updateHeld {
			skip[name] = true
			continue
		}
		a := &appC{}
		for _, c := range st.Containers {
			a.names = append(a.names, c.Name)
			owner[c.Name] = name
			all = append(all, c.Name)
		}
		apps[name] = a
	}
	m.mu.RUnlock()

	restarts := map[string]int64{}
	if len(all) > 0 {
		sort.Strings(all)
		args := append([]string{"inspect", "-f", "{{.Name}}|{{.RestartCount}}"}, all...)
		out, _ := m.execCommand("docker", args...) // a vanished container fails its own line only
		for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
			f := strings.SplitN(strings.TrimSpace(line), "|", 2)
			if len(f) != 2 {
				continue
			}
			n, err := strconv.ParseInt(strings.TrimSpace(f[1]), 10, 64)
			if err != nil {
				continue
			}
			if app, ok := owner[strings.TrimPrefix(f[0], "/")]; ok {
				restarts[app] += n
			}
		}
	}
	kills := map[string]int64{}
	for _, o := range ooms {
		if o.Kills >= 0 {
			kills[o.Stack] += o.Kills
		}
	}

	m.unhealthy.mu.Lock()
	defer m.unhealthy.mu.Unlock()
	if m.unhealthy.samples == nil {
		m.unhealthy.samples = map[string][]unhealthySample{}
	}
	for name := range m.unhealthy.samples {
		if _, live := apps[name]; !live {
			delete(m.unhealthy.samples, name) // removed, deploying, updating or held: history dropped
		}
	}
	var out []UnhealthyVerdict
	for name := range apps {
		if skip[name] {
			continue
		}
		k := int64(-1)
		if v, ok := kills[name]; ok {
			k = v
		} else {
			k = 0
		}
		ss := append(m.unhealthy.samples[name], unhealthySample{at: now, restarts: restarts[name], kills: k})
		// keep 35 minutes of history — the longer window plus a margin
		for len(ss) > 1 && now.Sub(ss[0].at) > OOMStormWindow+5*time.Minute {
			ss = ss[1:]
		}
		m.unhealthy.samples[name] = ss
		if kind, n, win := judgeUnhealthy(ss); kind != "" {
			out = append(out, UnhealthyVerdict{Stack: name, Kind: kind, Count: n, Window: win})
			delete(m.unhealthy.samples, name) // one verdict per episode
		}
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Stack < out[j].Stack })
	return out
}