felhom-controller/controller/cmd/controller/r386_intent_test.go

package main

import (
	"bytes"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/notify"
	"gitea.dooplex.hu/admin/felhom-controller/internal/stacks"
)

// R-386 — the dead-app classifier asked the STATE whether the customer wanted an app stopped, when
// the product already records the ANSWER.
//
// THE DEFECT. `userStopped` was `st.State == StateStopped && !failedRestart[...]` — every stopped
// stack was assumed to be a deliberate customer stop. Measured live on `demo-hp` 2026-08-23:
// `privatebin` stopped out of band, nine dead-app scans over four minutes, **zero events and zero
// banner lines**, against a positive control from the same box seventeen minutes earlier. The comment
// beside the code claimed an out-of-band stop "still alerts". It did not.
//
// THE LAYER. These sit at `classifyRunStates`, which is the single derivation point for "this app is
// down" and the only place the guess was made. `classifyRunStates` is pure, so the fixture is exact
// rather than a race against docker.
//
// RED-PROOF (observed, see REPORT.md): restore the pre-fix predicate
//
//	userStopped := st.State == stacks.StateStopped && !failedRestart[st.Name]
//
// and TestR386_OutOfBandStopWithRunningIntentAlarms fails with `dead-app banner = [], want privatebin`
// — which is exactly what the live box reported.

func stoppedStack(name, intent string) stacks.Stack {
	st := stacks.Stack{Name: name, Deployed: true, State: stacks.StateStopped}
	if intent != stacks.DesiredStateUnknown {
		st.AppConfig = &stacks.AppConfig{DesiredState: intent}
	}
	return st
}

// Scenario D — R-386's measured case. Intent says Running; something stopped it anyway.
func TestR386_OutOfBandStopWithRunningIntentAlarms(t *testing.T) {
	dead, states := classifyRunStates(
		[]stacks.Stack{stoppedStack("privatebin", stacks.DesiredStateRunning)}, nil, nil, time.Now())

	if len(dead) != 1 || dead[0].Name != "privatebin" {
		t.Fatalf("dead-app banner = %+v, want exactly privatebin — nobody asked for this app to be "+
			"stopped, so its being stopped is a fault (measured silent on demo-hp 2026-08-23)", dead)
	}
	if !states[0].Down {
		t.Fatalf("run state = %+v, want Down=true (this is what NotifyAppStartFailures reads)", states[0])
	}
	if states[0].IntentUnknown {
		t.Errorf("IntentUnknown set for an app with a RECORDED intent — the log line would name an " +
			"app that is not actually ambiguous")
	}
}

// Scenario C — the customer pressed Stop. Telling them their own action was a fault is the thing
// v0.164.0 was written to stop, and R-386 must not undo it.
func TestR386_CustomerStopStaysSilent(t *testing.T) {
	dead, states := classifyRunStates(
		[]stacks.Stack{stoppedStack("docmost", stacks.DesiredStateStopped)}, nil, nil, time.Now())

	if len(dead) != 0 {
		t.Fatalf("a customer's own Stop raised an alarm: %+v", dead)
	}
	if states[0].Down {
		t.Fatalf("run state = %+v, want Down=false — the customer asked for this", states[0])
	}
	if states[0].IntentUnknown {
		t.Errorf("IntentUnknown set for an app whose intent is RECORDED as stopped")
	}
}

// Scenario E — intent absent. §4's ruling: keep today's behaviour, and SAY SO.
func TestR386_AbsentIntentSuppressesButIsAnnounced(t *testing.T) {
	dead, states := classifyRunStates(
		[]stacks.Stack{stoppedStack("legacy-app", stacks.DesiredStateUnknown)}, nil, nil, time.Now())

	if len(dead) != 0 {
		t.Fatalf("an app with NO recorded intent alarmed: %+v — on the first cycle after upgrade "+
			"that is every app any owner ever deliberately stopped, fleet-wide", dead)
	}
	if states[0].Down {
		t.Fatalf("run state = %+v, want Down=false", states[0])
	}
	// THE HALF THAT MAKES THE GAP BOUNDED RATHER THAN SILENT.
	if !states[0].IntentUnknown {
		t.Fatalf("IntentUnknown = false — the suppression happened but nothing records it, so an " +
			"operator cannot answer 'how many apps am I blind to?'. A rule without a mechanism is " +
			"not a rule")
	}
}

// And the log line itself — the mechanism §4 demands, asserted as an OBSERVABLE, not as "the
// function ran". The heartbeat cadence is deliberate; drive the counter to a firing scan.
func TestR386_UnknownIntentIsLoggedWithTheAppName(t *testing.T) {
	var buf bytes.Buffer
	logger := log.New(&buf, "", 0)

	states := []notify.AppRunState{
		{Name: "zulip", IntentUnknown: true},
		{Name: "legacy-app", IntentUnknown: true},
		{Name: "docmost"}, // recorded intent — must NOT appear
	}

	saved := deadAppScans
	t.Cleanup(func() { deadAppScans = saved })

	// A non-firing scan says nothing (the 2880-lines/day lesson).
	deadAppScans = 1
	noteUnknownIntentSuppressions(logger, states)
	if buf.Len() != 0 {
		t.Fatalf("logged on a non-heartbeat scan: %q", buf.String())
	}

	// A firing scan names every ambiguous app, and only those.
	deadAppScans = deadAppHeartbeatEvery
	noteUnknownIntentSuppressions(logger, states)
	out := buf.String()
	if !strings.Contains(out, "[INFO]") {
		t.Errorf("not logged at INFO — it must survive a default logging.level box: %q", out)
	}
	for _, want := range []string{"zulip", "legacy-app", "2 stopped app(s)", "R-386"} {
		if !strings.Contains(out, want) {
			t.Errorf("log line %q does not contain %q", out, want)
		}
	}
	if strings.Contains(out, "docmost") {
		t.Errorf("log line names an app with a RECORDED intent: %q", out)
	}

	// Nothing ambiguous → nothing said.
	buf.Reset()
	noteUnknownIntentSuppressions(logger, []notify.AppRunState{{Name: "docmost"}})
	if buf.Len() != 0 {
		t.Errorf("logged with no ambiguous apps: %q", buf.String())
	}
}

// Scenario F — F-CRIT-1 must not re-open. The quiesce loop stops stacks by the same path a customer
// does, so a stack it stopped and could NOT restart must alarm WHATEVER the intent says — including
// when the intent is `Stopped`, which is the case that would silently swallow it.
func TestR386_FailedRestartStillLiftsAStoppedIntent(t *testing.T) {
	for _, intent := range []string{
		stacks.DesiredStateStopped, stacks.DesiredStateRunning, stacks.DesiredStateUnknown,
	} {
		name := intent
		if name == "" {
			name = "(absent)"
		}
		t.Run(name, func(t *testing.T) {
			dead, states := classifyRunStates(
				[]stacks.Stack{stoppedStack("bookstack", intent)},
				nil, map[string]bool{"bookstack": true}, time.Now())

			if len(dead) != 1 {
				t.Fatalf("intent %q: a stack the quiesce loop stopped and FAILED to restart did not "+
					"alarm — this is F-CRIT-1's indefinitely-silent dead app, back through R-386's "+
					"door: %+v", intent, dead)
			}
			if !states[0].Down {
				t.Fatalf("intent %q: run state = %+v, want Down=true", intent, states[0])
			}
		})
	}
}

// The quiesce suppression is unchanged and still wins inside its own window, for every intent.
func TestR386_QuiesceSuppressionIsUntouched(t *testing.T) {
	for _, intent := range []string{stacks.DesiredStateRunning, stacks.DesiredStateStopped} {
		dead, states := classifyRunStates(
			[]stacks.Stack{stoppedStack("bookstack", intent)},
			map[string]bool{"bookstack": true}, nil, time.Now())
		if len(dead) != 0 || states[0].Down {
			t.Fatalf("intent %q: a quiesced stack alarmed (%+v) — R-97b's exact defect: the customer "+
				"told their app broke during an outage the backup caused", intent, dead)
		}
	}
}

// A NON-stopped down state is unaffected by intent. A `degraded` stack (R-384) alarms regardless —
// intent only ever governed the StateStopped whitelist, and widening it would silence R-384.
func TestR386_IntentDoesNotReachNonStoppedDownStates(t *testing.T) {
	st := stacks.Stack{
		Name: "bookstack", Deployed: true, State: stacks.StateDegraded,
		AppConfig: &stacks.AppConfig{DesiredState: stacks.DesiredStateStopped},
	}
	dead, states := classifyRunStates([]stacks.Stack{st}, nil, nil, time.Now())
	if len(dead) != 1 || !states[0].Down {
		t.Fatalf("a degraded stack was silenced by a Stopped intent (%+v) — R-386 must not reach "+
			"past the StateStopped whitelist, or it undoes R-384", dead)
	}
}

// --- production wiring (§10) --------------------------------------------------------------------
//
// A correct classifier the scheduler does not call is R-106's defect. This walks the AST of
// main.go and proves the real job wires BOTH halves: the classification and the announcement.
func TestR386_TheSchedulerWiresBothHalves(t *testing.T) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "main.go", nil, 0)
	if err != nil {
		t.Fatalf("parse main.go: %v", err)
	}

	var scan, announce, desiredRead bool
	ast.Inspect(f, func(n ast.Node) bool {
		call, ok := n.(*ast.CallExpr)
		if !ok {
			return true
		}
		switch fn := call.Fun.(type) {
		case *ast.Ident:
			switch fn.Name {
			case "scanDeployedAppRunStates":
				scan = true
			case "noteUnknownIntentSuppressions":
				announce = true
			}
		case *ast.SelectorExpr:
			// stacks.DesiredStateOf(st) — the intent read itself, inside classifyRunStates
			if fn.Sel.Name == "DesiredStateOf" {
				desiredRead = true
			}
		}
		return true
	})

	if !scan {
		t.Error("scanDeployedAppRunStates is never called from main.go — the detector is not wired")
	}
	if !announce {
		t.Error("noteUnknownIntentSuppressions is never called from main.go — the unknown-intent gap " +
			"is silent again, which is the half §4 required to make it bounded")
	}
	if !desiredRead {
		t.Error("stacks.DesiredStateOf is never called from main.go — the classifier is back to " +
			"guessing from the state (R-386)")
	}
}

// THE FENCE (§12): DesiredState has exactly ONE writer — the customer's own action. This session
// must not have added a second. Twelve of StopStack's fourteen callers are machines, so a writer in
// the wrong place makes a nightly backup indistinguishable from the customer pressing Stop.
func TestR386_NoNewDesiredStateWriterInMain(t *testing.T) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "main.go", nil, 0)
	if err != nil {
		t.Fatal(err)
	}
	ast.Inspect(f, func(n ast.Node) bool {
		call, ok := n.(*ast.CallExpr)
		if !ok {
			return true
		}
		if sel, ok := call.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "SetDesiredState" {
			t.Errorf("%s: SetDesiredState is called from main.go — the field has ONE writer, the "+
				"customer's own action. Reading it is fine; writing it here is the fenced act",
				fset.Position(call.Pos()))
		}
		return true
	})
	_ = strings.TrimSpace
}