Files
felhom-controller/controller/internal/bootstrap/drift_test.go
T
admin 6fac245d24 R-270: the local_api drift detector also names a TOKEN-only divergence
A rotation that reached bootstrap.json but not controller.yaml left the
agent channel at 401 across restarts while the endpoint-only detector
stayed silent. The tokens are now compared (constant time, boolean only);
the token case gets its own banner key and operator message. Still
detection only - nothing is reconciled (R-78).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-05 23:12:59 +02:00

262 lines
10 KiB
Go

package bootstrap
import (
"bytes"
"encoding/json"
"io"
"log"
"os"
"path/filepath"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-controller/internal/config"
)
// R-77 endpoint-drift detection. The outage this guards is DIAG-agent-channel-2026-07-26: the island
// migration rewrote bootstrap.json, controller.yaml kept the pre-island LAN address, and the fleet's
// controllers dialled the wrong host for 17.5 h while alerting only "agent unreachable".
const driftYAML = `customer:
id: demo-hp
local_api:
endpoint: %ENDPOINT%
fingerprint: aaaa1111
token: tok-secret
`
// writeDriftFixture lays out a controller.yaml + bootstrap.json pair and points Path() at the latter.
func writeDriftFixture(t *testing.T, cfgEndpoint, bsEndpoint, bsFingerprint, bsToken string) (cfgPath string, cfg *config.Config) {
t.Helper()
dir := t.TempDir()
cfgPath = filepath.Join(dir, "controller.yaml")
if err := os.WriteFile(cfgPath, []byte(strings.ReplaceAll(driftYAML, "%ENDPOINT%", cfgEndpoint)), 0o600); err != nil {
t.Fatal(err)
}
bsPath := filepath.Join(dir, "bootstrap.json")
b := Bootstrap{}
b.LocalAPI = BootstrapLocalAPI{Endpoint: bsEndpoint, Fingerprint: bsFingerprint, Token: bsToken}
raw, _ := json.Marshal(b)
if err := os.WriteFile(bsPath, raw, 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("FELHOM_BOOTSTRAP_PATH", bsPath)
var err error
cfg, err = config.LoadPermissive(cfgPath)
if err != nil {
t.Fatal(err)
}
return cfgPath, cfg
}
func sha(t *testing.T, p string) []byte {
t.Helper()
b, err := os.ReadFile(p)
if err != nil {
t.Fatal(err)
}
return b
}
// Scenario A — drift is DETECTED, NAMED, and nothing is written.
//
// The "nothing is written" half is the load-bearing assertion: auto-reconcile is R-78 and would
// clobber a working channel on any guest whose controller.yaml is the correct one. "An error was
// logged" alone would be a hollow test.
func TestScenarioA_DriftDetectedAndNamed_NoWrite(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t,
"192.168.0.87:8443", // the live demo-hp value
"169.254.253.1:8443", // the island value the migration wrote
"aaaa1111", "tok-secret")
before := sha(t, cfgPath)
var buf bytes.Buffer
d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0))
if d == nil {
t.Fatal("drift must be DETECTED — this is the exact live shape from the 2026-07-25 outage")
}
if d.ConfigEndpoint != "192.168.0.87:8443" || d.BootstrapEndpoint != "169.254.253.1:8443" {
t.Errorf("wrong values captured: %+v", d)
}
if !d.FingerprintAgrees {
t.Error("fingerprints are identical in this fixture — must report agreement")
}
// The log line alone must diagnose it: BOTH values AND BOTH paths.
logged := buf.String()
for _, want := range []string{"192.168.0.87:8443", "169.254.253.1:8443", cfgPath, "bootstrap.json", "ERROR"} {
if !strings.Contains(logged, want) {
t.Errorf("the ERROR line must contain %q; got:\n%s", want, logged)
}
}
// Never log a secret.
if strings.Contains(logged, "tok-secret") {
t.Error("the token must NEVER be logged")
}
// THE assertion: controller.yaml is byte-identical.
if after := sha(t, cfgPath); !bytes.Equal(before, after) {
t.Errorf("controller.yaml was MODIFIED — detection must never write (that is R-78)\nbefore:\n%s\nafter:\n%s", before, after)
}
}
// Scenario B — agreement is silent. A spurious alert on every healthy boot would be worse than the
// bug: it trains the operator to ignore the banner.
func TestScenarioB_AgreementIsSilent(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "169.254.253.1:8443", "169.254.253.1:8443", "aaaa1111", "tok-secret")
before := sha(t, cfgPath)
var buf bytes.Buffer
if d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0)); d != nil {
t.Errorf("matching endpoints must not report drift: %+v", d)
}
if strings.Contains(buf.String(), "ERROR") {
t.Errorf("no ERROR on agreement; got: %s", buf.String())
}
if after := sha(t, cfgPath); !bytes.Equal(before, after) {
t.Error("controller.yaml must not be touched")
}
}
// Scenario D — an absent / unparseable / incomplete bootstrap is NOT drift. An unprovisioned or
// legacy guest must stay silent, not alarm.
func TestScenarioD_IncompleteBootstrapIsFailSafe(t *testing.T) {
t.Run("bootstrap absent", func(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "aaaa1111", "tok")
os.Remove(os.Getenv("FELHOM_BOOTSTRAP_PATH"))
if d := DetectEndpointDrift(cfgPath, cfg, log.New(io.Discard, "", 0)); d != nil {
t.Errorf("a missing bootstrap is not drift: %+v", d)
}
})
t.Run("bootstrap unparseable", func(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "aaaa1111", "tok")
os.WriteFile(os.Getenv("FELHOM_BOOTSTRAP_PATH"), []byte("{not json"), 0o600)
if d := DetectEndpointDrift(cfgPath, cfg, log.New(io.Discard, "", 0)); d != nil {
t.Errorf("an unparseable bootstrap is not drift: %+v", d)
}
})
// Incomplete = any of endpoint/fingerprint/token empty — the same completeness bar ensureLocalAPI
// applies before it will merge.
for _, tc := range []struct{ name, ep, fp, tok string }{
{"no endpoint", "", "aaaa1111", "tok"},
{"no fingerprint", "169.254.253.1:8443", "", "tok"},
{"no token", "169.254.253.1:8443", "aaaa1111", ""},
} {
t.Run(tc.name, func(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "192.168.0.87:8443", tc.ep, tc.fp, tc.tok)
if d := DetectEndpointDrift(cfgPath, cfg, log.New(io.Discard, "", 0)); d != nil {
t.Errorf("an incomplete bootstrap is not drift: %+v", d)
}
})
}
t.Run("controller.yaml has no endpoint — that is the fill-if-missing path, not drift", func(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, `""`, "169.254.253.1:8443", "aaaa1111", "tok")
if cfg.LocalAPI.Endpoint != "" {
t.Skipf("fixture did not produce an empty endpoint (got %q)", cfg.LocalAPI.Endpoint)
}
if d := DetectEndpointDrift(cfgPath, cfg, log.New(io.Discard, "", 0)); d != nil {
t.Errorf("an absent local_api endpoint is ensureLocalAPI's job, not drift: %+v", d)
}
})
t.Run("nil cfg", func(t *testing.T) {
if d := DetectEndpointDrift("/nonexistent", nil, log.New(io.Discard, "", 0)); d != nil {
t.Error("nil cfg must be silent")
}
})
}
// A moved PIN alongside a moved address is a materially different failure — fixing the endpoint
// alone would then fail closed on the pin. It must be surfaced, as a boolean, never as a value.
func TestDrift_FingerprintDisagreementIsSurfacedNotLeaked(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "bbbb2222", "tok-secret")
var buf bytes.Buffer
d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0))
if d == nil {
t.Fatal("drift expected")
}
if d.FingerprintAgrees {
t.Error("fingerprints differ in this fixture — must report DISagreement")
}
for _, secret := range []string{"aaaa1111", "bbbb2222", "tok-secret"} {
if strings.Contains(buf.String(), secret) {
t.Errorf("a fingerprint/token value leaked into the log: %q", secret)
}
if strings.Contains(d.EnglishMessage(), secret) {
t.Errorf("a fingerprint/token value leaked into the operator message: %q", secret)
}
}
if !strings.Contains(d.EnglishMessage(), "false") {
t.Errorf("the operator message must carry the pin-agreement boolean: %q", d.EnglishMessage())
}
}
// Scenario C — the fill-if-missing path is untouched by any of this (it is the ONLY writer).
func TestScenarioC_AbsentBlockStillMerges(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "controller.yaml")
if err := os.WriteFile(cfgPath, []byte("customer:\n id: demo-hp\n"), 0o600); err != nil {
t.Fatal(err)
}
bsPath := filepath.Join(dir, "bootstrap.json")
b := Bootstrap{}
b.LocalAPI = BootstrapLocalAPI{Endpoint: "169.254.253.1:8443", Fingerprint: "aaaa1111", Token: "tok"}
raw, _ := json.Marshal(b)
os.WriteFile(bsPath, raw, 0o600)
t.Setenv("FELHOM_BOOTSTRAP_PATH", bsPath)
cfg, err := config.LoadPermissive(cfgPath)
if err != nil {
t.Fatal(err)
}
if cfg.LocalAPI.Endpoint != "" {
t.Fatalf("precondition: fixture should have no local_api, got %q", cfg.LocalAPI.Endpoint)
}
got := ensureLocalAPI(cfgPath, cfg, log.New(io.Discard, "", 0))
if got == nil || got.LocalAPI.Endpoint != "169.254.253.1:8443" {
t.Fatalf("the fill-if-missing merge must still work; got %+v", got)
}
// And the drift check stays silent on the now-merged config.
if d := DetectEndpointDrift(cfgPath, got, log.New(io.Discard, "", 0)); d != nil {
t.Errorf("after a successful merge the two files agree — no drift: %+v", d)
}
}
// R-270 — a token rotation that reached bootstrap.json but not controller.yaml left the channel at
// HTTP 401 across restarts, and the endpoint-only detector stayed silent (proved live 2026-08-09).
// The consequence pinned: the token-only divergence is DETECTED, NAMED as a token (not an address)
// on the banner key, nothing is written, and the token value appears in no output.
func TestR270_TokenOnlyDriftIsNamedNotLeaked(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "169.254.253.1:8443", "169.254.253.1:8443", "aaaa1111", "tok-ROTATED")
before := sha(t, cfgPath)
var buf bytes.Buffer
d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0))
if d == nil {
t.Fatal("a token-only divergence must be DETECTED — this is the R-270 live shape (401 across restarts)")
}
if !d.EndpointAgrees || d.TokenAgrees {
t.Errorf("want EndpointAgrees=true TokenAgrees=false, got %+v", *d)
}
if d.AlertKey() != "alert.local_api_token_drift" {
t.Errorf("the banner must name the TOKEN, not the address: key %q", d.AlertKey())
}
if !strings.Contains(d.EnglishMessage(), "token drift") || !strings.Contains(buf.String(), "TOKEN drift") {
t.Errorf("operator message/log must name the token drift:\nmsg=%q\nlog=%q", d.EnglishMessage(), buf.String())
}
for _, secret := range []string{"tok-secret", "tok-ROTATED", "aaaa1111"} {
if strings.Contains(buf.String(), secret) || strings.Contains(d.EnglishMessage(), secret) || strings.Contains(d.HungarianMessage(), secret) {
t.Errorf("a secret value leaked: %q", secret)
}
}
if after := sha(t, cfgPath); !bytes.Equal(before, after) {
t.Error("controller.yaml was MODIFIED — detection must never write (R-78)")
}
// The address case keeps its own key (no regression of R-77's banner).
cfgPath2, cfg2 := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "aaaa1111", "tok-secret")
d2 := DetectEndpointDrift(cfgPath2, cfg2, log.New(io.Discard, "", 0))
if d2 == nil || d2.EndpointAgrees || !d2.TokenAgrees || d2.AlertKey() != "alert.endpoint_drift" {
t.Errorf("address drift must keep alert.endpoint_drift: %+v", d2)
}
}