R-270: the local_api drift detector also names a TOKEN-only divergence

A rotation that reached bootstrap.json but not controller.yaml left the
agent channel at 401 across restarts while the endpoint-only detector
stayed silent. The tokens are now compared (constant time, boolean only);
the token case gets its own banner key and operator message. Still
detection only - nothing is reconciled (R-78).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-05 21:36:34 +02:00
parent 33c397380e
commit 6fac245d24
6 changed files with 89 additions and 4 deletions
+3 -2
View File
@@ -2010,12 +2010,13 @@ func main() {
// local_api endpoint drift (R-77, from the 2026-07-25 outage): controller.yaml and bootstrap.json
// can disagree indefinitely and silently — the island migration rewrote the latter and the
// controller kept dialling the former for 17.5 h, alerting only "agent unreachable". This NAMES
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins).
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins). Since R-270 it
// also names a TOKEN-only divergence (a rotation that reached bootstrap.json only).
//
// Startup-only is sufficient and correct: both files are read at boot and neither changes under a
// running controller, so a periodic re-check would add noise without adding signal.
if d := bootstrap.DetectEndpointDrift(*configPath, cfg, logger); d != nil {
alertMgr.SetEndpointDriftAlert(true, "alert.endpoint_drift", d.HungarianMessage())
alertMgr.SetEndpointDriftAlert(true, d.AlertKey(), d.HungarianMessage())
if notifier != nil {
notifier.NotifyEndpointDrift(d.EnglishMessage(), d.FingerprintAgrees)
}
+45 -2
View File
@@ -12,6 +12,7 @@
package bootstrap
import (
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
@@ -351,6 +352,11 @@ type EndpointDrift struct {
// situation than a moved address — fixing the endpoint alone would then fail closed on the pin —
// so it is surfaced, as a boolean, never as a value.
FingerprintAgrees bool
// EndpointAgrees is true when only the TOKEN moved (R-270): the address is right, the agent
// answers 401, and before R-270 nothing named it. TokenAgrees is a BOOLEAN for the same reason
// FingerprintAgrees is one — the token value is never stored, logged or sent.
EndpointAgrees bool
TokenAgrees bool
}
// DetectEndpointDrift compares controller.yaml's live local_api.endpoint against bootstrap.json's.
@@ -362,7 +368,12 @@ type EndpointDrift struct {
// unprovisioned guest is not a drifted one;
// - the bootstrap local_api block is incomplete (any of endpoint/fingerprint/token empty) — the
// same completeness bar ensureLocalAPI applies before it will merge;
// - the endpoints agree.
// - the endpoints agree AND the tokens agree.
//
// R-270: a token rotation that rewrote bootstrap.json but not controller.yaml left the channel at
// HTTP 401 across restarts, and this detector — endpoint-only — stayed silent. It now also compares
// the token (constant-time, as a boolean only) and names that case; it still reconciles nothing
// (R-78). Pinned by TestR270_TokenOnlyDriftIsNamedNotLeaked.
//
// It reads two files and writes NOTHING. Emitting the ERROR here (rather than at the call site)
// keeps the diagnosis in one line of log even when the alert path is unavailable.
@@ -381,7 +392,9 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
if b.LocalAPI.Endpoint == "" || b.LocalAPI.Fingerprint == "" || b.LocalAPI.Token == "" {
return nil
}
if cfg.LocalAPI.Endpoint == b.LocalAPI.Endpoint {
endpointAgrees := cfg.LocalAPI.Endpoint == b.LocalAPI.Endpoint
tokenAgrees := subtle.ConstantTimeCompare([]byte(cfg.LocalAPI.Token), []byte(b.LocalAPI.Token)) == 1
if endpointAgrees && tokenAgrees {
return nil
}
d := &EndpointDrift{
@@ -390,6 +403,18 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
ConfigEndpoint: cfg.LocalAPI.Endpoint,
BootstrapEndpoint: b.LocalAPI.Endpoint,
FingerprintAgrees: cfg.LocalAPI.Fingerprint == b.LocalAPI.Fingerprint,
EndpointAgrees: endpointAgrees,
TokenAgrees: tokenAgrees,
}
if endpointAgrees {
if logger != nil {
logger.Printf("[ERROR] bootstrap: local_api TOKEN drift — %s and %s carry DIFFERENT tokens "+
"(endpoint %q agrees; pin agrees: %v). The controller presents %s's token, so the agent "+
"answers 401 if it was rotated. Not auto-corrected (R-78) — write the rotated token into "+
"%s too and restart the controller (R-270).",
d.ConfigPath, d.BootstrapPath, d.ConfigEndpoint, d.FingerprintAgrees, d.ConfigPath, d.ConfigPath)
}
return d
}
if logger != nil {
logger.Printf("[ERROR] bootstrap: local_api endpoint DRIFT — %s says %q but %s says %q; "+
@@ -402,6 +427,12 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
// EnglishMessage is the operator-tier alert body (operator events are English by convention).
func (d *EndpointDrift) EnglishMessage() string {
if d.EndpointAgrees {
return fmt.Sprintf("local_api token drift: controller.yaml and bootstrap.json carry different tokens "+
"(endpoint %s agrees, pin agrees: %v) — the controller presents controller.yaml's token; after a "+
"rotation write it there too and restart the controller (R-270).",
d.ConfigEndpoint, d.FingerprintAgrees)
}
return fmt.Sprintf("local_api endpoint drift: controller.yaml=%s bootstrap.json=%s (pin agrees: %v) "+
"— the controller is dialling controller.yaml's value; the agent may be listening on the other.",
d.ConfigEndpoint, d.BootstrapEndpoint, d.FingerprintAgrees)
@@ -410,5 +441,17 @@ func (d *EndpointDrift) EnglishMessage() string {
// HungarianMessage is the customer-facing dashboard line, matching channelhealth's tone (short,
// no addresses — the operator gets those in the event and the log).
func (d *EndpointDrift) HungarianMessage() string {
if d.EndpointAgrees {
return "A tárolókezelő ügynök hozzáférési kulcsa elavult a beállításokban."
}
return "A tárolókezelő ügynök címe elavult a beállításokban."
}
// AlertKey is the dashboard banner's bundle key: the address case keeps its key; the token-only case
// (R-270) has its own, so the banner never claims the ADDRESS is stale when it is not.
func (d *EndpointDrift) AlertKey() string {
if d.EndpointAgrees {
return "alert.local_api_token_drift"
}
return "alert.endpoint_drift"
}
@@ -221,3 +221,41 @@ func TestScenarioC_AbsentBlockStillMerges(t *testing.T) {
t.Errorf("after a successful merge the two files agree — no drift: %+v", d)
}
}
// R-270 — a token rotation that reached bootstrap.json but not controller.yaml left the channel at
// HTTP 401 across restarts, and the endpoint-only detector stayed silent (proved live 2026-08-09).
// The consequence pinned: the token-only divergence is DETECTED, NAMED as a token (not an address)
// on the banner key, nothing is written, and the token value appears in no output.
func TestR270_TokenOnlyDriftIsNamedNotLeaked(t *testing.T) {
cfgPath, cfg := writeDriftFixture(t, "169.254.253.1:8443", "169.254.253.1:8443", "aaaa1111", "tok-ROTATED")
before := sha(t, cfgPath)
var buf bytes.Buffer
d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0))
if d == nil {
t.Fatal("a token-only divergence must be DETECTED — this is the R-270 live shape (401 across restarts)")
}
if !d.EndpointAgrees || d.TokenAgrees {
t.Errorf("want EndpointAgrees=true TokenAgrees=false, got %+v", *d)
}
if d.AlertKey() != "alert.local_api_token_drift" {
t.Errorf("the banner must name the TOKEN, not the address: key %q", d.AlertKey())
}
if !strings.Contains(d.EnglishMessage(), "token drift") || !strings.Contains(buf.String(), "TOKEN drift") {
t.Errorf("operator message/log must name the token drift:\nmsg=%q\nlog=%q", d.EnglishMessage(), buf.String())
}
for _, secret := range []string{"tok-secret", "tok-ROTATED", "aaaa1111"} {
if strings.Contains(buf.String(), secret) || strings.Contains(d.EnglishMessage(), secret) || strings.Contains(d.HungarianMessage(), secret) {
t.Errorf("a secret value leaked: %q", secret)
}
}
if after := sha(t, cfgPath); !bytes.Equal(before, after) {
t.Error("controller.yaml was MODIFIED — detection must never write (R-78)")
}
// The address case keeps its own key (no regression of R-77's banner).
cfgPath2, cfg2 := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "aaaa1111", "tok-secret")
d2 := DetectEndpointDrift(cfgPath2, cfg2, log.New(io.Discard, "", 0))
if d2 == nil || d2.EndpointAgrees || !d2.TokenAgrees || d2.AlertKey() != "alert.endpoint_drift" {
t.Errorf("address drift must keep alert.endpoint_drift: %+v", d2)
}
}
+1
View File
@@ -13,6 +13,7 @@
"alert.deadapp.single": "An installed app is not running: %s",
"alert.deadapp.single_state": "An installed app is not running: %s (%s)",
"alert.endpoint_drift": "The storage agent's address in the settings is out of date.",
"alert.local_api_token_drift": "The access key of the storage agent in the settings is out of date.",
"alert.hub.disabled": "The hub connection is off — central monitoring is not running",
"alert.hub.unreachable": "The hub cannot be reached — last error: %s",
"alert.link.monitoring": "System monitor",
+1
View File
@@ -11,6 +11,7 @@
"alert.deadapp.single": "Telepített alkalmazás nem fut: %s",
"alert.deadapp.single_state": "Telepített alkalmazás nem fut: %s (%s)",
"alert.endpoint_drift": "A tárolókezelő ügynök címe elavult a beállításokban.",
"alert.local_api_token_drift": "A tárolókezelő ügynök hozzáférési kulcsa elavult a beállításokban.",
"alert.hub.disabled": "Hub kapcsolat kikapcsolva — a központi monitoring nem aktív",
"alert.hub.unreachable": "Hub nem elérhető — utolsó hiba: %s",
"alert.link.monitoring": "Rendszermonitor",
+1
View File
@@ -9,6 +9,7 @@
"event.backup_catchup_done": "BORN AS A KEY, v0.295.0 (R-871, `09` decision 109) -- the catch-up's NEW timeline line, never a Go literal; sent by Notifier.NotifyBackupCatchUp (wiring pinned by TestR871_CatchUpWiring).",
"badge.lifecycle.abandoned": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",
"app_export.network_drive_needs_password": "BORN AS A KEY (R-126, `09` decision 128) -- a NEW sentence, never a Go literal: the export refusal for a network drive without a bundle password. Pinned in Hungarian by TestExportStart_NetworkDriveNeedsPassword.",
"alert.local_api_token_drift": "BORN AS A KEY (R-270) -- a NEW banner sentence, never a Go literal: the local-API token in controller.yaml differs from bootstrap.json. Pinned by internal/bootstrap TestR270_TokenOnlyDriftIsNamedNotLeaked (key) and the bundle.",
"badge.lifecycle.abandoned.title": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",
"badge.update.ahead.title": "BORN AS A KEY, v0.260.0 (R-524) -- a NEW sentence, never a Go literal, so there is nothing in the base capture to measure it against. Pinned in both languages by TestUpdateBadgeFollowsTheLanguage.",
"badge.update.behind": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",