R-270: the local_api drift detector also names a TOKEN-only divergence
A rotation that reached bootstrap.json but not controller.yaml left the agent channel at 401 across restarts while the endpoint-only detector stayed silent. The tokens are now compared (constant time, boolean only); the token case gets its own banner key and operator message. Still detection only - nothing is reconciled (R-78). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -2010,12 +2010,13 @@ func main() {
|
||||
// local_api endpoint drift (R-77, from the 2026-07-25 outage): controller.yaml and bootstrap.json
|
||||
// can disagree indefinitely and silently — the island migration rewrote the latter and the
|
||||
// controller kept dialling the former for 17.5 h, alerting only "agent unreachable". This NAMES
|
||||
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins).
|
||||
// the fault; it deliberately does not reconcile the files (R-78 owns which one wins). Since R-270 it
|
||||
// also names a TOKEN-only divergence (a rotation that reached bootstrap.json only).
|
||||
//
|
||||
// Startup-only is sufficient and correct: both files are read at boot and neither changes under a
|
||||
// running controller, so a periodic re-check would add noise without adding signal.
|
||||
if d := bootstrap.DetectEndpointDrift(*configPath, cfg, logger); d != nil {
|
||||
alertMgr.SetEndpointDriftAlert(true, "alert.endpoint_drift", d.HungarianMessage())
|
||||
alertMgr.SetEndpointDriftAlert(true, d.AlertKey(), d.HungarianMessage())
|
||||
if notifier != nil {
|
||||
notifier.NotifyEndpointDrift(d.EnglishMessage(), d.FingerprintAgrees)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -351,6 +352,11 @@ type EndpointDrift struct {
|
||||
// situation than a moved address — fixing the endpoint alone would then fail closed on the pin —
|
||||
// so it is surfaced, as a boolean, never as a value.
|
||||
FingerprintAgrees bool
|
||||
// EndpointAgrees is true when only the TOKEN moved (R-270): the address is right, the agent
|
||||
// answers 401, and before R-270 nothing named it. TokenAgrees is a BOOLEAN for the same reason
|
||||
// FingerprintAgrees is one — the token value is never stored, logged or sent.
|
||||
EndpointAgrees bool
|
||||
TokenAgrees bool
|
||||
}
|
||||
|
||||
// DetectEndpointDrift compares controller.yaml's live local_api.endpoint against bootstrap.json's.
|
||||
@@ -362,7 +368,12 @@ type EndpointDrift struct {
|
||||
// unprovisioned guest is not a drifted one;
|
||||
// - the bootstrap local_api block is incomplete (any of endpoint/fingerprint/token empty) — the
|
||||
// same completeness bar ensureLocalAPI applies before it will merge;
|
||||
// - the endpoints agree.
|
||||
// - the endpoints agree AND the tokens agree.
|
||||
//
|
||||
// R-270: a token rotation that rewrote bootstrap.json but not controller.yaml left the channel at
|
||||
// HTTP 401 across restarts, and this detector — endpoint-only — stayed silent. It now also compares
|
||||
// the token (constant-time, as a boolean only) and names that case; it still reconciles nothing
|
||||
// (R-78). Pinned by TestR270_TokenOnlyDriftIsNamedNotLeaked.
|
||||
//
|
||||
// It reads two files and writes NOTHING. Emitting the ERROR here (rather than at the call site)
|
||||
// keeps the diagnosis in one line of log even when the alert path is unavailable.
|
||||
@@ -381,7 +392,9 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
|
||||
if b.LocalAPI.Endpoint == "" || b.LocalAPI.Fingerprint == "" || b.LocalAPI.Token == "" {
|
||||
return nil
|
||||
}
|
||||
if cfg.LocalAPI.Endpoint == b.LocalAPI.Endpoint {
|
||||
endpointAgrees := cfg.LocalAPI.Endpoint == b.LocalAPI.Endpoint
|
||||
tokenAgrees := subtle.ConstantTimeCompare([]byte(cfg.LocalAPI.Token), []byte(b.LocalAPI.Token)) == 1
|
||||
if endpointAgrees && tokenAgrees {
|
||||
return nil
|
||||
}
|
||||
d := &EndpointDrift{
|
||||
@@ -390,6 +403,18 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
|
||||
ConfigEndpoint: cfg.LocalAPI.Endpoint,
|
||||
BootstrapEndpoint: b.LocalAPI.Endpoint,
|
||||
FingerprintAgrees: cfg.LocalAPI.Fingerprint == b.LocalAPI.Fingerprint,
|
||||
EndpointAgrees: endpointAgrees,
|
||||
TokenAgrees: tokenAgrees,
|
||||
}
|
||||
if endpointAgrees {
|
||||
if logger != nil {
|
||||
logger.Printf("[ERROR] bootstrap: local_api TOKEN drift — %s and %s carry DIFFERENT tokens "+
|
||||
"(endpoint %q agrees; pin agrees: %v). The controller presents %s's token, so the agent "+
|
||||
"answers 401 if it was rotated. Not auto-corrected (R-78) — write the rotated token into "+
|
||||
"%s too and restart the controller (R-270).",
|
||||
d.ConfigPath, d.BootstrapPath, d.ConfigEndpoint, d.FingerprintAgrees, d.ConfigPath, d.ConfigPath)
|
||||
}
|
||||
return d
|
||||
}
|
||||
if logger != nil {
|
||||
logger.Printf("[ERROR] bootstrap: local_api endpoint DRIFT — %s says %q but %s says %q; "+
|
||||
@@ -402,6 +427,12 @@ func DetectEndpointDrift(configPath string, cfg *config.Config, logger *log.Logg
|
||||
|
||||
// EnglishMessage is the operator-tier alert body (operator events are English by convention).
|
||||
func (d *EndpointDrift) EnglishMessage() string {
|
||||
if d.EndpointAgrees {
|
||||
return fmt.Sprintf("local_api token drift: controller.yaml and bootstrap.json carry different tokens "+
|
||||
"(endpoint %s agrees, pin agrees: %v) — the controller presents controller.yaml's token; after a "+
|
||||
"rotation write it there too and restart the controller (R-270).",
|
||||
d.ConfigEndpoint, d.FingerprintAgrees)
|
||||
}
|
||||
return fmt.Sprintf("local_api endpoint drift: controller.yaml=%s bootstrap.json=%s (pin agrees: %v) "+
|
||||
"— the controller is dialling controller.yaml's value; the agent may be listening on the other.",
|
||||
d.ConfigEndpoint, d.BootstrapEndpoint, d.FingerprintAgrees)
|
||||
@@ -410,5 +441,17 @@ func (d *EndpointDrift) EnglishMessage() string {
|
||||
// HungarianMessage is the customer-facing dashboard line, matching channelhealth's tone (short,
|
||||
// no addresses — the operator gets those in the event and the log).
|
||||
func (d *EndpointDrift) HungarianMessage() string {
|
||||
if d.EndpointAgrees {
|
||||
return "A tárolókezelő ügynök hozzáférési kulcsa elavult a beállításokban."
|
||||
}
|
||||
return "A tárolókezelő ügynök címe elavult a beállításokban."
|
||||
}
|
||||
|
||||
// AlertKey is the dashboard banner's bundle key: the address case keeps its key; the token-only case
|
||||
// (R-270) has its own, so the banner never claims the ADDRESS is stale when it is not.
|
||||
func (d *EndpointDrift) AlertKey() string {
|
||||
if d.EndpointAgrees {
|
||||
return "alert.local_api_token_drift"
|
||||
}
|
||||
return "alert.endpoint_drift"
|
||||
}
|
||||
|
||||
@@ -221,3 +221,41 @@ func TestScenarioC_AbsentBlockStillMerges(t *testing.T) {
|
||||
t.Errorf("after a successful merge the two files agree — no drift: %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// R-270 — a token rotation that reached bootstrap.json but not controller.yaml left the channel at
|
||||
// HTTP 401 across restarts, and the endpoint-only detector stayed silent (proved live 2026-08-09).
|
||||
// The consequence pinned: the token-only divergence is DETECTED, NAMED as a token (not an address)
|
||||
// on the banner key, nothing is written, and the token value appears in no output.
|
||||
func TestR270_TokenOnlyDriftIsNamedNotLeaked(t *testing.T) {
|
||||
cfgPath, cfg := writeDriftFixture(t, "169.254.253.1:8443", "169.254.253.1:8443", "aaaa1111", "tok-ROTATED")
|
||||
before := sha(t, cfgPath)
|
||||
var buf bytes.Buffer
|
||||
d := DetectEndpointDrift(cfgPath, cfg, log.New(&buf, "", 0))
|
||||
if d == nil {
|
||||
t.Fatal("a token-only divergence must be DETECTED — this is the R-270 live shape (401 across restarts)")
|
||||
}
|
||||
if !d.EndpointAgrees || d.TokenAgrees {
|
||||
t.Errorf("want EndpointAgrees=true TokenAgrees=false, got %+v", *d)
|
||||
}
|
||||
if d.AlertKey() != "alert.local_api_token_drift" {
|
||||
t.Errorf("the banner must name the TOKEN, not the address: key %q", d.AlertKey())
|
||||
}
|
||||
if !strings.Contains(d.EnglishMessage(), "token drift") || !strings.Contains(buf.String(), "TOKEN drift") {
|
||||
t.Errorf("operator message/log must name the token drift:\nmsg=%q\nlog=%q", d.EnglishMessage(), buf.String())
|
||||
}
|
||||
for _, secret := range []string{"tok-secret", "tok-ROTATED", "aaaa1111"} {
|
||||
if strings.Contains(buf.String(), secret) || strings.Contains(d.EnglishMessage(), secret) || strings.Contains(d.HungarianMessage(), secret) {
|
||||
t.Errorf("a secret value leaked: %q", secret)
|
||||
}
|
||||
}
|
||||
if after := sha(t, cfgPath); !bytes.Equal(before, after) {
|
||||
t.Error("controller.yaml was MODIFIED — detection must never write (R-78)")
|
||||
}
|
||||
|
||||
// The address case keeps its own key (no regression of R-77's banner).
|
||||
cfgPath2, cfg2 := writeDriftFixture(t, "192.168.0.87:8443", "169.254.253.1:8443", "aaaa1111", "tok-secret")
|
||||
d2 := DetectEndpointDrift(cfgPath2, cfg2, log.New(io.Discard, "", 0))
|
||||
if d2 == nil || d2.EndpointAgrees || !d2.TokenAgrees || d2.AlertKey() != "alert.endpoint_drift" {
|
||||
t.Errorf("address drift must keep alert.endpoint_drift: %+v", d2)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
"alert.deadapp.single": "An installed app is not running: %s",
|
||||
"alert.deadapp.single_state": "An installed app is not running: %s (%s)",
|
||||
"alert.endpoint_drift": "The storage agent's address in the settings is out of date.",
|
||||
"alert.local_api_token_drift": "The access key of the storage agent in the settings is out of date.",
|
||||
"alert.hub.disabled": "The hub connection is off — central monitoring is not running",
|
||||
"alert.hub.unreachable": "The hub cannot be reached — last error: %s",
|
||||
"alert.link.monitoring": "System monitor",
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"alert.deadapp.single": "Telepített alkalmazás nem fut: %s",
|
||||
"alert.deadapp.single_state": "Telepített alkalmazás nem fut: %s (%s)",
|
||||
"alert.endpoint_drift": "A tárolókezelő ügynök címe elavult a beállításokban.",
|
||||
"alert.local_api_token_drift": "A tárolókezelő ügynök hozzáférési kulcsa elavult a beállításokban.",
|
||||
"alert.hub.disabled": "Hub kapcsolat kikapcsolva — a központi monitoring nem aktív",
|
||||
"alert.hub.unreachable": "Hub nem elérhető — utolsó hiba: %s",
|
||||
"alert.link.monitoring": "Rendszermonitor",
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
"event.backup_catchup_done": "BORN AS A KEY, v0.295.0 (R-871, `09` decision 109) -- the catch-up's NEW timeline line, never a Go literal; sent by Notifier.NotifyBackupCatchUp (wiring pinned by TestR871_CatchUpWiring).",
|
||||
"badge.lifecycle.abandoned": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",
|
||||
"app_export.network_drive_needs_password": "BORN AS A KEY (R-126, `09` decision 128) -- a NEW sentence, never a Go literal: the export refusal for a network drive without a bundle password. Pinned in Hungarian by TestExportStart_NetworkDriveNeedsPassword.",
|
||||
"alert.local_api_token_drift": "BORN AS A KEY (R-270) -- a NEW banner sentence, never a Go literal: the local-API token in controller.yaml differs from bootstrap.json. Pinned by internal/bootstrap TestR270_TokenOnlyDriftIsNamedNotLeaked (key) and the bundle.",
|
||||
"badge.lifecycle.abandoned.title": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",
|
||||
"badge.update.ahead.title": "BORN AS A KEY, v0.260.0 (R-524) -- a NEW sentence, never a Go literal, so there is nothing in the base capture to measure it against. Pinned in both languages by TestUpdateBadgeFollowsTheLanguage.",
|
||||
"badge.update.behind": "R-589 (v0.258.0) -- localeFuncs; the Hungarian form stays in templateFuncMap, pinned by TestLocaleFuncsHungarianBundleMatchesFuncMap",
|
||||
|
||||
Reference in New Issue
Block a user