b2ca63ee9f
Closes the agent half of R-39's fleet fix. Requires hub >=0.68.0 for the re-arm signal; that hub is safe for 0.90.0 agents (unknown key dropped), so it deploys first. Three compounding defects let a box report `applied` while every PBS request 401'd: 1. The re-key was INVISIBLE. An ep0 re-issue rotates the secret of an existing token, so token_id/fingerprint/datastore/namespace come back byte-identical and the descriptor content hash never moved — the converged agent short-circuited and never consumed the fresh secret. WirePBSDR.SecretGeneration (field-exact with the hub) is what moves the hash now, because descriptorHash marshals this struct. 2. The agent could not READ its own credential. It writes /etc/pve/priv/storage/<id>.pw through the root wrapper, but that dir is 0700 root:www-data and the wrapper had no read verb — so the target resolver got "permission denied" every cycle, warned, and skipped. The one loop that could have caught the 401 was blind BY CONSTRUCTION. Adds a narrow `read` verb (+ exactly one sudoers line, + a pbsdr-read capability row): one secret to stdout, no network, no mutation, never in argv (sudo logs argv), traversal refused by the id grammar, the dir allowlist AND a resolved-path prefix assertion. 3. Nothing probed AUTHENTICATION. pbs.ProbeAuth (GET /version + an ErrUnauthorized sentinel) runs on the 15-minute collect path and its verdict becomes a loud `auth_failed` the hub escalates to a fresh mint. /version needs no datastore, namespace or privilege, so a 401 means the CREDENTIAL is bad; 403 is deliberately NOT treated as unauthorized, since re-keying a too-narrow token would mint forever without fixing anything. A transport error is UNKNOWN, never a rejection — otherwise every network blip burns a credential. Recovery self-clears. R-50b(a): the report now carries the installed wrapper's sha256 so drift against the vouched manifest value is answerable. Empty = unknown, never drift. Three red-proofs, all at the assertion level. Removing SecretGeneration fails the re-arm test with "consume calls=1, want 2". Swallowing the probe result leaves State:applied AuthFailed:false — the July-18 shape exactly. Notably, deleting the wrapper's id charset guard alone does NOT open a traversal hole (readlink + the prefix assertion still catch it), so the isolating red-proof removes BOTH and shows the out-of-tree secret printed — the layering is real, and a single-guard red-proof would have passed vacuously.
176 lines
7.4 KiB
Go
176 lines
7.4 KiB
Go
package pbsdr
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"testing"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// jsonUnmarshal is aliased so the Scenario-C decode reads as intent, not plumbing.
|
|
var jsonUnmarshal = json.Unmarshal
|
|
|
|
// R-39, Scenario A — THE fix: a credential re-key must re-arm a converged agent.
|
|
//
|
|
// The 2026-07-18 N100 failure in one sentence: an ep0 re-issue re-keys the SECRET of an existing
|
|
// token, so token_id, fingerprint, datastore and namespace all come back byte-identical; the
|
|
// descriptor hash did not move; the converged agent short-circuited; the fresh secret was never
|
|
// consumed; and the box served a revoked credential while reporting `applied`. The hub now stamps a
|
|
// monotonic SecretGeneration into the descriptor, and because descriptorHash marshals THIS STRUCT,
|
|
// that is what finally moves the hash.
|
|
//
|
|
// COMPANION RED-PROOF (run + recorded): delete SecretGeneration from hub.WirePBSDR (or stop the hub
|
|
// from advancing it) → the two descriptors marshal identically, the marker short-circuit fires, and
|
|
// this test FAILS with consume calls stuck at 1. That reproduces the defect exactly.
|
|
func TestR39_ReKeyReArmsAConvergedAgent(t *testing.T) {
|
|
r := &fakeRunner{}
|
|
// found=false on the first apply (fresh create), then the entry exists but is NOT active — which
|
|
// is what PVE reports for a storage whose credential is rejected (storage_info catches the 401,
|
|
// leaves active=0). That is the state a re-key has to recover from.
|
|
st := &fakeStorage{found: false, active: []bool{true}}
|
|
c := &fakeConsumer{secret: "SECRET-GEN-1"}
|
|
m, _ := newTestManager(t, r, st, c)
|
|
|
|
block := testBlock()
|
|
block.SecretGeneration = 1
|
|
m.Apply(context.Background(), true, block)
|
|
if c.calls != 1 {
|
|
t.Fatalf("first apply consumed %d secrets, want 1", c.calls)
|
|
}
|
|
if s := m.Status(); s == nil || s.State != "applied" {
|
|
t.Fatalf("first apply status = %+v, want applied", s)
|
|
}
|
|
|
|
// A re-apply of the SAME descriptor must stay a no-op — the idempotency the marker exists for.
|
|
m.Apply(context.Background(), true, block)
|
|
if c.calls != 1 {
|
|
t.Fatalf("re-apply of an unchanged descriptor consumed again (calls=%d)", c.calls)
|
|
}
|
|
|
|
// THE RE-KEY. Everything an ep0 re-issue actually returns is unchanged; only the generation moves.
|
|
//
|
|
// The entry now EXISTS and reads INACTIVE — which is exactly what PVE reports for a PBS storage
|
|
// whose credential is rejected: storage_info wraps activate_storage/status in eval{}, warns, and
|
|
// leaves the pre-initialised active=0. (Verified against PVE's own source; it returns HTTP 200
|
|
// with active:0, never an API error — which is what lets Apply fall through to the recovery path
|
|
// instead of bailing out at the status probe.)
|
|
st.found = true
|
|
st.entry = &proxmox.StorageEntryConfig{Type: "pbs", Namespace: "peti"}
|
|
st.active = append(st.active, false, true) // rejected → inactive, healthy after the reconcile
|
|
c.secret = "SECRET-GEN-2"
|
|
rekeyed := testBlock()
|
|
|
|
m.Apply(context.Background(), true, rekeyed)
|
|
|
|
if c.calls != 2 {
|
|
t.Fatalf("the re-key did NOT re-arm the agent: consume calls=%d, want 2.\n"+
|
|
"The converged short-circuit fired because the descriptor hash did not move — this is the "+
|
|
"R-39 defect (N100, 2026-07-18).", c.calls)
|
|
}
|
|
if s := m.Status(); s == nil || (s.State != "applied" && s.State != "adopted") {
|
|
t.Fatalf("post-re-key status = %+v, want converged", s)
|
|
}
|
|
}
|
|
|
|
// The generation genuinely changes the hash — the mechanism the test above depends on. Stated
|
|
// separately so a failure points at the CAUSE rather than at the flow.
|
|
func TestR39_SecretGenerationMovesTheDescriptorHash(t *testing.T) {
|
|
a := testBlock()
|
|
a.SecretGeneration = 1
|
|
b := testBlock()
|
|
b.SecretGeneration = 2
|
|
|
|
if descriptorHash(a) == descriptorHash(b) {
|
|
t.Fatal("SecretGeneration does not move descriptorHash — a re-key stays invisible to a " +
|
|
"converged agent and the fresh secret is never consumed (R-39)")
|
|
}
|
|
// And an unchanged generation must NOT move it (or every report would re-apply).
|
|
c := testBlock()
|
|
c.SecretGeneration = 1
|
|
if descriptorHash(a) != descriptorHash(c) {
|
|
t.Fatal("identical descriptors hash differently — the agent would re-apply on every tick")
|
|
}
|
|
}
|
|
|
|
// Scenario C, from the agent side: a descriptor carrying an UNKNOWN field (what a pre-0.91.0 agent
|
|
// sees) must be ignored, not rejected. Asserted by decoding hub JSON that contains a key this build
|
|
// does not know about.
|
|
func TestR39_UnknownDescriptorFieldIsInert(t *testing.T) {
|
|
var b hub.WirePBSDR
|
|
raw := []byte(`{"enabled":true,"storage_id":"felhom-pbs","secret_generation":7,"some_future_key":"x"}`)
|
|
if err := jsonUnmarshal(raw, &b); err != nil {
|
|
t.Fatalf("a descriptor with an unknown key must decode, got %v", err)
|
|
}
|
|
if !b.Enabled || b.StorageID != "felhom-pbs" || b.SecretGeneration != 7 {
|
|
t.Fatalf("known fields lost while ignoring an unknown one: %+v", b)
|
|
}
|
|
}
|
|
|
|
// R-39 leg (c) — a rejected credential becomes a LOUD auth_failed, and recovers by itself.
|
|
//
|
|
// COMPANION RED-PROOF (run + recorded): make NoteAuthResult ignore `unauthorized` (the pre-fix
|
|
// Warn-and-skip shape) → the state stays `applied` and this test FAILS.
|
|
func TestR39_AuthFailureBecomesLoudAndSelfClears(t *testing.T) {
|
|
r := &fakeRunner{}
|
|
st := &fakeStorage{found: false, active: []bool{true}}
|
|
c := &fakeConsumer{secret: "S"}
|
|
m, _ := newTestManager(t, r, st, c)
|
|
block := testBlock()
|
|
block.SecretGeneration = 1
|
|
m.Apply(context.Background(), true, block)
|
|
if s := m.Status(); s.State != "applied" {
|
|
t.Fatalf("precondition: want applied, got %+v", s)
|
|
}
|
|
|
|
// PBS rejects the credential.
|
|
m.NoteAuthResult("felhom-pbs", true, "PBS rejected the stored credential (401)")
|
|
s := m.Status()
|
|
if s.State != "auth_failed" || !s.AuthFailed {
|
|
t.Fatalf("a rejected credential must be LOUD: status = %+v, want auth_failed", s)
|
|
}
|
|
if s.StorageID != "felhom-pbs" {
|
|
t.Errorf("auth_failed must name the storage, got %q", s.StorageID)
|
|
}
|
|
|
|
// A transport error is UNKNOWN, not dead — it must not clear a real fault either.
|
|
m.NoteAuthResult("felhom-pbs", false, "dial tcp: connection refused")
|
|
if m.Status().State != "auth_failed" {
|
|
t.Error("an unreachable PBS cleared a real 401 — a blip must not paper over a dead credential")
|
|
}
|
|
|
|
// A clean probe restores the converged state without operator action.
|
|
m.NoteAuthResult("felhom-pbs", false, "")
|
|
if got := m.Status().State; got != "applied" {
|
|
t.Errorf("recovery did not clear auth_failed: state = %q, want applied", got)
|
|
}
|
|
}
|
|
|
|
// Another host's storage must never move this bridge's state.
|
|
func TestR39_AuthResultForAnotherStorageIsIgnored(t *testing.T) {
|
|
r := &fakeRunner{}
|
|
st := &fakeStorage{found: false, active: []bool{true}}
|
|
m, _ := newTestManager(t, r, st, &fakeConsumer{secret: "S"})
|
|
block := testBlock()
|
|
block.SecretGeneration = 1
|
|
m.Apply(context.Background(), true, block)
|
|
|
|
m.NoteAuthResult("some-other-pbs", true, "401")
|
|
if got := m.Status().State; got != "applied" {
|
|
t.Errorf("an unrelated storage's 401 changed our state to %q", got)
|
|
}
|
|
}
|
|
|
|
// A bridge that has never seen a descriptor must not invent a state from a probe.
|
|
func TestR39_AuthResultBeforeAnyDescriptorIsIgnored(t *testing.T) {
|
|
m := NewManager(&fakeRunner{}, &fakeStorage{}, &fakeConsumer{}, t.TempDir(), "/etc/pve/priv/storage",
|
|
t.TempDir()+"/agent.json", slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
m.NoteAuthResult("felhom-pbs", true, "401")
|
|
if s := m.Status(); s != nil {
|
|
t.Errorf("status invented from a probe with no descriptor: %+v", s)
|
|
}
|
|
}
|