Files
felhom-agent/internal/pvetrust/pvetrust_test.go
T

490 lines
17 KiB
Go

package pvetrust
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/hex"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"math/big"
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)
// ---- a small Proxmox: a root CA that signs node certificates, and a server that can change its certificate ----
type pki struct {
t *testing.T
caKey *ecdsa.PrivateKey
caCert *x509.Certificate
caPEM []byte
}
func newPKI(t *testing.T, cn string) *pki {
t.Helper()
k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
return (&pki{t: t, caKey: k}).renewCA(cn)
}
// renewCA makes a new CA certificate with the SAME key and name (what Proxmox does when its root CA is renewed).
func (p *pki) renewCA(cn string) *pki {
tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: cn},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), IsCA: true, BasicConstraintsValid: true,
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign}
der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &p.caKey.PublicKey, p.caKey)
if err != nil {
p.t.Fatal(err)
}
p.caCert, _ = x509.ParseCertificate(der)
p.caPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
return p
}
func (p *pki) leaf() tls.Certificate {
k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "node"},
NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(12 * time.Hour), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}
der, err := x509.CreateCertificate(rand.Reader, tpl, p.caCert, &k.PublicKey, p.caKey)
if err != nil {
p.t.Fatal(err)
}
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: k}
}
func fp(c tls.Certificate) string {
s := sha256.Sum256(c.Certificate[0])
return hex.EncodeToString(s[:])
}
type pve struct {
srv *httptest.Server
mu sync.Mutex
cert tls.Certificate
}
func newPVE(t *testing.T, c tls.Certificate) *pve {
p := &pve{cert: c}
p.srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
fmt.Fprint(w, `{"data":{"version":"9.2.2","release":"9.2","repoid":"x"}}`)
}))
// GetConfigForClient, not GetCertificate: httptest adds its own certificate, and for a call by IP address (no
// server name) that one would win over GetCertificate.
p.srv.TLS = &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
p.mu.Lock()
defer p.mu.Unlock()
return &tls.Config{Certificates: []tls.Certificate{p.cert}}, nil
}}
p.srv.StartTLS()
t.Cleanup(p.srv.Close)
return p
}
// newCert is `pveupdate` making a new node certificate: served from the next connection on.
func (p *pve) newCert(c tls.Certificate) {
p.mu.Lock()
p.cert = c
p.mu.Unlock()
p.srv.CloseClientConnections()
}
// ---- the agent's side ----
type box struct {
t *testing.T
m *Manager
px *proxmox.Client
cfgPath string
copyPath string
source []byte // what the root step would copy (nil = the root step refuses)
events []string
applyErr error
}
func cfgJSON(fingerprint, caFile string) string {
return fmt.Sprintf(`{
"hub": {"url": "https://hub.example", "host_id": "box-1", "api_key": "KEEP-ME"},
"proxmox": {"endpoint": "X", "node": "n", "token": "u@pve!t=s", "tls": {"fingerprint": %q, "ca_file": %q, "insecure_skip_verify": false}, "custom": [1, 2, 3]},
"unknown_key": {"stays": true}
}`, fingerprint, caFile)
}
func newBox(t *testing.T, srv *pve, start proxmox.TLSConfig, source []byte) *box {
t.Helper()
dir := t.TempDir()
b := &box{t: t, cfgPath: filepath.Join(dir, "agent.json"), copyPath: filepath.Join(dir, "pve-root-ca.pem"), source: source}
if start.CAFile == "COPY" {
start.CAFile = b.copyPath
if err := os.WriteFile(b.copyPath, source, 0o644); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(b.cfgPath, []byte(cfgJSON(start.Fingerprint, start.CAFile)), 0o600); err != nil {
t.Fatal(err)
}
pcfg := proxmox.Config{Endpoint: srv.srv.URL, Node: "n", Token: "u@pve!t=s", TLS: start, HTTPTimeout: 5 * time.Second}
px, err := proxmox.NewClient(pcfg)
if err != nil {
t.Fatal(err)
}
b.px = px
b.m = &Manager{ConfigPath: b.cfgPath, StateDir: filepath.Join(dir, "state"),
EnsureCopy: func(context.Context) (Copy, error) {
b.events = append(b.events, "copy")
if b.source == nil {
return Copy{}, errors.New(`pve-ca-copy refused: {"code": "T3", "reason": "not a CA certificate"}`)
}
outcome := "same"
if cur, _ := os.ReadFile(b.copyPath); string(cur) != string(b.source) {
outcome = "written"
if err := os.WriteFile(b.copyPath, b.source, 0o644); err != nil {
t.Fatal(err)
}
}
return Copy{Path: b.copyPath, Outcome: outcome, NotAfter: "2036-08-17T15:44:42Z", DaysLeft: 3560}, nil
},
Probe: func(ctx context.Context, tl proxmox.TLSConfig) error {
b.events = append(b.events, fmt.Sprintf("probe(fp=%q,ca=%v) while the file says fp=%q", tl.Fingerprint, tl.CAFile != "", b.fileTLS().Fingerprint))
c := pcfg
c.TLS = tl
p, err := proxmox.NewClient(c)
if err != nil {
return err
}
_, err = p.Version(ctx)
return err
},
Apply: func(tl proxmox.TLSConfig) error {
b.events = append(b.events, fmt.Sprintf("apply(fp=%q,ca=%v)", tl.Fingerprint, tl.CAFile != ""))
if b.applyErr != nil {
return b.applyErr
}
return b.px.SetTLS(tl)
},
}
b.m.Start(start)
return b
}
type fileTLS struct {
Fingerprint string `json:"fingerprint"`
CAFile string `json:"ca_file"`
}
func (b *box) fileTLS() fileTLS {
var d struct {
Proxmox struct {
TLS fileTLS `json:"tls"`
} `json:"proxmox"`
}
raw, _ := os.ReadFile(b.cfgPath)
if err := json.Unmarshal(raw, &d); err != nil {
b.t.Fatalf("agent.json is not JSON any more: %v\n%s", err, raw)
}
return d.Proxmox.TLS
}
func (b *box) call() error {
_, err := b.px.Version(context.Background())
return err
}
func (b *box) has(prefix string) bool {
for _, e := range b.events {
if strings.HasPrefix(e, prefix) {
return true
}
}
return false
}
// Scenario: a NEW box — the installer wrote the copy and ca_file, no fingerprint. Nothing is proven again, nothing written.
func TestNewBox_CAOnly_NothingToMove(t *testing.T) {
p := newPKI(t, "PVE CA")
srv := newPVE(t, p.leaf())
b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
before, _ := os.ReadFile(b.cfgPath)
if b.m.Ensure(context.Background(), "start") {
t.Fatal("a box that already trusts by the copy changed its trust")
}
after, _ := os.ReadFile(b.cfgPath)
if string(before) != string(after) || b.has("probe") || b.has("apply") {
t.Fatalf("the config was touched or a proof ran: %v", b.events)
}
if st := b.m.Status(); st.Mode != "ca" || st.Problem != "" || st.CADaysLeft == nil || *st.CADaysLeft != 3560 {
t.Fatalf("status = %+v", st)
}
if err := b.call(); err != nil {
t.Fatalf("the call fails: %v", err)
}
}
// Scenario: an OLD box moves. The proof runs with the CA copy ALONE while the file still holds the fingerprint; only
// then ONE write sets ca_file and clears the fingerprint; every other key stays; a recovery copy is parked.
// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/A/redproof-agent.txt): write the config
// BEFORE the probe → the order check fails; skip the probe → TestProofFails_TheFingerprintStays fails.
func TestOldBox_MovesAndTheFingerprintIsClearedOnlyAfterTheProof(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
old, _ := os.ReadFile(b.cfgPath)
if !b.m.Ensure(context.Background(), "start") {
t.Fatalf("the box did not move: %+v %v", b.m.Status(), b.events)
}
want := []string{"copy", fmt.Sprintf("probe(fp=\"\",ca=true) while the file says fp=%q", fp(leaf)), "apply(fp=\"\",ca=true)"}
if strings.Join(b.events, " | ") != strings.Join(want, " | ") {
t.Fatalf("order = %v\nwant %v", b.events, want)
}
if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath {
t.Fatalf("the config's tls = %+v", got)
}
raw, _ := os.ReadFile(b.cfgPath)
for _, keep := range []string{`"KEEP-ME"`, `"unknown_key"`, `"stays"`, `"custom"`, `"u@pve!t=s"`, `"insecure_skip_verify"`} {
if !strings.Contains(string(raw), keep) {
t.Fatalf("the write lost %s:\n%s", keep, raw)
}
}
if parked, _ := os.ReadFile(filepath.Join(b.m.StateDir, "agent.json.pre-pve-trust")); string(parked) != string(old) {
t.Fatal("no recovery copy of the old config was parked")
}
if st := b.m.Status(); st.Mode != "ca" || st.MovedAt == "" || st.Problem != "" {
t.Fatalf("status = %+v", st)
}
// and now Proxmox makes a new certificate: the agent goes on, with no restart and no new client
srv.newCert(p.leaf())
if err := b.call(); err != nil {
t.Fatalf("after a new Proxmox certificate the moved box fails: %v", err)
}
}
// Scenario: the proof fails (the copy is another box's CA). The fingerprint stays — in the file, byte for byte, and in
// the running client — and the cause is named.
func TestProofFails_TheFingerprintStays(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
other := newPKI(t, "another box")
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM)
before, _ := os.ReadFile(b.cfgPath)
if b.m.Ensure(context.Background(), "start") {
t.Fatal("the trust changed although the proof failed")
}
after, _ := os.ReadFile(b.cfgPath)
if string(before) != string(after) {
t.Fatal("the config was written although the proof failed")
}
if b.has("apply") {
t.Fatalf("the running client was changed: %v", b.events)
}
if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemCopyNoProof || !strings.Contains(st.Detail, "did not sign") {
t.Fatalf("status = %+v", st)
}
if err := b.call(); err != nil {
t.Fatalf("the fingerprint no longer works: %v", err)
}
}
// Scenario: Proxmox makes a new certificate on a box that still PINS the old one. The failed call names the cause, the
// box moves to the CA copy right then, and the next call works — no restart.
func TestPinnedBox_NewCertificate_HealsWithNoRestart(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
srv.newCert(p.leaf())
err := b.call()
if proxmox.TrustProblem(err) != proxmox.TrustPinnedCertChanged {
t.Fatalf("fixture: want a pin mismatch, got %v", err)
}
b.m.OnCallError(context.Background(), err)
if err := b.call(); err != nil {
t.Fatalf("the box is still blind after the move: %v (%v)", err, b.events)
}
if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath {
t.Fatalf("the config's tls = %+v", got)
}
// a failure that is NOT one of trust does nothing
n := len(b.events)
b.m.OnCallError(context.Background(), errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused"))
if len(b.events) != n {
t.Fatalf("a refused connection started a trust run: %v", b.events[n:])
}
}
// Scenario: the root CA itself is renewed with the same key. The copy is refreshed, proven, and the running client
// reloads it; the config is not written again.
func TestCARenewedWithTheSameKey_TheCopyIsRefreshedAndReloaded(t *testing.T) {
p := newPKI(t, "PVE CA")
srv := newPVE(t, p.leaf())
b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
oldCA := string(p.caPEM)
p.renewCA("PVE CA")
srv.newCert(p.leaf())
b.source = p.caPEM
before, _ := os.ReadFile(b.cfgPath)
if !b.m.Ensure(context.Background(), "daily") {
t.Fatalf("a refreshed copy was not taken: %+v %v", b.m.Status(), b.events)
}
if cur, _ := os.ReadFile(b.copyPath); string(cur) == oldCA || string(cur) != string(p.caPEM) {
t.Fatal("the copy was not refreshed")
}
if !b.has("probe") || !b.has("apply") {
t.Fatalf("a refreshed copy must be proven and reloaded: %v", b.events)
}
if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
t.Fatal("the config was written again")
}
if err := b.call(); err != nil {
t.Fatalf("the call fails after the CA's renewal: %v", err)
}
}
// The root step refuses (the source is not a CA certificate): nothing changes, the cause is named.
func TestRootStepRefuses_NothingChanges(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, nil)
before, _ := os.ReadFile(b.cfgPath)
if b.m.Ensure(context.Background(), "start") || b.has("probe") || b.has("apply") {
t.Fatalf("something ran without a copy: %v", b.events)
}
if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
t.Fatal("the config was written")
}
if st := b.m.Status(); st.Problem != ProblemCopyUnavailable || !strings.Contains(st.Detail, "T3") {
t.Fatalf("status = %+v", st)
}
}
// The proof works but the config cannot be written: the fingerprint stays, in the file and in the running client.
func TestConfigCannotBeWritten_TheFingerprintStays(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("root writes a read-only file")
}
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
if err := os.Chmod(b.cfgPath, 0o400); err != nil {
t.Fatal(err)
}
if b.m.Ensure(context.Background(), "start") || b.has("apply") {
t.Fatalf("the trust changed without a saved config: %v", b.events)
}
if got := b.fileTLS(); got.Fingerprint != fp(leaf) {
t.Fatalf("the fingerprint is gone from the file: %+v", got)
}
if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemConfigNotSaved {
t.Fatalf("status = %+v", st)
}
}
// What is never moved: an operator's own CA file, a box with checks switched off, a config that is not a file.
func TestNeverMoved(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
own := filepath.Join(t.TempDir(), "own-ca.pem")
os.WriteFile(own, p.caPEM, 0o644)
for name, start := range map[string]proxmox.TLSConfig{"own CA file": {CAFile: own}, "insecure": {InsecureSkipVerify: true}} {
b := newBox(t, srv, start, p.caPEM)
before, _ := os.ReadFile(b.cfgPath)
if b.m.Ensure(context.Background(), "start") || b.has("apply") {
t.Fatalf("%s: moved (%v)", name, b.events)
}
if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
t.Fatalf("%s: the config was written", name)
}
}
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
b.m.ConfigPath = ""
if b.m.Ensure(context.Background(), "start") || b.has("apply") || b.m.Status().Problem != ProblemConfigNotSaved {
t.Fatalf("a config that is not a file was 'moved': %v %+v", b.events, b.m.Status())
}
}
// The copy ends soon: the cause is named (the box still works).
func TestCopyEndsSoon(t *testing.T) {
p := newPKI(t, "PVE CA")
srv := newPVE(t, p.leaf())
b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
inner := b.m.EnsureCopy
b.m.EnsureCopy = func(ctx context.Context) (Copy, error) {
c, err := inner(ctx)
c.DaysLeft, c.NotAfter = 12, "2026-11-26T00:00:00Z"
return c, err
}
b.m.Ensure(context.Background(), "daily")
if st := b.m.Status(); st.Problem != ProblemCopyEndsSoon || !strings.Contains(st.Detail, "12 day") {
t.Fatalf("status = %+v", st)
}
}
// A trust failure asks at most once per MinGap.
func TestOnCallError_Throttled(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
other := newPKI(t, "another box")
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM)
srv.newCert(p.leaf())
err := b.call()
now := time.Date(2026, 11, 15, 3, 0, 0, 0, time.UTC)
b.m.Now = func() time.Time { return now }
b.m.OnCallError(context.Background(), err)
n := len(b.events)
b.m.OnCallError(context.Background(), err)
if len(b.events) != n {
t.Fatalf("a second failure inside the gap ran again: %v", b.events[n:])
}
now = now.Add(11 * time.Minute)
b.m.OnCallError(context.Background(), err)
if len(b.events) == n {
t.Fatal("no new run after the gap")
}
if st := b.m.Status(); st.Problem == "" {
t.Fatalf("a blind box names no cause: %+v", st)
}
}
// Proxmox does not answer at all (it restarts): nothing is known about the copy — no cause is named, nothing is written,
// and the box counts as not moved, so the timer asks again soon.
func TestProofCannotBeMadeNow_NoCauseNamed(t *testing.T) {
p := newPKI(t, "PVE CA")
leaf := p.leaf()
srv := newPVE(t, leaf)
b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
b.m.Probe = func(context.Context, proxmox.TLSConfig) error {
return errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused")
}
before, _ := os.ReadFile(b.cfgPath)
if b.m.Ensure(context.Background(), "start") {
t.Fatal("moved without a proof")
}
if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
t.Fatal("the config was written")
}
if st := b.m.Status(); st.Problem != "" || st.Mode != "fingerprint" || b.m.Moved() {
t.Fatalf("status = %+v moved=%v", st, b.m.Moved())
}
}