felhom-agent/internal/pvetrust/pvetrust_test.go

package pvetrust

import (
	"context"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// ---- a small Proxmox: a root CA that signs node certificates, and a server that can change its certificate ----

type pki struct {
	t      *testing.T
	caKey  *ecdsa.PrivateKey
	caCert *x509.Certificate
	caPEM  []byte
}

func newPKI(t *testing.T, cn string) *pki {
	t.Helper()
	k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	return (&pki{t: t, caKey: k}).renewCA(cn)
}

// renewCA makes a new CA certificate with the SAME key and name (what Proxmox does when its root CA is renewed).
func (p *pki) renewCA(cn string) *pki {
	tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: cn},
		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), IsCA: true, BasicConstraintsValid: true,
		KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign}
	der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &p.caKey.PublicKey, p.caKey)
	if err != nil {
		p.t.Fatal(err)
	}
	p.caCert, _ = x509.ParseCertificate(der)
	p.caPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
	return p
}

func (p *pki) leaf() tls.Certificate {
	k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
	tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "node"},
		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(12 * time.Hour), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}
	der, err := x509.CreateCertificate(rand.Reader, tpl, p.caCert, &k.PublicKey, p.caKey)
	if err != nil {
		p.t.Fatal(err)
	}
	return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: k}
}

func fp(c tls.Certificate) string {
	s := sha256.Sum256(c.Certificate[0])
	return hex.EncodeToString(s[:])
}

type pve struct {
	srv  *httptest.Server
	mu   sync.Mutex
	cert tls.Certificate
}

func newPVE(t *testing.T, c tls.Certificate) *pve {
	p := &pve{cert: c}
	p.srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		fmt.Fprint(w, `{"data":{"version":"9.2.2","release":"9.2","repoid":"x"}}`)
	}))
	// GetConfigForClient, not GetCertificate: httptest adds its own certificate, and for a call by IP address (no
	// server name) that one would win over GetCertificate.
	p.srv.TLS = &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
		p.mu.Lock()
		defer p.mu.Unlock()
		return &tls.Config{Certificates: []tls.Certificate{p.cert}}, nil
	}}
	p.srv.StartTLS()
	t.Cleanup(p.srv.Close)
	return p
}

// newCert is `pveupdate` making a new node certificate: served from the next connection on.
func (p *pve) newCert(c tls.Certificate) {
	p.mu.Lock()
	p.cert = c
	p.mu.Unlock()
	p.srv.CloseClientConnections()
}

// ---- the agent's side ----

type box struct {
	t        *testing.T
	m        *Manager
	px       *proxmox.Client
	cfgPath  string
	copyPath string
	source   []byte // what the root step would copy (nil = the root step refuses)
	events   []string
	applyErr error
}

func cfgJSON(fingerprint, caFile string) string {
	return fmt.Sprintf(`{
  "hub": {"url": "https://hub.example", "host_id": "box-1", "api_key": "KEEP-ME"},
  "proxmox": {"endpoint": "X", "node": "n", "token": "u@pve!t=s", "tls": {"fingerprint": %q, "ca_file": %q, "insecure_skip_verify": false}, "custom": [1, 2, 3]},
  "unknown_key": {"stays": true}
}`, fingerprint, caFile)
}

func newBox(t *testing.T, srv *pve, start proxmox.TLSConfig, source []byte) *box {
	t.Helper()
	dir := t.TempDir()
	b := &box{t: t, cfgPath: filepath.Join(dir, "agent.json"), copyPath: filepath.Join(dir, "pve-root-ca.pem"), source: source}
	if start.CAFile == "COPY" {
		start.CAFile = b.copyPath
		if err := os.WriteFile(b.copyPath, source, 0o644); err != nil {
			t.Fatal(err)
		}
	}
	if err := os.WriteFile(b.cfgPath, []byte(cfgJSON(start.Fingerprint, start.CAFile)), 0o600); err != nil {
		t.Fatal(err)
	}
	pcfg := proxmox.Config{Endpoint: srv.srv.URL, Node: "n", Token: "u@pve!t=s", TLS: start, HTTPTimeout: 5 * time.Second}
	px, err := proxmox.NewClient(pcfg)
	if err != nil {
		t.Fatal(err)
	}
	b.px = px
	b.m = &Manager{ConfigPath: b.cfgPath, StateDir: filepath.Join(dir, "state"),
		EnsureCopy: func(context.Context) (Copy, error) {
			b.events = append(b.events, "copy")
			if b.source == nil {
				return Copy{}, errors.New(`pve-ca-copy refused: {"code": "T3", "reason": "not a CA certificate"}`)
			}
			outcome := "same"
			if cur, _ := os.ReadFile(b.copyPath); string(cur) != string(b.source) {
				outcome = "written"
				if err := os.WriteFile(b.copyPath, b.source, 0o644); err != nil {
					t.Fatal(err)
				}
			}
			return Copy{Path: b.copyPath, Outcome: outcome, NotAfter: "2036-08-17T15:44:42Z", DaysLeft: 3560}, nil
		},
		Probe: func(ctx context.Context, tl proxmox.TLSConfig) error {
			b.events = append(b.events, fmt.Sprintf("probe(fp=%q,ca=%v) while the file says fp=%q", tl.Fingerprint, tl.CAFile != "", b.fileTLS().Fingerprint))
			c := pcfg
			c.TLS = tl
			p, err := proxmox.NewClient(c)
			if err != nil {
				return err
			}
			_, err = p.Version(ctx)
			return err
		},
		Apply: func(tl proxmox.TLSConfig) error {
			b.events = append(b.events, fmt.Sprintf("apply(fp=%q,ca=%v)", tl.Fingerprint, tl.CAFile != ""))
			if b.applyErr != nil {
				return b.applyErr
			}
			return b.px.SetTLS(tl)
		},
	}
	b.m.Start(start)
	return b
}

type fileTLS struct {
	Fingerprint string `json:"fingerprint"`
	CAFile      string `json:"ca_file"`
}

func (b *box) fileTLS() fileTLS {
	var d struct {
		Proxmox struct {
			TLS fileTLS `json:"tls"`
		} `json:"proxmox"`
	}
	raw, _ := os.ReadFile(b.cfgPath)
	if err := json.Unmarshal(raw, &d); err != nil {
		b.t.Fatalf("agent.json is not JSON any more: %v\n%s", err, raw)
	}
	return d.Proxmox.TLS
}

func (b *box) call() error {
	_, err := b.px.Version(context.Background())
	return err
}

func (b *box) has(prefix string) bool {
	for _, e := range b.events {
		if strings.HasPrefix(e, prefix) {
			return true
		}
	}
	return false
}

// Scenario: a NEW box — the installer wrote the copy and ca_file, no fingerprint. Nothing is proven again, nothing written.
func TestNewBox_CAOnly_NothingToMove(t *testing.T) {
	p := newPKI(t, "PVE CA")
	srv := newPVE(t, p.leaf())
	b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
	before, _ := os.ReadFile(b.cfgPath)
	if b.m.Ensure(context.Background(), "start") {
		t.Fatal("a box that already trusts by the copy changed its trust")
	}
	after, _ := os.ReadFile(b.cfgPath)
	if string(before) != string(after) || b.has("probe") || b.has("apply") {
		t.Fatalf("the config was touched or a proof ran: %v", b.events)
	}
	if st := b.m.Status(); st.Mode != "ca" || st.Problem != "" || st.CADaysLeft == nil || *st.CADaysLeft != 3560 {
		t.Fatalf("status = %+v", st)
	}
	if err := b.call(); err != nil {
		t.Fatalf("the call fails: %v", err)
	}
}

// Scenario: an OLD box moves. The proof runs with the CA copy ALONE while the file still holds the fingerprint; only
// then ONE write sets ca_file and clears the fingerprint; every other key stays; a recovery copy is parked.
// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/A/redproof-agent.txt): write the config
// BEFORE the probe → the order check fails; skip the probe → TestProofFails_TheFingerprintStays fails.
func TestOldBox_MovesAndTheFingerprintIsClearedOnlyAfterTheProof(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
	old, _ := os.ReadFile(b.cfgPath)
	if !b.m.Ensure(context.Background(), "start") {
		t.Fatalf("the box did not move: %+v %v", b.m.Status(), b.events)
	}
	want := []string{"copy", fmt.Sprintf("probe(fp=\"\",ca=true) while the file says fp=%q", fp(leaf)), "apply(fp=\"\",ca=true)"}
	if strings.Join(b.events, " | ") != strings.Join(want, " | ") {
		t.Fatalf("order = %v\nwant    %v", b.events, want)
	}
	if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath {
		t.Fatalf("the config's tls = %+v", got)
	}
	raw, _ := os.ReadFile(b.cfgPath)
	for _, keep := range []string{`"KEEP-ME"`, `"unknown_key"`, `"stays"`, `"custom"`, `"u@pve!t=s"`, `"insecure_skip_verify"`} {
		if !strings.Contains(string(raw), keep) {
			t.Fatalf("the write lost %s:\n%s", keep, raw)
		}
	}
	if parked, _ := os.ReadFile(filepath.Join(b.m.StateDir, "agent.json.pre-pve-trust")); string(parked) != string(old) {
		t.Fatal("no recovery copy of the old config was parked")
	}
	if st := b.m.Status(); st.Mode != "ca" || st.MovedAt == "" || st.Problem != "" {
		t.Fatalf("status = %+v", st)
	}
	// and now Proxmox makes a new certificate: the agent goes on, with no restart and no new client
	srv.newCert(p.leaf())
	if err := b.call(); err != nil {
		t.Fatalf("after a new Proxmox certificate the moved box fails: %v", err)
	}
}

// Scenario: the proof fails (the copy is another box's CA). The fingerprint stays — in the file, byte for byte, and in
// the running client — and the cause is named.
func TestProofFails_TheFingerprintStays(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	other := newPKI(t, "another box")
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM)
	before, _ := os.ReadFile(b.cfgPath)
	if b.m.Ensure(context.Background(), "start") {
		t.Fatal("the trust changed although the proof failed")
	}
	after, _ := os.ReadFile(b.cfgPath)
	if string(before) != string(after) {
		t.Fatal("the config was written although the proof failed")
	}
	if b.has("apply") {
		t.Fatalf("the running client was changed: %v", b.events)
	}
	if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemCopyNoProof || !strings.Contains(st.Detail, "did not sign") {
		t.Fatalf("status = %+v", st)
	}
	if err := b.call(); err != nil {
		t.Fatalf("the fingerprint no longer works: %v", err)
	}
}

// Scenario: Proxmox makes a new certificate on a box that still PINS the old one. The failed call names the cause, the
// box moves to the CA copy right then, and the next call works — no restart.
func TestPinnedBox_NewCertificate_HealsWithNoRestart(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
	srv.newCert(p.leaf())
	err := b.call()
	if proxmox.TrustProblem(err) != proxmox.TrustPinnedCertChanged {
		t.Fatalf("fixture: want a pin mismatch, got %v", err)
	}
	b.m.OnCallError(context.Background(), err)
	if err := b.call(); err != nil {
		t.Fatalf("the box is still blind after the move: %v (%v)", err, b.events)
	}
	if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath {
		t.Fatalf("the config's tls = %+v", got)
	}
	// a failure that is NOT one of trust does nothing
	n := len(b.events)
	b.m.OnCallError(context.Background(), errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused"))
	if len(b.events) != n {
		t.Fatalf("a refused connection started a trust run: %v", b.events[n:])
	}
}

// Scenario: the root CA itself is renewed with the same key. The copy is refreshed, proven, and the running client
// reloads it; the config is not written again.
func TestCARenewedWithTheSameKey_TheCopyIsRefreshedAndReloaded(t *testing.T) {
	p := newPKI(t, "PVE CA")
	srv := newPVE(t, p.leaf())
	b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
	oldCA := string(p.caPEM)
	p.renewCA("PVE CA")
	srv.newCert(p.leaf())
	b.source = p.caPEM
	before, _ := os.ReadFile(b.cfgPath)
	if !b.m.Ensure(context.Background(), "daily") {
		t.Fatalf("a refreshed copy was not taken: %+v %v", b.m.Status(), b.events)
	}
	if cur, _ := os.ReadFile(b.copyPath); string(cur) == oldCA || string(cur) != string(p.caPEM) {
		t.Fatal("the copy was not refreshed")
	}
	if !b.has("probe") || !b.has("apply") {
		t.Fatalf("a refreshed copy must be proven and reloaded: %v", b.events)
	}
	if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
		t.Fatal("the config was written again")
	}
	if err := b.call(); err != nil {
		t.Fatalf("the call fails after the CA's renewal: %v", err)
	}
}

// The root step refuses (the source is not a CA certificate): nothing changes, the cause is named.
func TestRootStepRefuses_NothingChanges(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, nil)
	before, _ := os.ReadFile(b.cfgPath)
	if b.m.Ensure(context.Background(), "start") || b.has("probe") || b.has("apply") {
		t.Fatalf("something ran without a copy: %v", b.events)
	}
	if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
		t.Fatal("the config was written")
	}
	if st := b.m.Status(); st.Problem != ProblemCopyUnavailable || !strings.Contains(st.Detail, "T3") {
		t.Fatalf("status = %+v", st)
	}
}

// The proof works but the config cannot be written: the fingerprint stays, in the file and in the running client.
func TestConfigCannotBeWritten_TheFingerprintStays(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("root writes a read-only file")
	}
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
	if err := os.Chmod(b.cfgPath, 0o400); err != nil {
		t.Fatal(err)
	}
	if b.m.Ensure(context.Background(), "start") || b.has("apply") {
		t.Fatalf("the trust changed without a saved config: %v", b.events)
	}
	if got := b.fileTLS(); got.Fingerprint != fp(leaf) {
		t.Fatalf("the fingerprint is gone from the file: %+v", got)
	}
	if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemConfigNotSaved {
		t.Fatalf("status = %+v", st)
	}
}

// What is never moved: an operator's own CA file, a box with checks switched off, a config that is not a file.
func TestNeverMoved(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	own := filepath.Join(t.TempDir(), "own-ca.pem")
	os.WriteFile(own, p.caPEM, 0o644)
	for name, start := range map[string]proxmox.TLSConfig{"own CA file": {CAFile: own}, "insecure": {InsecureSkipVerify: true}} {
		b := newBox(t, srv, start, p.caPEM)
		before, _ := os.ReadFile(b.cfgPath)
		if b.m.Ensure(context.Background(), "start") || b.has("apply") {
			t.Fatalf("%s: moved (%v)", name, b.events)
		}
		if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
			t.Fatalf("%s: the config was written", name)
		}
	}
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
	b.m.ConfigPath = ""
	if b.m.Ensure(context.Background(), "start") || b.has("apply") || b.m.Status().Problem != ProblemConfigNotSaved {
		t.Fatalf("a config that is not a file was 'moved': %v %+v", b.events, b.m.Status())
	}
}

// The copy ends soon: the cause is named (the box still works).
func TestCopyEndsSoon(t *testing.T) {
	p := newPKI(t, "PVE CA")
	srv := newPVE(t, p.leaf())
	b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM)
	inner := b.m.EnsureCopy
	b.m.EnsureCopy = func(ctx context.Context) (Copy, error) {
		c, err := inner(ctx)
		c.DaysLeft, c.NotAfter = 12, "2026-11-26T00:00:00Z"
		return c, err
	}
	b.m.Ensure(context.Background(), "daily")
	if st := b.m.Status(); st.Problem != ProblemCopyEndsSoon || !strings.Contains(st.Detail, "12 day") {
		t.Fatalf("status = %+v", st)
	}
}

// A trust failure asks at most once per MinGap.
func TestOnCallError_Throttled(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	other := newPKI(t, "another box")
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM)
	srv.newCert(p.leaf())
	err := b.call()
	now := time.Date(2026, 11, 15, 3, 0, 0, 0, time.UTC)
	b.m.Now = func() time.Time { return now }
	b.m.OnCallError(context.Background(), err)
	n := len(b.events)
	b.m.OnCallError(context.Background(), err)
	if len(b.events) != n {
		t.Fatalf("a second failure inside the gap ran again: %v", b.events[n:])
	}
	now = now.Add(11 * time.Minute)
	b.m.OnCallError(context.Background(), err)
	if len(b.events) == n {
		t.Fatal("no new run after the gap")
	}
	if st := b.m.Status(); st.Problem == "" {
		t.Fatalf("a blind box names no cause: %+v", st)
	}
}

// Proxmox does not answer at all (it restarts): nothing is known about the copy — no cause is named, nothing is written,
// and the box counts as not moved, so the timer asks again soon.
func TestProofCannotBeMadeNow_NoCauseNamed(t *testing.T) {
	p := newPKI(t, "PVE CA")
	leaf := p.leaf()
	srv := newPVE(t, leaf)
	b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM)
	b.m.Probe = func(context.Context, proxmox.TLSConfig) error {
		return errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused")
	}
	before, _ := os.ReadFile(b.cfgPath)
	if b.m.Ensure(context.Background(), "start") {
		t.Fatal("moved without a proof")
	}
	if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) {
		t.Fatal("the config was written")
	}
	if st := b.m.Status(); st.Problem != "" || st.Mode != "fingerprint" || b.m.Moved() {
		t.Fatalf("status = %+v moved=%v", st, b.m.Moved())
	}
}