package pvetrust import ( "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" "math/big" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // ---- a small Proxmox: a root CA that signs node certificates, and a server that can change its certificate ---- type pki struct { t *testing.T caKey *ecdsa.PrivateKey caCert *x509.Certificate caPEM []byte } func newPKI(t *testing.T, cn string) *pki { t.Helper() k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) return (&pki{t: t, caKey: k}).renewCA(cn) } // renewCA makes a new CA certificate with the SAME key and name (what Proxmox does when its root CA is renewed). func (p *pki) renewCA(cn string) *pki { tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: cn}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &p.caKey.PublicKey, p.caKey) if err != nil { p.t.Fatal(err) } p.caCert, _ = x509.ParseCertificate(der) p.caPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) return p } func (p *pki) leaf() tls.Certificate { k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) tpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: "node"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(12 * time.Hour), IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}} der, err := x509.CreateCertificate(rand.Reader, tpl, p.caCert, &k.PublicKey, p.caKey) if err != nil { p.t.Fatal(err) } return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: k} } func fp(c tls.Certificate) string { s := sha256.Sum256(c.Certificate[0]) return hex.EncodeToString(s[:]) } type pve struct { srv *httptest.Server mu sync.Mutex cert tls.Certificate } func newPVE(t *testing.T, c tls.Certificate) *pve { p := &pve{cert: c} p.srv = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { fmt.Fprint(w, `{"data":{"version":"9.2.2","release":"9.2","repoid":"x"}}`) })) // GetConfigForClient, not GetCertificate: httptest adds its own certificate, and for a call by IP address (no // server name) that one would win over GetCertificate. p.srv.TLS = &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { p.mu.Lock() defer p.mu.Unlock() return &tls.Config{Certificates: []tls.Certificate{p.cert}}, nil }} p.srv.StartTLS() t.Cleanup(p.srv.Close) return p } // newCert is `pveupdate` making a new node certificate: served from the next connection on. func (p *pve) newCert(c tls.Certificate) { p.mu.Lock() p.cert = c p.mu.Unlock() p.srv.CloseClientConnections() } // ---- the agent's side ---- type box struct { t *testing.T m *Manager px *proxmox.Client cfgPath string copyPath string source []byte // what the root step would copy (nil = the root step refuses) events []string applyErr error } func cfgJSON(fingerprint, caFile string) string { return fmt.Sprintf(`{ "hub": {"url": "https://hub.example", "host_id": "box-1", "api_key": "KEEP-ME"}, "proxmox": {"endpoint": "X", "node": "n", "token": "u@pve!t=s", "tls": {"fingerprint": %q, "ca_file": %q, "insecure_skip_verify": false}, "custom": [1, 2, 3]}, "unknown_key": {"stays": true} }`, fingerprint, caFile) } func newBox(t *testing.T, srv *pve, start proxmox.TLSConfig, source []byte) *box { t.Helper() dir := t.TempDir() b := &box{t: t, cfgPath: filepath.Join(dir, "agent.json"), copyPath: filepath.Join(dir, "pve-root-ca.pem"), source: source} if start.CAFile == "COPY" { start.CAFile = b.copyPath if err := os.WriteFile(b.copyPath, source, 0o644); err != nil { t.Fatal(err) } } if err := os.WriteFile(b.cfgPath, []byte(cfgJSON(start.Fingerprint, start.CAFile)), 0o600); err != nil { t.Fatal(err) } pcfg := proxmox.Config{Endpoint: srv.srv.URL, Node: "n", Token: "u@pve!t=s", TLS: start, HTTPTimeout: 5 * time.Second} px, err := proxmox.NewClient(pcfg) if err != nil { t.Fatal(err) } b.px = px b.m = &Manager{ConfigPath: b.cfgPath, StateDir: filepath.Join(dir, "state"), EnsureCopy: func(context.Context) (Copy, error) { b.events = append(b.events, "copy") if b.source == nil { return Copy{}, errors.New(`pve-ca-copy refused: {"code": "T3", "reason": "not a CA certificate"}`) } outcome := "same" if cur, _ := os.ReadFile(b.copyPath); string(cur) != string(b.source) { outcome = "written" if err := os.WriteFile(b.copyPath, b.source, 0o644); err != nil { t.Fatal(err) } } return Copy{Path: b.copyPath, Outcome: outcome, NotAfter: "2036-08-17T15:44:42Z", DaysLeft: 3560}, nil }, Probe: func(ctx context.Context, tl proxmox.TLSConfig) error { b.events = append(b.events, fmt.Sprintf("probe(fp=%q,ca=%v) while the file says fp=%q", tl.Fingerprint, tl.CAFile != "", b.fileTLS().Fingerprint)) c := pcfg c.TLS = tl p, err := proxmox.NewClient(c) if err != nil { return err } _, err = p.Version(ctx) return err }, Apply: func(tl proxmox.TLSConfig) error { b.events = append(b.events, fmt.Sprintf("apply(fp=%q,ca=%v)", tl.Fingerprint, tl.CAFile != "")) if b.applyErr != nil { return b.applyErr } return b.px.SetTLS(tl) }, } b.m.Start(start) return b } type fileTLS struct { Fingerprint string `json:"fingerprint"` CAFile string `json:"ca_file"` } func (b *box) fileTLS() fileTLS { var d struct { Proxmox struct { TLS fileTLS `json:"tls"` } `json:"proxmox"` } raw, _ := os.ReadFile(b.cfgPath) if err := json.Unmarshal(raw, &d); err != nil { b.t.Fatalf("agent.json is not JSON any more: %v\n%s", err, raw) } return d.Proxmox.TLS } func (b *box) call() error { _, err := b.px.Version(context.Background()) return err } func (b *box) has(prefix string) bool { for _, e := range b.events { if strings.HasPrefix(e, prefix) { return true } } return false } // Scenario: a NEW box — the installer wrote the copy and ca_file, no fingerprint. Nothing is proven again, nothing written. func TestNewBox_CAOnly_NothingToMove(t *testing.T) { p := newPKI(t, "PVE CA") srv := newPVE(t, p.leaf()) b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM) before, _ := os.ReadFile(b.cfgPath) if b.m.Ensure(context.Background(), "start") { t.Fatal("a box that already trusts by the copy changed its trust") } after, _ := os.ReadFile(b.cfgPath) if string(before) != string(after) || b.has("probe") || b.has("apply") { t.Fatalf("the config was touched or a proof ran: %v", b.events) } if st := b.m.Status(); st.Mode != "ca" || st.Problem != "" || st.CADaysLeft == nil || *st.CADaysLeft != 3560 { t.Fatalf("status = %+v", st) } if err := b.call(); err != nil { t.Fatalf("the call fails: %v", err) } } // Scenario: an OLD box moves. The proof runs with the CA copy ALONE while the file still holds the fingerprint; only // then ONE write sets ca_file and clears the fingerprint; every other key stays; a recovery copy is parked. // COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/A/redproof-agent.txt): write the config // BEFORE the probe → the order check fails; skip the probe → TestProofFails_TheFingerprintStays fails. func TestOldBox_MovesAndTheFingerprintIsClearedOnlyAfterTheProof(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM) old, _ := os.ReadFile(b.cfgPath) if !b.m.Ensure(context.Background(), "start") { t.Fatalf("the box did not move: %+v %v", b.m.Status(), b.events) } want := []string{"copy", fmt.Sprintf("probe(fp=\"\",ca=true) while the file says fp=%q", fp(leaf)), "apply(fp=\"\",ca=true)"} if strings.Join(b.events, " | ") != strings.Join(want, " | ") { t.Fatalf("order = %v\nwant %v", b.events, want) } if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath { t.Fatalf("the config's tls = %+v", got) } raw, _ := os.ReadFile(b.cfgPath) for _, keep := range []string{`"KEEP-ME"`, `"unknown_key"`, `"stays"`, `"custom"`, `"u@pve!t=s"`, `"insecure_skip_verify"`} { if !strings.Contains(string(raw), keep) { t.Fatalf("the write lost %s:\n%s", keep, raw) } } if parked, _ := os.ReadFile(filepath.Join(b.m.StateDir, "agent.json.pre-pve-trust")); string(parked) != string(old) { t.Fatal("no recovery copy of the old config was parked") } if st := b.m.Status(); st.Mode != "ca" || st.MovedAt == "" || st.Problem != "" { t.Fatalf("status = %+v", st) } // and now Proxmox makes a new certificate: the agent goes on, with no restart and no new client srv.newCert(p.leaf()) if err := b.call(); err != nil { t.Fatalf("after a new Proxmox certificate the moved box fails: %v", err) } } // Scenario: the proof fails (the copy is another box's CA). The fingerprint stays — in the file, byte for byte, and in // the running client — and the cause is named. func TestProofFails_TheFingerprintStays(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) other := newPKI(t, "another box") b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM) before, _ := os.ReadFile(b.cfgPath) if b.m.Ensure(context.Background(), "start") { t.Fatal("the trust changed although the proof failed") } after, _ := os.ReadFile(b.cfgPath) if string(before) != string(after) { t.Fatal("the config was written although the proof failed") } if b.has("apply") { t.Fatalf("the running client was changed: %v", b.events) } if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemCopyNoProof || !strings.Contains(st.Detail, "did not sign") { t.Fatalf("status = %+v", st) } if err := b.call(); err != nil { t.Fatalf("the fingerprint no longer works: %v", err) } } // Scenario: Proxmox makes a new certificate on a box that still PINS the old one. The failed call names the cause, the // box moves to the CA copy right then, and the next call works — no restart. func TestPinnedBox_NewCertificate_HealsWithNoRestart(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM) srv.newCert(p.leaf()) err := b.call() if proxmox.TrustProblem(err) != proxmox.TrustPinnedCertChanged { t.Fatalf("fixture: want a pin mismatch, got %v", err) } b.m.OnCallError(context.Background(), err) if err := b.call(); err != nil { t.Fatalf("the box is still blind after the move: %v (%v)", err, b.events) } if got := b.fileTLS(); got.Fingerprint != "" || got.CAFile != b.copyPath { t.Fatalf("the config's tls = %+v", got) } // a failure that is NOT one of trust does nothing n := len(b.events) b.m.OnCallError(context.Background(), errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused")) if len(b.events) != n { t.Fatalf("a refused connection started a trust run: %v", b.events[n:]) } } // Scenario: the root CA itself is renewed with the same key. The copy is refreshed, proven, and the running client // reloads it; the config is not written again. func TestCARenewedWithTheSameKey_TheCopyIsRefreshedAndReloaded(t *testing.T) { p := newPKI(t, "PVE CA") srv := newPVE(t, p.leaf()) b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM) oldCA := string(p.caPEM) p.renewCA("PVE CA") srv.newCert(p.leaf()) b.source = p.caPEM before, _ := os.ReadFile(b.cfgPath) if !b.m.Ensure(context.Background(), "daily") { t.Fatalf("a refreshed copy was not taken: %+v %v", b.m.Status(), b.events) } if cur, _ := os.ReadFile(b.copyPath); string(cur) == oldCA || string(cur) != string(p.caPEM) { t.Fatal("the copy was not refreshed") } if !b.has("probe") || !b.has("apply") { t.Fatalf("a refreshed copy must be proven and reloaded: %v", b.events) } if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) { t.Fatal("the config was written again") } if err := b.call(); err != nil { t.Fatalf("the call fails after the CA's renewal: %v", err) } } // The root step refuses (the source is not a CA certificate): nothing changes, the cause is named. func TestRootStepRefuses_NothingChanges(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, nil) before, _ := os.ReadFile(b.cfgPath) if b.m.Ensure(context.Background(), "start") || b.has("probe") || b.has("apply") { t.Fatalf("something ran without a copy: %v", b.events) } if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) { t.Fatal("the config was written") } if st := b.m.Status(); st.Problem != ProblemCopyUnavailable || !strings.Contains(st.Detail, "T3") { t.Fatalf("status = %+v", st) } } // The proof works but the config cannot be written: the fingerprint stays, in the file and in the running client. func TestConfigCannotBeWritten_TheFingerprintStays(t *testing.T) { if os.Geteuid() == 0 { t.Skip("root writes a read-only file") } p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM) if err := os.Chmod(b.cfgPath, 0o400); err != nil { t.Fatal(err) } if b.m.Ensure(context.Background(), "start") || b.has("apply") { t.Fatalf("the trust changed without a saved config: %v", b.events) } if got := b.fileTLS(); got.Fingerprint != fp(leaf) { t.Fatalf("the fingerprint is gone from the file: %+v", got) } if st := b.m.Status(); st.Mode != "fingerprint" || st.Problem != ProblemConfigNotSaved { t.Fatalf("status = %+v", st) } } // What is never moved: an operator's own CA file, a box with checks switched off, a config that is not a file. func TestNeverMoved(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) own := filepath.Join(t.TempDir(), "own-ca.pem") os.WriteFile(own, p.caPEM, 0o644) for name, start := range map[string]proxmox.TLSConfig{"own CA file": {CAFile: own}, "insecure": {InsecureSkipVerify: true}} { b := newBox(t, srv, start, p.caPEM) before, _ := os.ReadFile(b.cfgPath) if b.m.Ensure(context.Background(), "start") || b.has("apply") { t.Fatalf("%s: moved (%v)", name, b.events) } if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) { t.Fatalf("%s: the config was written", name) } } b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM) b.m.ConfigPath = "" if b.m.Ensure(context.Background(), "start") || b.has("apply") || b.m.Status().Problem != ProblemConfigNotSaved { t.Fatalf("a config that is not a file was 'moved': %v %+v", b.events, b.m.Status()) } } // The copy ends soon: the cause is named (the box still works). func TestCopyEndsSoon(t *testing.T) { p := newPKI(t, "PVE CA") srv := newPVE(t, p.leaf()) b := newBox(t, srv, proxmox.TLSConfig{CAFile: "COPY"}, p.caPEM) inner := b.m.EnsureCopy b.m.EnsureCopy = func(ctx context.Context) (Copy, error) { c, err := inner(ctx) c.DaysLeft, c.NotAfter = 12, "2026-11-26T00:00:00Z" return c, err } b.m.Ensure(context.Background(), "daily") if st := b.m.Status(); st.Problem != ProblemCopyEndsSoon || !strings.Contains(st.Detail, "12 day") { t.Fatalf("status = %+v", st) } } // A trust failure asks at most once per MinGap. func TestOnCallError_Throttled(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) other := newPKI(t, "another box") b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, other.caPEM) srv.newCert(p.leaf()) err := b.call() now := time.Date(2026, 11, 15, 3, 0, 0, 0, time.UTC) b.m.Now = func() time.Time { return now } b.m.OnCallError(context.Background(), err) n := len(b.events) b.m.OnCallError(context.Background(), err) if len(b.events) != n { t.Fatalf("a second failure inside the gap ran again: %v", b.events[n:]) } now = now.Add(11 * time.Minute) b.m.OnCallError(context.Background(), err) if len(b.events) == n { t.Fatal("no new run after the gap") } if st := b.m.Status(); st.Problem == "" { t.Fatalf("a blind box names no cause: %+v", st) } } // Proxmox does not answer at all (it restarts): nothing is known about the copy — no cause is named, nothing is written, // and the box counts as not moved, so the timer asks again soon. func TestProofCannotBeMadeNow_NoCauseNamed(t *testing.T) { p := newPKI(t, "PVE CA") leaf := p.leaf() srv := newPVE(t, leaf) b := newBox(t, srv, proxmox.TLSConfig{Fingerprint: fp(leaf)}, p.caPEM) b.m.Probe = func(context.Context, proxmox.TLSConfig) error { return errors.New("proxmox: GET /version: dial tcp 127.0.0.1:8006: connect: connection refused") } before, _ := os.ReadFile(b.cfgPath) if b.m.Ensure(context.Background(), "start") { t.Fatal("moved without a proof") } if after, _ := os.ReadFile(b.cfgPath); string(after) != string(before) { t.Fatal("the config was written") } if st := b.m.Status(); st.Problem != "" || st.Mode != "fingerprint" || b.m.Moved() { t.Fatalf("status = %+v moved=%v", st, b.m.Moved()) } }