// Package pvetrust moves the agent's trust in Proxmox from ONE pinned certificate to the box's own root CA, and names // the cause when a call fails on trust (R-1380, D80, felhom.eu `09` §3 decision 303; design `03-host-agent.md` §12a). // // WHY. Proxmox makes a new node certificate by itself, at most every two years. An agent that pins the old one is then // blind: every call fails, no host report reaches the hub, and the only alarm names a symptom. A copy of the box's own // root CA verifies every certificate that CA signs. // // THE ORDER, and it is the whole point (a box must never be left with neither way working): // // 1. the root step makes or refreshes the copy (it refuses anything that is not one CA certificate); // 2. a REAL call to Proxmox is made with the copy ALONE — while the fingerprint still is the configured trust; // 3. only after that call worked, ONE write of the agent's config sets `ca_file` and clears the fingerprint; // 4. the running client takes the new trust (no restart). // // If 1 or 2 fails nothing is written and the fingerprint stays. If 3 fails the fingerprint stays, in the file and in // the running client. `proxmox.tls` keeps its order in code — a set fingerprint wins over a CA file — so the // fingerprint is cleared LAST, in the same write that sets `ca_file`. // // WHO WRITES THE CONFIG. The agent itself: `agent.json` is the agent's own file (0600) in a root-owned folder, so it is // rewritten in place after a recovery copy was parked in the agent's state folder — the same way the backup-target move // and the PBS seed write it. No root step touches it. package pvetrust import ( "context" "encoding/json" "fmt" "gitea.dooplex.hu/admin/felhom-agent/internal/config" "log/slog" "os" "path/filepath" "sync" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // EndsSoonDays: the copy's certificate ends within this many days → the problem „ca_copy_ends_soon". Proxmox renews its // root CA when less than two years are left, so a healthy box never comes near it. const EndsSoonDays = 30 // Problems this package names beyond proxmox.Trust*. const ( ProblemCopyUnavailable = "ca_copy_unavailable" // the root step refused or did not answer ProblemCopyNoProof = "ca_copy_does_not_verify" // a call with the copy alone failed — the fingerprint stays ProblemCopyEndsSoon = "ca_copy_ends_soon" ProblemConfigNotSaved = "config_not_written" // the proof worked but the config could not be written — the fingerprint stays ) // Copy is what the root step answered. type Copy struct { Path string Outcome string // written | same NotAfter string DaysLeft int } // Status is the stanza that rides the host report (`proxmox_trust`). No secret: a mode, a cause, a date. type Status struct { Mode string `json:"mode"` // ca | fingerprint | ca_file_own | insecure | system Problem string `json:"problem,omitempty"` // "" = none Detail string `json:"detail,omitempty"` CANotAfter string `json:"ca_not_after,omitempty"` CADaysLeft *int `json:"ca_days_left,omitempty"` MovedAt string `json:"moved_at,omitempty"` // when this box moved from the pinned certificate to the CA copy CheckedAt string `json:"checked_at,omitempty"` } // Manager holds the trust state of one agent process. type Manager struct { // EnsureCopy runs the root step (osupdate.Leg.PVECACopy). EnsureCopy func(ctx context.Context) (Copy, error) // Probe makes one real call to Proxmox with EXACTLY this trust (a throwaway client). Probe func(ctx context.Context, t proxmox.TLSConfig) error // Apply gives the running client the new trust (proxmox.Client.SetTLS). Apply func(t proxmox.TLSConfig) error // ConfigPath is agent.json ("" = the config came from the environment: nothing can be saved, so nothing is moved). ConfigPath string StateDir string Logger *slog.Logger Now func() time.Time // MinGap between two runs that a failed call asked for (default 10 minutes). MinGap time.Duration runMu sync.Mutex // serialises Ensure mu sync.Mutex cur proxmox.TLSConfig status Status lastRun time.Time } // copyStepTimeout bounds the root step that makes the CA copy. const copyStepTimeout = 90 * time.Second func (m *Manager) now() time.Time { if m.Now != nil { return m.Now() } return time.Now() } func (m *Manager) log() *slog.Logger { if m.Logger != nil { return m.Logger } return slog.Default() } // Start records the trust the process started with. Call once, before Ensure. func (m *Manager) Start(t proxmox.TLSConfig) { m.mu.Lock() defer m.mu.Unlock() m.cur = t m.status = Status{Mode: modeOf(t)} } func modeOf(t proxmox.TLSConfig) string { switch { case t.InsecureSkipVerify: return "insecure" case t.Fingerprint != "": return "fingerprint" case t.CAFile != "": return "ca" } return "system" } // Status is the current stanza (nil-safe). func (m *Manager) Status() *Status { if m == nil { return nil } m.mu.Lock() defer m.mu.Unlock() s := m.status return &s } // Moved reports whether the box trusts Proxmox by the CA copy (nothing is left to move). func (m *Manager) Moved() bool { m.mu.Lock() defer m.mu.Unlock() return m.cur.Fingerprint == "" && m.cur.CAFile != "" } // Run asks again on a timer until ctx ends: every `pending` while the box is not moved yet (a wrapper that did not // know the mode, a Proxmox that did not answer), every `settled` after (a renewed CA refreshes the copy). func (m *Manager) Run(ctx context.Context, pending, settled time.Duration) { for { d := settled if !m.Moved() { d = pending } select { case <-ctx.Done(): return case <-time.After(d): m.Ensure(ctx, "timer") } } } // Current is the trust the running client uses. func (m *Manager) Current() proxmox.TLSConfig { m.mu.Lock() defer m.mu.Unlock() return m.cur } // OnCallError is the hook for a failed Proxmox call: when the failure is one of TRUST, it names the cause and runs // Ensure (at most once per MinGap) — a pinned box whose certificate changed moves to the CA copy right then, and a box // whose CA copy was refreshed reloads it. A failure of another kind does nothing. func (m *Manager) OnCallError(ctx context.Context, err error) { cause := proxmox.TrustProblem(err) if cause == "" { return } m.mu.Lock() gap := m.MinGap if gap <= 0 { gap = 10 * time.Minute } tooSoon := !m.lastRun.IsZero() && m.now().Sub(m.lastRun) < gap if m.status.Problem == "" { m.status.Problem, m.status.Detail = cause, causeText(cause) } m.mu.Unlock() m.log().Error("pvetrust: a call to Proxmox failed on TRUST — "+causeText(cause), "cause", cause, "err", err) if tooSoon { return } m.Ensure(ctx, cause) } func causeText(cause string) string { switch cause { case proxmox.TrustPinnedCertChanged: return "the pinned certificate is not the one Proxmox serves any more (Proxmox made a new certificate)" case proxmox.TrustCAMismatch: return "the CA copy did not sign the certificate Proxmox serves (the box's root CA changed, or the copy is not this box's)" case proxmox.TrustCertExpired: return "the certificate Proxmox serves, or its CA, has ended" case proxmox.TrustCAFileUnreadable: return "the CA copy is missing or holds no certificate" case proxmox.TrustNameMismatch: return "the certificate Proxmox serves does not name the address the agent calls" } return cause } // Ensure runs the order of the package comment once. why is a log word ("start", "daily", a trust cause). It returns // true when the running trust changed. func (m *Manager) Ensure(ctx context.Context, why string) bool { // One run at a time. Ensure is called from the start, the timer, a failed collect and after a bundle install; two // overlapping runs would both read the config, and the second would park the ALREADY-MOVED file as the "before" // copy (losing the fingerprint from the recovery copy). Found by review, 2026-11-14. m.runMu.Lock() defer m.runMu.Unlock() m.mu.Lock() cur := m.cur m.lastRun = m.now() m.mu.Unlock() lg := m.log().With("why", why, "mode", modeOf(cur)) set := func(mode, problem, detail string, c *Copy) { m.mu.Lock() defer m.mu.Unlock() moved := m.status.MovedAt m.status = Status{Mode: mode, Problem: problem, Detail: detail, MovedAt: moved, CheckedAt: m.now().UTC().Format(time.RFC3339)} if c != nil { d := c.DaysLeft m.status.CANotAfter, m.status.CADaysLeft = c.NotAfter, &d } } if cur.InsecureSkipVerify { lg.Warn("pvetrust: certificate checks are switched off in the config (insecure_skip_verify) — nothing is moved") set("insecure", "", "", nil) return false } if m.EnsureCopy == nil || m.Probe == nil { return false } // 1. the copy. The root step gets a deadline of its own: Ensure runs on the daemon's start path and inside the // report loop, and a wrapper that hangs (sudo, a stuck read of /etc/pve) must not stop either. cctx, ccancel := context.WithTimeout(ctx, copyStepTimeout) c, err := m.EnsureCopy(cctx) ccancel() if err != nil { lg.Warn("pvetrust: the root step gave no CA copy — the trust stays as it is", "err", err) set(modeOf(cur), ProblemCopyUnavailable, truncate(err.Error(), 240), nil) return false } ca := proxmox.TLSConfig{CAFile: c.Path} onCopy := cur.Fingerprint == "" && cur.CAFile == c.Path if cur.Fingerprint == "" && cur.CAFile != "" && cur.CAFile != c.Path { // an operator's own CA file: never replaced lg.Info("pvetrust: the config names its own CA file — left as it is", "ca_file", cur.CAFile) set("ca_file_own", "", "", &c) return false } if onCopy && c.Outcome != "written" && why != proxmox.TrustCAMismatch && why != proxmox.TrustCAFileUnreadable && why != proxmox.TrustCertExpired { // the box already trusts by the copy and the copy did not change: nothing to prove again problem, detail := "", "" if c.DaysLeft < EndsSoonDays { problem, detail = ProblemCopyEndsSoon, fmt.Sprintf("the CA copy ends in %d day(s), on %s", c.DaysLeft, c.NotAfter) lg.Error("pvetrust: "+detail, "days_left", c.DaysLeft) } set("ca", problem, detail, &c) return false } // 2. the proof: a real call with the copy ALONE — the fingerprint is still the configured trust pctx, cancel := context.WithTimeout(ctx, 30*time.Second) perr := m.Probe(pctx, ca) cancel() if perr != nil { cause := proxmox.TrustProblem(perr) detail := "a call to Proxmox with the CA copy alone failed" if cause != "" { detail += ": " + causeText(cause) } else { detail += " (not a trust failure: " + truncate(perr.Error(), 160) + ")" } if onCopy { lg.Error("pvetrust: the box trusts Proxmox by the CA copy and the copy does not verify the served certificate", "err", perr) set("ca", firstNonEmpty(cause, ProblemCopyNoProof), detail, &c) return false } if cause == "" { // Proxmox did not answer at all (it restarts, the box just booted): nothing is known about the copy, so no // cause is named; the next run asks again. lg.Warn("pvetrust: the proof could not be made now (not a trust failure) — the trust stays as it is; asked again later", "err", perr) set(modeOf(cur), "", "", &c) return false } lg.Error("pvetrust: the proof with the CA copy FAILED — the fingerprint STAYS, nothing was written", "err", perr) set(modeOf(cur), ProblemCopyNoProof, detail+" — the fingerprint stays", &c) return false } if onCopy { // the copy was refreshed (or a trust failure asked): the running client reloads it if m.Apply != nil { if aerr := m.Apply(ca); aerr != nil { set("ca", proxmox.TrustCAFileUnreadable, truncate(aerr.Error(), 200), &c) return false } } lg.Warn("pvetrust: the CA copy was refreshed and proven — the running client uses it now", "outcome", c.Outcome) set("ca", "", "", &c) return true } // 3. the config: ONE write sets ca_file and clears the fingerprint — only now, after the proof if m.ConfigPath == "" { lg.Warn("pvetrust: the proof worked, but the config did not come from a file — nothing can be saved, so the trust stays as it is") set(modeOf(cur), ProblemConfigNotSaved, "the config is not a file; the move was not made", &c) return false } if werr := m.writeConfig(c.Path); werr != nil { lg.Error("pvetrust: the proof worked but the config could NOT be written — the fingerprint stays", "err", werr) set(modeOf(cur), ProblemConfigNotSaved, truncate(werr.Error(), 200), &c) return false } // 4. the running client if m.Apply != nil { if aerr := m.Apply(ca); aerr != nil { lg.Error("pvetrust: the config is written but the running client could not take the CA copy — it keeps the old trust until the next start", "err", aerr) } } m.mu.Lock() m.cur = ca m.mu.Unlock() lg.Warn("pvetrust: MOVED — the agent trusts Proxmox by the box's own root CA now; the pinned fingerprint was cleared AFTER a working call with the copy", "ca_file", c.Path, "ca_not_after", c.NotAfter) set("ca", "", "", &c) m.mu.Lock() m.status.MovedAt = m.now().UTC().Format(time.RFC3339) m.mu.Unlock() return true } func firstNonEmpty(a, b string) string { if a != "" { return a } return b } func truncate(s string, n int) string { if len(s) <= n { return s } return s[:n] } // writeConfig sets proxmox.tls.ca_file and clears proxmox.tls.fingerprint in ONE write of agent.json. Every other key is // kept byte for byte. A recovery copy is parked in the state folder first; the result is read back and checked. func (m *Manager) writeConfig(caPath string) error { config.FileMu.Lock() // one writer of agent.json at a time in this process (config.FileMu) defer config.FileMu.Unlock() raw, err := os.ReadFile(m.ConfigPath) if err != nil { return err } var doc map[string]json.RawMessage if err := json.Unmarshal(raw, &doc); err != nil { return fmt.Errorf("parse %s: %w", m.ConfigPath, err) } px := map[string]json.RawMessage{} if cur, ok := doc["proxmox"]; ok { if err := json.Unmarshal(cur, &px); err != nil { return fmt.Errorf("parse the proxmox section: %w", err) } } tlsSec := map[string]json.RawMessage{} if cur, ok := px["tls"]; ok { if err := json.Unmarshal(cur, &tlsSec); err != nil { return fmt.Errorf("parse the proxmox.tls section: %w", err) } } caJSON, _ := json.Marshal(caPath) tlsSec["ca_file"] = caJSON tlsSec["fingerprint"] = json.RawMessage(`""`) b, _ := json.Marshal(tlsSec) px["tls"] = b b, _ = json.Marshal(px) doc["proxmox"] = b out, err := json.MarshalIndent(doc, "", " ") if err != nil { return err } out = append(out, '\n') st, err := os.Stat(m.ConfigPath) if err != nil { return err } if m.StateDir != "" { if err := os.MkdirAll(m.StateDir, 0o700); err != nil { return err } if err := os.WriteFile(filepath.Join(m.StateDir, "agent.json.pre-pve-trust"), raw, 0o600); err != nil { return fmt.Errorf("parking the recovery copy: %w", err) } } // In place (O_TRUNC), not tmp+rename: the folder is root-owned and the file is the agent's (the same reason as the // backup-target move and the PBS seed). One small write, then fsync. f, err := os.OpenFile(m.ConfigPath, os.O_WRONLY|os.O_TRUNC, st.Mode().Perm()) if err != nil { return err } if _, err := f.Write(out); err != nil { f.Close() _ = os.WriteFile(m.ConfigPath, raw, st.Mode().Perm()) // put the old bytes back: the fingerprint must not be lost return err } if err := f.Sync(); err != nil { f.Close() return err } if err := f.Close(); err != nil { return err } // read back: the file must say exactly what was meant var check struct { Proxmox struct { TLS struct { CAFile string `json:"ca_file"` Fingerprint string `json:"fingerprint"` } `json:"tls"` } `json:"proxmox"` } back, err := os.ReadFile(m.ConfigPath) if err != nil || json.Unmarshal(back, &check) != nil || check.Proxmox.TLS.CAFile != caPath || check.Proxmox.TLS.Fingerprint != "" { _ = os.WriteFile(m.ConfigPath, raw, st.Mode().Perm()) return fmt.Errorf("the config did not read back as written (%v) — the old config was put back", err) } return nil }