felhom-agent/internal/pvetrust/pvetrust.go

// Package pvetrust moves the agent's trust in Proxmox from ONE pinned certificate to the box's own root CA, and names
// the cause when a call fails on trust (R-1380, D80, felhom.eu `09` §3 decision 303; design `03-host-agent.md` §12a).
//
// WHY. Proxmox makes a new node certificate by itself, at most every two years. An agent that pins the old one is then
// blind: every call fails, no host report reaches the hub, and the only alarm names a symptom. A copy of the box's own
// root CA verifies every certificate that CA signs.
//
// THE ORDER, and it is the whole point (a box must never be left with neither way working):
//
//  1. the root step makes or refreshes the copy (it refuses anything that is not one CA certificate);
//  2. a REAL call to Proxmox is made with the copy ALONE — while the fingerprint still is the configured trust;
//  3. only after that call worked, ONE write of the agent's config sets `ca_file` and clears the fingerprint;
//  4. the running client takes the new trust (no restart).
//
// If 1 or 2 fails nothing is written and the fingerprint stays. If 3 fails the fingerprint stays, in the file and in
// the running client. `proxmox.tls` keeps its order in code — a set fingerprint wins over a CA file — so the
// fingerprint is cleared LAST, in the same write that sets `ca_file`.
//
// WHO WRITES THE CONFIG. The agent itself: `agent.json` is the agent's own file (0600) in a root-owned folder, so it is
// rewritten in place after a recovery copy was parked in the agent's state folder — the same way the backup-target move
// and the PBS seed write it. No root step touches it.
package pvetrust

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"os"
	"path/filepath"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// EndsSoonDays: the copy's certificate ends within this many days → the problem „ca_copy_ends_soon". Proxmox renews its
// root CA when less than two years are left, so a healthy box never comes near it.
const EndsSoonDays = 30

// Problems this package names beyond proxmox.Trust*.
const (
	ProblemCopyUnavailable = "ca_copy_unavailable"     // the root step refused or did not answer
	ProblemCopyNoProof     = "ca_copy_does_not_verify" // a call with the copy alone failed — the fingerprint stays
	ProblemCopyEndsSoon    = "ca_copy_ends_soon"
	ProblemConfigNotSaved  = "config_not_written" // the proof worked but the config could not be written — the fingerprint stays
)

// Copy is what the root step answered.
type Copy struct {
	Path     string
	Outcome  string // written | same
	NotAfter string
	DaysLeft int
}

// Status is the stanza that rides the host report (`proxmox_trust`). No secret: a mode, a cause, a date.
type Status struct {
	Mode       string `json:"mode"`              // ca | fingerprint | ca_file_own | insecure | system
	Problem    string `json:"problem,omitempty"` // "" = none
	Detail     string `json:"detail,omitempty"`
	CANotAfter string `json:"ca_not_after,omitempty"`
	CADaysLeft *int   `json:"ca_days_left,omitempty"`
	MovedAt    string `json:"moved_at,omitempty"` // when this box moved from the pinned certificate to the CA copy
	CheckedAt  string `json:"checked_at,omitempty"`
}

// Manager holds the trust state of one agent process.
type Manager struct {
	// EnsureCopy runs the root step (osupdate.Leg.PVECACopy).
	EnsureCopy func(ctx context.Context) (Copy, error)
	// Probe makes one real call to Proxmox with EXACTLY this trust (a throwaway client).
	Probe func(ctx context.Context, t proxmox.TLSConfig) error
	// Apply gives the running client the new trust (proxmox.Client.SetTLS).
	Apply func(t proxmox.TLSConfig) error
	// ConfigPath is agent.json ("" = the config came from the environment: nothing can be saved, so nothing is moved).
	ConfigPath string
	StateDir   string
	Logger     *slog.Logger
	Now        func() time.Time
	// MinGap between two runs that a failed call asked for (default 10 minutes).
	MinGap time.Duration

	mu      sync.Mutex
	cur     proxmox.TLSConfig
	status  Status
	lastRun time.Time
}

func (m *Manager) now() time.Time {
	if m.Now != nil {
		return m.Now()
	}
	return time.Now()
}

func (m *Manager) log() *slog.Logger {
	if m.Logger != nil {
		return m.Logger
	}
	return slog.Default()
}

// Start records the trust the process started with. Call once, before Ensure.
func (m *Manager) Start(t proxmox.TLSConfig) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.cur = t
	m.status = Status{Mode: modeOf(t)}
}

func modeOf(t proxmox.TLSConfig) string {
	switch {
	case t.InsecureSkipVerify:
		return "insecure"
	case t.Fingerprint != "":
		return "fingerprint"
	case t.CAFile != "":
		return "ca"
	}
	return "system"
}

// Status is the current stanza (nil-safe).
func (m *Manager) Status() *Status {
	if m == nil {
		return nil
	}
	m.mu.Lock()
	defer m.mu.Unlock()
	s := m.status
	return &s
}

// Moved reports whether the box trusts Proxmox by the CA copy (nothing is left to move).
func (m *Manager) Moved() bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.cur.Fingerprint == "" && m.cur.CAFile != ""
}

// Run asks again on a timer until ctx ends: every `pending` while the box is not moved yet (a wrapper that did not
// know the mode, a Proxmox that did not answer), every `settled` after (a renewed CA refreshes the copy).
func (m *Manager) Run(ctx context.Context, pending, settled time.Duration) {
	for {
		d := settled
		if !m.Moved() {
			d = pending
		}
		select {
		case <-ctx.Done():
			return
		case <-time.After(d):
			m.Ensure(ctx, "timer")
		}
	}
}

// Current is the trust the running client uses.
func (m *Manager) Current() proxmox.TLSConfig {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.cur
}

// OnCallError is the hook for a failed Proxmox call: when the failure is one of TRUST, it names the cause and runs
// Ensure (at most once per MinGap) — a pinned box whose certificate changed moves to the CA copy right then, and a box
// whose CA copy was refreshed reloads it. A failure of another kind does nothing.
func (m *Manager) OnCallError(ctx context.Context, err error) {
	cause := proxmox.TrustProblem(err)
	if cause == "" {
		return
	}
	m.mu.Lock()
	gap := m.MinGap
	if gap <= 0 {
		gap = 10 * time.Minute
	}
	tooSoon := !m.lastRun.IsZero() && m.now().Sub(m.lastRun) < gap
	if m.status.Problem == "" {
		m.status.Problem, m.status.Detail = cause, causeText(cause)
	}
	m.mu.Unlock()
	m.log().Error("pvetrust: a call to Proxmox failed on TRUST — "+causeText(cause), "cause", cause, "err", err)
	if tooSoon {
		return
	}
	m.Ensure(ctx, cause)
}

func causeText(cause string) string {
	switch cause {
	case proxmox.TrustPinnedCertChanged:
		return "the pinned certificate is not the one Proxmox serves any more (Proxmox made a new certificate)"
	case proxmox.TrustCAMismatch:
		return "the CA copy did not sign the certificate Proxmox serves (the box's root CA changed, or the copy is not this box's)"
	case proxmox.TrustCertExpired:
		return "the certificate Proxmox serves, or its CA, has ended"
	case proxmox.TrustCAFileUnreadable:
		return "the CA copy is missing or holds no certificate"
	case proxmox.TrustNameMismatch:
		return "the certificate Proxmox serves does not name the address the agent calls"
	}
	return cause
}

// Ensure runs the order of the package comment once. why is a log word ("start", "daily", a trust cause). It returns
// true when the running trust changed.
func (m *Manager) Ensure(ctx context.Context, why string) bool {
	m.mu.Lock()
	cur := m.cur
	m.lastRun = m.now()
	m.mu.Unlock()
	lg := m.log().With("why", why, "mode", modeOf(cur))
	set := func(mode, problem, detail string, c *Copy) {
		m.mu.Lock()
		defer m.mu.Unlock()
		moved := m.status.MovedAt
		m.status = Status{Mode: mode, Problem: problem, Detail: detail, MovedAt: moved, CheckedAt: m.now().UTC().Format(time.RFC3339)}
		if c != nil {
			d := c.DaysLeft
			m.status.CANotAfter, m.status.CADaysLeft = c.NotAfter, &d
		}
	}
	if cur.InsecureSkipVerify {
		lg.Warn("pvetrust: certificate checks are switched off in the config (insecure_skip_verify) — nothing is moved")
		set("insecure", "", "", nil)
		return false
	}
	if m.EnsureCopy == nil || m.Probe == nil {
		return false
	}
	// 1. the copy
	c, err := m.EnsureCopy(ctx)
	if err != nil {
		lg.Warn("pvetrust: the root step gave no CA copy — the trust stays as it is", "err", err)
		set(modeOf(cur), ProblemCopyUnavailable, truncate(err.Error(), 240), nil)
		return false
	}
	ca := proxmox.TLSConfig{CAFile: c.Path}
	onCopy := cur.Fingerprint == "" && cur.CAFile == c.Path
	if cur.Fingerprint == "" && cur.CAFile != "" && cur.CAFile != c.Path {
		// an operator's own CA file: never replaced
		lg.Info("pvetrust: the config names its own CA file — left as it is", "ca_file", cur.CAFile)
		set("ca_file_own", "", "", &c)
		return false
	}
	if onCopy && c.Outcome != "written" && why != proxmox.TrustCAMismatch && why != proxmox.TrustCAFileUnreadable && why != proxmox.TrustCertExpired {
		// the box already trusts by the copy and the copy did not change: nothing to prove again
		problem, detail := "", ""
		if c.DaysLeft < EndsSoonDays {
			problem, detail = ProblemCopyEndsSoon, fmt.Sprintf("the CA copy ends in %d day(s), on %s", c.DaysLeft, c.NotAfter)
			lg.Error("pvetrust: "+detail, "days_left", c.DaysLeft)
		}
		set("ca", problem, detail, &c)
		return false
	}
	// 2. the proof: a real call with the copy ALONE — the fingerprint is still the configured trust
	pctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	perr := m.Probe(pctx, ca)
	cancel()
	if perr != nil {
		cause := proxmox.TrustProblem(perr)
		detail := "a call to Proxmox with the CA copy alone failed"
		if cause != "" {
			detail += ": " + causeText(cause)
		} else {
			detail += " (not a trust failure: " + truncate(perr.Error(), 160) + ")"
		}
		if onCopy {
			lg.Error("pvetrust: the box trusts Proxmox by the CA copy and the copy does not verify the served certificate", "err", perr)
			set("ca", firstNonEmpty(cause, ProblemCopyNoProof), detail, &c)
			return false
		}
		if cause == "" {
			// Proxmox did not answer at all (it restarts, the box just booted): nothing is known about the copy, so no
			// cause is named; the next run asks again.
			lg.Warn("pvetrust: the proof could not be made now (not a trust failure) — the trust stays as it is; asked again later", "err", perr)
			set(modeOf(cur), "", "", &c)
			return false
		}
		lg.Error("pvetrust: the proof with the CA copy FAILED — the fingerprint STAYS, nothing was written", "err", perr)
		set(modeOf(cur), ProblemCopyNoProof, detail+" — the fingerprint stays", &c)
		return false
	}
	if onCopy {
		// the copy was refreshed (or a trust failure asked): the running client reloads it
		if m.Apply != nil {
			if aerr := m.Apply(ca); aerr != nil {
				set("ca", proxmox.TrustCAFileUnreadable, truncate(aerr.Error(), 200), &c)
				return false
			}
		}
		lg.Warn("pvetrust: the CA copy was refreshed and proven — the running client uses it now", "outcome", c.Outcome)
		set("ca", "", "", &c)
		return true
	}
	// 3. the config: ONE write sets ca_file and clears the fingerprint — only now, after the proof
	if m.ConfigPath == "" {
		lg.Warn("pvetrust: the proof worked, but the config did not come from a file — nothing can be saved, so the trust stays as it is")
		set(modeOf(cur), ProblemConfigNotSaved, "the config is not a file; the move was not made", &c)
		return false
	}
	if werr := m.writeConfig(c.Path); werr != nil {
		lg.Error("pvetrust: the proof worked but the config could NOT be written — the fingerprint stays", "err", werr)
		set(modeOf(cur), ProblemConfigNotSaved, truncate(werr.Error(), 200), &c)
		return false
	}
	// 4. the running client
	if m.Apply != nil {
		if aerr := m.Apply(ca); aerr != nil {
			lg.Error("pvetrust: the config is written but the running client could not take the CA copy — it keeps the old trust until the next start", "err", aerr)
		}
	}
	m.mu.Lock()
	m.cur = ca
	m.mu.Unlock()
	lg.Warn("pvetrust: MOVED — the agent trusts Proxmox by the box's own root CA now; the pinned fingerprint was cleared AFTER a working call with the copy",
		"ca_file", c.Path, "ca_not_after", c.NotAfter)
	set("ca", "", "", &c)
	m.mu.Lock()
	m.status.MovedAt = m.now().UTC().Format(time.RFC3339)
	m.mu.Unlock()
	return true
}

func firstNonEmpty(a, b string) string {
	if a != "" {
		return a
	}
	return b
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n]
}

// writeConfig sets proxmox.tls.ca_file and clears proxmox.tls.fingerprint in ONE write of agent.json. Every other key is
// kept byte for byte. A recovery copy is parked in the state folder first; the result is read back and checked.
func (m *Manager) writeConfig(caPath string) error {
	raw, err := os.ReadFile(m.ConfigPath)
	if err != nil {
		return err
	}
	var doc map[string]json.RawMessage
	if err := json.Unmarshal(raw, &doc); err != nil {
		return fmt.Errorf("parse %s: %w", m.ConfigPath, err)
	}
	px := map[string]json.RawMessage{}
	if cur, ok := doc["proxmox"]; ok {
		if err := json.Unmarshal(cur, &px); err != nil {
			return fmt.Errorf("parse the proxmox section: %w", err)
		}
	}
	tlsSec := map[string]json.RawMessage{}
	if cur, ok := px["tls"]; ok {
		if err := json.Unmarshal(cur, &tlsSec); err != nil {
			return fmt.Errorf("parse the proxmox.tls section: %w", err)
		}
	}
	caJSON, _ := json.Marshal(caPath)
	tlsSec["ca_file"] = caJSON
	tlsSec["fingerprint"] = json.RawMessage(`""`)
	b, _ := json.Marshal(tlsSec)
	px["tls"] = b
	b, _ = json.Marshal(px)
	doc["proxmox"] = b
	out, err := json.MarshalIndent(doc, "", "  ")
	if err != nil {
		return err
	}
	out = append(out, '\n')
	st, err := os.Stat(m.ConfigPath)
	if err != nil {
		return err
	}
	if m.StateDir != "" {
		if err := os.MkdirAll(m.StateDir, 0o700); err != nil {
			return err
		}
		if err := os.WriteFile(filepath.Join(m.StateDir, "agent.json.pre-pve-trust"), raw, 0o600); err != nil {
			return fmt.Errorf("parking the recovery copy: %w", err)
		}
	}
	// In place (O_TRUNC), not tmp+rename: the folder is root-owned and the file is the agent's (the same reason as the
	// backup-target move and the PBS seed). One small write, then fsync.
	f, err := os.OpenFile(m.ConfigPath, os.O_WRONLY|os.O_TRUNC, st.Mode().Perm())
	if err != nil {
		return err
	}
	if _, err := f.Write(out); err != nil {
		f.Close()
		_ = os.WriteFile(m.ConfigPath, raw, st.Mode().Perm()) // put the old bytes back: the fingerprint must not be lost
		return err
	}
	if err := f.Sync(); err != nil {
		f.Close()
		return err
	}
	if err := f.Close(); err != nil {
		return err
	}
	// read back: the file must say exactly what was meant
	var check struct {
		Proxmox struct {
			TLS struct {
				CAFile      string `json:"ca_file"`
				Fingerprint string `json:"fingerprint"`
			} `json:"tls"`
		} `json:"proxmox"`
	}
	back, err := os.ReadFile(m.ConfigPath)
	if err != nil || json.Unmarshal(back, &check) != nil || check.Proxmox.TLS.CAFile != caPath || check.Proxmox.TLS.Fingerprint != "" {
		_ = os.WriteFile(m.ConfigPath, raw, st.Mode().Perm())
		return fmt.Errorf("the config did not read back as written (%v) — the old config was put back", err)
	}
	return nil
}