felhom-agent/internal/thincheck/thincheck.go

// Package thincheck finds an ORPHAN thin device: one the LVM thin pool holds that LVM itself no longer knows (R-1012).
//
// A full host root while vzdump takes its storage snapshot can leave such a device behind (demo-hp 2026-10-14: the
// pool held devices 1, 2 and 4; LVM knew 1 and 2). From then on every snapshot backup of that guest fails at the
// snapshot step („Creation of new snapshot 4 of device 2 failed") and nothing in the product said why. This package
// NAMES the orphan, in plain words, in the backup's failure — it never removes one: the repair
// (`dmsetup message <pool>-tpool 0 "delete <id>"`) stays a human step, decided by the operator (`03` whole-guest
// backup; runbook `whole-guest-backup-orphan-thin.md`).
//
// The device list comes from the root helper `felhom-thin-check <vg> <pool>` (config bundle), which reads the pool
// through a metadata snapshot and LVM's list before and after. An id is an orphan only when the pool holds it and
// NEITHER LVM list names it, on two reads a few seconds apart — so a volume being created at that moment is never
// reported.
package thincheck

import (
	"context"
	"fmt"
	"log/slog"
	"sort"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// HelperPath is where the config bundle installs the root helper.
const HelperPath = "/usr/local/sbin/felhom-thin-check"

// Pool names one LVM thin pool.
type Pool struct{ VG, Pool string }

func (p Pool) String() string { return p.VG + "/" + p.Pool }

// Reading is one helper run, parsed.
type Reading struct {
	PoolIDs   []int
	LVMBefore map[int]string
	LVMAfter  map[int]string
}

// Parse reads the helper's output. ok=false unless the closing "ok" line is there (a cut-off run is no reading).
func Parse(out string) (Reading, bool) {
	r := Reading{LVMBefore: map[int]string{}, LVMAfter: map[int]string{}}
	done := false
	for _, line := range strings.Split(out, "\n") {
		f := strings.Fields(line)
		switch {
		case len(f) == 1 && f[0] == "ok":
			done = true
		case len(f) == 2 && f[0] == "pool":
			if id, err := strconv.Atoi(f[1]); err == nil {
				r.PoolIDs = append(r.PoolIDs, id)
			}
		case len(f) == 3 && (f[0] == "lvm-before" || f[0] == "lvm-after"):
			id, err := strconv.Atoi(f[1])
			if err != nil {
				continue
			}
			if f[0] == "lvm-before" {
				r.LVMBefore[id] = f[2]
			} else {
				r.LVMAfter[id] = f[2]
			}
		}
	}
	return r, done
}

// Orphans returns the ids the pool holds that neither LVM list names, sorted.
func (r Reading) Orphans() []int {
	var out []int
	for _, id := range r.PoolIDs {
		_, b := r.LVMBefore[id]
		_, a := r.LVMAfter[id]
		if !b && !a {
			out = append(out, id)
		}
	}
	sort.Ints(out)
	return out
}

// Runner is the privileged exec surface (proxmox.ExecRunner in sudo mode).
type Runner interface {
	Run(ctx context.Context, name string, args ...string) ([]byte, []byte, error)
}

// StorageLister is the read the checker needs to map a guest's volumes to their thin pools (*proxmox.Client).
type StorageLister interface {
	GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
	ListStorage(ctx context.Context) ([]proxmox.Storage, error)
}

// Checker runs the check for one guest's pools.
type Checker struct {
	API    StorageLister
	Runner Runner
	Helper string        // "" → HelperPath
	Pause  time.Duration // between the two reads when the first finds an orphan; 0 → 3 s
	Logger *slog.Logger
}

// Finding is one pool with orphans.
type Finding struct {
	Pool    Pool
	Orphans []int
}

// PoolsOf maps the guest's volume-backed disks (rootfs, mpN) to the lvmthin pools they live on.
func (c *Checker) PoolsOf(ctx context.Context, vmid int) ([]Pool, error) {
	cfg, err := c.API.GuestConfig(ctx, vmid)
	if err != nil {
		return nil, err
	}
	stores, err := c.API.ListStorage(ctx)
	if err != nil {
		return nil, err
	}
	thin := map[string]Pool{}
	for _, s := range stores {
		if s.Type == "lvmthin" && s.VGName != "" && s.ThinPool != "" {
			thin[s.Storage] = Pool{VG: s.VGName, Pool: s.ThinPool}
		}
	}
	vols := []string{cfg.RootFS}
	mps := cfg.MountPoints()
	keys := make([]string, 0, len(mps))
	for k := range mps {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	for _, k := range keys {
		vols = append(vols, mps[k])
	}
	seen := map[Pool]bool{}
	var out []Pool
	for _, v := range vols {
		store, _, ok := strings.Cut(strings.Split(v, ",")[0], ":")
		if !ok || strings.HasPrefix(store, "/") {
			continue // a bind mount (a path) is no volume
		}
		if p, isThin := thin[store]; isThin && !seen[p] {
			seen[p] = true
			out = append(out, p)
		}
	}
	return out, nil
}

// read runs the helper once.
func (c *Checker) read(ctx context.Context, p Pool) (Reading, error) {
	helper := c.Helper
	if helper == "" {
		helper = HelperPath
	}
	out, stderr, err := c.Runner.Run(ctx, helper, p.VG, p.Pool)
	if err != nil {
		return Reading{}, fmt.Errorf("felhom-thin-check %s: %w: %s", p, err, strings.TrimSpace(string(stderr)))
	}
	r, ok := Parse(string(out))
	if !ok {
		return Reading{}, fmt.Errorf("felhom-thin-check %s: the output has no closing ok line", p)
	}
	return r, nil
}

// Check returns the pools of vmid that hold an orphan (two reads, intersected). A pool that cannot be read is logged
// and left out: the check FAILS OPEN — it explains a failure, it never stops a backup.
func (c *Checker) Check(ctx context.Context, vmid int) []Finding {
	logger := c.Logger
	if logger == nil {
		logger = slog.Default()
	}
	pools, err := c.PoolsOf(ctx, vmid)
	if err != nil {
		logger.Warn("thincheck: cannot map the guest's disks to thin pools — not checked", "vmid", vmid, "err", err)
		return nil
	}
	pause := c.Pause
	if pause <= 0 {
		pause = 3 * time.Second
	}
	var found []Finding
	for _, p := range pools {
		first, err := c.read(ctx, p)
		if err != nil {
			logger.Warn("thincheck: thin pool not read — not checked", "vmid", vmid, "pool", p.String(), "err", err)
			continue
		}
		o1 := first.Orphans()
		if len(o1) == 0 {
			logger.Debug("thincheck: thin pool and LVM agree", "vmid", vmid, "pool", p.String(), "devices", len(first.PoolIDs))
			continue
		}
		select {
		case <-ctx.Done():
			return found
		case <-time.After(pause):
		}
		second, err := c.read(ctx, p)
		if err != nil {
			logger.Warn("thincheck: second read failed — not reported", "vmid", vmid, "pool", p.String(), "err", err)
			continue
		}
		in2 := map[int]bool{}
		for _, id := range second.Orphans() {
			in2[id] = true
		}
		var both []int
		for _, id := range o1 {
			if in2[id] {
				both = append(both, id)
			}
		}
		if len(both) > 0 {
			found = append(found, Finding{Pool: p, Orphans: both})
			logger.Warn("thincheck: ORPHAN thin device(s) in the pool — LVM does not know them; NOT removed (a human step)",
				"vmid", vmid, "pool", p.String(), "orphans", both)
		}
	}
	return found
}

// Describe is the operator sentence for a backup failure. Plain words; names the pool and the ids; says nothing was
// removed and where the repair is written down.
func Describe(f []Finding) string {
	if len(f) == 0 {
		return ""
	}
	var parts []string
	for _, x := range f {
		ids := make([]string, len(x.Orphans))
		for i, id := range x.Orphans {
			ids[i] = strconv.Itoa(id)
		}
		parts = append(parts, fmt.Sprintf("thin pool %s holds device %s that LVM does not know", x.Pool, strings.Join(ids, ", ")))
	}
	return "probable cause: " + strings.Join(parts, "; ") +
		" (an orphan snapshot, usually left when the host disk filled during a backup). Snapshot backups on this pool " +
		"fail until an operator removes it; Felhom did not remove anything. Repair: runbook whole-guest-backup-orphan-thin.md"
}