// Package thincheck finds an ORPHAN thin device: one the LVM thin pool holds that LVM itself no longer knows (R-1012). // // A full host root while vzdump takes its storage snapshot can leave such a device behind (demo-hp 2026-10-14: the // pool held devices 1, 2 and 4; LVM knew 1 and 2). From then on every snapshot backup of that guest fails at the // snapshot step („Creation of new snapshot 4 of device 2 failed") and nothing in the product said why. This package // NAMES the orphan, in plain words, in the backup's failure — it never removes one: the repair // (`dmsetup message -tpool 0 "delete "`) stays a human step, decided by the operator (`03` whole-guest // backup; runbook `whole-guest-backup-orphan-thin.md`). // // The device list comes from the root helper `felhom-thin-check ` (config bundle), which reads the pool // through a metadata snapshot and LVM's list before and after. An id is an orphan only when the pool holds it and // NEITHER LVM list names it, on two reads a few seconds apart — so a volume being created at that moment is never // reported. package thincheck import ( "context" "fmt" "log/slog" "sort" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // HelperPath is where the config bundle installs the root helper. const HelperPath = "/usr/local/sbin/felhom-thin-check" // Pool names one LVM thin pool. type Pool struct{ VG, Pool string } func (p Pool) String() string { return p.VG + "/" + p.Pool } // Reading is one helper run, parsed. type Reading struct { PoolIDs []int LVMBefore map[int]string LVMAfter map[int]string } // Parse reads the helper's output. ok=false unless the closing "ok" line is there (a cut-off run is no reading). func Parse(out string) (Reading, bool) { r := Reading{LVMBefore: map[int]string{}, LVMAfter: map[int]string{}} done := false for _, line := range strings.Split(out, "\n") { f := strings.Fields(line) switch { case len(f) == 1 && f[0] == "ok": done = true case len(f) == 2 && f[0] == "pool": if id, err := strconv.Atoi(f[1]); err == nil { r.PoolIDs = append(r.PoolIDs, id) } case len(f) == 3 && (f[0] == "lvm-before" || f[0] == "lvm-after"): id, err := strconv.Atoi(f[1]) if err != nil { continue } if f[0] == "lvm-before" { r.LVMBefore[id] = f[2] } else { r.LVMAfter[id] = f[2] } } } return r, done } // Orphans returns the ids the pool holds that neither LVM list names, sorted. func (r Reading) Orphans() []int { var out []int for _, id := range r.PoolIDs { _, b := r.LVMBefore[id] _, a := r.LVMAfter[id] if !b && !a { out = append(out, id) } } sort.Ints(out) return out } // Runner is the privileged exec surface (proxmox.ExecRunner in sudo mode). type Runner interface { Run(ctx context.Context, name string, args ...string) ([]byte, []byte, error) } // StorageLister is the read the checker needs to map a guest's volumes to their thin pools (*proxmox.Client). type StorageLister interface { GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error) ListStorage(ctx context.Context) ([]proxmox.Storage, error) } // Checker runs the check for one guest's pools. type Checker struct { API StorageLister Runner Runner Helper string // "" → HelperPath Pause time.Duration // between the two reads when the first finds an orphan; 0 → 3 s Logger *slog.Logger } // Finding is one pool with orphans. type Finding struct { Pool Pool Orphans []int } // PoolsOf maps the guest's volume-backed disks (rootfs, mpN) to the lvmthin pools they live on. func (c *Checker) PoolsOf(ctx context.Context, vmid int) ([]Pool, error) { cfg, err := c.API.GuestConfig(ctx, vmid) if err != nil { return nil, err } stores, err := c.API.ListStorage(ctx) if err != nil { return nil, err } thin := map[string]Pool{} for _, s := range stores { if s.Type == "lvmthin" && s.VGName != "" && s.ThinPool != "" { thin[s.Storage] = Pool{VG: s.VGName, Pool: s.ThinPool} } } vols := []string{cfg.RootFS} mps := cfg.MountPoints() keys := make([]string, 0, len(mps)) for k := range mps { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { vols = append(vols, mps[k]) } seen := map[Pool]bool{} var out []Pool for _, v := range vols { store, _, ok := strings.Cut(strings.Split(v, ",")[0], ":") if !ok || strings.HasPrefix(store, "/") { continue // a bind mount (a path) is no volume } if p, isThin := thin[store]; isThin && !seen[p] { seen[p] = true out = append(out, p) } } return out, nil } // read runs the helper once. func (c *Checker) read(ctx context.Context, p Pool) (Reading, error) { helper := c.Helper if helper == "" { helper = HelperPath } out, stderr, err := c.Runner.Run(ctx, helper, p.VG, p.Pool) if err != nil { return Reading{}, fmt.Errorf("felhom-thin-check %s: %w: %s", p, err, strings.TrimSpace(string(stderr))) } r, ok := Parse(string(out)) if !ok { return Reading{}, fmt.Errorf("felhom-thin-check %s: the output has no closing ok line", p) } return r, nil } // Check returns the pools of vmid that hold an orphan (two reads, intersected). A pool that cannot be read is logged // and left out: the check FAILS OPEN — it explains a failure, it never stops a backup. func (c *Checker) Check(ctx context.Context, vmid int) []Finding { logger := c.Logger if logger == nil { logger = slog.Default() } pools, err := c.PoolsOf(ctx, vmid) if err != nil { logger.Warn("thincheck: cannot map the guest's disks to thin pools — not checked", "vmid", vmid, "err", err) return nil } pause := c.Pause if pause <= 0 { pause = 3 * time.Second } var found []Finding for _, p := range pools { first, err := c.read(ctx, p) if err != nil { logger.Warn("thincheck: thin pool not read — not checked", "vmid", vmid, "pool", p.String(), "err", err) continue } o1 := first.Orphans() if len(o1) == 0 { logger.Debug("thincheck: thin pool and LVM agree", "vmid", vmid, "pool", p.String(), "devices", len(first.PoolIDs)) continue } select { case <-ctx.Done(): return found case <-time.After(pause): } second, err := c.read(ctx, p) if err != nil { logger.Warn("thincheck: second read failed — not reported", "vmid", vmid, "pool", p.String(), "err", err) continue } in2 := map[int]bool{} for _, id := range second.Orphans() { in2[id] = true } var both []int for _, id := range o1 { if in2[id] { both = append(both, id) } } if len(both) > 0 { found = append(found, Finding{Pool: p, Orphans: both}) logger.Warn("thincheck: ORPHAN thin device(s) in the pool — LVM does not know them; NOT removed (a human step)", "vmid", vmid, "pool", p.String(), "orphans", both) } } return found } // Describe is the operator sentence for a backup failure. Plain words; names the pool and the ids; says nothing was // removed and where the repair is written down. func Describe(f []Finding) string { if len(f) == 0 { return "" } var parts []string for _, x := range f { ids := make([]string, len(x.Orphans)) for i, id := range x.Orphans { ids[i] = strconv.Itoa(id) } parts = append(parts, fmt.Sprintf("thin pool %s holds device %s that LVM does not know", x.Pool, strings.Join(ids, ", "))) } return "probable cause: " + strings.Join(parts, "; ") + " (an orphan snapshot, usually left when the host disk filled during a backup). Snapshot backups on this pool " + "fail until an operator removes it; Felhom did not remove anything. Repair: runbook whole-guest-backup-orphan-thin.md" }