209 lines
7.2 KiB
Go
209 lines
7.2 KiB
Go
// Package vzpartial removes a dead, half-written vzdump backup of a guest from a local dir storage (R-1006).
|
|
//
|
|
// A power cut during the night's whole-guest backup leaves `vzdump-lxc-<vmid>-<date>.tar.dat` and its `.tmp/` directory
|
|
// on the target (demo-hp 2026-10-14: 12.2 GB on the host root). Nothing removes them: Proxmox does not list them as
|
|
// backups, every later backup is skipped „not enough space", and a host root that fills up breaks LVM's own metadata
|
|
// writes (audits/power-cut-fixes-2026-10-14 C4). This package finds such a leftover and asks the root helper
|
|
// /usr/local/sbin/felhom-vzdump-partial-clean to remove it; the helper re-checks every rule as root.
|
|
//
|
|
// Rules here (the helper repeats them, and adds the open-file check the agent cannot do):
|
|
// - only a `dir` storage; only names of the exact vzdump pattern for that vmid (NamePattern) — never a finished
|
|
// archive, a log, another guest;
|
|
// - only when no vzdump for that vmid is in flight (the R-1008 answer: a task from before the boot is not in flight);
|
|
// - only when dead: last modified before this boot, or untouched for QuietFor.
|
|
//
|
|
// Any doubt leaves the entry and logs why. Pinned by vzpartial_test.go.
|
|
package vzpartial
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
|
)
|
|
|
|
// Helper is the root helper's installed path (configs/felhom-vzdump-partial-clean, delivered by the config bundle).
|
|
const Helper = "/usr/local/sbin/felhom-vzdump-partial-clean"
|
|
|
|
// QuietFor is how long an entry written in THIS boot must be untouched before it counts as dead (the helper's QUIET_S).
|
|
const QuietFor = 120 * time.Second
|
|
|
|
// NamePattern matches the half-written entries of one guest: the archive being written (.tar.dat / .vma.dat) and the
|
|
// temporary directory (.tmp). The timestamp is vzdump's own (host local time).
|
|
func NamePattern(vmid int) *regexp.Regexp {
|
|
return regexp.MustCompile(`^vzdump-(lxc|qemu)-` + strconv.Itoa(vmid) +
|
|
`-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$`)
|
|
}
|
|
|
|
// Entry is one leftover found in the dump directory.
|
|
type Entry struct {
|
|
Name string
|
|
Dir bool
|
|
ModTime time.Time
|
|
Size int64
|
|
}
|
|
|
|
// Candidates lists the entries of dumpDir that match vmid's half-written pattern. A missing directory is no error.
|
|
func Candidates(dumpDir string, vmid int) ([]Entry, error) {
|
|
ents, err := os.ReadDir(dumpDir)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, nil
|
|
}
|
|
return nil, err
|
|
}
|
|
re := NamePattern(vmid)
|
|
var out []Entry
|
|
for _, e := range ents {
|
|
if !re.MatchString(e.Name()) || e.Type()&os.ModeSymlink != 0 {
|
|
continue
|
|
}
|
|
info, err := e.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if strings.HasSuffix(e.Name(), ".tmp") != info.IsDir() {
|
|
continue // a .tmp that is not a directory, or an archive name that is one: not vzdump's
|
|
}
|
|
mt := info.ModTime()
|
|
if info.IsDir() {
|
|
mt = newestUnder(filepath.Join(dumpDir, e.Name()), mt)
|
|
}
|
|
out = append(out, Entry{Name: e.Name(), Dir: info.IsDir(), ModTime: mt, Size: info.Size()})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func newestUnder(dir string, mt time.Time) time.Time {
|
|
_ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
if i, e := d.Info(); e == nil && i.ModTime().After(mt) {
|
|
mt = i.ModTime()
|
|
}
|
|
return nil
|
|
})
|
|
return mt
|
|
}
|
|
|
|
// Dead decides whether an entry is dead: written before this boot, or untouched for QuietFor. bootOK=false (the boot
|
|
// time could not be read) leaves only the quiet rule.
|
|
func Dead(e Entry, boot time.Time, bootOK bool, now time.Time) (bool, string) {
|
|
if bootOK && e.ModTime.Before(boot) {
|
|
return true, "written before this boot"
|
|
}
|
|
if age := now.Sub(e.ModTime); age >= QuietFor {
|
|
return true, "untouched for " + age.Round(time.Second).String()
|
|
}
|
|
return false, "modified " + now.Sub(e.ModTime).Round(time.Second).String() + " ago — may still be written"
|
|
}
|
|
|
|
// StorageAPI is the one Proxmox read the cleaner needs (the storage's type and path).
|
|
type StorageAPI interface {
|
|
ListStorage(ctx context.Context) ([]proxmox.Storage, error)
|
|
}
|
|
|
|
// Cleaner removes dead leftovers. All fields but Logger and Now are required; a nil Cleaner is a no-op.
|
|
type Cleaner struct {
|
|
API StorageAPI
|
|
Runner proxmox.Runner // the fenced sudo runner
|
|
InFlight func(ctx context.Context, vmid int) (bool, error)
|
|
Boot func() (int64, bool)
|
|
Logger *slog.Logger
|
|
Now func() time.Time
|
|
// dumpDir maps a storage path to its dump directory (tests); nil = <path>/dump.
|
|
dumpDir func(path string) string
|
|
}
|
|
|
|
// Result is one removal or one entry left in place.
|
|
type Result struct {
|
|
Entry Entry
|
|
Removed bool
|
|
Why string
|
|
}
|
|
|
|
// Clean removes the dead leftovers of vmid on storage. It never returns an error for „nothing to do"; every entry it
|
|
// finds is logged with what was done and why.
|
|
func (c *Cleaner) Clean(ctx context.Context, storage string, vmid int) []Result {
|
|
if c == nil || c.API == nil || c.Runner == nil || c.InFlight == nil || c.Boot == nil {
|
|
return nil
|
|
}
|
|
log := c.Logger
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
now := time.Now
|
|
if c.Now != nil {
|
|
now = c.Now
|
|
}
|
|
stores, err := c.API.ListStorage(ctx)
|
|
if err != nil {
|
|
log.Warn("vzdump-partial: storage list unreadable — not looking for leftovers", "storage", storage, "err", err)
|
|
return nil
|
|
}
|
|
var st *proxmox.Storage
|
|
for i := range stores {
|
|
if stores[i].Storage == storage {
|
|
st = &stores[i]
|
|
break
|
|
}
|
|
}
|
|
if st == nil || st.Type != "dir" || st.Path == "" {
|
|
return nil // PBS, LVM, NFS …: vzdump does not leave a .tar.dat there that this agent may remove
|
|
}
|
|
dump := filepath.Join(st.Path, "dump")
|
|
if c.dumpDir != nil {
|
|
dump = c.dumpDir(st.Path)
|
|
}
|
|
ents, err := Candidates(dump, vmid)
|
|
if err != nil {
|
|
log.Warn("vzdump-partial: dump directory unreadable", "dir", dump, "err", err)
|
|
return nil
|
|
}
|
|
if len(ents) == 0 {
|
|
return nil
|
|
}
|
|
var out []Result
|
|
inFlight, err := c.InFlight(ctx, vmid)
|
|
if err != nil || inFlight {
|
|
why := "a backup of this guest is in flight"
|
|
if err != nil {
|
|
why = "cannot tell whether a backup is in flight: " + err.Error()
|
|
}
|
|
for _, e := range ents {
|
|
log.Info("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why)
|
|
out = append(out, Result{Entry: e, Why: why})
|
|
}
|
|
return out
|
|
}
|
|
b, bootOK := c.Boot()
|
|
boot := time.Unix(b, 0)
|
|
for _, e := range ents {
|
|
dead, why := Dead(e, boot, bootOK, now())
|
|
if !dead {
|
|
log.Info("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why)
|
|
out = append(out, Result{Entry: e, Why: why})
|
|
continue
|
|
}
|
|
stdout, stderr, err := c.Runner.Run(ctx, Helper, storage, strconv.Itoa(vmid), e.Name)
|
|
if err != nil {
|
|
why = fmt.Sprintf("the root helper refused: %s", strings.TrimSpace(string(stderr)))
|
|
log.Warn("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why, "err", err)
|
|
out = append(out, Result{Entry: e, Why: why})
|
|
continue
|
|
}
|
|
log.Warn("vzdump-partial: removed a dead half-written backup", "storage", storage, "vmid", vmid, "name", e.Name,
|
|
"bytes", e.Size, "why", why, "helper", strings.TrimSpace(string(stdout)))
|
|
out = append(out, Result{Entry: e, Removed: true, Why: why})
|
|
}
|
|
return out
|
|
}
|