// Package vzpartial removes a dead, half-written vzdump backup of a guest from a local dir storage (R-1006). // // A power cut during the night's whole-guest backup leaves `vzdump-lxc--.tar.dat` and its `.tmp/` directory // on the target (demo-hp 2026-10-14: 12.2 GB on the host root). Nothing removes them: Proxmox does not list them as // backups, every later backup is skipped „not enough space", and a host root that fills up breaks LVM's own metadata // writes (audits/power-cut-fixes-2026-10-14 C4). This package finds such a leftover and asks the root helper // /usr/local/sbin/felhom-vzdump-partial-clean to remove it; the helper re-checks every rule as root. // // Rules here (the helper repeats them, and adds the open-file check the agent cannot do): // - only a `dir` storage; only names of the exact vzdump pattern for that vmid (NamePattern) — never a finished // archive, a log, another guest; // - only when no vzdump for that vmid is in flight (the R-1008 answer: a task from before the boot is not in flight); // - only when dead: last modified before this boot, or untouched for QuietFor. // // Any doubt leaves the entry and logs why. Pinned by vzpartial_test.go. package vzpartial import ( "context" "fmt" "log/slog" "os" "path/filepath" "regexp" "strconv" "strings" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // Helper is the root helper's installed path (configs/felhom-vzdump-partial-clean, delivered by the config bundle). const Helper = "/usr/local/sbin/felhom-vzdump-partial-clean" // QuietFor is how long an entry written in THIS boot must be untouched before it counts as dead (the helper's QUIET_S). const QuietFor = 120 * time.Second // NamePattern matches the half-written entries of one guest: the archive being written (.tar.dat / .vma.dat) and the // temporary directory (.tmp). The timestamp is vzdump's own (host local time). func NamePattern(vmid int) *regexp.Regexp { return regexp.MustCompile(`^vzdump-(lxc|qemu)-` + strconv.Itoa(vmid) + `-[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}\.(tar\.dat|vma\.dat|tmp)$`) } // Entry is one leftover found in the dump directory. type Entry struct { Name string Dir bool ModTime time.Time Size int64 } // Candidates lists the entries of dumpDir that match vmid's half-written pattern. A missing directory is no error. func Candidates(dumpDir string, vmid int) ([]Entry, error) { ents, err := os.ReadDir(dumpDir) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } re := NamePattern(vmid) var out []Entry for _, e := range ents { if !re.MatchString(e.Name()) || e.Type()&os.ModeSymlink != 0 { continue } info, err := e.Info() if err != nil { continue } if strings.HasSuffix(e.Name(), ".tmp") != info.IsDir() { continue // a .tmp that is not a directory, or an archive name that is one: not vzdump's } mt := info.ModTime() if info.IsDir() { mt = newestUnder(filepath.Join(dumpDir, e.Name()), mt) } out = append(out, Entry{Name: e.Name(), Dir: info.IsDir(), ModTime: mt, Size: info.Size()}) } return out, nil } func newestUnder(dir string, mt time.Time) time.Time { _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { if err != nil { return nil } if i, e := d.Info(); e == nil && i.ModTime().After(mt) { mt = i.ModTime() } return nil }) return mt } // Dead decides whether an entry is dead: written before this boot, or untouched for QuietFor. bootOK=false (the boot // time could not be read) leaves only the quiet rule. func Dead(e Entry, boot time.Time, bootOK bool, now time.Time) (bool, string) { if bootOK && e.ModTime.Before(boot) { return true, "written before this boot" } if age := now.Sub(e.ModTime); age >= QuietFor { return true, "untouched for " + age.Round(time.Second).String() } return false, "modified " + now.Sub(e.ModTime).Round(time.Second).String() + " ago — may still be written" } // StorageAPI is the one Proxmox read the cleaner needs (the storage's type and path). type StorageAPI interface { ListStorage(ctx context.Context) ([]proxmox.Storage, error) } // Cleaner removes dead leftovers. All fields but Logger and Now are required; a nil Cleaner is a no-op. type Cleaner struct { API StorageAPI Runner proxmox.Runner // the fenced sudo runner InFlight func(ctx context.Context, vmid int) (bool, error) Boot func() (int64, bool) Logger *slog.Logger Now func() time.Time // dumpDir maps a storage path to its dump directory (tests); nil = /dump. dumpDir func(path string) string } // Result is one removal or one entry left in place. type Result struct { Entry Entry Removed bool Why string } // Clean removes the dead leftovers of vmid on storage. It never returns an error for „nothing to do"; every entry it // finds is logged with what was done and why. func (c *Cleaner) Clean(ctx context.Context, storage string, vmid int) []Result { if c == nil || c.API == nil || c.Runner == nil || c.InFlight == nil || c.Boot == nil { return nil } log := c.Logger if log == nil { log = slog.Default() } now := time.Now if c.Now != nil { now = c.Now } stores, err := c.API.ListStorage(ctx) if err != nil { log.Warn("vzdump-partial: storage list unreadable — not looking for leftovers", "storage", storage, "err", err) return nil } var st *proxmox.Storage for i := range stores { if stores[i].Storage == storage { st = &stores[i] break } } if st == nil || st.Type != "dir" || st.Path == "" { return nil // PBS, LVM, NFS …: vzdump does not leave a .tar.dat there that this agent may remove } dump := filepath.Join(st.Path, "dump") if c.dumpDir != nil { dump = c.dumpDir(st.Path) } ents, err := Candidates(dump, vmid) if err != nil { log.Warn("vzdump-partial: dump directory unreadable", "dir", dump, "err", err) return nil } if len(ents) == 0 { return nil } var out []Result inFlight, err := c.InFlight(ctx, vmid) if err != nil || inFlight { why := "a backup of this guest is in flight" if err != nil { why = "cannot tell whether a backup is in flight: " + err.Error() } for _, e := range ents { log.Info("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why) out = append(out, Result{Entry: e, Why: why}) } return out } b, bootOK := c.Boot() boot := time.Unix(b, 0) for _, e := range ents { dead, why := Dead(e, boot, bootOK, now()) if !dead { log.Info("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why) out = append(out, Result{Entry: e, Why: why}) continue } stdout, stderr, err := c.Runner.Run(ctx, Helper, storage, strconv.Itoa(vmid), e.Name) if err != nil { why = fmt.Sprintf("the root helper refused: %s", strings.TrimSpace(string(stderr))) log.Warn("vzdump-partial: leaving a half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "why", why, "err", err) out = append(out, Result{Entry: e, Why: why}) continue } log.Warn("vzdump-partial: removed a dead half-written backup", "storage", storage, "vmid", vmid, "name", e.Name, "bytes", e.Size, "why", why, "helper", strings.TrimSpace(string(stdout))) out = append(out, Result{Entry: e, Removed: true, Why: why}) } return out }