D62: restore copies kept 30 days, listed with their date, deletable by hand (typed name, never a needed copy, operator event); D63: a test-build-only fault switch for the R-1256 H4/H5 live proofs — tests red first
gates / gates (push) Successful in 33s

This commit is contained in:
admin committed 2026-10-29 08:50:59 +01:00
1 parent a815b37fbd
commit e3fce09d96
24 files changed
+1992 -11

No files matched your search

@@ -0,0 +1,185 @@
package backup
import (
"errors"
"os"
"path/filepath"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// D62 (operator ruling 2026-10-29): an unused restore copy is kept 30 days; the household sees each copy (app, the
// restore's date, size, the day it goes) and may delete one earlier. A copy a hold or a pending put-back needs is never
// deleted, by the sweep or by hand.
// RED-PROOF: with the age at 14 days the 20-day copy is swept → FAILS; at 30 days only the 31-day copy goes.
func TestD62_AnUnusedCopyIsKept30Days(t *testing.T) {
m, _ := newOffboxManager(t)
now := time.Now()
day29 := r1256Copy(t, m, "a", 29*24*time.Hour, now)
day20 := r1256Copy(t, m, "b", 20*24*time.Hour, now)
day31 := r1256Copy(t, m, "c", 31*24*time.Hour, now)
got := m.SweepUnusedRestoreCopies(now)
if len(got) != 1 || got[0] != day31 {
t.Fatalf("swept %v — want only the 31-day copy %s", got, day31)
}
if !r1256Exists(day29) || !r1256Exists(day20) {
t.Fatal("a copy younger than 30 days was swept")
}
if restoreCopyUnusedAge != 30*24*time.Hour {
t.Fatalf("restoreCopyUnusedAge = %s, D62 says 30 days", restoreCopyUnusedAge)
}
}
// The list shows each copy with its restore date, its size and the day it goes; a needed copy has no such day and
// says why. RED-PROOF: drop the hold check from restoreCopyNeededFor → the held copy is offered → FAILS.
func TestD62_TheListShowsDateSizeAndWhyACopyIsNeeded(t *testing.T) {
m, sett := newOffboxManager(t)
now := time.Now().UTC().Truncate(time.Second)
free := r1256Copy(t, m, "free", 2*24*time.Hour, now)
r1256Copy(t, m, "held", 2*24*time.Hour, now)
r1256Copy(t, m, "waiting", 2*24*time.Hour, now)
r1256Copy(t, m, "named", 2*24*time.Hour, now)
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "held", At: now.Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}); err != nil {
t.Fatal(err)
}
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "waiting", At: now.Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending}); err != nil {
t.Fatal(err)
}
if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil {
t.Fatal(err)
}
list := m.RestoreCopies()
if len(list) != 4 {
t.Fatalf("listed %d copies, want 4: %+v", len(list), list)
}
want := map[string]string{"free": "", "held": RestoreCopyNeededHold, "waiting": RestoreCopyNeededPutBack, "named": RestoreCopyNeededPutBack}
for _, c := range list {
if c.Needed != want[c.App] {
t.Errorf("%s: needed %q, want %q", c.App, c.Needed, want[c.App])
}
if !c.At.Equal(now.Add(-2 * 24 * time.Hour)) {
t.Errorf("%s: restore date %s, want the stamp's %s", c.App, c.At, now.Add(-2*24*time.Hour))
}
if c.SizeBytes != int64(len("LIVE")) {
t.Errorf("%s: size %d", c.App, c.SizeBytes)
}
if c.Needed == "" && !c.DeleteOn.Equal(c.At.Add(30*24*time.Hour)) {
t.Errorf("%s: deleted on %s, want 30 days after the restore", c.App, c.DeleteOn)
}
if c.Needed != "" && !c.DeleteOn.IsZero() {
t.Errorf("%s: a needed copy shows a delete day %s — the sweep never takes it", c.App, c.DeleteOn)
}
if c.App == "free" && c.Path != free {
t.Errorf("path %s, want %s", c.Path, free)
}
}
}
// A held or waiting copy cannot be deleted by hand. RED-PROOF: drop the Needed check in DeleteRestoreCopy → FAILS.
func TestD62_AHeldCopyCannotBeDeleted(t *testing.T) {
m, sett := newOffboxManager(t)
now := time.Now()
for _, tc := range []struct{ app, reason string }{
{"held", settings.HoldReasonRestoreMixed},
{"waiting", settings.HoldReasonPutBackPending},
{"failed", settings.HoldReasonRestoreFailed},
} {
dir := r1256Copy(t, m, tc.app, time.Hour, now)
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: tc.app, At: now.Format(time.RFC3339), Reason: tc.reason}); err != nil {
t.Fatal(err)
}
if _, err := m.DeleteRestoreCopy(tc.app, filepath.Base(dir)); !errors.Is(err, ErrRestoreCopyNeeded) {
t.Errorf("%s: err %v, want ErrRestoreCopyNeeded", tc.app, err)
}
if !r1256Exists(dir) {
t.Errorf("%s: a held copy was deleted", tc.app)
}
}
// a copy a cut restore names (the marker) is needed too
named := r1256Copy(t, m, "named", time.Hour, now)
if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil {
t.Fatal(err)
}
if _, err := m.DeleteRestoreCopy("named", filepath.Base(named)); !errors.Is(err, ErrRestoreCopyNeeded) || !r1256Exists(named) {
t.Errorf("a copy the marker names: err %v, exists %v", err, r1256Exists(named))
}
// an unreadable marker offers nothing
mustWrite(t, m.appStop.path, "{not json")
free := r1256Copy(t, m, "free", time.Hour, now)
if _, err := m.DeleteRestoreCopy("free", filepath.Base(free)); !errors.Is(err, ErrRestoreCopyNeeded) || !r1256Exists(free) {
t.Errorf("an unreadable marker: err %v, exists %v", err, r1256Exists(free))
}
}
// A delete removes ONLY that copy: the app's other copy, another app's copy and anything outside the copy tree stay;
// a name that is not one path element, or a link in the copy's place, is refused.
// RED-PROOF: remove the validCopyName check → "../.." reaches outside → FAILS; remove the Lstat link check → the
// link is followed/removed → FAILS.
func TestD62_ADeleteRemovesOnlyThatCopy(t *testing.T) {
m, _ := newOffboxManager(t)
now := time.Now()
one := r1256Copy(t, m, "app", 3*time.Hour, now)
two := r1256Copy(t, m, "app", 2*time.Hour, now)
other := r1256Copy(t, m, "other", 3*time.Hour, now)
got, err := m.DeleteRestoreCopy("app", filepath.Base(one))
if err != nil {
t.Fatal(err)
}
if got.Path != one || got.SizeBytes != int64(len("LIVE")) || got.App != "app" {
t.Fatalf("returned %+v", got)
}
if r1256Exists(one) || !r1256Exists(two) || !r1256Exists(other) {
t.Fatalf("one gone=%v, two kept=%v, other kept=%v", !r1256Exists(one), r1256Exists(two), r1256Exists(other))
}
// path guard
outside := filepath.Join(m.namespaceRoot(m.systemDataPath), "backups", "primary", "keep.txt")
mustWrite(t, outside, "UNIT")
for _, bad := range [][2]string{{"app", ".."}, {"..", "primary"}, {"app", "../../primary"}, {"", "x"}, {"app", ""}, {"app/x", "y"}} {
if _, err := m.DeleteRestoreCopy(bad[0], bad[1]); !errors.Is(err, ErrRestoreCopyNotFound) {
t.Errorf("DeleteRestoreCopy(%q,%q) = %v, want not found", bad[0], bad[1], err)
}
}
if !r1256Exists(outside) || !r1256Exists(two) {
t.Fatal("a bad name reached outside the copy")
}
// a link in a copy's place is not a copy
target := t.TempDir()
mustWrite(t, filepath.Join(target, "precious"), "X")
link := filepath.Join(filepath.Dir(two), "20260101T000000Z")
if err := os.Symlink(target, link); err != nil {
t.Fatal(err)
}
if _, err := m.DeleteRestoreCopy("app", "20260101T000000Z"); !errors.Is(err, ErrRestoreCopyNotFound) {
t.Errorf("a link: %v, want not found", err)
}
if !r1256Exists(filepath.Join(target, "precious")) {
t.Fatal("the link's target was removed")
}
if _, err := m.DeleteRestoreCopy("app", "20990101T000000Z"); !errors.Is(err, ErrRestoreCopyNotFound) {
t.Errorf("a missing copy: %v", err)
}
}
// While a backup or restore runs, a delete is refused and nothing moves. RED-PROOF: drop the busy check → FAILS.
func TestD62_NoDeleteWhileAnOperationRuns(t *testing.T) {
m, _ := newOffboxManager(t)
c := r1256Copy(t, m, "app", time.Hour, time.Now())
m.BeginRestoreOp("restore", "other")
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); !errors.Is(err, ErrRestoreCopyBusy) || !r1256Exists(c) {
t.Fatalf("during a restore: err %v, exists %v", err, r1256Exists(c))
}
m.EndRestoreOp(true, "ok")
if err := m.acquireRunning(); err != nil {
t.Fatalf("could not take the backup single-flight: %v", err)
}
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); !errors.Is(err, ErrRestoreCopyBusy) || !r1256Exists(c) {
t.Fatalf("during a backup run: err %v, exists %v", err, r1256Exists(c))
}
m.ReleaseRunningForTest()
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); err != nil || r1256Exists(c) {
t.Fatalf("control: with nothing running the delete goes through: %v", err)
}
}
@@ -300,7 +300,7 @@ func r1256Exists(p string) bool { _, err := os.Stat(p); return err == nil }
func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) {
m, sett := newOffboxManager(t)
now := time.Now()
old := 20 * 24 * time.Hour
old := 40 * 24 * time.Hour
unused := r1256Copy(t, m, "unused", old, now)
young := r1256Copy(t, m, "young", 24*time.Hour, now)
held := r1256Copy(t, m, "held", old, now)
@@ -339,7 +339,7 @@ func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) {
func TestR1256_H8_NothingIsRemovedDuringARestoreOrWithAnUnreadableMarker(t *testing.T) {
m, _ := newOffboxManager(t)
now := time.Now()
c := r1256Copy(t, m, "app", 20*24*time.Hour, now)
c := r1256Copy(t, m, "app", 40*24*time.Hour, now)
m.BeginRestoreOp("restore", "other")
if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) {
t.Fatalf("removed %v during a restore", got)
@@ -0,0 +1,182 @@
package backup
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
)
// D62 (operator ruling 2026-10-29; `07` §6.3): the household sees its restore copies in the kept-copies list
// („Megőrzött másolatok") on the restore page — the app, the date of the restore the copy protects, its size, and the
// day the box deletes it by itself (restoreCopyUnusedAge after the restore) — and may delete one earlier with
// „Törlés most", after a warning and the app's name typed.
//
// A copy a hold or a pending put-back needs is NEVER deletable (the same three states the sweep leaves alone, R-1256
// H8): the row says why in one line and the button is off; the handler refuses it too. While any backup or restore
// operation runs on the box, a delete is refused (409). The check and the removal are ONE step under m.mu, so a
// restore cannot begin in between (the R-1256 H6 shape). Pinned by TestD62_*.
// Why a restore copy is needed — the row's one line, and the refusal's reason. "" = not needed: deletable.
const (
RestoreCopyNeededHold = "hold" // the app is held (a failed put-back keeps its copy, D20) — support decides
RestoreCopyNeededPutBack = "putback" // a put-back waits for its drive (R-1251) or a cut restore names it (D22)
RestoreCopyNeededUnknown = "unknown" // the app-stop marker cannot be read: nothing is offered (fail closed)
)
var (
ErrRestoreCopyNotFound = errors.New("restore copy not found")
ErrRestoreCopyNeeded = errors.New("restore copy is needed by a hold or a pending put-back")
ErrRestoreCopyBusy = errors.New("a backup or restore operation runs")
)
// RestoreCopy is one restore copy (<namespace>/backups/restore-undo/<app>/<stamp>).
type RestoreCopy struct {
App string
Stamp string // the folder's name; with App, the copy's identity on the page
At time.Time // when the restore began (the stamp; the folder's time when the stamp does not parse)
SizeBytes int64
Path string
Needed string // "" or one of RestoreCopyNeeded*
DeleteOn time.Time // when the sweep removes it; zero when Needed != "" (a needed copy is never swept)
}
// restoreCopyAt is a copy's restore time: its stamp, or the folder's time when the stamp does not parse.
func restoreCopyAt(dir, name string) (time.Time, bool) {
if at, err := time.Parse(restoreCopyStampLayout, name); err == nil {
return at, true
}
fi, err := os.Stat(dir)
if err != nil {
return time.Time{}, false
}
return fi.ModTime(), true
}
// restoreCopyNeededFor is why app's copies are needed ("" = deletable). named is appStopNamedApps' answer, ok its ok.
func (m *Manager) restoreCopyNeededFor(app string, named map[string]bool, ok bool) string {
if !ok || m.settings == nil {
return RestoreCopyNeededUnknown
}
if named[app] {
return RestoreCopyNeededPutBack
}
if h, held := m.settings.GetRestoreHold(app); held {
if h.Reason == settings.HoldReasonPutBackPending {
return RestoreCopyNeededPutBack
}
return RestoreCopyNeededHold
}
return ""
}
func dirSize(dir string) int64 {
var n int64
_ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
if err == nil && d.Type().IsRegular() {
if fi, e := d.Info(); e == nil {
n += fi.Size()
}
}
return nil
})
return n
}
// RestoreCopies lists every restore copy on the box's drives, oldest first.
func (m *Manager) RestoreCopies() []RestoreCopy {
named, ok := m.appStopNamedApps()
var out []RestoreCopy
for _, nsRoot := range m.primaryUnitRoots() {
base := filepath.Join(nsRoot, "backups", "restore-undo")
apps, err := os.ReadDir(base)
if err != nil {
continue
}
for _, a := range apps {
if !a.IsDir() {
continue
}
app := a.Name()
needed := m.restoreCopyNeededFor(app, named, ok)
copies, err := os.ReadDir(filepath.Join(base, app))
if err != nil {
continue
}
for _, c := range copies {
if !c.IsDir() {
continue
}
dir := filepath.Join(base, app, c.Name())
at, ok := restoreCopyAt(dir, c.Name())
if !ok {
continue
}
rc := RestoreCopy{App: app, Stamp: c.Name(), At: at, SizeBytes: dirSize(dir), Path: dir, Needed: needed}
if needed == "" {
rc.DeleteOn = at.Add(restoreCopyUnusedAge)
}
out = append(out, rc)
}
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
return out
}
// validCopyName is a single path element: no separator, not empty, not "." or "..".
func validCopyName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, `/\`) && filepath.Base(s) == s
}
// DeleteRestoreCopy removes ONE restore copy, named by its app and its folder's name, and nothing else (the path
// guard: both names are single path elements, the folder is a real directory — not a link — directly under
// <namespace>/backups/restore-undo/<app>/). Refused while any backup or restore operation runs (ErrRestoreCopyBusy)
// and for a copy a hold or a pending put-back needs (ErrRestoreCopyNeeded). The check and the removal hold m.mu, so
// no restore can begin in between. Returns the copy as it was (for the log line and the operator event).
func (m *Manager) DeleteRestoreCopy(app, stamp string) (RestoreCopy, error) {
if !validCopyName(app) || !validCopyName(stamp) {
return RestoreCopy{}, ErrRestoreCopyNotFound
}
m.mu.Lock()
defer m.mu.Unlock()
if busy, _ := m.restoreBusyLocked(); busy != "" {
return RestoreCopy{}, ErrRestoreCopyBusy
}
var found *RestoreCopy
for _, nsRoot := range m.primaryUnitRoots() {
appDir := filepath.Join(nsRoot, "backups", "restore-undo", app)
dir := filepath.Join(appDir, stamp)
fi, err := os.Lstat(dir)
if err != nil || !fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 {
continue
}
if filepath.Dir(dir) != appDir || filepath.Dir(appDir) != filepath.Join(nsRoot, "backups", "restore-undo") {
continue
}
at, ok := restoreCopyAt(dir, stamp)
if !ok {
continue
}
found = &RestoreCopy{App: app, Stamp: stamp, At: at, SizeBytes: dirSize(dir), Path: dir}
break
}
if found == nil {
return RestoreCopy{}, ErrRestoreCopyNotFound
}
named, ok := m.appStopNamedApps()
if found.Needed = m.restoreCopyNeededFor(app, named, ok); found.Needed != "" {
return *found, ErrRestoreCopyNeeded
}
if err := os.RemoveAll(found.Path); err != nil {
return *found, fmt.Errorf("removing the restore copy %s: %w", found.Path, err)
}
_ = os.Remove(filepath.Dir(found.Path)) // the per-app folder, when it is now empty
return *found, nil
}
@@ -15,7 +15,8 @@ import (
// stable start (R-1270). Every one of those is still needed. A copy outside all of them — the app runs again, or was
// removed, and nothing names the copy — is disk the household cannot see or use.
//
// The rule (safest reading, decided in the 2026-10-28 night run; operator may change the age): a copy is removed only
// The rule (the age: operator ruling D62, 2026-10-29 — 30 days, and the household may delete a copy earlier with
// „Törlés most", restore_copies.go): a copy is removed only
// when ALL of these hold, and the sweep removes NOTHING when it cannot answer one of them:
// - it is older than restoreCopyUnusedAge (its folder's stamp; the folder's time when the stamp does not parse);
// - the app has no restore hold of any kind (held apps keep every copy — support decides);
@@ -24,8 +25,8 @@ import (
//
// Run every 6 hours from main.go (TestR1256_H8_TheSweepIsWiredAtStartup). Pinned by TestR1256_H8_*.
// restoreCopyUnusedAge is how long an unused copy stays for support after its restore.
const restoreCopyUnusedAge = 14 * 24 * time.Hour
// restoreCopyUnusedAge is how long an unused copy stays after its restore (D62: 30 days; TestD62_AnUnusedCopyIsKept30Days).
const restoreCopyUnusedAge = 30 * 24 * time.Hour
// restoreCopyStampLayout is copyLiveVolumes' folder name.
const restoreCopyStampLayout = "20060102T150405Z"