D62: restore copies kept 30 days, listed with their date, deletable by hand (typed name, never a needed copy, operator event); D63: a test-build-only fault switch for the R-1256 H4/H5 live proofs — tests red first
gates / gates (push) Successful in 33s
gates / gates (push) Successful in 33s
This commit is contained in:
1 parent
a815b37fbd
commit
e3fce09d96
24 files changed
+1992
-11
No files matched your search
@@ -0,0 +1,185 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// D62 (operator ruling 2026-10-29): an unused restore copy is kept 30 days; the household sees each copy (app, the
|
||||
// restore's date, size, the day it goes) and may delete one earlier. A copy a hold or a pending put-back needs is never
|
||||
// deleted, by the sweep or by hand.
|
||||
|
||||
// RED-PROOF: with the age at 14 days the 20-day copy is swept → FAILS; at 30 days only the 31-day copy goes.
|
||||
func TestD62_AnUnusedCopyIsKept30Days(t *testing.T) {
|
||||
m, _ := newOffboxManager(t)
|
||||
now := time.Now()
|
||||
day29 := r1256Copy(t, m, "a", 29*24*time.Hour, now)
|
||||
day20 := r1256Copy(t, m, "b", 20*24*time.Hour, now)
|
||||
day31 := r1256Copy(t, m, "c", 31*24*time.Hour, now)
|
||||
got := m.SweepUnusedRestoreCopies(now)
|
||||
if len(got) != 1 || got[0] != day31 {
|
||||
t.Fatalf("swept %v — want only the 31-day copy %s", got, day31)
|
||||
}
|
||||
if !r1256Exists(day29) || !r1256Exists(day20) {
|
||||
t.Fatal("a copy younger than 30 days was swept")
|
||||
}
|
||||
if restoreCopyUnusedAge != 30*24*time.Hour {
|
||||
t.Fatalf("restoreCopyUnusedAge = %s, D62 says 30 days", restoreCopyUnusedAge)
|
||||
}
|
||||
}
|
||||
|
||||
// The list shows each copy with its restore date, its size and the day it goes; a needed copy has no such day and
|
||||
// says why. RED-PROOF: drop the hold check from restoreCopyNeededFor → the held copy is offered → FAILS.
|
||||
func TestD62_TheListShowsDateSizeAndWhyACopyIsNeeded(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
free := r1256Copy(t, m, "free", 2*24*time.Hour, now)
|
||||
r1256Copy(t, m, "held", 2*24*time.Hour, now)
|
||||
r1256Copy(t, m, "waiting", 2*24*time.Hour, now)
|
||||
r1256Copy(t, m, "named", 2*24*time.Hour, now)
|
||||
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "held", At: now.Format(time.RFC3339), Reason: settings.HoldReasonRestoreMixed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: "waiting", At: now.Format(time.RFC3339), Reason: settings.HoldReasonPutBackPending}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list := m.RestoreCopies()
|
||||
if len(list) != 4 {
|
||||
t.Fatalf("listed %d copies, want 4: %+v", len(list), list)
|
||||
}
|
||||
want := map[string]string{"free": "", "held": RestoreCopyNeededHold, "waiting": RestoreCopyNeededPutBack, "named": RestoreCopyNeededPutBack}
|
||||
for _, c := range list {
|
||||
if c.Needed != want[c.App] {
|
||||
t.Errorf("%s: needed %q, want %q", c.App, c.Needed, want[c.App])
|
||||
}
|
||||
if !c.At.Equal(now.Add(-2 * 24 * time.Hour)) {
|
||||
t.Errorf("%s: restore date %s, want the stamp's %s", c.App, c.At, now.Add(-2*24*time.Hour))
|
||||
}
|
||||
if c.SizeBytes != int64(len("LIVE")) {
|
||||
t.Errorf("%s: size %d", c.App, c.SizeBytes)
|
||||
}
|
||||
if c.Needed == "" && !c.DeleteOn.Equal(c.At.Add(30*24*time.Hour)) {
|
||||
t.Errorf("%s: deleted on %s, want 30 days after the restore", c.App, c.DeleteOn)
|
||||
}
|
||||
if c.Needed != "" && !c.DeleteOn.IsZero() {
|
||||
t.Errorf("%s: a needed copy shows a delete day %s — the sweep never takes it", c.App, c.DeleteOn)
|
||||
}
|
||||
if c.App == "free" && c.Path != free {
|
||||
t.Errorf("path %s, want %s", c.Path, free)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A held or waiting copy cannot be deleted by hand. RED-PROOF: drop the Needed check in DeleteRestoreCopy → FAILS.
|
||||
func TestD62_AHeldCopyCannotBeDeleted(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
now := time.Now()
|
||||
for _, tc := range []struct{ app, reason string }{
|
||||
{"held", settings.HoldReasonRestoreMixed},
|
||||
{"waiting", settings.HoldReasonPutBackPending},
|
||||
{"failed", settings.HoldReasonRestoreFailed},
|
||||
} {
|
||||
dir := r1256Copy(t, m, tc.app, time.Hour, now)
|
||||
if err := sett.SetRestoreHold(settings.RestoreHold{Stack: tc.app, At: now.Format(time.RFC3339), Reason: tc.reason}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy(tc.app, filepath.Base(dir)); !errors.Is(err, ErrRestoreCopyNeeded) {
|
||||
t.Errorf("%s: err %v, want ErrRestoreCopyNeeded", tc.app, err)
|
||||
}
|
||||
if !r1256Exists(dir) {
|
||||
t.Errorf("%s: a held copy was deleted", tc.app)
|
||||
}
|
||||
}
|
||||
// a copy a cut restore names (the marker) is needed too
|
||||
named := r1256Copy(t, m, "named", time.Hour, now)
|
||||
if err := m.appStop.Begin("unit-restore:named", ReasonUnitRestore, []string{"named"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy("named", filepath.Base(named)); !errors.Is(err, ErrRestoreCopyNeeded) || !r1256Exists(named) {
|
||||
t.Errorf("a copy the marker names: err %v, exists %v", err, r1256Exists(named))
|
||||
}
|
||||
// an unreadable marker offers nothing
|
||||
mustWrite(t, m.appStop.path, "{not json")
|
||||
free := r1256Copy(t, m, "free", time.Hour, now)
|
||||
if _, err := m.DeleteRestoreCopy("free", filepath.Base(free)); !errors.Is(err, ErrRestoreCopyNeeded) || !r1256Exists(free) {
|
||||
t.Errorf("an unreadable marker: err %v, exists %v", err, r1256Exists(free))
|
||||
}
|
||||
}
|
||||
|
||||
// A delete removes ONLY that copy: the app's other copy, another app's copy and anything outside the copy tree stay;
|
||||
// a name that is not one path element, or a link in the copy's place, is refused.
|
||||
// RED-PROOF: remove the validCopyName check → "../.." reaches outside → FAILS; remove the Lstat link check → the
|
||||
// link is followed/removed → FAILS.
|
||||
func TestD62_ADeleteRemovesOnlyThatCopy(t *testing.T) {
|
||||
m, _ := newOffboxManager(t)
|
||||
now := time.Now()
|
||||
one := r1256Copy(t, m, "app", 3*time.Hour, now)
|
||||
two := r1256Copy(t, m, "app", 2*time.Hour, now)
|
||||
other := r1256Copy(t, m, "other", 3*time.Hour, now)
|
||||
got, err := m.DeleteRestoreCopy("app", filepath.Base(one))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Path != one || got.SizeBytes != int64(len("LIVE")) || got.App != "app" {
|
||||
t.Fatalf("returned %+v", got)
|
||||
}
|
||||
if r1256Exists(one) || !r1256Exists(two) || !r1256Exists(other) {
|
||||
t.Fatalf("one gone=%v, two kept=%v, other kept=%v", !r1256Exists(one), r1256Exists(two), r1256Exists(other))
|
||||
}
|
||||
// path guard
|
||||
outside := filepath.Join(m.namespaceRoot(m.systemDataPath), "backups", "primary", "keep.txt")
|
||||
mustWrite(t, outside, "UNIT")
|
||||
for _, bad := range [][2]string{{"app", ".."}, {"..", "primary"}, {"app", "../../primary"}, {"", "x"}, {"app", ""}, {"app/x", "y"}} {
|
||||
if _, err := m.DeleteRestoreCopy(bad[0], bad[1]); !errors.Is(err, ErrRestoreCopyNotFound) {
|
||||
t.Errorf("DeleteRestoreCopy(%q,%q) = %v, want not found", bad[0], bad[1], err)
|
||||
}
|
||||
}
|
||||
if !r1256Exists(outside) || !r1256Exists(two) {
|
||||
t.Fatal("a bad name reached outside the copy")
|
||||
}
|
||||
// a link in a copy's place is not a copy
|
||||
target := t.TempDir()
|
||||
mustWrite(t, filepath.Join(target, "precious"), "X")
|
||||
link := filepath.Join(filepath.Dir(two), "20260101T000000Z")
|
||||
if err := os.Symlink(target, link); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy("app", "20260101T000000Z"); !errors.Is(err, ErrRestoreCopyNotFound) {
|
||||
t.Errorf("a link: %v, want not found", err)
|
||||
}
|
||||
if !r1256Exists(filepath.Join(target, "precious")) {
|
||||
t.Fatal("the link's target was removed")
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy("app", "20990101T000000Z"); !errors.Is(err, ErrRestoreCopyNotFound) {
|
||||
t.Errorf("a missing copy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// While a backup or restore runs, a delete is refused and nothing moves. RED-PROOF: drop the busy check → FAILS.
|
||||
func TestD62_NoDeleteWhileAnOperationRuns(t *testing.T) {
|
||||
m, _ := newOffboxManager(t)
|
||||
c := r1256Copy(t, m, "app", time.Hour, time.Now())
|
||||
m.BeginRestoreOp("restore", "other")
|
||||
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); !errors.Is(err, ErrRestoreCopyBusy) || !r1256Exists(c) {
|
||||
t.Fatalf("during a restore: err %v, exists %v", err, r1256Exists(c))
|
||||
}
|
||||
m.EndRestoreOp(true, "ok")
|
||||
if err := m.acquireRunning(); err != nil {
|
||||
t.Fatalf("could not take the backup single-flight: %v", err)
|
||||
}
|
||||
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); !errors.Is(err, ErrRestoreCopyBusy) || !r1256Exists(c) {
|
||||
t.Fatalf("during a backup run: err %v, exists %v", err, r1256Exists(c))
|
||||
}
|
||||
m.ReleaseRunningForTest()
|
||||
if _, err := m.DeleteRestoreCopy("app", filepath.Base(c)); err != nil || r1256Exists(c) {
|
||||
t.Fatalf("control: with nothing running the delete goes through: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -300,7 +300,7 @@ func r1256Exists(p string) bool { _, err := os.Stat(p); return err == nil }
|
||||
func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
now := time.Now()
|
||||
old := 20 * 24 * time.Hour
|
||||
old := 40 * 24 * time.Hour
|
||||
unused := r1256Copy(t, m, "unused", old, now)
|
||||
young := r1256Copy(t, m, "young", 24*time.Hour, now)
|
||||
held := r1256Copy(t, m, "held", old, now)
|
||||
@@ -339,7 +339,7 @@ func TestR1256_H8_OnlyAnUnusedOldCopyIsRemoved(t *testing.T) {
|
||||
func TestR1256_H8_NothingIsRemovedDuringARestoreOrWithAnUnreadableMarker(t *testing.T) {
|
||||
m, _ := newOffboxManager(t)
|
||||
now := time.Now()
|
||||
c := r1256Copy(t, m, "app", 20*24*time.Hour, now)
|
||||
c := r1256Copy(t, m, "app", 40*24*time.Hour, now)
|
||||
m.BeginRestoreOp("restore", "other")
|
||||
if got := m.SweepUnusedRestoreCopies(now); len(got) != 0 || !r1256Exists(c) {
|
||||
t.Fatalf("removed %v during a restore", got)
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// D62 (operator ruling 2026-10-29; `07` §6.3): the household sees its restore copies in the kept-copies list
|
||||
// („Megőrzött másolatok") on the restore page — the app, the date of the restore the copy protects, its size, and the
|
||||
// day the box deletes it by itself (restoreCopyUnusedAge after the restore) — and may delete one earlier with
|
||||
// „Törlés most", after a warning and the app's name typed.
|
||||
//
|
||||
// A copy a hold or a pending put-back needs is NEVER deletable (the same three states the sweep leaves alone, R-1256
|
||||
// H8): the row says why in one line and the button is off; the handler refuses it too. While any backup or restore
|
||||
// operation runs on the box, a delete is refused (409). The check and the removal are ONE step under m.mu, so a
|
||||
// restore cannot begin in between (the R-1256 H6 shape). Pinned by TestD62_*.
|
||||
|
||||
// Why a restore copy is needed — the row's one line, and the refusal's reason. "" = not needed: deletable.
|
||||
const (
|
||||
RestoreCopyNeededHold = "hold" // the app is held (a failed put-back keeps its copy, D20) — support decides
|
||||
RestoreCopyNeededPutBack = "putback" // a put-back waits for its drive (R-1251) or a cut restore names it (D22)
|
||||
RestoreCopyNeededUnknown = "unknown" // the app-stop marker cannot be read: nothing is offered (fail closed)
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRestoreCopyNotFound = errors.New("restore copy not found")
|
||||
ErrRestoreCopyNeeded = errors.New("restore copy is needed by a hold or a pending put-back")
|
||||
ErrRestoreCopyBusy = errors.New("a backup or restore operation runs")
|
||||
)
|
||||
|
||||
// RestoreCopy is one restore copy (<namespace>/backups/restore-undo/<app>/<stamp>).
|
||||
type RestoreCopy struct {
|
||||
App string
|
||||
Stamp string // the folder's name; with App, the copy's identity on the page
|
||||
At time.Time // when the restore began (the stamp; the folder's time when the stamp does not parse)
|
||||
SizeBytes int64
|
||||
Path string
|
||||
Needed string // "" or one of RestoreCopyNeeded*
|
||||
DeleteOn time.Time // when the sweep removes it; zero when Needed != "" (a needed copy is never swept)
|
||||
}
|
||||
|
||||
// restoreCopyAt is a copy's restore time: its stamp, or the folder's time when the stamp does not parse.
|
||||
func restoreCopyAt(dir, name string) (time.Time, bool) {
|
||||
if at, err := time.Parse(restoreCopyStampLayout, name); err == nil {
|
||||
return at, true
|
||||
}
|
||||
fi, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return fi.ModTime(), true
|
||||
}
|
||||
|
||||
// restoreCopyNeededFor is why app's copies are needed ("" = deletable). named is appStopNamedApps' answer, ok its ok.
|
||||
func (m *Manager) restoreCopyNeededFor(app string, named map[string]bool, ok bool) string {
|
||||
if !ok || m.settings == nil {
|
||||
return RestoreCopyNeededUnknown
|
||||
}
|
||||
if named[app] {
|
||||
return RestoreCopyNeededPutBack
|
||||
}
|
||||
if h, held := m.settings.GetRestoreHold(app); held {
|
||||
if h.Reason == settings.HoldReasonPutBackPending {
|
||||
return RestoreCopyNeededPutBack
|
||||
}
|
||||
return RestoreCopyNeededHold
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func dirSize(dir string) int64 {
|
||||
var n int64
|
||||
_ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
|
||||
if err == nil && d.Type().IsRegular() {
|
||||
if fi, e := d.Info(); e == nil {
|
||||
n += fi.Size()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return n
|
||||
}
|
||||
|
||||
// RestoreCopies lists every restore copy on the box's drives, oldest first.
|
||||
func (m *Manager) RestoreCopies() []RestoreCopy {
|
||||
named, ok := m.appStopNamedApps()
|
||||
var out []RestoreCopy
|
||||
for _, nsRoot := range m.primaryUnitRoots() {
|
||||
base := filepath.Join(nsRoot, "backups", "restore-undo")
|
||||
apps, err := os.ReadDir(base)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range apps {
|
||||
if !a.IsDir() {
|
||||
continue
|
||||
}
|
||||
app := a.Name()
|
||||
needed := m.restoreCopyNeededFor(app, named, ok)
|
||||
copies, err := os.ReadDir(filepath.Join(base, app))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range copies {
|
||||
if !c.IsDir() {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Join(base, app, c.Name())
|
||||
at, ok := restoreCopyAt(dir, c.Name())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
rc := RestoreCopy{App: app, Stamp: c.Name(), At: at, SizeBytes: dirSize(dir), Path: dir, Needed: needed}
|
||||
if needed == "" {
|
||||
rc.DeleteOn = at.Add(restoreCopyUnusedAge)
|
||||
}
|
||||
out = append(out, rc)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
|
||||
return out
|
||||
}
|
||||
|
||||
// validCopyName is a single path element: no separator, not empty, not "." or "..".
|
||||
func validCopyName(s string) bool {
|
||||
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, `/\`) && filepath.Base(s) == s
|
||||
}
|
||||
|
||||
// DeleteRestoreCopy removes ONE restore copy, named by its app and its folder's name, and nothing else (the path
|
||||
// guard: both names are single path elements, the folder is a real directory — not a link — directly under
|
||||
// <namespace>/backups/restore-undo/<app>/). Refused while any backup or restore operation runs (ErrRestoreCopyBusy)
|
||||
// and for a copy a hold or a pending put-back needs (ErrRestoreCopyNeeded). The check and the removal hold m.mu, so
|
||||
// no restore can begin in between. Returns the copy as it was (for the log line and the operator event).
|
||||
func (m *Manager) DeleteRestoreCopy(app, stamp string) (RestoreCopy, error) {
|
||||
if !validCopyName(app) || !validCopyName(stamp) {
|
||||
return RestoreCopy{}, ErrRestoreCopyNotFound
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if busy, _ := m.restoreBusyLocked(); busy != "" {
|
||||
return RestoreCopy{}, ErrRestoreCopyBusy
|
||||
}
|
||||
var found *RestoreCopy
|
||||
for _, nsRoot := range m.primaryUnitRoots() {
|
||||
appDir := filepath.Join(nsRoot, "backups", "restore-undo", app)
|
||||
dir := filepath.Join(appDir, stamp)
|
||||
fi, err := os.Lstat(dir)
|
||||
if err != nil || !fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 {
|
||||
continue
|
||||
}
|
||||
if filepath.Dir(dir) != appDir || filepath.Dir(appDir) != filepath.Join(nsRoot, "backups", "restore-undo") {
|
||||
continue
|
||||
}
|
||||
at, ok := restoreCopyAt(dir, stamp)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
found = &RestoreCopy{App: app, Stamp: stamp, At: at, SizeBytes: dirSize(dir), Path: dir}
|
||||
break
|
||||
}
|
||||
if found == nil {
|
||||
return RestoreCopy{}, ErrRestoreCopyNotFound
|
||||
}
|
||||
named, ok := m.appStopNamedApps()
|
||||
if found.Needed = m.restoreCopyNeededFor(app, named, ok); found.Needed != "" {
|
||||
return *found, ErrRestoreCopyNeeded
|
||||
}
|
||||
if err := os.RemoveAll(found.Path); err != nil {
|
||||
return *found, fmt.Errorf("removing the restore copy %s: %w", found.Path, err)
|
||||
}
|
||||
_ = os.Remove(filepath.Dir(found.Path)) // the per-app folder, when it is now empty
|
||||
return *found, nil
|
||||
}
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
// stable start (R-1270). Every one of those is still needed. A copy outside all of them — the app runs again, or was
|
||||
// removed, and nothing names the copy — is disk the household cannot see or use.
|
||||
//
|
||||
// The rule (safest reading, decided in the 2026-10-28 night run; operator may change the age): a copy is removed only
|
||||
// The rule (the age: operator ruling D62, 2026-10-29 — 30 days, and the household may delete a copy earlier with
|
||||
// „Törlés most", restore_copies.go): a copy is removed only
|
||||
// when ALL of these hold, and the sweep removes NOTHING when it cannot answer one of them:
|
||||
// - it is older than restoreCopyUnusedAge (its folder's stamp; the folder's time when the stamp does not parse);
|
||||
// - the app has no restore hold of any kind (held apps keep every copy — support decides);
|
||||
@@ -24,8 +25,8 @@ import (
|
||||
//
|
||||
// Run every 6 hours from main.go (TestR1256_H8_TheSweepIsWiredAtStartup). Pinned by TestR1256_H8_*.
|
||||
|
||||
// restoreCopyUnusedAge is how long an unused copy stays for support after its restore.
|
||||
const restoreCopyUnusedAge = 14 * 24 * time.Hour
|
||||
// restoreCopyUnusedAge is how long an unused copy stays after its restore (D62: 30 days; TestD62_AnUnusedCopyIsKept30Days).
|
||||
const restoreCopyUnusedAge = 30 * 24 * time.Hour
|
||||
|
||||
// restoreCopyStampLayout is copyLiveVolumes' folder name.
|
||||
const restoreCopyStampLayout = "20060102T150405Z"
|
||||
|
||||
Reference in new issue
Block a user