382 lines
16 KiB
Go
382 lines
16 KiB
Go
package backup
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// ── D47 (operator ruling 2026-10-25 08:38; R-1269): a new install never writes into an earlier install's unit ──────
|
|
//
|
|
// MEASURED BEFORE (catalog sweep 2026-10-24, three times; `audits/catalog-sweep-2026-10-24/_unitprobe/tandoor/`):
|
|
// install Tandoor, one account, „Mentés most", remove keeping data and backups, install again. Five minutes later the
|
|
// periodic refresh (captureRecoveryUnit, dataRun=false) rewrote ONLY the unit's compose/app.yaml — the new install's
|
|
// database password — beside the OLD install's data. Its one „same definition" test is the pins (`frozen` below), and
|
|
// a reinstall of the same version has the same pins, so it passed. A restore then paired the new password with the old
|
|
// data directory: „password authentication failed", the app never started, the account gone from view. The next
|
|
// night's data legs would have overwritten the old data too.
|
|
//
|
|
// THE RULE. A unit carries the identity of the install its data belongs to (its app.yaml `install_id:`, minted at a
|
|
// fresh install, taken over by a restore — stacks.AppConfig.InstallID). EVERY writer of a unit — the capture and
|
|
// refresh, the database dump legs, the volume dump leg, the update's pre-backup dump, the second drive's mirror, the
|
|
// stale-drive prune — asks guardUnitWrite first. A unit of ANOTHER install is SET ASIDE WHOLE (a rename on the same
|
|
// drive, never a copy, never an edit) into <namespace>/backups/set-aside/<app>/<UTC stamp>/, recorded as a kept copy
|
|
// (D25's record: listed on the restore page with „Visszaállítás" and „Eltávolítás"), and the writer then writes the new
|
|
// install's own unit. A set-aside that cannot be done REFUSES the write and leaves the old unit exactly as it was, with
|
|
// an ERROR line naming the app and both identities. Nothing deletes a set-aside unit but the household: its
|
|
// „Eltávolítás", or the app's remove with „delete its backups too" (the D25 / R-1252 H9 rule for kept copies).
|
|
//
|
|
// A UNIT WRITTEN BEFORE D47 records no install. Then the secrets decide: a reinstall mints new ones, a restore writes
|
|
// the unit's into the app, so a carried secret that differs on both sides means another install. With no differing
|
|
// secret it is this install's (the old behaviour) and the next write records the identity.
|
|
|
|
// setAsideDirName is the folder beside backups/primary that holds the set-aside units.
|
|
const setAsideDirName = "set-aside"
|
|
|
|
// KeptReasonEarlierInstall is the kept-copy reason of a set-aside unit (the page's „kept.reason.earlier_install").
|
|
const KeptReasonEarlierInstall = "earlier_install"
|
|
|
|
// KeptKindUnit is the kept-copy kind of a set-aside recovery unit (restorable from the restore page).
|
|
const KeptKindUnit = "unit"
|
|
|
|
// errUnitOfAnotherInstall is the refusal of a write into another install's unit that could not be set aside.
|
|
var errUnitOfAnotherInstall = errors.New("the recovery unit belongs to another install and could not be set aside")
|
|
|
|
// setAsideRoot is <nsRoot>/backups/set-aside.
|
|
func setAsideRoot(nsRoot string) string {
|
|
return filepath.Join(nsRoot, "backups", setAsideDirName)
|
|
}
|
|
|
|
// installIdentity is the optional provider method that mints an identity for an app installed before D47.
|
|
type installIdentity interface {
|
|
EnsureInstallID(name string) (string, error)
|
|
}
|
|
|
|
type unitOwner int
|
|
|
|
const (
|
|
ownerNoUnit unitOwner = iota // nothing to protect
|
|
ownerSame // this install's unit (or no signal that it is not)
|
|
ownerOther // another install's unit — set it aside before writing
|
|
)
|
|
|
|
// unitOwnerVerdict is the rule as a pure function. Both identities known → they decide. Otherwise (a unit or an app
|
|
// from before D47) the carried secrets decide: a name both sides hold with different values means another install.
|
|
// why names the reason, never a value.
|
|
func unitOwnerVerdict(unitID, liveID string, unitSecrets, liveSecrets map[string]string) (unitOwner, string) {
|
|
if unitID != "" && liveID != "" {
|
|
if unitID == liveID {
|
|
return ownerSame, "same install id"
|
|
}
|
|
return ownerOther, "the install ids differ"
|
|
}
|
|
var differ []string
|
|
for n, uv := range unitSecrets {
|
|
if lv := liveSecrets[n]; uv != "" && lv != "" && uv != lv {
|
|
differ = append(differ, n)
|
|
}
|
|
}
|
|
if len(differ) > 0 {
|
|
sort.Strings(differ)
|
|
return ownerOther, "carried secret(s) " + strings.Join(differ, ", ") + " differ (a unit from before D47)"
|
|
}
|
|
return ownerSame, "no install id on one side and no differing secret"
|
|
}
|
|
|
|
// unitOwnerOf reads a unit's recorded install and the secrets it carries (names from its manifest). exists=false when
|
|
// the directory holds nothing a restore could use (no manifest, no compose/, no data folder).
|
|
func unitOwnerOf(unitDir string) (exists bool, installID string, secrets map[string]string) {
|
|
secrets = map[string]string{}
|
|
any := false
|
|
for _, p := range []string{UnitManifestFile(unitDir), UnitComposeDir(unitDir), UnitDBDumpDir(unitDir), UnitVolumeDumpDir(unitDir)} {
|
|
if _, err := os.Stat(p); err == nil {
|
|
any = true
|
|
break
|
|
}
|
|
}
|
|
if !any {
|
|
return false, "", secrets
|
|
}
|
|
var portable []string
|
|
if man := readManifest(UnitManifestFile(unitDir)); man != nil {
|
|
portable = man.PortableSecretEnvVars
|
|
}
|
|
appy := filepath.Join(UnitComposeDir(unitDir), "app.yaml")
|
|
_, secrets = readUnitEnv(appy, portable)
|
|
if b, err := os.ReadFile(appy); err == nil {
|
|
var doc struct {
|
|
InstallID string `yaml:"install_id"`
|
|
}
|
|
if yaml.Unmarshal(b, &doc) == nil {
|
|
installID = doc.InstallID
|
|
}
|
|
}
|
|
return true, installID, secrets
|
|
}
|
|
|
|
// liveInstall is the app's identity and carried secrets now; it mints an identity for an app from before D47 when the
|
|
// provider can. err = the identity could not be read or written (the caller refuses to decide).
|
|
func (m *Manager) liveInstall(stack string) (string, map[string]string, error) {
|
|
info, ok := m.stackProvider.GetStackRecoveryInfo(stack)
|
|
if !ok {
|
|
// No record to compare at all (a provider that does not know the app): no signal — the old behaviour.
|
|
return "", nil, nil
|
|
}
|
|
id := info.InstallID
|
|
if id == "" {
|
|
if e, ok := m.stackProvider.(installIdentity); ok {
|
|
got, err := e.EnsureInstallID(stack)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
id = got
|
|
}
|
|
}
|
|
return id, info.PortableSecrets, nil
|
|
}
|
|
|
|
// guardUnitWrite is asked by EVERY writer before it writes into <nsRoot>/backups/primary/<stack> (what names the
|
|
// writer, for the log). nil = write; an error = do not write — the unit is untouched.
|
|
func (m *Manager) guardUnitWrite(stack, nsRoot, what string) error {
|
|
if m.stackProvider == nil {
|
|
return nil
|
|
}
|
|
return m.guardUnitDir(stack, RecoveryUnitPath(nsRoot, stack), nsRoot, what)
|
|
}
|
|
|
|
// guardUnitDir is guardUnitWrite for any unit directory (the second drive's mirror too); setAsideNs is the namespace
|
|
// root whose backups/set-aside receives it (the unit's own drive, so the move is a rename).
|
|
func (m *Manager) guardUnitDir(stack, unitDir, setAsideNs, what string) error {
|
|
exists, unitID, unitSecrets := unitOwnerOf(unitDir)
|
|
if !exists {
|
|
return nil
|
|
}
|
|
liveID, liveSecrets, err := m.liveInstall(stack)
|
|
if err != nil {
|
|
m.logger.Printf("[ERROR] [backup] %s: the %s is NOT written into %s — this install's identity cannot be read (%v); the unit is untouched (D47)", stack, what, unitDir, err)
|
|
return fmt.Errorf("%w: %v", errUnitOfAnotherInstall, err)
|
|
}
|
|
verdict, why := unitOwnerVerdict(unitID, liveID, unitSecrets, liveSecrets)
|
|
if verdict != ownerOther {
|
|
return nil
|
|
}
|
|
// A restore of this app reads that unit right now (its install becomes the unit's when the definition is written):
|
|
// never move it under the restore — refuse this write instead.
|
|
if m.restoreRunningFor(stack) {
|
|
m.logger.Printf("[ERROR] [backup] %s: the %s is NOT written into the unit of install %s (this install %s): a restore of the app runs — the unit is untouched (D47)", stack, what, orUnknown(unitID), orUnknown(liveID))
|
|
return errUnitOfAnotherInstall
|
|
}
|
|
dest, err := m.setAsideUnit(stack, unitDir, setAsideNs)
|
|
if err != nil {
|
|
m.logger.Printf("[ERROR] [backup] %s: REFUSED to write the %s into the recovery unit of another install (unit install %s, this install %s; %s): it could not be set aside: %v — the unit is untouched (D47)",
|
|
stack, what, orUnknown(unitID), orUnknown(liveID), why, err)
|
|
return fmt.Errorf("%w: %v", errUnitOfAnotherInstall, err)
|
|
}
|
|
m.logger.Printf("[INFO] [backup] %s: the recovery unit of an earlier install (%s; %s) is SET ASIDE whole → %s; this install (%s) gets its own unit from its first backup (D47)",
|
|
stack, orUnknown(unitID), why, dest, orUnknown(liveID))
|
|
if m.unitSetAsideNotify != nil {
|
|
m.unitSetAsideNotify(stack, orUnknown(unitID), orUnknown(liveID), dest)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetUnitSetAsideNotify wires the operator line for a set-aside unit (main.go).
|
|
func (m *Manager) SetUnitSetAsideNotify(fn func(stack, unitID, liveID, dest string)) {
|
|
m.unitSetAsideNotify = fn
|
|
}
|
|
|
|
func orUnknown(id string) string {
|
|
if id == "" {
|
|
return "unrecorded"
|
|
}
|
|
return id
|
|
}
|
|
|
|
// renameUnitFn is the set-aside's move (tests inject a failure); production is os.Rename.
|
|
var renameUnitFn = os.Rename
|
|
|
|
// setAsideUnit MOVES unitDir whole to <setAsideNs>/backups/set-aside/<stack>/<UTC stamp> and records it as a kept
|
|
// copy. A rename only: never a copy, never across drives (EXDEV refuses, nothing moved).
|
|
func (m *Manager) setAsideUnit(stack, unitDir, setAsideNs string) (string, error) {
|
|
base := filepath.Join(setAsideRoot(setAsideNs), stack)
|
|
if err := os.MkdirAll(base, 0o755); err != nil {
|
|
return "", err
|
|
}
|
|
dest := filepath.Join(base, time.Now().UTC().Format("20060102T150405Z"))
|
|
for i := 1; ; i++ {
|
|
if _, err := os.Lstat(dest); os.IsNotExist(err) {
|
|
break
|
|
}
|
|
dest = filepath.Join(base, fmt.Sprintf("%s-%d", time.Now().UTC().Format("20060102T150405Z"), i))
|
|
}
|
|
if err := renameUnitFn(unitDir, dest); err != nil {
|
|
_ = os.Remove(base) // only when empty
|
|
return "", err
|
|
}
|
|
m.keepCopy(KeptKindUnit, stack, m.storageLabel(m.driveOfNamespace(setAsideNs)), KeptReasonEarlierInstall, setAsideRoot(setAsideNs), dest)
|
|
return dest, nil
|
|
}
|
|
|
|
// driveOfNamespace names the drive a namespace root sits on (for the record's label): the registered storage path
|
|
// whose namespace it is, else the namespace root itself.
|
|
func (m *Manager) driveOfNamespace(nsRoot string) string {
|
|
if m.settings != nil {
|
|
for _, sp := range m.settings.GetStoragePaths() {
|
|
if sp.Path != "" && filepath.Clean(NamespaceRootFor(sp.Path, m.systemDataPath)) == filepath.Clean(nsRoot) {
|
|
return sp.Path
|
|
}
|
|
}
|
|
}
|
|
if m.systemDataPath != "" && filepath.Clean(NamespaceRootFor(m.systemDataPath, m.systemDataPath)) == filepath.Clean(nsRoot) {
|
|
return m.systemDataPath
|
|
}
|
|
return nsRoot
|
|
}
|
|
|
|
// KeptBackupsFor (R-1278) counts the backups of an EARLIER install of stack that the box keeps: its set-aside units
|
|
// (D47) and the unit „remove — keep backups" left (a removed app's). The install page names them beside a fresh start.
|
|
func (m *Manager) KeptBackupsFor(stack string) int {
|
|
if m == nil || !validMirrorStackName(stack) {
|
|
return 0
|
|
}
|
|
n := 0
|
|
if m.settings != nil {
|
|
for _, k := range m.settings.GetKeptCopies() {
|
|
if k.Kind == KeptKindUnit && k.Source == stack {
|
|
n++
|
|
}
|
|
}
|
|
}
|
|
if _, ok := m.RemovedAppUnitFor(stack); ok {
|
|
n++
|
|
}
|
|
return n
|
|
}
|
|
|
|
// SetAsideDirsForApp lists the app's set-aside folders (<root>/backups/set-aside/<stack>) that exist, on every
|
|
// namespace root a unit of it can sit on.
|
|
func (m *Manager) SetAsideDirsForApp(stack string) []string {
|
|
if m == nil || !validMirrorStackName(stack) {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for _, root := range m.tier2MirrorRoots(stack) {
|
|
d := filepath.Join(setAsideRoot(root), stack)
|
|
if fi, err := os.Stat(d); err == nil && fi.IsDir() {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// RemoveSetAsideUnits is the app's remove with „delete its backups too": each dir only if it is exactly
|
|
// <root>/backups/set-aside/<stack> for one of the app's roots, and the kept-copy records inside it are dropped.
|
|
func (m *Manager) RemoveSetAsideUnits(stack string, dirs []string) []string {
|
|
if m == nil || !validMirrorStackName(stack) {
|
|
return nil
|
|
}
|
|
allowed := map[string]bool{}
|
|
for _, root := range m.tier2MirrorRoots(stack) {
|
|
allowed[filepath.Join(setAsideRoot(root), stack)] = true
|
|
}
|
|
var removed []string
|
|
for _, d := range dirs {
|
|
if !allowed[d] {
|
|
m.logger.Printf("[WARN] [backup] remove %s: refusing to delete %q — not this app's set-aside folder", stack, d)
|
|
continue
|
|
}
|
|
size := humanizeBytes(dirSizeBytes(d))
|
|
if err := os.RemoveAll(d); err != nil {
|
|
m.logger.Printf("[ERROR] [backup] remove %s: deleting the set-aside units %s failed: %v", stack, d, err)
|
|
continue
|
|
}
|
|
m.logger.Printf("[INFO] [backup] remove %s: the set-aside units of earlier installs deleted with its backups: %s (%s)", stack, d, size)
|
|
m.dropKeptCopiesUnder(d)
|
|
removed = append(removed, fmt.Sprintf("%s (%s)", d, size))
|
|
}
|
|
return removed
|
|
}
|
|
|
|
// SetAsideUnitFor returns the kept-copy record of a set-aside unit by id, still the recorded directory (device +
|
|
// inode) and still inside its set-aside tree — the only way the restore page names one.
|
|
func (m *Manager) SetAsideUnitFor(id string) (settings.KeptCopy, error) {
|
|
if m.settings == nil {
|
|
return settings.KeptCopy{}, ErrKeptCopyNotFound
|
|
}
|
|
for _, k := range m.settings.GetKeptCopies() {
|
|
if k.ID != id {
|
|
continue
|
|
}
|
|
if k.Kind != KeptKindUnit || !validMirrorStackName(k.Source) {
|
|
return settings.KeptCopy{}, ErrKeptCopyNotFound
|
|
}
|
|
rel, err := filepath.Rel(filepath.Clean(k.DestBase), filepath.Clean(k.Path))
|
|
if err != nil || rel == "." || strings.HasPrefix(rel, "..") || filepath.Base(k.DestBase) != setAsideDirName {
|
|
return settings.KeptCopy{}, ErrKeptCopyChanged
|
|
}
|
|
st, err := os.Lstat(k.Path)
|
|
if err != nil || !st.IsDir() {
|
|
return settings.KeptCopy{}, ErrKeptCopyChanged
|
|
}
|
|
if dev, ino := statIdentity(st); dev != k.Dev || ino != k.Ino {
|
|
return settings.KeptCopy{}, ErrKeptCopyChanged
|
|
}
|
|
return k, nil
|
|
}
|
|
return settings.KeptCopy{}, ErrKeptCopyNotFound
|
|
}
|
|
|
|
// RestoreSetAsideUnit restores the app from a set-aside unit (the restore page's „Visszaállítás" on a kept copy):
|
|
// the one unit-restore body, with the unit named explicitly. The data then belongs to that unit's install, so its
|
|
// backups follow it (the app's own unit, of the install it replaced, is set aside at the next backup — never lost).
|
|
func (m *Manager) RestoreSetAsideUnit(id string) (string, UnitRestoreResult, error) {
|
|
k, err := m.SetAsideUnitFor(id)
|
|
if err != nil {
|
|
return "", UnitRestoreResult{}, err
|
|
}
|
|
m.logger.Printf("[INFO] [backup] restore %s from the set-aside unit %s (an earlier install, D47)", k.Source, k.Path)
|
|
res, err := m.RestoreFromRecoveryUnitAt(k.Source, k.Path)
|
|
if err == nil || errors.Is(err, ErrRestoreNotStarted) {
|
|
// The restored data stays in both cases (D49 = A), so the app is now the set-aside unit's install.
|
|
m.retireReplacedUnit(k.Source)
|
|
}
|
|
return k.Source, res, err
|
|
}
|
|
|
|
// retireReplacedUnit (R-1288): after the household restored an earlier install's set-aside unit, the app's primary
|
|
// unit belongs to the install the restore REPLACED. It is set aside here, at the restore — moved whole, never deleted
|
|
// (D47), one log line and NO operator alarm: the household chose this restore, so nothing unexpected happened. Before,
|
|
// the next backup's writer found it and set it aside with a second alarm and a second kept copy that looked like a
|
|
// reinstall. A unit that is already this install's (or none) is left alone; a move that fails leaves it for the next
|
|
// writer, whose guard sets it aside as before.
|
|
func (m *Manager) retireReplacedUnit(stack string) {
|
|
nsRoot := m.namespaceRoot(m.GetAppDrivePath(stack))
|
|
unitDir := RecoveryUnitPath(nsRoot, stack)
|
|
exists, unitID, unitSecrets := unitOwnerOf(unitDir)
|
|
if !exists {
|
|
return
|
|
}
|
|
liveID, liveSecrets, err := m.liveInstall(stack)
|
|
if err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: after the restore of an earlier install, this install's identity cannot be read (%v) — the replaced unit is left for the next backup's check (D47)", stack, err)
|
|
return
|
|
}
|
|
if v, _ := unitOwnerVerdict(unitID, liveID, unitSecrets, liveSecrets); v != ownerOther {
|
|
return
|
|
}
|
|
dest, err := m.setAsideUnit(stack, unitDir, nsRoot)
|
|
if err != nil {
|
|
m.logger.Printf("[WARN] [backup] %s: the unit of the install the restore replaced (%s) could not be set aside: %v — the next backup's check sets it aside (D47)", stack, orUnknown(unitID), err)
|
|
return
|
|
}
|
|
m.logger.Printf("[INFO] [backup] %s: the unit of the install the restore replaced (%s) is set aside whole → %s — replaced by the restored install %s; no alarm (R-1288, D47)",
|
|
stack, orUnknown(unitID), dest, orUnknown(liveID))
|
|
}
|