package backup import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/system" "gopkg.in/yaml.v3" ) // RecoveryManifest describes an app's self-contained recovery unit. // // The unit on a drive is `/backups/primary//` and contains: // // compose/ docker-compose.yml + .felhom.yml + app.yaml (0600; carries the PORTABLE secrets) // db-dumps/ app-consistent DB dump(s) (written by the dump flow) // volume-dumps/ named-volume tars (written by the dump flow) // manifest.json this file // // D5 (schema 2) changed what the unit holds. Before it held NO secret at all, which made // "restore from the drive alone" false: the fast, local, customer-doable Tier-1/2 restore secretly // depended on the slow, operator-driven whole-guest restore, because a data-encrypting key or a DB // password absent from the guest cannot be regenerated without rendering the restored data // unreachable. The unit now carries the PORTABLE secret class (stacks.PortableSecretEnvVars) in its // 0600 app.yaml, and Tier-1/2 needs the DRIVE AND NOTHING ELSE. // // It still holds NO `type: password` admin login (those are internet-reachable, so their blast radius // is not bounded by the drive — they stay in the guest and are regenerated on restore) and NOT the // Docker image, only the pinned tag(s), re-pulled on restore. SecretSource records the split. // // A schema-1 unit carries no secrets: the restore degrades to the pre-D5 guest-only behaviour rather // than failing, and the next capture rewrites it (the app.yaml checksum changes). type RecoveryManifest struct { SchemaVersion int `json:"schema_version"` AppName string `json:"app_name"` DisplayName string `json:"display_name"` ControllerVer string `json:"controller_version"` CreatedAt string `json:"created_at"` Drive string `json:"drive"` // HDD_PATH (in-guest mount) NamespaceRoot string `json:"namespace_root"` // resolved felhom-data namespace root ImagePins []string `json:"image_pins"` // image NOT stored — re-pulled on restore SecretEnvVars []string `json:"secret_env_vars"` // NAMES of every secret/password field DataKeyEnvVars []string `json:"data_key_env_vars"` // fail-closed gate on restore SecretSource string `json:"secret_source"` // human note: where secrets come from ConfigFiles []string `json:"config_files"` // captured into compose/ DBDumps []string `json:"db_dumps"` VolumeDumps []string `json:"volume_dumps"` Checksums map[string]string `json:"checksums"` // sha256 of captured compose/ files // PortableSecretEnvVars (D5) are the NAMES of the secrets this unit's app.yaml CARRIES. Names only // — the manifest is 0644 and never holds a value. The restore reads it to know which app.yaml env // entries are secrets rather than plain config; absent (schema 1) ⇒ the unit carries none. PortableSecretEnvVars []string `json:"portable_secret_env_vars,omitempty"` // R-43/R-44 (v0.148.0): the coherence stamp. An offsite run refreshes the dumps FIRST and then // captures the unit, so a manifest carrying an OffsiteRunID asserts "the db-dumps/ in this unit // were taken by that run" — i.e. the snapshot is an internally coherent {DB@T, files@T} pair. // A manifest WITHOUT these fields is a pre-v0.148 unit whose dump age is unknown and may skew // arbitrarily from the files beside it (the DIAG-immich-restore-2026-07-19 failure); the restore // confirm surfaces that honestly rather than blocking. Empty on the periodic refresh, which must // never claim a coherence it did not establish — it carries the prior stamp forward instead. OffsiteRunID string `json:"offsite_run_id,omitempty"` DumpsAt string `json:"dumps_at,omitempty"` // RFC3339 UTC — when this run's dump leg finished } // SetVersion records the controller version stamped into recovery-unit manifests. func (m *Manager) SetVersion(v string) { m.mu.Lock() m.version = v m.mu.Unlock() } // SetTier2Notifier wires the notification callback invoked after each Tier 2 copy. func (m *Manager) SetTier2Notifier(fn func(stackName, destLabel string, dur time.Duration, err error)) { m.tier2Notify = fn } // CaptureRecoveryUnit writes/refreshes an app's recovery unit: it captures the compose + metadata + // an app.yaml carrying the PORTABLE secret class (D5) into compose/, enumerates the DB/volume dumps // already present, and writes manifest.json. It never writes the Docker image (only the pinned tag), // and never writes a WITHHELD secret — the split is decided in buildUnitAppYaml, pinned by // TestCaptureRecoveryUnitCarriesPortableSecretsOnly. // // Idempotent: it builds the captured content in memory first and SKIPS all writes when the unit is // already current (same config checksums, same dump set, same controller version) — so it can run on // the periodic status refresh without thrashing a spinning USB drive. func (m *Manager) CaptureRecoveryUnit(stackName string) error { if m.stackProvider == nil { return fmt.Errorf("no stack provider") } info, ok := m.stackProvider.GetStackRecoveryInfo(stackName) if !ok { return fmt.Errorf("stack %q not found", stackName) } drivePath := m.GetAppDrivePath(stackName) if drivePath == "" || !filepath.IsAbs(drivePath) { return fmt.Errorf("cannot determine absolute drive path for %s", stackName) } nsRoot := m.namespaceRoot(drivePath) // Build the captured config CONTENT in memory (no writes yet) so we can checksum-compare. type capFile struct { name string data []byte perm os.FileMode } var files []capFile checksums := make(map[string]string) var configFiles []string for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} { data, err := os.ReadFile(filepath.Join(info.StackDir, fname)) if err != nil { continue // optional — capture whichever exist } files = append(files, capFile{fname, data, 0644}) checksums[fname] = sha256Hex(data) configFiles = append(configFiles, fname) } appYaml := buildUnitAppYaml(info) files = append(files, capFile{"app.yaml", appYaml, 0600}) checksums["app.yaml"] = sha256Hex(appYaml) configFiles = append(configFiles, "app.yaml") dbDumps := listFileNames(AppDBDumpPath(nsRoot, stackName), ".sql") volDumps := listFileNames(AppVolumeDumpPath(nsRoot, stackName), ".tar") version := m.versionLocked() manifestPath := RecoveryUnitManifestPath(nsRoot, stackName) cur := readManifest(manifestPath) // R-43/R-44: the coherence stamp of the offsite run currently in flight ("" on the periodic // refresh and on the local dump run). When empty we CARRY THE PRIOR STAMP FORWARD rather than // blanking it — a periodic refresh must neither claim a coherence it did not establish nor // destroy the record of one that a real run did. runID, dumpsAt := m.offsiteRunStamp() if runID == "" && cur != nil { runID, dumpsAt = cur.OffsiteRunID, cur.DumpsAt } // Skip if the unit is already current — avoids needless drive writes on the periodic refresh. // The run-id is part of "current": an offsite run must re-stamp the manifest even when nothing // else changed, because the stamp is exactly the claim the restore path reads. if cur != nil && cur.ControllerVer == version && stringMapEqual(cur.Checksums, checksums) && stringSliceEqual(cur.DBDumps, dbDumps) && stringSliceEqual(cur.VolumeDumps, volDumps) && cur.OffsiteRunID == runID { return nil } composeDir := RecoveryUnitComposePath(nsRoot, stackName) if err := os.MkdirAll(composeDir, 0755); err != nil { return fmt.Errorf("creating recovery-unit compose dir: %w", err) } for _, f := range files { if err := atomicWrite(filepath.Join(composeDir, f.name), f.data, f.perm); err != nil { return fmt.Errorf("capturing %s: %w", f.name, err) } } manifest := &RecoveryManifest{ SchemaVersion: 2, // D5: compose/app.yaml carries the portable secret class AppName: stackName, DisplayName: info.DisplayName, ControllerVer: version, CreatedAt: time.Now().UTC().Format(time.RFC3339), Drive: drivePath, NamespaceRoot: nsRoot, ImagePins: info.ImagePins, SecretEnvVars: info.SecretEnvVars, DataKeyEnvVars: info.DataKeyEnvVars, PortableSecretEnvVars: info.PortableSecretEnvVars, SecretSource: "portable secrets (data keys, DB passwords, internal signing secrets) are IN this unit's compose/app.yaml (0600); internet-reachable admin logins are NOT, and come from the guest's app.yaml or are regenerated on restore", ConfigFiles: configFiles, DBDumps: dbDumps, VolumeDumps: volDumps, Checksums: checksums, OffsiteRunID: runID, DumpsAt: dumpsAt, } if err := writeManifest(manifestPath, manifest); err != nil { return fmt.Errorf("writing manifest: %w", err) } // Counts and NAMES only — never a value (D5 puts more secrets through this path than before). m.logger.Printf("[INFO] [backup] Recovery unit captured for %s → %s (images=%d, secrets-referenced=%d, data_keys=%d, portable-carried=%d/%d, withheld=%d)", stackName, RecoveryUnitPath(nsRoot, stackName), len(info.ImagePins), len(info.SecretEnvVars), len(info.DataKeyEnvVars), len(info.PortableSecrets), len(info.PortableSecretEnvVars), len(withheldSecretNames(info))) return nil } // UnitSpace is the target filesystem's occupancy at the moment a capture failed — the numbers that // answer "why" without an operator logging in. Nil when the filesystem could not be read at all // (system.GetDiskUsage returns nil on error), which is reported as unknown rather than as full. type UnitSpace struct { Path string UsedGB float64 AvailGB float64 TotalGB float64 UsedPercent float64 } // String renders the space figures for an operator, or says plainly that they are unknown. An absent // reading must never render as zeros — "0 GB free" and "we could not look" are opposite diagnoses. func (u *UnitSpace) String() string { if u == nil { return "target filesystem usage unavailable" } return fmt.Sprintf("%s: %.1f/%.1f GB used (%.0f%%), %.1f GB free", u.Path, u.UsedGB, u.TotalGB, u.UsedPercent, u.AvailGB) } // SetUnitNotify wires the per-app recovery-unit capture failure alert (R-158 / R-167). INIT-ONLY — // call once at startup, in main.go, alongside SetOffboxNotify. Nil-safe: an unwired seam is silently // the pre-v0.191.0 behaviour, which is a `[WARN]` line and nothing else. func (m *Manager) SetUnitNotify(fn func(stackName string, err error, usage *UnitSpace)) { m.unitNotify = fn } // unitTargetSpace reads the occupancy of the filesystem a unit for `stackName` would be written to. // Nil on an unreadable path — never a fabricated zero (§8.4: an unreadable filesystem is not a full // one, and the drive gate already owns the absent-drive case). func (m *Manager) unitTargetSpace(stackName string) *UnitSpace { path := m.GetAppDrivePath(stackName) if path == "" { return nil } di := system.GetDiskUsage(path) if di == nil { return nil } return &UnitSpace{ Path: path, UsedGB: di.UsedGB, AvailGB: di.AvailGB, TotalGB: di.TotalGB, UsedPercent: di.UsedPercent, } } // ── The capture floor (R-165 / decision B2) ────────────────────────────────────────────────────── // // WHAT IT REPLACES. Until the `mp1`→`mp0` merge, the 20 G backup partition was a BULKHEAD as well as // a ceiling: an app whose unit outgrew it was refused per app, its last good unit preserved // byte-identical, and the overflow **could not reach `/var/lib/docker`** because that was a different // filesystem. After the merge it can, and a full Docker data-root is a stopped box, not a slow one. // This floor is that bulkhead, done deliberately instead of by accident. // // IT IS ABOUT THE FILESYSTEM'S HEADROOM, NEVER THE UNIT'S SIZE. A per-unit size cap would be R-163 // rebuilt inside one volume — the wall moved rather than removed — so a large unit on a filesystem // with ample room is captured, whatever its size. // // IT REFUSES; IT NEVER DELETES. Nothing on this filesystem is generational: a unit is ONE fixed path // per app (`backups/primary/`) refreshed in place, and a DB dump is `-.sql`, also // fixed. So "prune the oldest" could only mean deleting a DIFFERENT app's only local recovery unit to // make room for this one, and that is not a trade this system makes. `pruneStalePrimaryDirs` is NOT a // retention policy — it removes ORPHANED directories left when an app moves drives, and has no notion // of age — so it must never be repurposed here. const ( // FloorUsedPercent / FloorFreeGiB — the reserve. Two terms, whichever binds first, the same shape // as `internal/fillwatch` (proven live on 2026-08-02: the critical alert fired on the free-byte // term at 91% used, where a percent-only rule stayed silent). // // THEY SIT DELIBERATELY BEYOND fillwatch's CRITICAL BAND (95% / 2 GiB), so the customer is ALWAYS // warned before a refusal can happen. A floor that fires before its own warning is a silent // failure wearing a threshold; `TestFloorSitsBelowTheCriticalWarningBand` pins the ordering. // // 1 GiB is the reserve, not a working budget: §7.5 measures a DB-backed app's unit at up to ~2× its // data, so no fixed number can guarantee a capture fits. What this guarantees is different and is // the bulkhead's actual job — that a capture cannot consume the last of the space the container // runtime needs to keep running. FloorUsedPercent = 97.0 FloorFreeGiB = 1.0 ) // ErrCaptureFloor marks an app's backup refused for headroom. It is a REFUSAL, not a failure of the // backup machinery — the distinction matters to a reader of the alert, which is why the message names // the reserve rather than reporting an I/O error. // // R-181 widened what it covers: it now refuses the app's DB dump, volume dump and capture together // (see admission.go), not the capture alone. The sentinel keeps its name because callers match it and // "capture" still reads correctly for "capturing this app's backup"; the MESSAGE is what changed, and // the message is what an operator sees. var ErrCaptureFloor = errors.New("refused: backing up this app would leave the filesystem below the reserve") // floorVerdict is the PURE predicate: given a reading and this app's estimated write, does the floor // refuse, and on which term? Separated so the thresholds are unit-testable without a filesystem, a // stack provider or a clock. // // TWO QUESTIONS, NOT ONE (R-181). "Is the filesystem already below the reserve?" is the headroom term // and was all B2 asked. "Would THIS app's write take it below?" is the size term, and its absence is // how an app was admitted at 96% used and then allowed to write 2 GB. Both terms are evaluated // against BOTH thresholds — a large write can cross the percentage bound on a small volume and the // free-byte bound on a large one, which is the same reason the reserve has two terms at all. // // §8.4 — A NIL READING NEITHER REFUSES NOR WARNS. An unreadable filesystem is the drive gate's // business and has its own alert; refusing on it would block every backup on a box whose drive merely // blipped, and warning on it would be a false alarm with a misleading cause. // // estGiB == 0 (no previous dump to estimate from) degrades to the headroom term alone, deliberately: // refusing an app that has never been backed up would make the FIRST backup the one that can never // happen (Scenario E). func (m *Manager) floorVerdict(u *UnitSpace, estGiB float64) (*UnitSpace, floorReason) { if u == nil { return nil, floorAdmit } if u.UsedPercent >= FloorUsedPercent || u.AvailGB < FloorFreeGiB { return u, floorHeadroom } if estGiB > 0 { availAfter := u.AvailGB - estGiB usedAfter := u.UsedPercent if u.TotalGB > 0 { usedAfter = (u.UsedGB + estGiB) / u.TotalGB * 100 } if availAfter < FloorFreeGiB || usedAfter >= FloorUsedPercent { return u, floorSize } } return u, floorAdmit } // readUnitSpace goes through the seam when one is injected, so a test can state the filesystem's // occupancy as an input instead of manufacturing it on a real disk. Nil seam → the real statfs. func (m *Manager) readUnitSpace(stackName string) *UnitSpace { if m.unitSpaceFn != nil { return m.unitSpaceFn(stackName) } return m.unitTargetSpace(stackName) } // captureAllRecoveryUnits refreshes the recovery unit for every deployed stack. Best-effort: // a per-app failure is logged, NOTIFIED (R-158), and does not abort the others. // // R-181: the reserve is consulted through `admitApp`, which is the SAME verdict the DB-dump and // volume-dump legs of this run already consulted for this app. When a run is in flight the answer // here is a memo lookup — an app refused before its first write is refused here too, silently, // because it was already alerted once. Outside a run (the periodic status refresh) it decides fresh. func (m *Manager) captureAllRecoveryUnits() { if m.stackProvider == nil { return } for _, stack := range m.stackProvider.ListDeployedStacks() { drivePath := m.GetAppDrivePath(stack.Name) if m.settings != nil && (m.settings.IsDisconnected(drivePath) || m.settings.IsDecommissioned(drivePath)) { continue // drive not writable — skip, the existing unit stays as-is } m.noteAttempted(stack.Name) // The reserve, checked BEFORE anything is written. Per app, and the loop continues. if !m.admitApp(stack.Name) { continue } if err := m.CaptureRecoveryUnit(stack.Name); err != nil { m.noteFailure(stack.Name, "recovery-unit capture", err.Error()) m.logger.Printf("[WARN] [backup] Recovery unit capture failed for %s: %v", stack.Name, err) // R-158: per app, and the loop CONTINUES — one app's failure must not silence the // others, and it must not abort their captures either. The space figures are read at // the moment of failure, because the point is to answer "why" (usually: no room). if m.unitNotify != nil { m.unitNotify(stack.Name, err, m.unitTargetSpace(stack.Name)) } } } } func (m *Manager) versionLocked() string { m.mu.Lock() defer m.mu.Unlock() return m.version } // strippedAppYaml is the on-disk shape of the app.yaml captured into the unit. The name is historical: // since D5 the `env` map carries the PORTABLE secrets alongside the plain config (see buildUnitAppYaml). type strippedAppYaml struct { Deployed bool `yaml:"deployed"` Env map[string]string `yaml:"env"` } // buildUnitAppYaml renders the unit's app.yaml as bytes: the non-secret env PLUS the portable secret // values (D5). Deterministic: yaml.v3 sorts map keys and the name lists come in stable metadata order, // so identical input yields identical bytes (needed for the checksum-skip guard). // // This is the ONE place the capture side decides what does and does not reach the drive — there is no // second path that writes a unit app.yaml. The caller writes the result 0600. func buildUnitAppYaml(info RecoveryInfo) []byte { env := make(map[string]string, len(info.NonSecretEnv)+len(info.PortableSecrets)) for k, v := range info.NonSecretEnv { env[k] = v } // Portable secrets last: NonSecretEnv is disjoint from the secret set by construction // (GetStackRecoveryInfo), so this cannot shadow a plain config value. for k, v := range info.PortableSecrets { env[k] = v } body, err := yaml.Marshal(strippedAppYaml{Deployed: true, Env: env}) if err != nil { body = []byte("deployed: true\nenv: {}\n") } header := "# Captured by felhom-controller recovery unit.\n" + "# This file CARRIES SECRETS (D5) so a Tier-1/2 restore needs the drive and nothing else:\n" + "# data-encrypting keys, database passwords and internal signing secrets. Mode 0600.\n" if len(info.PortableSecretEnvVars) > 0 { header += "# Carried: " + strings.Join(info.PortableSecretEnvVars, ", ") + "\n" } // The withheld class is named, not valued — an operator reading the unit must be able to see WHY a // credential is missing rather than suspecting a capture bug. if withheld := withheldSecretNames(info); len(withheld) > 0 { header += "# WITHHELD (internet-reachable logins — stay in the guest, regenerated on restore): " + strings.Join(withheld, ", ") + "\n" } return []byte(header + string(body)) } // withheldSecretNames returns the secret names deliberately NOT carried by the unit, in stable order. func withheldSecretNames(info RecoveryInfo) []string { portable := make(map[string]bool, len(info.PortableSecretEnvVars)) for _, n := range info.PortableSecretEnvVars { portable[n] = true } var out []string for _, n := range info.SecretEnvVars { if !portable[n] { out = append(out, n) } } return out } // writeManifest writes the manifest JSON atomically. func writeManifest(dst string, manifest *RecoveryManifest) error { data, err := json.MarshalIndent(manifest, "", " ") if err != nil { return err } return atomicWrite(dst, append(data, '\n'), 0644) } // readManifest reads an existing recovery-unit manifest (nil if absent or unparseable). func readManifest(path string) *RecoveryManifest { data, err := os.ReadFile(path) if err != nil { return nil } var m RecoveryManifest if json.Unmarshal(data, &m) != nil { return nil } return &m } func sha256Hex(data []byte) string { sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) } func stringMapEqual(a, b map[string]string) bool { if len(a) != len(b) { return false } for k, v := range a { if b[k] != v { return false } } return true } func stringSliceEqual(a, b []string) bool { if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } // listFileNames returns the names of files with the given suffix in dir (sorted, none if absent). func listFileNames(dir, suffix string) []string { entries, err := os.ReadDir(dir) if err != nil { return nil } var names []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(e.Name(), suffix) { names = append(names, e.Name()) } } sort.Strings(names) return names } // atomicWrite writes data to path via a .tmp file + rename. func atomicWrite(path string, data []byte, perm os.FileMode) error { tmp := path + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) if err != nil { return err } if _, err := io.Copy(f, strings.NewReader(string(data))); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Close(); err != nil { os.Remove(tmp) return err } if err := os.Rename(tmp, path); err != nil { os.Remove(tmp) return err } return nil }