package backup import ( "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strings" "time" "gopkg.in/yaml.v3" ) // RecoveryManifest describes an app's self-contained recovery unit. // // The unit on a drive is `/backups/primary//` and contains: // // compose/ docker-compose.yml + .felhom.yml + app.yaml (0600; carries the PORTABLE secrets) // db-dumps/ app-consistent DB dump(s) (written by the dump flow) // volume-dumps/ named-volume tars (written by the dump flow) // manifest.json this file // // D5 (schema 2) changed what the unit holds. Before it held NO secret at all, which made // "restore from the drive alone" false: the fast, local, customer-doable Tier-1/2 restore secretly // depended on the slow, operator-driven whole-guest restore, because a data-encrypting key or a DB // password absent from the guest cannot be regenerated without rendering the restored data // unreachable. The unit now carries the PORTABLE secret class (stacks.PortableSecretEnvVars) in its // 0600 app.yaml, and Tier-1/2 needs the DRIVE AND NOTHING ELSE. // // It still holds NO `type: password` admin login (those are internet-reachable, so their blast radius // is not bounded by the drive — they stay in the guest and are regenerated on restore) and NOT the // Docker image, only the pinned tag(s), re-pulled on restore. SecretSource records the split. // // A schema-1 unit carries no secrets: the restore degrades to the pre-D5 guest-only behaviour rather // than failing, and the next capture rewrites it (the app.yaml checksum changes). type RecoveryManifest struct { SchemaVersion int `json:"schema_version"` AppName string `json:"app_name"` DisplayName string `json:"display_name"` ControllerVer string `json:"controller_version"` CreatedAt string `json:"created_at"` Drive string `json:"drive"` // HDD_PATH (in-guest mount) NamespaceRoot string `json:"namespace_root"` // resolved felhom-data namespace root ImagePins []string `json:"image_pins"` // image NOT stored — re-pulled on restore SecretEnvVars []string `json:"secret_env_vars"` // NAMES of every secret/password field DataKeyEnvVars []string `json:"data_key_env_vars"` // fail-closed gate on restore SecretSource string `json:"secret_source"` // human note: where secrets come from ConfigFiles []string `json:"config_files"` // captured into compose/ DBDumps []string `json:"db_dumps"` VolumeDumps []string `json:"volume_dumps"` Checksums map[string]string `json:"checksums"` // sha256 of captured compose/ files // PortableSecretEnvVars (D5) are the NAMES of the secrets this unit's app.yaml CARRIES. Names only // — the manifest is 0644 and never holds a value. The restore reads it to know which app.yaml env // entries are secrets rather than plain config; absent (schema 1) ⇒ the unit carries none. PortableSecretEnvVars []string `json:"portable_secret_env_vars,omitempty"` // R-43/R-44 (v0.148.0): the coherence stamp. An offsite run refreshes the dumps FIRST and then // captures the unit, so a manifest carrying an OffsiteRunID asserts "the db-dumps/ in this unit // were taken by that run" — i.e. the snapshot is an internally coherent {DB@T, files@T} pair. // A manifest WITHOUT these fields is a pre-v0.148 unit whose dump age is unknown and may skew // arbitrarily from the files beside it (the DIAG-immich-restore-2026-07-19 failure); the restore // confirm surfaces that honestly rather than blocking. Empty on the periodic refresh, which must // never claim a coherence it did not establish — it carries the prior stamp forward instead. OffsiteRunID string `json:"offsite_run_id,omitempty"` DumpsAt string `json:"dumps_at,omitempty"` // RFC3339 UTC — when this run's dump leg finished } // SetVersion records the controller version stamped into recovery-unit manifests. func (m *Manager) SetVersion(v string) { m.mu.Lock() m.version = v m.mu.Unlock() } // SetTier2Notifier wires the notification callback invoked after each Tier 2 copy. func (m *Manager) SetTier2Notifier(fn func(stackName, destLabel string, dur time.Duration, err error)) { m.tier2Notify = fn } // CaptureRecoveryUnit writes/refreshes an app's recovery unit: it captures the compose + metadata + // an app.yaml carrying the PORTABLE secret class (D5) into compose/, enumerates the DB/volume dumps // already present, and writes manifest.json. It never writes the Docker image (only the pinned tag), // and never writes a WITHHELD secret — the split is decided in buildUnitAppYaml, pinned by // TestCaptureRecoveryUnitCarriesPortableSecretsOnly. // // Idempotent: it builds the captured content in memory first and SKIPS all writes when the unit is // already current (same config checksums, same dump set, same controller version) — so it can run on // the periodic status refresh without thrashing a spinning USB drive. func (m *Manager) CaptureRecoveryUnit(stackName string) error { if m.stackProvider == nil { return fmt.Errorf("no stack provider") } info, ok := m.stackProvider.GetStackRecoveryInfo(stackName) if !ok { return fmt.Errorf("stack %q not found", stackName) } drivePath := m.GetAppDrivePath(stackName) if drivePath == "" || !filepath.IsAbs(drivePath) { return fmt.Errorf("cannot determine absolute drive path for %s", stackName) } nsRoot := m.namespaceRoot(drivePath) // Build the captured config CONTENT in memory (no writes yet) so we can checksum-compare. type capFile struct { name string data []byte perm os.FileMode } var files []capFile checksums := make(map[string]string) var configFiles []string for _, fname := range []string{"docker-compose.yml", ".felhom.yml"} { data, err := os.ReadFile(filepath.Join(info.StackDir, fname)) if err != nil { continue // optional — capture whichever exist } files = append(files, capFile{fname, data, 0644}) checksums[fname] = sha256Hex(data) configFiles = append(configFiles, fname) } appYaml := buildUnitAppYaml(info) files = append(files, capFile{"app.yaml", appYaml, 0600}) checksums["app.yaml"] = sha256Hex(appYaml) configFiles = append(configFiles, "app.yaml") dbDumps := listFileNames(AppDBDumpPath(nsRoot, stackName), ".sql") volDumps := listFileNames(AppVolumeDumpPath(nsRoot, stackName), ".tar") version := m.versionLocked() manifestPath := RecoveryUnitManifestPath(nsRoot, stackName) cur := readManifest(manifestPath) // R-43/R-44: the coherence stamp of the offsite run currently in flight ("" on the periodic // refresh and on the local dump run). When empty we CARRY THE PRIOR STAMP FORWARD rather than // blanking it — a periodic refresh must neither claim a coherence it did not establish nor // destroy the record of one that a real run did. runID, dumpsAt := m.offsiteRunStamp() if runID == "" && cur != nil { runID, dumpsAt = cur.OffsiteRunID, cur.DumpsAt } // Skip if the unit is already current — avoids needless drive writes on the periodic refresh. // The run-id is part of "current": an offsite run must re-stamp the manifest even when nothing // else changed, because the stamp is exactly the claim the restore path reads. if cur != nil && cur.ControllerVer == version && stringMapEqual(cur.Checksums, checksums) && stringSliceEqual(cur.DBDumps, dbDumps) && stringSliceEqual(cur.VolumeDumps, volDumps) && cur.OffsiteRunID == runID { return nil } composeDir := RecoveryUnitComposePath(nsRoot, stackName) if err := os.MkdirAll(composeDir, 0755); err != nil { return fmt.Errorf("creating recovery-unit compose dir: %w", err) } for _, f := range files { if err := atomicWrite(filepath.Join(composeDir, f.name), f.data, f.perm); err != nil { return fmt.Errorf("capturing %s: %w", f.name, err) } } manifest := &RecoveryManifest{ SchemaVersion: 2, // D5: compose/app.yaml carries the portable secret class AppName: stackName, DisplayName: info.DisplayName, ControllerVer: version, CreatedAt: time.Now().UTC().Format(time.RFC3339), Drive: drivePath, NamespaceRoot: nsRoot, ImagePins: info.ImagePins, SecretEnvVars: info.SecretEnvVars, DataKeyEnvVars: info.DataKeyEnvVars, PortableSecretEnvVars: info.PortableSecretEnvVars, SecretSource: "portable secrets (data keys, DB passwords, internal signing secrets) are IN this unit's compose/app.yaml (0600); internet-reachable admin logins are NOT, and come from the guest's app.yaml or are regenerated on restore", ConfigFiles: configFiles, DBDumps: dbDumps, VolumeDumps: volDumps, Checksums: checksums, OffsiteRunID: runID, DumpsAt: dumpsAt, } if err := writeManifest(manifestPath, manifest); err != nil { return fmt.Errorf("writing manifest: %w", err) } // Counts and NAMES only — never a value (D5 puts more secrets through this path than before). m.logger.Printf("[INFO] [backup] Recovery unit captured for %s → %s (images=%d, secrets-referenced=%d, data_keys=%d, portable-carried=%d/%d, withheld=%d)", stackName, RecoveryUnitPath(nsRoot, stackName), len(info.ImagePins), len(info.SecretEnvVars), len(info.DataKeyEnvVars), len(info.PortableSecrets), len(info.PortableSecretEnvVars), len(withheldSecretNames(info))) return nil } // captureAllRecoveryUnits refreshes the recovery unit for every deployed stack. Best-effort: // a per-app failure is logged and does not abort the others. func (m *Manager) captureAllRecoveryUnits() { if m.stackProvider == nil { return } for _, stack := range m.stackProvider.ListDeployedStacks() { drivePath := m.GetAppDrivePath(stack.Name) if m.settings != nil && (m.settings.IsDisconnected(drivePath) || m.settings.IsDecommissioned(drivePath)) { continue // drive not writable — skip, the existing unit stays as-is } if err := m.CaptureRecoveryUnit(stack.Name); err != nil { m.logger.Printf("[WARN] [backup] Recovery unit capture failed for %s: %v", stack.Name, err) } } } func (m *Manager) versionLocked() string { m.mu.Lock() defer m.mu.Unlock() return m.version } // strippedAppYaml is the on-disk shape of the app.yaml captured into the unit. The name is historical: // since D5 the `env` map carries the PORTABLE secrets alongside the plain config (see buildUnitAppYaml). type strippedAppYaml struct { Deployed bool `yaml:"deployed"` Env map[string]string `yaml:"env"` } // buildUnitAppYaml renders the unit's app.yaml as bytes: the non-secret env PLUS the portable secret // values (D5). Deterministic: yaml.v3 sorts map keys and the name lists come in stable metadata order, // so identical input yields identical bytes (needed for the checksum-skip guard). // // This is the ONE place the capture side decides what does and does not reach the drive — there is no // second path that writes a unit app.yaml. The caller writes the result 0600. func buildUnitAppYaml(info RecoveryInfo) []byte { env := make(map[string]string, len(info.NonSecretEnv)+len(info.PortableSecrets)) for k, v := range info.NonSecretEnv { env[k] = v } // Portable secrets last: NonSecretEnv is disjoint from the secret set by construction // (GetStackRecoveryInfo), so this cannot shadow a plain config value. for k, v := range info.PortableSecrets { env[k] = v } body, err := yaml.Marshal(strippedAppYaml{Deployed: true, Env: env}) if err != nil { body = []byte("deployed: true\nenv: {}\n") } header := "# Captured by felhom-controller recovery unit.\n" + "# This file CARRIES SECRETS (D5) so a Tier-1/2 restore needs the drive and nothing else:\n" + "# data-encrypting keys, database passwords and internal signing secrets. Mode 0600.\n" if len(info.PortableSecretEnvVars) > 0 { header += "# Carried: " + strings.Join(info.PortableSecretEnvVars, ", ") + "\n" } // The withheld class is named, not valued — an operator reading the unit must be able to see WHY a // credential is missing rather than suspecting a capture bug. if withheld := withheldSecretNames(info); len(withheld) > 0 { header += "# WITHHELD (internet-reachable logins — stay in the guest, regenerated on restore): " + strings.Join(withheld, ", ") + "\n" } return []byte(header + string(body)) } // withheldSecretNames returns the secret names deliberately NOT carried by the unit, in stable order. func withheldSecretNames(info RecoveryInfo) []string { portable := make(map[string]bool, len(info.PortableSecretEnvVars)) for _, n := range info.PortableSecretEnvVars { portable[n] = true } var out []string for _, n := range info.SecretEnvVars { if !portable[n] { out = append(out, n) } } return out } // writeManifest writes the manifest JSON atomically. func writeManifest(dst string, manifest *RecoveryManifest) error { data, err := json.MarshalIndent(manifest, "", " ") if err != nil { return err } return atomicWrite(dst, append(data, '\n'), 0644) } // readManifest reads an existing recovery-unit manifest (nil if absent or unparseable). func readManifest(path string) *RecoveryManifest { data, err := os.ReadFile(path) if err != nil { return nil } var m RecoveryManifest if json.Unmarshal(data, &m) != nil { return nil } return &m } func sha256Hex(data []byte) string { sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) } func stringMapEqual(a, b map[string]string) bool { if len(a) != len(b) { return false } for k, v := range a { if b[k] != v { return false } } return true } func stringSliceEqual(a, b []string) bool { if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } // listFileNames returns the names of files with the given suffix in dir (sorted, none if absent). func listFileNames(dir, suffix string) []string { entries, err := os.ReadDir(dir) if err != nil { return nil } var names []string for _, e := range entries { if !e.IsDir() && strings.HasSuffix(e.Name(), suffix) { names = append(names, e.Name()) } } sort.Strings(names) return names } // atomicWrite writes data to path via a .tmp file + rename. func atomicWrite(path string, data []byte, perm os.FileMode) error { tmp := path + ".tmp" f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) if err != nil { return err } if _, err := io.Copy(f, strings.NewReader(string(data))); err != nil { f.Close() os.Remove(tmp) return err } if err := f.Close(); err != nil { os.Remove(tmp) return err } if err := os.Rename(tmp, path); err != nil { os.Remove(tmp) return err } return nil }