package backup import ( "encoding/json" "os" "path/filepath" "sort" "strings" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/settings" ) // ── The backup's data and its version travel together (controller v0.275.0, R-696, `07` §6.6) ──────── // // WHAT WENT WRONG. The recovery unit's DEFINITION (compose/, image_pins) was re-captured by the periodic // status refresh as soon as the app's pin moved, while its DATA (db-dumps/, volume-dumps/) is written // only by the backup legs. So for up to a day after an update the unit said "new version" and held the // old version's data. Measured on 9202 2026-09-26 (`audits/version-travel-2026-09-26/A1/`): after a // PostgreSQL 16 → 18 step the unit held the 18 definition over a 16 dump and a 16 datadir tar; a restore // poured the 16 datadir back, `postgres:18` refused it and the app was left down. And Tier 1's "proven // at" was the manifest's refresh time, so the kept pre-conversion copy was released on a backup taken // BEFORE the conversion (demo-hp, 2026-09-26 night). // // THE SHAPE. Every data file a backup leg writes gets a STAMP beside it, at the moment it is written: // its size and mtime (so a file rewritten by anything else is recognised as unstamped), the definition's // image pins, and what each service was running (`installed_images`, ref@digest). The unit capture folds // the stamps into the manifest's `data` block — the time of the unit's data (its OLDEST stamped file: // the copy is as fresh as its stalest part) and the versions that wrote it — and KEEPS the definition // the data belongs to: when the pins have moved since the data was written, compose/ is not rewritten // until the next data run replaces the data. The manifest's `image_pins` stays the app's CURRENT pins, // so it says both. Restores read `data` to start the data with its own definition (restore_unit.go), // and the update/release read `data.at` as the unit's time (restore_points.go). // // UNKNOWN IS NEVER CURRENT. A unit whose data files are not all validly stamped (written before // v0.275.0, or by a path that does not stamp) has NO `data` block, restores as before with a WARN, and // its time is the newest DATA file's mtime — never the manifest's. // dataStampsFile sits in the unit root, beside manifest.json, so it travels with every copy of the unit // (the Tier-2 mirror copies the whole unit, the off-site snapshot includes it). const dataStampsFile = "data-stamps.json" // DataStamp is one data file's record, written by the leg that wrote the file. type DataStamp struct { // At is the file's mtime when it was stamped, RFC3339Nano UTC — the stamp is valid only while the // file still has exactly this mtime and Size. At string `json:"at"` Size int64 `json:"size"` // Pins are the definition's image pins (the compose `image:` lines) when the file was written. Pins []string `json:"pins"` // Images is service -> ref@digest running when the file was written. Empty when not observed. Images map[string]string `json:"images,omitempty"` } // UnitData is the manifest's account of the data the unit holds. type UnitData struct { // At is RFC3339 UTC: the unit's data time — the oldest data file's stamp, or, for a unit with no data // files, the data run that confirmed it. This is the unit's "proven at", never the manifest's time. At string `json:"at"` // ImagePins are the definition pins the data belongs to — what compose/ holds. Empty when Mixed. ImagePins []string `json:"image_pins"` // Images is the running set (service -> ref@digest) when the oldest file was written. Images map[string]string `json:"images,omitempty"` // Files are the stamps, keyed by the path relative to the unit ("db-dumps/x.sql"). Files map[string]DataStamp `json:"files,omitempty"` // Mixed: the files were written under DIFFERENT pins — no one definition fits all of them, so a // restore of this unit is refused (a restore never starts data with a definition it does not belong to). Mixed bool `json:"mixed,omitempty"` } // DataTime parses At. False when absent or unreadable. func (d *UnitData) DataTime() (time.Time, bool) { if d == nil || d.At == "" { return time.Time{}, false } t, err := time.Parse(time.RFC3339, d.At) if err != nil { return time.Time{}, false } return t, true } func readDataStamps(unitDir string) map[string]DataStamp { data, err := os.ReadFile(filepath.Join(unitDir, dataStampsFile)) if err != nil { return map[string]DataStamp{} } var out map[string]DataStamp if json.Unmarshal(data, &out) != nil || out == nil { return map[string]DataStamp{} } return out } // stampDataFile records the file at / as written NOW by the current definition and // running images. Called by the legs right after a dump or a tar is promoted to its final name. A failure // to stamp is a WARN and leaves the file unstamped — the unit then reads as "versions unknown", which is // the pre-v0.275.0 behaviour, never a false claim. func (m *Manager) stampDataFile(stackName, unitDir, rel string) { fi, err := os.Stat(filepath.Join(unitDir, rel)) if err != nil { m.logger.Printf("[WARN] [backup] %s: cannot stamp %s (%v) — its versions will read as unknown", stackName, rel, err) return } var pins []string var images map[string]string if m.stackProvider != nil { if info, ok := m.stackProvider.GetStackRecoveryInfo(stackName); ok { pins, images = definitionPins(info), info.InstalledImages } } m.stampMu.Lock() defer m.stampMu.Unlock() stamps := readDataStamps(unitDir) stamps[rel] = DataStamp{At: fi.ModTime().UTC().Format(time.RFC3339Nano), Size: fi.Size(), Pins: pins, Images: images} // Entries whose file is gone (a volume the app no longer has) are dropped here, so the file stays small. for k := range stamps { if _, err := os.Stat(filepath.Join(unitDir, k)); err != nil { delete(stamps, k) } } body, err := json.MarshalIndent(stamps, "", " ") if err == nil { err = atomicWrite(filepath.Join(unitDir, dataStampsFile), append(body, '\n'), 0644) } if err != nil { m.logger.Printf("[WARN] [backup] %s: writing the data stamp for %s failed (%v) — its versions will read as unknown", stackName, rel, err) return } if m.isDebug() { m.logger.Printf("[DEBUG] [backup] %s: stamped %s (%d B) with pins %v", stackName, rel, fi.Size(), pins) } } // foldUnitData builds the manifest's `data` block from the unit's data files and their stamps. // // - no data files: a data run confirms the unit NOW under the current pins; a refresh keeps `prev`; // - every file validly stamped: the oldest stamp's time; its pins when all agree, else Mixed; // - any file unstamped or re-written since its stamp: nil — the versions are UNKNOWN. func foldUnitData(unitDir string, dbDumps, volDumps []string, dataRun bool, now time.Time, pins []string, images map[string]string, prev *UnitData) *UnitData { var rels []string for _, n := range dbDumps { rels = append(rels, "db-dumps/"+n) } for _, n := range volDumps { rels = append(rels, "volume-dumps/"+n) } if len(rels) == 0 { if dataRun { return &UnitData{At: now.UTC().Format(time.RFC3339), ImagePins: pins, Images: images} } return prev } stamps := readDataStamps(unitDir) out := &UnitData{Files: map[string]DataStamp{}} var oldest time.Time for _, rel := range rels { st, ok := stamps[rel] if !ok { return nil } fi, err := os.Stat(filepath.Join(unitDir, rel)) if err != nil || fi.Size() != st.Size || fi.ModTime().UTC().Format(time.RFC3339Nano) != st.At { return nil } t := fi.ModTime().UTC() if oldest.IsZero() || t.Before(oldest) { oldest = t out.ImagePins, out.Images = st.Pins, st.Images } out.Files[rel] = st } for _, st := range out.Files { if !samePins(st.Pins, out.ImagePins) { out.Mixed = true } } if out.Mixed { out.ImagePins, out.Images = nil, nil } out.At = oldest.Format(time.RFC3339) return out } // samePins compares two pin lists as SETS (compose order is file order on both sides, but a set compare // cannot be fooled by it). func samePins(a, b []string) bool { if len(a) != len(b) { return false } x := append([]string(nil), a...) y := append([]string(nil), b...) sort.Strings(x) sort.Strings(y) return stringSliceEqual(x, y) } // unitDataEqual is the manifest-rewrite check's view of `data`. func unitDataEqual(a, b *UnitData) bool { if a == nil || b == nil { return a == nil && b == nil } if a.At != b.At || a.Mixed != b.Mixed || !stringSliceEqual(a.ImagePins, b.ImagePins) || len(a.Files) != len(b.Files) { return false } for k, v := range a.Files { w, ok := b.Files[k] if !ok || v.At != w.At || v.Size != w.Size { return false } } return true } // PinsVersion is the household's name for a set of pins: every image as `name:tag`, registry path and // digest stripped, in compose order ("docmost:0.96.0, postgres:16-alpine, redis:7-alpine"). All of them, // because a version step can move ANY service — a PostgreSQL 16 → 18 step leaves the app's own tag as it // was, and a label naming only the first image would read the same on both sides of it (seen on the // first draft of the mismatch sentence: „(0.96.0) … (0.96.0)"). func PinsVersion(pins []string) string { out := make([]string, 0, len(pins)) for _, ref := range pins { if i := strings.Index(ref, "@"); i >= 0 { ref = ref[:i] } if i := strings.LastIndex(ref, "/"); i >= 0 { ref = ref[i+1:] } out = append(out, ref) } return strings.Join(out, ", ") } // recordOffsiteDataAt remembers the data time of the unit just pushed off-site (v0.275.0, R-696). func (m *Manager) recordOffsiteDataAt(stackName, unitDir string) { if m.settings == nil { return } t, ok := unitNewestArtifact(unitDir) if !ok { return } now := time.Now().UTC().Format(time.RFC3339) if err := m.settings.SetOffsiteDataAt(stackName, settings.OffsiteDataRecord{PushedAt: now, DataAt: t.UTC().Format(time.RFC3339)}); err != nil { m.logger.Printf("[WARN] [offbox] %s: recording the pushed copy's data time failed: %v — the off-site copy is dated by its snapshot", stackName, err) } } // offsiteDataTime caps an off-site snapshot's time with the data time this box recorded when it pushed // it. A snapshot NEWER than the last recorded push (taken by another box, or a record lost) keeps its // own time: the cap applies only to a copy this box knows the content of. func (m *Manager) offsiteDataTime(stackName string, snapshotAt time.Time) time.Time { if m.settings == nil { return snapshotAt } rec, ok := m.settings.GetOffsiteDataAt(stackName) if !ok { return snapshotAt } pushed, perr := time.Parse(time.RFC3339, rec.PushedAt) data, derr := time.Parse(time.RFC3339, rec.DataAt) if perr != nil || derr != nil || snapshotAt.After(pushed) || !data.Before(snapshotAt) { return snapshotAt } return data } // UnitDumpStamp is one stamped database dump of an app's own unit (v0.275.0). type UnitDumpStamp struct { File string At time.Time Images map[string]string } // UnitDumpStamps returns the app's OWN unit's database dumps as its manifest records them — only when // the unit's data is known (a `data` block); an unstamped unit returns nothing, and a release that needs // one waits (fail closed). func (m *Manager) UnitDumpStamps(stackName string) []UnitDumpStamp { man := readManifest(UnitManifestFile(m.primaryUnitDirFor(stackName))) if man == nil || man.Data == nil { return nil } var out []UnitDumpStamp for rel, st := range man.Data.Files { if !strings.HasPrefix(rel, "db-dumps/") || !strings.HasSuffix(rel, ".sql") { continue } t, err := time.Parse(time.RFC3339Nano, st.At) if err != nil { continue } out = append(out, UnitDumpStamp{File: rel, At: t, Images: st.Images}) } sort.Slice(out, func(i, j int) bool { return out[i].File < out[j].File }) return out } // definitionPins are the pins of the definition a capture copies into compose/: the stack dir's own // docker-compose.yml, parsed. ONE source for the stamps, the fold and the freeze, so "the data's pins" // and "the pins compose/ holds" are read from the same file the restore will start (production's // RecoveryInfo.ImagePins is that parse too; a provider that says otherwise cannot make them disagree). func definitionPins(info RecoveryInfo) []string { if info.StackDir != "" { if p := ParseComposeImages(filepath.Join(info.StackDir, "docker-compose.yml")); len(p) > 0 { return p } } return info.ImagePins }