diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 9867185d..925cd2e7 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -19,6 +19,7 @@ box's controller). The checker now stands down for that declared state (the wait is the household's, it can take days); an undeclared consumed shape alarms as before. `TestR1230_AHeldBoxIsNotStuck`, red-proved. - `scripts/wire_contract_gate.py`: `wireguard.last_handshake_age_s` left the unread list (the hub reads it since 0.166.0). +- **D34 (operator yes 2026-10-23, R-232 G5) — the database snapshot every 6 hours.** The hub writes its `VACUUM INTO` snapshot at 02:00, 08:00, 14:00 and 20:00 Budapest (`dbsnap.Slots`, keep 2 unchanged), and the start-up catch-up fires when the newest is older than 6 h (`dbsnap.CatchUpAge`, was 24 h). DooPlex's push now runs :30 after each slot (`scripts/hub-db-backup`, refuses a newest copy older than 2 h, was 26 h), so the off-site hub copy is at most about 6.5 h old instead of a day. Added the same day as the build, on the operator's word (not soaked). Tests `TestD34_SlotsAreEverySixHours`, `TestR173_MainSchedulesTheDBSnapshot` (now the loop), and the script's `test_schedules` (the timer read against the hub's slots) and `test_d34_a_missed_slot_is_not_pushed_again` — each red-proved. ## v0.166.0 (unreleased — built 2026-10-22 night, NOT deployed; the deploy is the operator's yes) — the clock-ahead tunnel alarm (R-1204) diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index e8f1375d..7a30aae2 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -944,9 +944,9 @@ func main() { } }() - // Nightly DB snapshot (R-173, v0.136.0, `05` §16.3): VACUUM INTO /snapshots/, keep 2. DooPlex's - // felhom-hub-db-backup.service picks up the newest at 02:30, checks, encrypts and pushes it to ep0. A start-up - // catch-up runs one when the newest is older than 24 h (a pod that was down at 02:00). + // DB snapshot every 6 hours (R-173, v0.136.0, `05` §16.3; D34 v0.167.0): VACUUM INTO /snapshots/, keep 2. + // DooPlex's felhom-hub-db-backup.service picks up the newest at :30 after each slot, checks, encrypts and pushes it + // to ep0. A start-up catch-up runs one when the newest is older than one slot gap (a pod that was down at a slot). snapMaker := &dbsnap.Maker{Store: dataStore, Dir: filepath.Join(cfg.Server.DataDir, "snapshots"), Logger: logger} runSnapshot := func() { sctx, cancel := context.WithTimeout(ctx, 10*time.Minute) @@ -955,10 +955,12 @@ func main() { logger.Printf("[ERROR] db snapshot failed: %v", err) } } - if dbsnap.NeedsCatchUp(snapMaker.Dir, time.Now(), 24*time.Hour) { + if dbsnap.NeedsCatchUp(snapMaker.Dir, time.Now(), dbsnap.CatchUpAge) { go runSnapshot() } - go scheduleDaily(ctx, "db-snapshot", "02:00", runSnapshot, logger) + for _, at := range dbsnap.Slots { + go scheduleDaily(ctx, "db-snapshot "+at, at, runSnapshot, logger) + } // Backup deadline checker — runs daily at 05:00 Budapest go scheduleDaily(ctx, "deadline-check", "05:00", func() { diff --git a/hub/cmd/hub/r173_wiring_test.go b/hub/cmd/hub/r173_wiring_test.go index 55d65e1f..41c16119 100644 --- a/hub/cmd/hub/r173_wiring_test.go +++ b/hub/cmd/hub/r173_wiring_test.go @@ -4,13 +4,13 @@ import ( "go/ast" "go/parser" "go/token" - "strconv" "testing" ) -// R-173 seam wiring: main() must schedule the nightly db snapshot — a scheduleDaily call named "db-snapshot" at -// "02:00", before DooPlex's 02:30 push — and must run the start-up catch-up (dbsnap.NeedsCatchUp). Without them the -// dbsnap package exists and DooPlex pushes nothing new. RED-PROOF: delete either call → this test fails. +// R-173 seam wiring, D34: main() must schedule a db snapshot at every dbsnap.Slots time — a `for _, at := range +// dbsnap.Slots` loop calling scheduleDaily(ctx, , at, …) — and must run the start-up catch-up +// (dbsnap.NeedsCatchUp with dbsnap.CatchUpAge). Without them the dbsnap package exists and DooPlex pushes nothing +// new. RED-PROOF: delete the loop, schedule a literal time instead, or pass 24*time.Hour → this test fails. func TestR173_MainSchedulesTheDBSnapshot(t *testing.T) { f, err := parser.ParseFile(token.NewFileSet(), "main.go", nil, 0) if err != nil { @@ -18,30 +18,39 @@ func TestR173_MainSchedulesTheDBSnapshot(t *testing.T) { } scheduled, catchUp := false, false ast.Inspect(f, func(n ast.Node) bool { + if r, ok := n.(*ast.RangeStmt); ok { + sel, ok := r.X.(*ast.SelectorExpr) + v, vok := r.Value.(*ast.Ident) + if ok && vok && sel.Sel.Name == "Slots" { + ast.Inspect(r.Body, func(m ast.Node) bool { + c, ok := m.(*ast.CallExpr) + if !ok { + return true + } + if id, ok := c.Fun.(*ast.Ident); ok && id.Name == "scheduleDaily" && len(c.Args) >= 3 { + if a, ok := c.Args[2].(*ast.Ident); ok && a.Name == v.Name { + scheduled = true + } + } + return true + }) + } + } c, ok := n.(*ast.CallExpr) if !ok { return true } - if id, ok := c.Fun.(*ast.Ident); ok && id.Name == "scheduleDaily" && len(c.Args) >= 3 { - name, _ := c.Args[1].(*ast.BasicLit) - at, _ := c.Args[2].(*ast.BasicLit) - if name != nil && at != nil { - n, _ := strconv.Unquote(name.Value) - a, _ := strconv.Unquote(at.Value) - if n == "db-snapshot" && a == "02:00" { - scheduled = true - } + if sel, ok := c.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "NeedsCatchUp" && len(c.Args) == 3 { + if a, ok := c.Args[2].(*ast.SelectorExpr); ok && a.Sel.Name == "CatchUpAge" { + catchUp = true } } - if sel, ok := c.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "NeedsCatchUp" { - catchUp = true - } return true }) if !scheduled { - t.Error(`cmd/hub/main.go never calls scheduleDaily(ctx, "db-snapshot", "02:00", …) — no nightly snapshot`) + t.Error("cmd/hub/main.go never schedules a snapshot at each dbsnap.Slots time — no 6-hourly snapshot (D34)") } if !catchUp { - t.Error("cmd/hub/main.go never calls dbsnap.NeedsCatchUp — a pod down at 02:00 leaves a stale snapshot") + t.Error("cmd/hub/main.go never calls dbsnap.NeedsCatchUp(…, dbsnap.CatchUpAge) — a pod down at a slot leaves a stale snapshot") } } diff --git a/hub/internal/dbsnap/dbsnap.go b/hub/internal/dbsnap/dbsnap.go index 238be62d..cbc2a317 100644 --- a/hub/internal/dbsnap/dbsnap.go +++ b/hub/internal/dbsnap/dbsnap.go @@ -26,6 +26,15 @@ import ( // grown to 2 GiB for it, operator choice 2026-10-05). const Keep = 2 +// Slots are the Budapest times the hub writes a snapshot — every 6 hours (D34, operator yes 2026-10-23, R-232 G5). +// DooPlex's felhom-hub-db-backup.timer pushes the newest at :30 after each slot, so the off-site copy is at most +// about 6.5 h old instead of a day. Pinned by TestD34_SlotsAreEverySixHours and cmd/hub TestR173_MainSchedulesTheDBSnapshot. +var Slots = []string{"02:00", "08:00", "14:00", "20:00"} + +// CatchUpAge is the start-up catch-up limit: a pod that was down at a slot writes one at start when the newest is +// older than one slot gap. Pinned by TestD34_SlotsAreEverySixHours. +const CatchUpAge = 6 * time.Hour + // Prefix and Suffix make a snapshot's file name: hub-20261005T020000Z.db. const ( Prefix = "hub-" diff --git a/hub/internal/dbsnap/dbsnap_test.go b/hub/internal/dbsnap/dbsnap_test.go index 96411ec5..ad3d442f 100644 --- a/hub/internal/dbsnap/dbsnap_test.go +++ b/hub/internal/dbsnap/dbsnap_test.go @@ -206,3 +206,43 @@ func TestNeedsCatchUp(t *testing.T) { t.Error("25 h old: want catch-up") } } + +// D34 (operator yes 2026-10-23): a snapshot every 6 hours. The consequence pinned: no gap between two slots — the +// wrap past midnight included — is longer than CatchUpAge, so the off-site copy (pushed :30 after each slot) is never +// more than one slot gap behind; 02:00 stays a slot (the nightly restore test and the runbook read it); the start-up +// catch-up fires for a pod that missed exactly one slot. RED-PROOF: drop a slot, or set CatchUpAge to 24 h → fails. +func TestD34_SlotsAreEverySixHours(t *testing.T) { + if len(Slots) != 4 { + t.Fatalf("Slots = %v, want 4 (every 6 h)", Slots) + } + var mins []int + has0200 := false + for _, s := range Slots { + tm, err := time.Parse("15:04", s) + if err != nil { + t.Fatalf("slot %q: %v", s, err) + } + mins = append(mins, tm.Hour()*60+tm.Minute()) + has0200 = has0200 || s == "02:00" + } + if !has0200 { + t.Error("02:00 is not a slot") + } + for i := range mins { + next := mins[(i+1)%len(mins)] + if i == len(mins)-1 { + next += 24 * 60 + } + if gap := time.Duration(next-mins[i]) * time.Minute; gap <= 0 || gap > CatchUpAge { + t.Errorf("gap after %s is %s (want 0 < gap <= %s)", Slots[i], gap, CatchUpAge) + } + } + dir := t.TempDir() + _ = os.WriteFile(filepath.Join(dir, "hub-20261023T000000Z.db"), nil, 0o600) // 02:00 Budapest (CEST) + if !NeedsCatchUp(dir, time.Date(2026, 10, 23, 7, 0, 1, 0, time.UTC), CatchUpAge) { + t.Error("a pod up at 09:00 CEST that missed the 08:00 slot (newest 7 h old): want catch-up") + } + if NeedsCatchUp(dir, time.Date(2026, 10, 23, 5, 0, 0, 0, time.UTC), CatchUpAge) { + t.Error("newest 5 h old: want no catch-up") + } +} diff --git a/scripts/hub-db-backup/felhom-hub-db-backup b/scripts/hub-db-backup/felhom-hub-db-backup index db051eaf..177a5968 100755 --- a/scripts/hub-db-backup/felhom-hub-db-backup +++ b/scripts/hub-db-backup/felhom-hub-db-backup @@ -1,6 +1,7 @@ #!/bin/sh -# felhom-hub-db-backup — push the hub's newest nightly snapshot to ep0's PBS, encrypted (R-173, decision A). -# Runs on DooPlex as root from felhom-hub-db-backup.timer (02:30; the hub writes the snapshot at 02:00). +# felhom-hub-db-backup — push the hub's newest snapshot to ep0's PBS, encrypted (R-173, decision A). +# Runs on DooPlex as root from felhom-hub-db-backup.timer (02:30, 08:30, 14:30, 20:30; the hub writes a snapshot at +# 02:00, 08:00, 14:00 and 20:00 since hub 0.167.0 — D34). # Runbook: documentation/runbooks/RUNBOOK-hub-db-offsite-backup.md Step 4. Pinned by test_hub_db_backup.py. # # Refuses to push — and so never writes the success signal — when: no snapshot exists; the newest is older than @@ -11,7 +12,7 @@ set -eu CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup} STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup} TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector} -MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-26} +MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-2} # D34: the newest is ~30 min old at each push; 2 h refuses a missed slot NOW=${FELHOM_HUBBK_NOW:-$(date +%s)} . "$CONF/env" # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file) diff --git a/scripts/hub-db-backup/felhom-hub-db-backup.timer b/scripts/hub-db-backup/felhom-hub-db-backup.timer index 004726bd..76d29916 100644 --- a/scripts/hub-db-backup/felhom-hub-db-backup.timer +++ b/scripts/hub-db-backup/felhom-hub-db-backup.timer @@ -1,9 +1,13 @@ # Versioned in felhom.eu/scripts/hub-db-backup/ (R-231); installed by install.sh. [Unit] -Description=Felhom: push the hub DB snapshot to ep0 (R-173) — schedule +Description=Felhom: push the hub DB snapshot to ep0 (R-173, D34) — every 6 hours [Timer] +# D34 (operator yes 2026-10-23): every 6 h, :30 after each hub snapshot slot (hub dbsnap.Slots 02/08/14/20:00). OnCalendar=*-*-* 02:30:00 +OnCalendar=*-*-* 08:30:00 +OnCalendar=*-*-* 14:30:00 +OnCalendar=*-*-* 20:30:00 Persistent=true RandomizedDelaySec=2min diff --git a/scripts/hub-db-backup/test_hub_db_backup.py b/scripts/hub-db-backup/test_hub_db_backup.py index fdbabf33..78070693 100644 --- a/scripts/hub-db-backup/test_hub_db_backup.py +++ b/scripts/hub-db-backup/test_hub_db_backup.py @@ -8,6 +8,7 @@ was written. Run: python3 scripts/hub-db-backup/test_hub_db_backup.py """ import json import os +import re import shutil import sqlite3 import stat @@ -184,6 +185,15 @@ class Push(Base): self.assertEqual(self.pushed(), []) self.assertFalse(self.signal("felhom_hub_db_backup.prom")) + def test_d34_a_missed_slot_is_not_pushed_again(self): + # D34: snapshots every 6 h, so a newest copy 3 h old means the hub missed a slot — refuse, as for a day + self.snapshot(age_s=3 * 3600) + r = self.run_script(PUSH) + self.assertNotEqual(r.returncode, 0) + self.assertIn("stopped snapshotting", r.stderr) + self.assertEqual(self.pushed(), []) + self.assertFalse(self.signal("felhom_hub_db_backup.prom")) + def test_truncated_copy_is_not_pushed(self): self.snapshot() r = self.run_script(PUSH, FAKE_TRUNCATE="1") @@ -268,7 +278,14 @@ class Units(unittest.TestCase): return open(os.path.join(HERE, n)).read() def test_schedules(self): - self.assertIn("OnCalendar=*-*-* 02:30:00", self.read("felhom-hub-db-backup.timer")) + # D34: the push runs :30 after EVERY hub snapshot slot — read from the hub's own source, so a slot moved on + # one side only fails here (RED-PROOF: drop one OnCalendar line, or change a hub slot → fails). + src = open(os.path.join(HERE, "..", "..", "hub", "internal", "dbsnap", "dbsnap.go")).read() + slots = re.findall(r'"(\d\d):(\d\d)"', re.search(r"var Slots = \[\]string\{([^}]*)\}", src).group(1)) + self.assertEqual(len(slots), 4) + want = sorted("OnCalendar=*-*-* %02d:%02d:00" % (int(h), int(m) + 30) for h, m in slots) + got = sorted(l.strip() for l in self.read("felhom-hub-db-backup.timer").splitlines() if l.startswith("OnCalendar=")) + self.assertEqual(got, want) self.assertIn("OnCalendar=Sun *-*-* 04:30:00", self.read("felhom-hub-db-restore-test.timer")) def test_units_run_the_installed_scripts(self):