hub-db-backup: wait for a snapshot being written, limit 7 h (one slot gap + 1 h) — the 2 h limit refused the hub's own catch-up and mailed a false failure at 10:28 (R-1234, closed); tests red-proved

scripts/hub-db-backup/felhom-hub-db-backup

@@ -12,7 +12,13 @@ set -eu
 CONF=${FELHOM_HUBBK_CONF:-/etc/felhom-hub-backup}
 STATE=${FELHOM_HUBBK_STATE:-/var/lib/felhom-hub-backup}
 TEXTFILE_DIR=${FELHOM_HUBBK_TEXTFILE_DIR:-/var/lib/node_exporter/textfile_collector}
-MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-2}   # D34: the newest is ~30 min old at each push; 2 h refuses a missed slot
+# D34: one slot gap + 1 h. The hub writes a catch-up snapshot at start when its newest is older than 6 h
+# (dbsnap.CatchUpAge), so after any restart the newest is either < 7 h old or being written (waited for below). A hub
+# that really stopped is refused at the second missed slot. (2 h was tried first, 2026-10-23: it refused the timer's own
+# catch-up run while the hub's catch-up snapshot was still being written — a false "stopped snapshotting" mail.)
+MAX_AGE_H=${FELHOM_HUBBK_MAX_AGE_H:-7}
+WAIT_S=${FELHOM_HUBBK_WAIT_S:-1200}
+POLL_S=${FELHOM_HUBBK_POLL_S:-15}
 NOW=${FELHOM_HUBBK_NOW:-$(date +%s)}
 . "$CONF/env"   # PBS_REPOSITORY_PUSH, PBS_FINGERPRINT (no secrets in this file)
 
@@ -25,6 +31,13 @@ mkdir -p "$STAGE"; chmod 700 "$STATE" "$STAGE"
 rm -f "$STAGE"/*
 trap 'if [ -f "$STAGE/hub.db" ]; then shred -u "$STAGE/hub.db" 2>/dev/null || rm -f "$STAGE/hub.db"; fi' EXIT
 
+# A snapshot being written (hub-*.db.tmp) is waited for, so a push never takes the older one meanwhile (D34).
+WAITED=0
+while kubectl -n felhom-system exec deploy/hub -- sh -c 'ls /data/snapshots/hub-*.db.tmp >/dev/null 2>&1'; do
+  [ "$WAITED" -ge "$WAIT_S" ] && { log "a snapshot is still being written after ${WAIT_S} s — taking the newest finished one"; break; }
+  [ "$WAITED" -eq 0 ] && log "a snapshot is being written in the hub pod — waiting for it (up to ${WAIT_S} s)"
+  sleep "$POLL_S"; WAITED=$((WAITED + POLL_S))
+done
 SNAP=$(kubectl -n felhom-system exec deploy/hub -- sh -c 'ls -1 /data/snapshots/hub-*.db 2>/dev/null | tail -n 1') || die "listing snapshots in the hub pod"
 [ -n "$SNAP" ] || die "no snapshot in the hub pod's /data/snapshots"
 NAME=${SNAP##*/}

scripts/hub-db-backup/test_hub_db_backup.py

@@ -185,15 +185,42 @@ class Push(Base):
         self.assertEqual(self.pushed(), [])
         self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
 
-    def test_d34_a_missed_slot_is_not_pushed_again(self):
-        # D34: snapshots every 6 h, so a newest copy 3 h old means the hub missed a slot — refuse, as for a day
-        self.snapshot(age_s=3 * 3600)
+    def test_d34_two_missed_slots_are_refused(self):
+        # D34: snapshots every 6 h; the limit is one slot gap + 1 h (7 h) — a newest copy 8 h old means the hub missed
+        # two slots, or stopped. RED-PROOF: MAX_AGE_H back to 26 → this pushes.
+        self.snapshot(age_s=8 * 3600)
         r = self.run_script(PUSH)
         self.assertNotEqual(r.returncode, 0)
         self.assertIn("stopped snapshotting", r.stderr)
         self.assertEqual(self.pushed(), [])
         self.assertFalse(self.signal("felhom_hub_db_backup.prom"))
 
+    def test_d34_a_restart_within_one_slot_is_not_an_alarm(self):
+        # the hub's catch-up fires only past 6 h, so after a restart the newest can be up to 6 h old — that is pushed,
+        # not refused (2 h refused it: the false mail of 2026-10-23 10:28). RED-PROOF: MAX_AGE_H 2 → this fails.
+        self.snapshot(age_s=int(5.9 * 3600))
+        r = self.run_script(PUSH)
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(len(self.pushed()), 1)
+
+    def test_d34_waits_for_a_snapshot_being_written(self):
+        # the 2026-10-23 10:28 case: the hub's catch-up snapshot is being written (hub-*.db.tmp) and the only finished
+        # one is 8 h old. The push must wait for the new one, not refuse the old. RED-PROOF: drop the wait loop → fails.
+        import threading
+        self.snapshot(age_s=8 * 3600)
+        snaps = self.j("pod", "snapshots")
+        tmp = os.path.join(snaps, "hub-%s.db.tmp" % stamp(int(time.time())))
+        src = [os.path.join(snaps, n) for n in os.listdir(snaps) if n.endswith(".db")][0]
+        shutil.copy(src, tmp)
+        def finish():
+            time.sleep(1.5)
+            os.rename(tmp, tmp[: -len(".tmp")])
+        threading.Thread(target=finish).start()
+        r = self.run_script(PUSH, FELHOM_HUBBK_POLL_S="1", FELHOM_HUBBK_WAIT_S="20")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("being written", r.stdout)
+        self.assertEqual(len(self.pushed()), 1)
+
     def test_truncated_copy_is_not_pushed(self):
         self.snapshot()
         r = self.run_script(PUSH, FAKE_TRUNCATE="1")