v0.168.0: customer-configurable backup window (Mentési időablak)
ONE setting (window start W) drives every nightly leg at fixed, never-stored offsets: DB dump at W, tier-2 at W+60m, off-box at W+105m (wrap-safe). Precedence settings > controller.yaml db_dump_schedule > 02:30. - scheduler.UpdateDaily: retime a daily job at runtime (no restart) via a per-job buffered resched chan + a select case in runDailyJob. - new pure package internal/backupwindow (LegTimes/GateWindow/EffectiveWindow). - quiesce disk-tier window gate: scheduled cycles run only inside [W+2h,W+6h) with a safety valve (age>cadence+24h runs regardless); manual TriggerNow never gated. Backend.Due now also returns the backup age (from the agent's own /backup/due). - backup page: Mentési időablak card (time input + derived leg/gate rows); POST /backups/window validates -> saves -> UpdateDaily x3 -> flash. Tests: 5 groups, all red-proofed. Agent/cadence//backup/due untouched.
This commit is contained in:
@@ -20,6 +20,8 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/backupwindow"
|
||||
)
|
||||
|
||||
// ErrBackupInProgress is returned by TriggerNow when a scheduled or manual quiesce cycle is already
|
||||
@@ -27,9 +29,11 @@ import (
|
||||
var ErrBackupInProgress = errors.New("quiesce: a backup cycle is already in progress")
|
||||
|
||||
// Backend is the agent local-API surface the loop needs (satisfied by an adapter over
|
||||
// *agentapi.Client). Kept minimal (bool/string) so the loop is testable with plain fakes.
|
||||
// *agentapi.Client). Kept minimal (bool/int/string) so the loop is testable with plain fakes.
|
||||
// Due also returns the age of the newest successful backup in seconds (nil = none yet) — the
|
||||
// window gate's safety valve reads it so a box powered on only outside its window never starves.
|
||||
type Backend interface {
|
||||
Due(ctx context.Context) (bool, error)
|
||||
Due(ctx context.Context) (due bool, ageSecs *int64, err error)
|
||||
StartBackup(ctx context.Context) (jobID string, err error)
|
||||
BackupStatus(ctx context.Context) (phase string, err error)
|
||||
}
|
||||
@@ -68,6 +72,13 @@ type Options struct {
|
||||
StatusPoll time.Duration // how often to poll /backup/status while quiesced
|
||||
MaxQuiesce time.Duration // hard bound on app downtime (unquiesce no matter what)
|
||||
Logger *log.Logger
|
||||
// WindowStartFn returns the CURRENT effective backup-window start "HH:MM" (customer-configurable,
|
||||
// so it is read fresh each poll — a window change must take effect without restart). When nil the
|
||||
// window gate is disabled and a due cycle runs whenever the agent says due (pre-v0.168.0 behavior).
|
||||
WindowStartFn func() string
|
||||
// Cadence is the agent's backup cadence, used only by the gate's safety valve (run regardless of
|
||||
// the window once the last successful backup is older than Cadence+24h). Defaults to 24h.
|
||||
Cadence time.Duration
|
||||
}
|
||||
|
||||
// Loop is the quiesce background loop.
|
||||
@@ -80,6 +91,9 @@ type Loop struct {
|
||||
maxQuiesce time.Duration
|
||||
logger *log.Logger
|
||||
now func() time.Time
|
||||
// windowStartFn (nil = gate disabled) + cadence drive the scheduled-cycle window gate (Part 3).
|
||||
windowStartFn func() string
|
||||
cadence time.Duration
|
||||
// mu single-flights the quiesce cycle across the scheduled loop AND the manual trigger, so the
|
||||
// two can never stop the same stacks concurrently (the persisted marker covers crash-safety across
|
||||
// restarts; this covers concurrency within the process — which a manual trigger introduces).
|
||||
@@ -100,10 +114,14 @@ func New(o Options) *Loop {
|
||||
if o.Logger == nil {
|
||||
o.Logger = log.Default()
|
||||
}
|
||||
if o.Cadence <= 0 {
|
||||
o.Cadence = 24 * time.Hour
|
||||
}
|
||||
return &Loop{
|
||||
backend: o.Backend, stacks: o.Stacks, markerPath: o.MarkerPath,
|
||||
poll: o.Poll, statusPoll: o.StatusPoll, maxQuiesce: o.MaxQuiesce,
|
||||
logger: o.Logger, now: time.Now,
|
||||
windowStartFn: o.WindowStartFn, cadence: o.Cadence,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,7 +177,7 @@ func (l *Loop) runOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
due, err := l.backend.Due(ctx)
|
||||
due, ageSecs, err := l.backend.Due(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check due: %w", err)
|
||||
}
|
||||
@@ -167,6 +185,17 @@ func (l *Loop) runOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Window gate (Part 3) — SCHEDULED path only. TriggerNow calls quiesceAndPoll directly and is
|
||||
// never gated. Disabled when no window fn is wired (pre-v0.168.0 behavior).
|
||||
if l.windowStartFn != nil {
|
||||
window := l.windowStartFn()
|
||||
if !scheduledRunAllowed(l.now().In(budapestLocation()), window, ageSecs, l.cadence) {
|
||||
from, to := gateBounds(window)
|
||||
l.logger.Printf("[DEBUG] [quiesce] scheduled backup due but outside the backup window [%s–%s) — deferring to the next poll inside it", from, to)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return l.quiesceAndPoll(ctx)
|
||||
}
|
||||
|
||||
@@ -286,6 +315,63 @@ func (l *Loop) quiesceAndPoll(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// ---- window gate (Part 3, v0.168.0) -----------------------------------------------------
|
||||
|
||||
var (
|
||||
quiesceBudapest *time.Location
|
||||
quiesceBudapestOnce sync.Once
|
||||
)
|
||||
|
||||
func budapestLocation() *time.Location {
|
||||
quiesceBudapestOnce.Do(func() {
|
||||
loc, err := time.LoadLocation("Europe/Budapest")
|
||||
if err != nil {
|
||||
quiesceBudapest = time.UTC
|
||||
return
|
||||
}
|
||||
quiesceBudapest = loc
|
||||
})
|
||||
return quiesceBudapest
|
||||
}
|
||||
|
||||
const (
|
||||
gateOpenOffsetMin = 120 // gate opens at W+2h
|
||||
gateSpanMin = 240 // 4h span → [W+2h, W+6h)
|
||||
)
|
||||
|
||||
// scheduledRunAllowed decides whether a DUE, scheduled whole-guest backup may run at `now` (passed by
|
||||
// the caller as Budapest wall-clock — only its hour/minute are read). True when now is inside the gate
|
||||
// window [W+2h, W+6h); otherwise true ONLY if the safety valve holds — the newest successful backup is
|
||||
// missing (nil) or older than cadence+24h — so a box powered on only outside its window never starves.
|
||||
// An unparseable window fails OPEN (allow) rather than block backups forever.
|
||||
func scheduledRunAllowed(now time.Time, windowStart string, lastAgeSecs *int64, cadence time.Duration) bool {
|
||||
startMin, err := backupwindow.ParseHHMM(windowStart)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
nowMin := now.Hour()*60 + now.Minute()
|
||||
if within(nowMin, mod1440(startMin+gateOpenOffsetMin), gateSpanMin) {
|
||||
return true
|
||||
}
|
||||
// Outside the window: only the safety valve may run it.
|
||||
if lastAgeSecs == nil {
|
||||
return true // no recorded backup yet — never withhold the first one
|
||||
}
|
||||
return time.Duration(*lastAgeSecs)*time.Second > cadence+24*time.Hour
|
||||
}
|
||||
|
||||
// gateBounds returns the gate window [W+2h, W+6h) as HH:MM for the deferral log line.
|
||||
func gateBounds(windowStart string) (from, to string) {
|
||||
return backupwindow.GateWindow(windowStart)
|
||||
}
|
||||
|
||||
func mod1440(m int) int { return ((m % 1440) + 1440) % 1440 }
|
||||
|
||||
// within reports whether minute-of-day p falls in [start, start+span) modulo 24h (wrap-safe).
|
||||
func within(p, start, span int) bool {
|
||||
return mod1440(p-start) < span
|
||||
}
|
||||
|
||||
func (l *Loop) restartAll(stacks []string) {
|
||||
for _, s := range stacks {
|
||||
if err := l.stacks.StartStack(s); err != nil {
|
||||
|
||||
@@ -34,8 +34,8 @@ type eventBackend struct {
|
||||
i int
|
||||
}
|
||||
|
||||
func (b *eventBackend) Due(context.Context) (bool, error) { return true, nil }
|
||||
func (b *eventBackend) StartBackup(context.Context) (string, error) { return "job-1", nil }
|
||||
func (b *eventBackend) Due(context.Context) (bool, *int64, error) { return true, nil, nil }
|
||||
func (b *eventBackend) StartBackup(context.Context) (string, error) { return "job-1", nil }
|
||||
func (b *eventBackend) BackupStatus(context.Context) (string, error) {
|
||||
ph := b.phases[len(b.phases)-1]
|
||||
if b.i < len(b.phases) {
|
||||
|
||||
@@ -54,6 +54,7 @@ func (f *fakeStacks) stoppedNames() []string {
|
||||
// fakeBackend drives the agent-side responses.
|
||||
type fakeBackend struct {
|
||||
due bool
|
||||
dueAge *int64 // age of newest successful backup (nil = none); gate safety-valve input
|
||||
dueErr error
|
||||
startErr error
|
||||
jobID string
|
||||
@@ -64,7 +65,7 @@ type fakeBackend struct {
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (b *fakeBackend) Due(context.Context) (bool, error) { return b.due, b.dueErr }
|
||||
func (b *fakeBackend) Due(context.Context) (bool, *int64, error) { return b.due, b.dueAge, b.dueErr }
|
||||
func (b *fakeBackend) StartBackup(context.Context) (string, error) {
|
||||
b.mu.Lock()
|
||||
b.startCalls++
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package quiesce
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func i64(v int64) *int64 { return &v }
|
||||
|
||||
// atBudapest builds a time whose Budapest wall-clock hour/minute are h:m (the predicate reads only
|
||||
// those; the caller in the loop passes now.In(Budapest)).
|
||||
func atBudapest(h, m int) time.Time {
|
||||
return time.Date(2026, 7, 24, h, m, 0, 0, budapestLocation())
|
||||
}
|
||||
|
||||
const cadence24 = 24 * time.Hour
|
||||
|
||||
// Group D — scheduledRunAllowed truth table: inside gate / outside / outside+valve / wrap / nil age.
|
||||
// Window 02:30 → gate [04:30, 08:30). Valve threshold = cadence+24h = 48h.
|
||||
// Red-proof: invert the valve comparison (`<` instead of `>`) and the starving-box case (age 49h,
|
||||
// outside window) flips to false.
|
||||
func TestScheduledRunAllowed(t *testing.T) {
|
||||
h := func(hours int64) *int64 { return i64(hours * 3600) }
|
||||
cases := []struct {
|
||||
name string
|
||||
now time.Time
|
||||
window string
|
||||
age *int64
|
||||
want bool
|
||||
}{
|
||||
{"inside gate, recent backup", atBudapest(5, 0), "02:30", h(20), true},
|
||||
{"gate open boundary (inclusive)", atBudapest(4, 30), "02:30", h(20), true},
|
||||
{"gate close boundary (exclusive)", atBudapest(8, 30), "02:30", h(20), false},
|
||||
{"outside gate, no valve", atBudapest(12, 0), "02:30", h(20), false},
|
||||
{"outside gate, valve (age > 48h)", atBudapest(12, 0), "02:30", h(49), true},
|
||||
{"outside gate, nil age (no backup yet)", atBudapest(12, 0), "02:30", nil, true},
|
||||
{"wrap: inside gate across midnight", atBudapest(2, 0), "23:00", h(20), true},
|
||||
{"wrap: outside gate across midnight", atBudapest(12, 0), "23:00", h(20), false},
|
||||
{"unparseable window fails open", atBudapest(12, 0), "nonsense", h(20), true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := scheduledRunAllowed(c.now, c.window, c.age, cadence24); got != c.want {
|
||||
t.Errorf("%s: scheduledRunAllowed(%s, %q, age, cadence) = %v, want %v",
|
||||
c.name, c.now.Format("15:04"), c.window, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// windowLoop builds a Loop with the gate wired and now/window overridden for deterministic tests.
|
||||
func windowLoop(t *testing.T, be Backend, st Stacks, window string, now time.Time) *Loop {
|
||||
t.Helper()
|
||||
l := testLoop(t, be, st)
|
||||
l.windowStartFn = func() string { return window }
|
||||
l.cadence = cadence24
|
||||
l.now = func() time.Time { return now }
|
||||
return l
|
||||
}
|
||||
|
||||
// Group D (integration) — a DUE scheduled cycle outside the window with a recent backup is deferred:
|
||||
// no StartBackup, no stacks stopped.
|
||||
func TestRunOnce_GateDefersOutsideWindow(t *testing.T) {
|
||||
be := &fakeBackend{due: true, dueAge: i64(20 * 3600)}
|
||||
st := &fakeStacks{running: []string{"nextcloud"}}
|
||||
l := windowLoop(t, be, st, "02:30", atBudapest(12, 0)) // gate [04:30,08:30), 12:00 is outside
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatalf("runOnce: %v", err)
|
||||
}
|
||||
if be.startCalls != 0 {
|
||||
t.Errorf("gate should have deferred, but StartBackup was called %d time(s)", be.startCalls)
|
||||
}
|
||||
if len(st.stoppedNames()) != 0 {
|
||||
t.Errorf("gate should have deferred, but stacks were stopped: %v", st.stoppedNames())
|
||||
}
|
||||
}
|
||||
|
||||
// Group D (integration) — inside the window the cycle runs normally.
|
||||
func TestRunOnce_GateRunsInsideWindow(t *testing.T) {
|
||||
be := &fakeBackend{due: true, dueAge: i64(20 * 3600), phases: []string{"done"}}
|
||||
st := &fakeStacks{running: []string{"nextcloud"}}
|
||||
l := windowLoop(t, be, st, "02:30", atBudapest(5, 0)) // 05:00 is inside [04:30,08:30)
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatalf("runOnce: %v", err)
|
||||
}
|
||||
if be.startCalls != 1 {
|
||||
t.Errorf("inside the window the cycle should run; StartBackup calls = %d", be.startCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// Group D (integration) — outside the window but the safety valve holds (box was off during its
|
||||
// window; last backup older than cadence+24h): the cycle runs regardless of the window.
|
||||
func TestRunOnce_ValveRunsOutsideWindow(t *testing.T) {
|
||||
be := &fakeBackend{due: true, dueAge: i64(49 * 3600), phases: []string{"done"}}
|
||||
st := &fakeStacks{running: []string{"nextcloud"}}
|
||||
l := windowLoop(t, be, st, "02:30", atBudapest(12, 0)) // outside, but age 49h > 48h valve
|
||||
|
||||
if err := l.runOnce(context.Background()); err != nil {
|
||||
t.Fatalf("runOnce: %v", err)
|
||||
}
|
||||
if be.startCalls != 1 {
|
||||
t.Errorf("safety valve should have run the cycle; StartBackup calls = %d", be.startCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// Group D (integration) — the MANUAL trigger path is never gated: TriggerNow runs quiesceAndPoll
|
||||
// directly (bypassing runOnce), so an outside-window manual backup still fires.
|
||||
func TestTriggerNow_NotGated(t *testing.T) {
|
||||
be := &fakeBackend{due: false, phases: []string{"done"}} // not due — only the manual path can run it
|
||||
st := &fakeStacks{running: []string{"nextcloud"}}
|
||||
l := windowLoop(t, be, st, "02:30", atBudapest(12, 0)) // outside the window
|
||||
|
||||
if err := l.TriggerNow(); err != nil {
|
||||
t.Fatalf("TriggerNow: %v", err)
|
||||
}
|
||||
// TriggerNow runs asynchronously — wait for the backup to be started.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
be.mu.Lock()
|
||||
n := be.startCalls
|
||||
be.mu.Unlock()
|
||||
if n == 1 {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("manual TriggerNow did not start a backup — it must never be gated by the window")
|
||||
}
|
||||
Reference in New Issue
Block a user