147 lines
5.9 KiB
Go
147 lines
5.9 KiB
Go
package localapi
|
||
|
||
import (
|
||
"fmt"
|
||
"net/http"
|
||
"sort"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||
)
|
||
|
||
// The whole-guest backup cadence cap (R-973, security review 2026-10-12).
|
||
//
|
||
// POST /backup is reachable from inside the guest, and every successful run prunes the tier
|
||
// (`--prune-backups keep-last=N` on a local tier; ep0's server-side keep-last 2 on the off-site tier). So
|
||
// a compromised controller calling it in a loop pushed every pre-compromise whole-guest restore point out
|
||
// within minutes — the threat `07` §8 row 10 counts the off-site tier as the answer to. The agent now
|
||
// refuses a run that the schedule in `07` §6 would never ask for:
|
||
//
|
||
// - a tier whose cadence is at most 24 h (the local tier): at most 3 successful runs in any rolling
|
||
// 24 h. The schedule is one night run; a „Mentés most" press makes a LOCAL copy and, since R-899, the
|
||
// night still takes its own after it, which can land inside the same 24 h as the previous night (the
|
||
// window opens anywhere in [W+2h, W+6h)). So a normal day needs at most 3: two nights plus one press.
|
||
// A second press inside 24 h is refused — the margin is spent on the night, which must never be refused
|
||
// because of the household's presses.
|
||
// - a longer tier (the 7-day off-site tier): at most 1 successful run per max(cadence − 24 h, cadence/2)
|
||
// — 6 days for 7 days. The controller asks /backup/due first and runs only when the newest copy is older
|
||
// than the cadence, and the button never targets this tier (`07` §6, controller v0.301.0), so a
|
||
// legitimate run never comes sooner; 24 h of slack absorbs clock and archive-time differences.
|
||
//
|
||
// Only SUCCESSES count — a failed run prunes nothing and must stay retryable. The count survives an agent
|
||
// restart: it reads the success history saved on disk (LastKnownBackupStore, the R-894 file) as well as
|
||
// this process's own record. A refusal answers 429, is logged at WARN, is saved, and raises the operator
|
||
// signal BackupCapRefusal feeds (a critical capability in the host report → the hub's operator-only
|
||
// `whole_guest_backup_refused` event since hub v0.159.0, R-1150). Pinned by TestBackupCap_* (backup_cap_test.go).
|
||
|
||
// backupCapRefusalHold is how long one refusal keeps the operator signal raised.
|
||
const backupCapRefusalHold = 24 * time.Hour
|
||
|
||
// backupCadenceCap returns the rolling window and the most successful runs allowed in it for a tier.
|
||
func backupCadenceCap(cadence time.Duration) (window time.Duration, max int) {
|
||
if cadence <= 0 {
|
||
cadence = defaultBackupCadence
|
||
}
|
||
if cadence <= 24*time.Hour {
|
||
return 24 * time.Hour, 3
|
||
}
|
||
w := cadence - 24*time.Hour
|
||
if w < cadence/2 {
|
||
w = cadence / 2
|
||
}
|
||
return w, 1
|
||
}
|
||
|
||
// recentCapRuns returns the start times of the successful runs on this tier for this guest inside the
|
||
// window ending now — this process's record and the saved one, each counted once. Caller holds no lock.
|
||
func (s *Server) recentCapRuns(key backupJobKey, since time.Time) []time.Time {
|
||
seen := map[int64]bool{}
|
||
var out []time.Time
|
||
add := func(t time.Time) {
|
||
if t.Before(since) || seen[t.Unix()] {
|
||
return
|
||
}
|
||
seen[t.Unix()] = true
|
||
out = append(out, t)
|
||
}
|
||
s.capMu.Lock()
|
||
for _, t := range s.capRuns[key] {
|
||
add(t)
|
||
}
|
||
s.capMu.Unlock()
|
||
if s.lastKnown != nil {
|
||
for _, t := range s.lastKnown.RecentBackupSuccesses(key.target, key.vmid, since) {
|
||
add(t)
|
||
}
|
||
}
|
||
sort.Slice(out, func(i, j int) bool { return out[i].Before(out[j]) })
|
||
return out
|
||
}
|
||
|
||
// recordCapRun notes a successful run for the cap (the saved half is written by RecordBackupSuccess).
|
||
func (s *Server) recordCapRun(key backupJobKey, b hub.Backup) {
|
||
t, err := time.Parse(time.RFC3339, b.StartedAt)
|
||
if err != nil {
|
||
t = s.now()
|
||
}
|
||
s.capMu.Lock()
|
||
defer s.capMu.Unlock()
|
||
if s.capRuns == nil {
|
||
s.capRuns = map[backupJobKey][]time.Time{}
|
||
}
|
||
runs := append(s.capRuns[key], t.UTC())
|
||
if len(runs) > 8 {
|
||
runs = runs[len(runs)-8:]
|
||
}
|
||
s.capRuns[key] = runs
|
||
}
|
||
|
||
// capRefuses applies the cap to a new run request. It writes the 429 and returns true when refused.
|
||
func (s *Server) capRefuses(w http.ResponseWriter, key backupJobKey, tier BackupTier) bool {
|
||
window, max := backupCadenceCap(tier.Cadence)
|
||
now := s.now()
|
||
runs := s.recentCapRuns(key, now.Add(-window))
|
||
if len(runs) < max {
|
||
return false
|
||
}
|
||
next := runs[len(runs)-max].Add(window)
|
||
s.capMu.Lock()
|
||
s.capRefusal = capRefusal{target: key.target, vmid: key.vmid, at: now.UTC()}
|
||
s.capMu.Unlock()
|
||
if s.lastKnown != nil {
|
||
if err := s.lastKnown.RecordBackupRefusal(key.target, key.vmid, now); err != nil {
|
||
s.logger.Warn("local-api: could not save the backup refusal on disk (R-973)", "vmid", key.vmid, "target", key.target, "err", err)
|
||
}
|
||
}
|
||
s.logger.Warn("local-api: backup REFUSED — more whole-guest backups than the schedule allows (R-973 cadence cap; check the guest)",
|
||
"vmid", key.vmid, "target", key.target, "runs_in_window", len(runs), "allowed", max,
|
||
"window_h", int(window.Hours()), "next_allowed", next.UTC().Format(time.RFC3339))
|
||
writeStatus(w, http.StatusTooManyRequests, false, nil, fmt.Sprintf(
|
||
"too many whole-guest backups on target %q: %d successful in the last %d h, the schedule allows %d — refused (the next one is allowed after %s)",
|
||
key.target, len(runs), int(window.Hours()), max, next.UTC().Format(time.RFC3339)))
|
||
return true
|
||
}
|
||
|
||
type capRefusal struct {
|
||
target string
|
||
vmid int
|
||
at time.Time
|
||
}
|
||
|
||
// BackupCapRefusal reports the newest cadence-cap refusal still inside the 24 h hold (this process's or
|
||
// the saved one) — the source of the operator signal in the host report (R-973). ok=false = none.
|
||
func (s *Server) BackupCapRefusal() (target string, vmid int, at time.Time, ok bool) {
|
||
s.capMu.Lock()
|
||
r := s.capRefusal
|
||
s.capMu.Unlock()
|
||
if s.lastKnown != nil {
|
||
if t, v, a, found := s.lastKnown.LastBackupRefusal(); found && a.After(r.at) {
|
||
r = capRefusal{target: t, vmid: v, at: a}
|
||
}
|
||
}
|
||
if r.at.IsZero() || s.now().Sub(r.at) > backupCapRefusalHold {
|
||
return "", 0, time.Time{}, false
|
||
}
|
||
return r.target, r.vmid, r.at, true
|
||
}
|