admin
c1a73b24b3
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
251 lines
11 KiB
Go
251 lines
11 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ── Decision 68 (v0.289.0): the box prunes its own repository ONLY inside a hub-opened window ──────────
|
|
//
|
|
// The hub-provisioned tier's key is append-only (decision 69): every delete is refused by the provider.
|
|
// To keep retention working, the box ASKS the hub for a clean-up window after its run. The hub grants
|
|
// one at most weekly (or on an operator's one-shot grant) by PREPENDING a deleting line for the box's own
|
|
// key — OpenSSH uses the first matching line, measured on the provider — and closes it when the box
|
|
// reports, or after 20 minutes on its own.
|
|
//
|
|
// THE FAKE-SNAPSHOT GUARD (R-822) runs BEFORE any forget, inside the window. Measured in the lab: 13
|
|
// future-dated empty snapshots added through an add-only key make the box's own policy select EVERY real
|
|
// snapshot for removal. So, refuse when:
|
|
// - any snapshot is dated in the future (beyond offsiteGuardSkew), or after the hub's newest-allowed
|
|
// bound (the moment the window opened, plus the same skew);
|
|
// - the plan would remove a snapshot younger than offsiteGuardMinAge — the honest policy
|
|
// (--keep-daily 7) never removes the newest snapshot of any of the last 7 days, while a poisoning
|
|
// shape does exactly that.
|
|
// And a plan larger than MaxRemove (the hub's number for one week) REFUSES (v0.290.0, per the 2026-10-04
|
|
// brief, replacing v0.289's cap). The cost, recorded (R-96 rule 4): after a long gap without windows the
|
|
// honest backlog exceeds a week and the guard refuses until the operator grants a window by hand — R-833.
|
|
//
|
|
// The NAS tier (Transport "") is unchanged: the household's own disk, pruned by the box as before.
|
|
//
|
|
// Pinned by TestOffsiteGuard_* (offbox_window_test.go), including the lab's 13-fake shape.
|
|
|
|
const (
|
|
offsiteGuardSkew = time.Hour
|
|
offsiteGuardMinAge = 8 * 24 * time.Hour
|
|
)
|
|
|
|
// OffsiteWindow is the hub's answer to "may I prune now?".
|
|
type OffsiteWindow struct {
|
|
Granted bool
|
|
ID int64
|
|
NewestAllowed time.Time
|
|
MaxRemove int
|
|
Reason string // why not granted (logged)
|
|
}
|
|
|
|
// OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it).
|
|
type OffsiteWindowResult struct {
|
|
ID int64 `json:"window_id"`
|
|
CountBefore int `json:"count_before"`
|
|
CountAfter int `json:"count_after"`
|
|
Removed int `json:"removed"`
|
|
Outcome string `json:"outcome"` // pruned | nothing | guard-refused | error
|
|
Reason string `json:"reason,omitempty"`
|
|
}
|
|
|
|
// OffsiteWindowClient is the hub side (offsiteapply.HubWindowClient in production).
|
|
type OffsiteWindowClient interface {
|
|
Open(ctx context.Context, countBefore int) (OffsiteWindow, error)
|
|
Close(ctx context.Context, r OffsiteWindowResult) error
|
|
}
|
|
|
|
// SetOffsiteWindowClient wires the hub's window (decision 68). nil → no box-side retention on the pinned tier.
|
|
func (m *Manager) SetOffsiteWindowClient(c OffsiteWindowClient) { m.offsiteWindow = c }
|
|
|
|
// retentionPolicy is the ruled policy, unchanged since SP-2 (`--group-by host,tags`).
|
|
var retentionPolicy = []string{"--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6"}
|
|
|
|
type guardSnap struct {
|
|
ID string `json:"id"`
|
|
ShortID string `json:"short_id"`
|
|
Time time.Time `json:"time"`
|
|
Hostname string `json:"hostname"`
|
|
Tags []string `json:"tags"`
|
|
}
|
|
|
|
// group is the `--group-by host,tags` key.
|
|
func (g guardSnap) group() string {
|
|
t := append([]string{}, g.Tags...)
|
|
sort.Strings(t)
|
|
return g.Hostname + "|" + strings.Join(t, ",")
|
|
}
|
|
|
|
// supersededSameDay: a NEWER snapshot of the same group exists on the same UTC day (a manual run after
|
|
// the night's) — the one benign reason the honest policy removes a young snapshot (R-824, measured on
|
|
// demo-hp 2026-10-03).
|
|
func supersededSameDay(s guardSnap, all []guardSnap) bool {
|
|
day := s.Time.UTC().Format("2006-01-02")
|
|
for _, o := range all {
|
|
if o.ID != s.ID && o.group() == s.group() && o.Time.After(s.Time) && o.Time.UTC().Format("2006-01-02") == day {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// offsiteGuard is the PURE decision: from all snapshots and the policy's remove-plan, either the ids to
|
|
// remove (oldest first) or a refusal reason. v0.290.0 (R-824): a YOUNG snapshot that a newer same-day
|
|
// snapshot of its group supersedes is EXCLUDED (kept for a later window, when it is old) instead of
|
|
// refusing the run — v0.289.x refused every window after any manual run. A young removal WITHOUT that
|
|
// explanation still refuses: it is the poisoning signature. Future-dated snapshots, snapshots newer than
|
|
// the hub allows, and a plan larger than a week's removal (maxRemove) refuse.
|
|
func offsiteGuard(all, plan []guardSnap, now, newestAllowed time.Time, maxRemove int) ([]string, string) {
|
|
for _, s := range all {
|
|
if s.Time.After(now.Add(offsiteGuardSkew)) {
|
|
return nil, fmt.Sprintf("snapshot %s is dated in the future (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339))
|
|
}
|
|
if !newestAllowed.IsZero() && s.Time.After(newestAllowed.Add(offsiteGuardSkew)) {
|
|
return nil, fmt.Sprintf("snapshot %s (%s) is newer than the hub allows (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339), newestAllowed.UTC().Format(time.RFC3339))
|
|
}
|
|
}
|
|
var keep []guardSnap
|
|
for _, s := range plan {
|
|
if now.Sub(s.Time) < offsiteGuardMinAge {
|
|
if supersededSameDay(s, all) {
|
|
continue // excluded: removed in a later window, once older than offsiteGuardMinAge
|
|
}
|
|
return nil, fmt.Sprintf("the policy would remove snapshot %s from %s — younger than %d days and not superseded the same day, which honest retention never does",
|
|
s.ShortID, s.Time.UTC().Format(time.RFC3339), int(offsiteGuardMinAge.Hours()/24))
|
|
}
|
|
keep = append(keep, s)
|
|
}
|
|
if maxRemove >= 0 && len(keep) > maxRemove {
|
|
return nil, fmt.Sprintf("the plan would remove %d snapshots, more than one week's retention may (%d)", len(keep), maxRemove)
|
|
}
|
|
sort.Slice(keep, func(i, j int) bool { return keep[i].Time.Before(keep[j].Time) })
|
|
ids := make([]string, 0, len(keep))
|
|
for _, s := range keep {
|
|
ids = append(ids, s.ID)
|
|
}
|
|
return ids, ""
|
|
}
|
|
|
|
func (m *Manager) listGuardSnaps(ctx context.Context, base, env []string) ([]guardSnap, error) {
|
|
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
|
defer cancel()
|
|
out, err := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "--json")...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list snapshots: %w: %s", err, truncate(out))
|
|
}
|
|
var snaps []guardSnap
|
|
if err := json.Unmarshal(out, &snaps); err != nil {
|
|
return nil, fmt.Errorf("parse snapshots: %w", err)
|
|
}
|
|
return snaps, nil
|
|
}
|
|
|
|
func (m *Manager) planRemovals(ctx context.Context, base, env []string) ([]guardSnap, error) {
|
|
pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
|
defer cancel()
|
|
args := append(append(append([]string{}, base...), "forget"), retentionPolicy...)
|
|
args = append(args, "--dry-run", "--json")
|
|
out, err := m.runner()(pctx, env, args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("forget --dry-run: %w: %s", err, truncate(out))
|
|
}
|
|
// restic 0.14.0 prints the JSON array on stdout; the runner may combine stderr — take the array.
|
|
js := string(out)
|
|
if i := strings.Index(js, "["); i > 0 {
|
|
js = js[i:]
|
|
}
|
|
var groups []struct {
|
|
Remove []guardSnap `json:"remove"`
|
|
}
|
|
if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &groups); err != nil {
|
|
return nil, fmt.Errorf("parse forget plan: %w", err)
|
|
}
|
|
var plan []guardSnap
|
|
for _, g := range groups {
|
|
plan = append(plan, g.Remove...)
|
|
}
|
|
return plan, nil
|
|
}
|
|
|
|
// offsiteWindowRetention is the ONE retention step for both callers (after a run, over quota).
|
|
func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string, why string) {
|
|
t := m.settings.GetOffboxTarget()
|
|
if !t.Pinned() {
|
|
// The household's own SFTP NAS: the box prunes as it always did (SP-2 policy).
|
|
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
|
defer cancel()
|
|
args := append(append([]string{"forget"}, retentionPolicy...), "--prune")
|
|
if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
|
|
m.logger.Printf("[WARN] [offbox] forget --prune failed (%s; backups are safe): %v: %s", why, ferr, truncate(out))
|
|
}
|
|
return
|
|
}
|
|
if m.offsiteWindow == nil {
|
|
m.logger.Printf("[INFO] [offbox] retention skipped (%s): the off-site key is append-only and no clean-up window client is wired — nothing deleted (decision 68)", why)
|
|
return
|
|
}
|
|
snaps, err := m.listGuardSnaps(ctx, base, env)
|
|
if err != nil {
|
|
m.logger.Printf("[WARN] [offbox] retention skipped (%s): %v", why, err)
|
|
return
|
|
}
|
|
w, err := m.offsiteWindow.Open(ctx, len(snaps))
|
|
if err != nil {
|
|
m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err)
|
|
return
|
|
}
|
|
if !w.Granted {
|
|
m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason)
|
|
return
|
|
}
|
|
start := time.Now()
|
|
res := OffsiteWindowResult{ID: w.ID, CountBefore: len(snaps), CountAfter: len(snaps)}
|
|
defer func() {
|
|
cctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
if cerr := m.offsiteWindow.Close(cctx, res); cerr != nil {
|
|
m.logger.Printf("[WARN] [offbox] closing clean-up window %d with the hub failed (the hub closes it by itself in 20 min): %v", w.ID, cerr)
|
|
}
|
|
}()
|
|
plan, err := m.planRemovals(ctx, base, env)
|
|
if err != nil {
|
|
res.Outcome, res.Reason = "error", err.Error()
|
|
m.logger.Printf("[WARN] [offbox] clean-up window %d: %v", w.ID, err)
|
|
return
|
|
}
|
|
ids, refuse := offsiteGuard(snaps, plan, time.Now(), w.NewestAllowed, w.MaxRemove)
|
|
if refuse != "" {
|
|
res.Outcome, res.Reason = "guard-refused", refuse
|
|
m.logger.Printf("[ERROR] [offbox] clean-up window %d: the fake-snapshot guard REFUSED — nothing deleted: %s (R-822)", w.ID, refuse)
|
|
return
|
|
}
|
|
if len(ids) == 0 {
|
|
res.Outcome = "nothing"
|
|
m.logger.Printf("[INFO] [offbox] clean-up window %d: the policy removes nothing", w.ID)
|
|
return
|
|
}
|
|
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
|
defer cancel()
|
|
args := append(append([]string{"forget"}, ids...), "--prune")
|
|
if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
|
|
res.Outcome, res.Reason = "error", truncate(out)
|
|
m.logger.Printf("[WARN] [offbox] clean-up window %d: forget --prune failed (backups are safe): %v: %s", w.ID, ferr, truncate(out))
|
|
} else {
|
|
res.Outcome = "pruned"
|
|
}
|
|
if after, lerr := m.listGuardSnaps(ctx, base, env); lerr == nil {
|
|
res.CountAfter = len(after)
|
|
}
|
|
res.Removed = res.CountBefore - res.CountAfter
|
|
m.logger.Printf("[INFO] [offbox] clean-up window %d (%s): %d of %d planned snapshot(s) removed, %d -> %d, in %s",
|
|
w.ID, why, res.Removed, len(plan), res.CountBefore, res.CountAfter, time.Since(start).Round(time.Second))
|
|
}
|