felhom-controller/controller/internal/backup/offbox_window.go

package backup

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"
)

// ── Decision 68 (v0.289.0): the box prunes its own repository ONLY inside a hub-opened window ──────────
//
// The hub-provisioned tier's key is append-only (decision 69): every delete is refused by the provider.
// To keep retention working, the box ASKS the hub for a clean-up window after its run. The hub grants
// one at most weekly (or on an operator's one-shot grant) by PREPENDING a deleting line for the box's own
// key — OpenSSH uses the first matching line, measured on the provider — and closes it when the box
// reports, or after 20 minutes on its own.
//
// THE FAKE-SNAPSHOT GUARD (R-822) runs BEFORE any forget, inside the window. Measured in the lab: 13
// future-dated empty snapshots added through an add-only key make the box's own policy select EVERY real
// snapshot for removal. So, refuse when:
//   - any snapshot is dated in the future (beyond offsiteGuardSkew), or after the hub's newest-allowed
//     bound (the moment the window opened, plus the same skew);
//   - the plan would remove a snapshot younger than offsiteGuardMinAge — the honest policy
//     (--keep-daily 7) never removes the newest snapshot of any of the last 7 days, while a poisoning
//     shape does exactly that.
// And bound the damage of anything the guard cannot see: at most MaxRemove (the hub's number) snapshots
// per window, OLDEST first. DISAGREEMENT RECORDED (R-96 rule 4): the brief asked to ABORT when the plan
// exceeds a week's removal; the first window after the interim legitimately exceeds it (weeks of
// unpruned history), so an abort would never prune at all. Capping and taking the oldest gives the
// same bound on loss per window and still converges.
//
// The NAS tier (Transport "") is unchanged: the household's own disk, pruned by the box as before.
//
// Pinned by TestOffsiteGuard_* (offbox_window_test.go), including the lab's 13-fake shape.

const (
	offsiteGuardSkew   = time.Hour
	offsiteGuardMinAge = 8 * 24 * time.Hour
)

// OffsiteWindow is the hub's answer to "may I prune now?".
type OffsiteWindow struct {
	Granted       bool
	ID            int64
	NewestAllowed time.Time
	MaxRemove     int
	Reason        string // why not granted (logged)
}

// OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it).
type OffsiteWindowResult struct {
	ID          int64  `json:"window_id"`
	CountBefore int    `json:"count_before"`
	CountAfter  int    `json:"count_after"`
	Removed     int    `json:"removed"`
	Outcome     string `json:"outcome"` // pruned | nothing | guard-refused | error
	Reason      string `json:"reason,omitempty"`
}

// OffsiteWindowClient is the hub side (offsiteapply.HubWindowClient in production).
type OffsiteWindowClient interface {
	Open(ctx context.Context, countBefore int) (OffsiteWindow, error)
	Close(ctx context.Context, r OffsiteWindowResult) error
}

// SetOffsiteWindowClient wires the hub's window (decision 68). nil → no box-side retention on the pinned tier.
func (m *Manager) SetOffsiteWindowClient(c OffsiteWindowClient) { m.offsiteWindow = c }

// retentionPolicy is the ruled policy, unchanged since SP-2 (`--group-by host,tags`).
var retentionPolicy = []string{"--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6"}

type guardSnap struct {
	ID      string    `json:"id"`
	ShortID string    `json:"short_id"`
	Time    time.Time `json:"time"`
}

// offsiteGuard is the PURE decision: from all snapshots and the policy's remove-plan, either the ids to
// remove (oldest first, at most maxRemove) or a refusal reason.
func offsiteGuard(all, plan []guardSnap, now, newestAllowed time.Time, maxRemove int) ([]string, string) {
	for _, s := range all {
		if s.Time.After(now.Add(offsiteGuardSkew)) {
			return nil, fmt.Sprintf("snapshot %s is dated in the future (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339))
		}
		if !newestAllowed.IsZero() && s.Time.After(newestAllowed.Add(offsiteGuardSkew)) {
			return nil, fmt.Sprintf("snapshot %s (%s) is newer than the hub allows (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339), newestAllowed.UTC().Format(time.RFC3339))
		}
	}
	for _, s := range plan {
		if now.Sub(s.Time) < offsiteGuardMinAge {
			return nil, fmt.Sprintf("the policy would remove snapshot %s from %s — younger than %d days, which honest retention never does",
				s.ShortID, s.Time.UTC().Format(time.RFC3339), int(offsiteGuardMinAge.Hours()/24))
		}
	}
	sorted := append([]guardSnap{}, plan...)
	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Time.Before(sorted[j].Time) })
	if maxRemove >= 0 && len(sorted) > maxRemove {
		sorted = sorted[:maxRemove]
	}
	ids := make([]string, 0, len(sorted))
	for _, s := range sorted {
		ids = append(ids, s.ID)
	}
	return ids, ""
}

func (m *Manager) listGuardSnaps(ctx context.Context, base, env []string) ([]guardSnap, error) {
	sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
	defer cancel()
	out, err := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "--json")...)
	if err != nil {
		return nil, fmt.Errorf("list snapshots: %w: %s", err, truncate(out))
	}
	var snaps []guardSnap
	if err := json.Unmarshal(out, &snaps); err != nil {
		return nil, fmt.Errorf("parse snapshots: %w", err)
	}
	return snaps, nil
}

func (m *Manager) planRemovals(ctx context.Context, base, env []string) ([]guardSnap, error) {
	pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
	defer cancel()
	args := append(append(append([]string{}, base...), "forget"), retentionPolicy...)
	args = append(args, "--dry-run", "--json")
	out, err := m.runner()(pctx, env, args...)
	if err != nil {
		return nil, fmt.Errorf("forget --dry-run: %w: %s", err, truncate(out))
	}
	// restic 0.14.0 prints the JSON array on stdout; the runner may combine stderr — take the array.
	js := string(out)
	if i := strings.Index(js, "["); i > 0 {
		js = js[i:]
	}
	var groups []struct {
		Remove []guardSnap `json:"remove"`
	}
	if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &groups); err != nil {
		return nil, fmt.Errorf("parse forget plan: %w", err)
	}
	var plan []guardSnap
	for _, g := range groups {
		plan = append(plan, g.Remove...)
	}
	return plan, nil
}

// offsiteWindowRetention is the ONE retention step for both callers (after a run, over quota).
func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string, why string) {
	t := m.settings.GetOffboxTarget()
	if !t.Pinned() {
		// The household's own SFTP NAS: the box prunes as it always did (SP-2 policy).
		fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
		defer cancel()
		args := append(append([]string{"forget"}, retentionPolicy...), "--prune")
		if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
			m.logger.Printf("[WARN] [offbox] forget --prune failed (%s; backups are safe): %v: %s", why, ferr, truncate(out))
		}
		return
	}
	if m.offsiteWindow == nil {
		m.logger.Printf("[INFO] [offbox] retention skipped (%s): the off-site key is append-only and no clean-up window client is wired — nothing deleted (decision 68)", why)
		return
	}
	snaps, err := m.listGuardSnaps(ctx, base, env)
	if err != nil {
		m.logger.Printf("[WARN] [offbox] retention skipped (%s): %v", why, err)
		return
	}
	w, err := m.offsiteWindow.Open(ctx, len(snaps))
	if err != nil {
		m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err)
		return
	}
	if !w.Granted {
		m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason)
		return
	}
	start := time.Now()
	res := OffsiteWindowResult{ID: w.ID, CountBefore: len(snaps), CountAfter: len(snaps)}
	defer func() {
		cctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
		defer cancel()
		if cerr := m.offsiteWindow.Close(cctx, res); cerr != nil {
			m.logger.Printf("[WARN] [offbox] closing clean-up window %d with the hub failed (the hub closes it by itself in 20 min): %v", w.ID, cerr)
		}
	}()
	plan, err := m.planRemovals(ctx, base, env)
	if err != nil {
		res.Outcome, res.Reason = "error", err.Error()
		m.logger.Printf("[WARN] [offbox] clean-up window %d: %v", w.ID, err)
		return
	}
	ids, refuse := offsiteGuard(snaps, plan, time.Now(), w.NewestAllowed, w.MaxRemove)
	if refuse != "" {
		res.Outcome, res.Reason = "guard-refused", refuse
		m.logger.Printf("[ERROR] [offbox] clean-up window %d: the fake-snapshot guard REFUSED — nothing deleted: %s (R-822)", w.ID, refuse)
		return
	}
	if len(ids) == 0 {
		res.Outcome = "nothing"
		m.logger.Printf("[INFO] [offbox] clean-up window %d: the policy removes nothing", w.ID)
		return
	}
	fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
	defer cancel()
	args := append(append([]string{"forget"}, ids...), "--prune")
	if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
		res.Outcome, res.Reason = "error", truncate(out)
		m.logger.Printf("[WARN] [offbox] clean-up window %d: forget --prune failed (backups are safe): %v: %s", w.ID, ferr, truncate(out))
	} else {
		res.Outcome = "pruned"
	}
	if after, lerr := m.listGuardSnaps(ctx, base, env); lerr == nil {
		res.CountAfter = len(after)
	}
	res.Removed = res.CountBefore - res.CountAfter
	m.logger.Printf("[INFO] [offbox] clean-up window %d (%s): %d of %d planned snapshot(s) removed, %d -> %d, in %s",
		w.ID, why, res.Removed, len(plan), res.CountBefore, res.CountAfter, time.Since(start).Round(time.Second))
}