v0.289.0: off-site key cannot delete — append-only rclone transport, box sends only its public key (hub registrar), retention only inside a hub window behind the fake-snapshot guard (decisions 68-69, R-820, R-822)
gates / gates (push) Successful in 27s
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -113,6 +113,10 @@ type Manager struct {
|
||||
// offboxSSH (v0.142.0) is the raw-ssh exec seam for the orphaned-repo move-aside (restic has no
|
||||
// rename); tests inject a fake. Nil → the real ssh invocation (defaultOffboxSSH).
|
||||
offboxSSH func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)
|
||||
// offsiteMoveAside (v0.289.0, decision 69) asks the hub to set the repository aside.
|
||||
offsiteMoveAside func(ctx context.Context) (string, error)
|
||||
// offsiteWindow (v0.289.0, decision 68) asks the hub for a clean-up window. nil → no box retention.
|
||||
offsiteWindow OffsiteWindowClient
|
||||
|
||||
// offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable
|
||||
// in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb).
|
||||
|
||||
@@ -69,6 +69,11 @@ func (m *Manager) SetOffboxOrphanEvent(fn func(eventType, renamedTo string)) {
|
||||
}
|
||||
|
||||
// SetOffboxSSH overrides the raw-ssh exec used for the orphaned-repo move-aside (tests).
|
||||
// SetOffsiteMoveAside wires the hub's move-aside (decision 69): the orphan reset asks the hub.
|
||||
func (m *Manager) SetOffsiteMoveAside(fn func(ctx context.Context) (string, error)) {
|
||||
m.offsiteMoveAside = fn
|
||||
}
|
||||
|
||||
func (m *Manager) SetOffboxSSH(fn func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)) {
|
||||
m.offboxSSH = fn
|
||||
}
|
||||
@@ -208,7 +213,8 @@ func ClassifyOffsiteFailure(err error) OffsiteFailureClass {
|
||||
strings.Contains(s, "no route to host"), strings.Contains(s, "i/o timeout"),
|
||||
strings.Contains(s, "timed out"), strings.Contains(s, "permission denied"),
|
||||
strings.Contains(s, "host key"), strings.Contains(s, "handshake"),
|
||||
strings.Contains(s, "could not resolve"), strings.Contains(s, "network is unreachable"):
|
||||
strings.Contains(s, "could not resolve"), strings.Contains(s, "network is unreachable"),
|
||||
strings.Contains(s, "error talking http to rclone"): // measured: the rclone: backend's dead-ssh shape
|
||||
return OffsiteFailTransport
|
||||
default:
|
||||
return OffsiteFailUnknown
|
||||
@@ -311,27 +317,44 @@ func (m *Manager) resetOrphanedRepo(ctx context.Context, base, env []string, rea
|
||||
if t == nil {
|
||||
return fmt.Errorf("no offsite target configured")
|
||||
}
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
|
||||
date := time.Now().UTC().Format("20060102")
|
||||
base1 := t.RepoPath + ".orphaned-" + date
|
||||
newPath := base1
|
||||
for i := 2; i <= 20; i++ {
|
||||
// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
|
||||
// error also lands here; we then just try the mv and let it fail loudly rather than loop.
|
||||
out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
|
||||
if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
|
||||
break // absent (test -e exit 1) → free name
|
||||
var newPath string
|
||||
if t.Pinned() {
|
||||
// v0.289.0 (decision 69): the HUB sets the repository aside — the box's append-only key reaches only
|
||||
// the pinned rclone server and cannot rename a directory. The hub renames, never deletes, and picks a
|
||||
// name that never overwrites an earlier set-aside copy (`<repo>.orphaned-<date>[-n]`).
|
||||
if m.offsiteMoveAside == nil {
|
||||
return fmt.Errorf("offbox move-aside: the hub's key registrar is not configured on this box")
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): asking the hub to set %s aside, then re-init", reason, t.RepoPath)
|
||||
np, err := m.offsiteMoveAside(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w", err)
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] the hub set the orphaned repo aside: %s -> %s (nothing deleted)", t.RepoPath, newPath)
|
||||
newPath = np
|
||||
} else {
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
|
||||
date := time.Now().UTC().Format("20060102")
|
||||
base1 := t.RepoPath + ".orphaned-" + date
|
||||
newPath = base1
|
||||
for i := 2; i <= 20; i++ {
|
||||
// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
|
||||
// error also lands here; we then just try the mv and let it fail loudly rather than loop.
|
||||
out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
|
||||
if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
|
||||
break // absent (test -e exit 1) → free name
|
||||
}
|
||||
newPath = fmt.Sprintf("%s-%d", base1, i)
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
|
||||
if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
|
||||
fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
|
||||
}
|
||||
newPath = fmt.Sprintf("%s-%d", base1, i)
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
|
||||
if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
|
||||
fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
|
||||
}
|
||||
// Fresh init under the current passphrase.
|
||||
ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
@@ -628,6 +651,10 @@ func (m *Manager) ApplyOffsiteTarget(ctx context.Context, tgt *settings.OffboxTa
|
||||
if tgt.EscrowState != "escrowed" {
|
||||
tgt.EscrowState = "pending"
|
||||
}
|
||||
// Decision 69 (v0.289.0): a HUB-provisioned target is reached only through the append-only key the
|
||||
// registrar pinned — the apply-bridge proved that before calling here. The household's own NAS
|
||||
// (offboxConfigHandler) never comes through this function and stays Transport "".
|
||||
tgt.Transport = settings.TransportRclonePinned
|
||||
if err := m.settings.SetOffboxTarget(tgt); err != nil {
|
||||
return fmt.Errorf("apply offsite target: %w", err)
|
||||
}
|
||||
@@ -733,13 +760,32 @@ func (m *Manager) OffboxEscrowState() string {
|
||||
return t.EscrowState
|
||||
}
|
||||
|
||||
// offboxBaseArgs builds the restic global args (repo + sftp.args carrying the ConnectTimeout, key, pinned
|
||||
// known_hosts, port) and the env (RESTIC_PASSWORD_FILE). The ConnectTimeout is MANDATORY (fail-fast).
|
||||
// offboxBaseArgs builds the restic global args (the rclone: repo + rclone.program carrying the
|
||||
// ConnectTimeout, key, pinned known_hosts, port) and the env (RESTIC_PASSWORD_FILE).
|
||||
func (m *Manager) offboxBaseArgs(t *settings.OffboxTarget) ([]string, []string) {
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// v0.289.0 (decision 69, R-820): the `rclone:` backend over the box's APPEND-ONLY key. The hub's key
|
||||
// registrar pins this key in the sub-account's authorized_keys to
|
||||
// `command="rclone serve restic --stdio --append-only <repo>",restrict`, so whatever we ask for, the
|
||||
// provider runs that server: backups, restores, `check` and lock removal work; every delete is
|
||||
// refused (403) — measured on the provider 2026-10-03 (audits/offsite-append-only-2026-10-03/,
|
||||
// audits/offsite-lock-build-2026-10-03/partA: an sftp-written repo reads, extends, restores and
|
||||
// `check --read-data`s through it). restic 0.14.0 is enough; rclone is NOT needed in the image (it
|
||||
// runs on the provider). The program is split on whitespace by restic; the trailing "rclone" is the
|
||||
// remote command an UNPINNED key would run (and the pin ignores). ConnectTimeout stays MANDATORY.
|
||||
if t.Pinned() {
|
||||
if t.Port == 0 {
|
||||
port = 23 // the provider accepts OpenSSH keys on 23 only (measured: 22 refuses them)
|
||||
}
|
||||
rcloneProg := fmt.Sprintf("ssh -p %d -oBatchMode=yes -oConnectTimeout=%d -oStrictHostKeyChecking=yes -oUserKnownHostsFile=%s -oIdentitiesOnly=yes -i %s %s@%s rclone",
|
||||
port, offboxConnectTimeoutSec, m.offboxKnownHosts(), m.offboxKeyPath(), t.User, t.Host)
|
||||
return []string{"-r", "rclone:" + t.RepoPath, "-o", "rclone.program=" + rcloneProg},
|
||||
[]string{"RESTIC_PASSWORD_FILE=" + m.offboxPwPath()}
|
||||
}
|
||||
// The household's own SFTP NAS target (Transport ""): unchanged since v0.142.0.
|
||||
// restic's sftp backend connects via the `-o sftp.command` SSH invocation (the portable form across
|
||||
// restic versions — `sftp.args` is not recognized by restic 0.14). The ConnectTimeout makes a dead NAS
|
||||
// fail in ~N s (the load-bearing spike Q8 knob); StrictHostKeyChecking + a pinned known_hosts avoid
|
||||
@@ -1412,17 +1458,12 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
if firstErr != nil {
|
||||
return res, firstErr
|
||||
}
|
||||
// Retention: keep a sane window, prune the rest. SP-2: `--group-by host,tags` so an app's OLD
|
||||
// unit-only-shape snapshots share a group with its NEW enlarged shape (same <stack> tag) and age
|
||||
// out naturally — the default host,paths grouping would strand old-shape snapshots in their own
|
||||
// permanently-retained group. prune takes an EXCLUSIVE lock (the C2 stale-lock step) → resticStep.
|
||||
// Retention (decision 68, v0.289.0): the box's key is append-only, so it cannot prune on its own. It
|
||||
// asks the hub for a clean-up window; only inside one, and only past the fake-snapshot guard (R-822),
|
||||
// does it forget/prune. No window → nothing is deleted (the interim, option 3). SP-2's
|
||||
// `--group-by host,tags` policy is unchanged — it lives in offsiteWindowRetention now.
|
||||
m.offboxProgress.setPhase(OffboxPhaseRetention)
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
|
||||
m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
|
||||
}
|
||||
m.offsiteWindowRetention(ctx, base, env, "after-run")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -1778,24 +1819,16 @@ func OffboxQuotaPercent(t *settings.OffboxTarget) int {
|
||||
return pct
|
||||
}
|
||||
|
||||
// offboxPruneOnly runs ONLY the retention/prune step (the over-quota path: new backups are refused but
|
||||
// pruning must stay available — it is the only way back under the quota). Repo-ensure first so a fresh
|
||||
// target still fails loudly; errors are non-fatal (same as the regular run's prune).
|
||||
// offboxPruneOnly is the over-quota path: new backups are refused. Pruning is the only way back under the
|
||||
// quota — and since decision 68 it happens ONLY inside a hub-opened window, behind the R-822 guard. When
|
||||
// the hub grants none, nothing is deleted and the household's quota sentence stays (no delete attempt —
|
||||
// a refused delete costs ~48 s of restic retries per file and writes an index, measured).
|
||||
func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
|
||||
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune: repo unreachable: %v", rerr)
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
// SP-2: `--group-by host,tags` (mirrors runOffboxInternal's forget) so old unit-only-shape snapshots
|
||||
// age out with the enlarged shape instead of stranding in a permanently-retained host,paths group.
|
||||
fargs := append(append([]string{}, base...), "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] over-quota: prune executed (new backups refused until under quota)")
|
||||
}
|
||||
m.offsiteWindowRetention(ctx, base, env, "over-quota")
|
||||
}
|
||||
|
||||
// offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.
|
||||
|
||||
@@ -192,6 +192,19 @@ func (m *Manager) AbandonSweep(ctx context.Context) (bool, error) {
|
||||
m.CancelAbandon("no set-aside path recorded")
|
||||
return false, fmt.Errorf("abandonment due with no recorded path")
|
||||
}
|
||||
if t.Pinned() {
|
||||
// Decision 69 (v0.289.0): the box's off-site key is append-only and cannot delete — by design,
|
||||
// so that a broken-into box cannot erase history. The set-aside copy STAYS; the operator removes
|
||||
// it (R-823). The schedule is closed so the sweep stops; nothing was deleted.
|
||||
if uerr := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.AbandonAt = "" }); uerr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] abandonment: could not close the schedule: %v", uerr)
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] abandonment DUE for %s, but the off-site key is append-only (decision 69) — NOTHING deleted; the set-aside copy stays until the operator removes it", t.AbandonRepoPath)
|
||||
if m.offboxOrphanEvent != nil {
|
||||
m.offboxOrphanEvent("offbox_abandon_deferred", t.AbandonRepoPath)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ── Decision 68 (v0.289.0): the box prunes its own repository ONLY inside a hub-opened window ──────────
|
||||
//
|
||||
// The hub-provisioned tier's key is append-only (decision 69): every delete is refused by the provider.
|
||||
// To keep retention working, the box ASKS the hub for a clean-up window after its run. The hub grants
|
||||
// one at most weekly (or on an operator's one-shot grant) by PREPENDING a deleting line for the box's own
|
||||
// key — OpenSSH uses the first matching line, measured on the provider — and closes it when the box
|
||||
// reports, or after 20 minutes on its own.
|
||||
//
|
||||
// THE FAKE-SNAPSHOT GUARD (R-822) runs BEFORE any forget, inside the window. Measured in the lab: 13
|
||||
// future-dated empty snapshots added through an add-only key make the box's own policy select EVERY real
|
||||
// snapshot for removal. So, refuse when:
|
||||
// - any snapshot is dated in the future (beyond offsiteGuardSkew), or after the hub's newest-allowed
|
||||
// bound (the moment the window opened, plus the same skew);
|
||||
// - the plan would remove a snapshot younger than offsiteGuardMinAge — the honest policy
|
||||
// (--keep-daily 7) never removes the newest snapshot of any of the last 7 days, while a poisoning
|
||||
// shape does exactly that.
|
||||
// And bound the damage of anything the guard cannot see: at most MaxRemove (the hub's number) snapshots
|
||||
// per window, OLDEST first. DISAGREEMENT RECORDED (R-96 rule 4): the brief asked to ABORT when the plan
|
||||
// exceeds a week's removal; the first window after the interim legitimately exceeds it (weeks of
|
||||
// unpruned history), so an abort would never prune at all. Capping and taking the oldest gives the
|
||||
// same bound on loss per window and still converges.
|
||||
//
|
||||
// The NAS tier (Transport "") is unchanged: the household's own disk, pruned by the box as before.
|
||||
//
|
||||
// Pinned by TestOffsiteGuard_* (offbox_window_test.go), including the lab's 13-fake shape.
|
||||
|
||||
const (
|
||||
offsiteGuardSkew = time.Hour
|
||||
offsiteGuardMinAge = 8 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// OffsiteWindow is the hub's answer to "may I prune now?".
|
||||
type OffsiteWindow struct {
|
||||
Granted bool
|
||||
ID int64
|
||||
NewestAllowed time.Time
|
||||
MaxRemove int
|
||||
Reason string // why not granted (logged)
|
||||
}
|
||||
|
||||
// OffsiteWindowResult is what the box reports when it is done (the hub closes the window on it).
|
||||
type OffsiteWindowResult struct {
|
||||
ID int64 `json:"window_id"`
|
||||
CountBefore int `json:"count_before"`
|
||||
CountAfter int `json:"count_after"`
|
||||
Removed int `json:"removed"`
|
||||
Outcome string `json:"outcome"` // pruned | nothing | guard-refused | error
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// OffsiteWindowClient is the hub side (offsiteapply.HubWindowClient in production).
|
||||
type OffsiteWindowClient interface {
|
||||
Open(ctx context.Context, countBefore int) (OffsiteWindow, error)
|
||||
Close(ctx context.Context, r OffsiteWindowResult) error
|
||||
}
|
||||
|
||||
// SetOffsiteWindowClient wires the hub's window (decision 68). nil → no box-side retention on the pinned tier.
|
||||
func (m *Manager) SetOffsiteWindowClient(c OffsiteWindowClient) { m.offsiteWindow = c }
|
||||
|
||||
// retentionPolicy is the ruled policy, unchanged since SP-2 (`--group-by host,tags`).
|
||||
var retentionPolicy = []string{"--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6"}
|
||||
|
||||
type guardSnap struct {
|
||||
ID string `json:"id"`
|
||||
ShortID string `json:"short_id"`
|
||||
Time time.Time `json:"time"`
|
||||
}
|
||||
|
||||
// offsiteGuard is the PURE decision: from all snapshots and the policy's remove-plan, either the ids to
|
||||
// remove (oldest first, at most maxRemove) or a refusal reason.
|
||||
func offsiteGuard(all, plan []guardSnap, now, newestAllowed time.Time, maxRemove int) ([]string, string) {
|
||||
for _, s := range all {
|
||||
if s.Time.After(now.Add(offsiteGuardSkew)) {
|
||||
return nil, fmt.Sprintf("snapshot %s is dated in the future (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339))
|
||||
}
|
||||
if !newestAllowed.IsZero() && s.Time.After(newestAllowed.Add(offsiteGuardSkew)) {
|
||||
return nil, fmt.Sprintf("snapshot %s (%s) is newer than the hub allows (%s)", s.ShortID, s.Time.UTC().Format(time.RFC3339), newestAllowed.UTC().Format(time.RFC3339))
|
||||
}
|
||||
}
|
||||
for _, s := range plan {
|
||||
if now.Sub(s.Time) < offsiteGuardMinAge {
|
||||
return nil, fmt.Sprintf("the policy would remove snapshot %s from %s — younger than %d days, which honest retention never does",
|
||||
s.ShortID, s.Time.UTC().Format(time.RFC3339), int(offsiteGuardMinAge.Hours()/24))
|
||||
}
|
||||
}
|
||||
sorted := append([]guardSnap{}, plan...)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].Time.Before(sorted[j].Time) })
|
||||
if maxRemove >= 0 && len(sorted) > maxRemove {
|
||||
sorted = sorted[:maxRemove]
|
||||
}
|
||||
ids := make([]string, 0, len(sorted))
|
||||
for _, s := range sorted {
|
||||
ids = append(ids, s.ID)
|
||||
}
|
||||
return ids, ""
|
||||
}
|
||||
|
||||
func (m *Manager) listGuardSnaps(ctx context.Context, base, env []string) ([]guardSnap, error) {
|
||||
sctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
out, err := m.runner()(sctx, env, append(append([]string{}, base...), "snapshots", "--json")...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list snapshots: %w: %s", err, truncate(out))
|
||||
}
|
||||
var snaps []guardSnap
|
||||
if err := json.Unmarshal(out, &snaps); err != nil {
|
||||
return nil, fmt.Errorf("parse snapshots: %w", err)
|
||||
}
|
||||
return snaps, nil
|
||||
}
|
||||
|
||||
func (m *Manager) planRemovals(ctx context.Context, base, env []string) ([]guardSnap, error) {
|
||||
pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
defer cancel()
|
||||
args := append(append(append([]string{}, base...), "forget"), retentionPolicy...)
|
||||
args = append(args, "--dry-run", "--json")
|
||||
out, err := m.runner()(pctx, env, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("forget --dry-run: %w: %s", err, truncate(out))
|
||||
}
|
||||
// restic 0.14.0 prints the JSON array on stdout; the runner may combine stderr — take the array.
|
||||
js := string(out)
|
||||
if i := strings.Index(js, "["); i > 0 {
|
||||
js = js[i:]
|
||||
}
|
||||
var groups []struct {
|
||||
Remove []guardSnap `json:"remove"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(strings.TrimSpace(js)), &groups); err != nil {
|
||||
return nil, fmt.Errorf("parse forget plan: %w", err)
|
||||
}
|
||||
var plan []guardSnap
|
||||
for _, g := range groups {
|
||||
plan = append(plan, g.Remove...)
|
||||
}
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
// offsiteWindowRetention is the ONE retention step for both callers (after a run, over quota).
|
||||
func (m *Manager) offsiteWindowRetention(ctx context.Context, base, env []string, why string) {
|
||||
t := m.settings.GetOffboxTarget()
|
||||
if !t.Pinned() {
|
||||
// The household's own SFTP NAS: the box prunes as it always did (SP-2 policy).
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
args := append(append([]string{"forget"}, retentionPolicy...), "--prune")
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] forget --prune failed (%s; backups are safe): %v: %s", why, ferr, truncate(out))
|
||||
}
|
||||
return
|
||||
}
|
||||
if m.offsiteWindow == nil {
|
||||
m.logger.Printf("[INFO] [offbox] retention skipped (%s): the off-site key is append-only and no clean-up window client is wired — nothing deleted (decision 68)", why)
|
||||
return
|
||||
}
|
||||
snaps, err := m.listGuardSnaps(ctx, base, env)
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] retention skipped (%s): %v", why, err)
|
||||
return
|
||||
}
|
||||
w, err := m.offsiteWindow.Open(ctx, len(snaps))
|
||||
if err != nil {
|
||||
m.logger.Printf("[WARN] [offbox] retention skipped (%s): asking the hub for a window failed: %v", why, err)
|
||||
return
|
||||
}
|
||||
if !w.Granted {
|
||||
m.logger.Printf("[INFO] [offbox] retention skipped (%s): no clean-up window now (%s) — nothing deleted (decision 68)", why, w.Reason)
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
res := OffsiteWindowResult{ID: w.ID, CountBefore: len(snaps), CountAfter: len(snaps)}
|
||||
defer func() {
|
||||
cctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
if cerr := m.offsiteWindow.Close(cctx, res); cerr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] closing clean-up window %d with the hub failed (the hub closes it by itself in 20 min): %v", w.ID, cerr)
|
||||
}
|
||||
}()
|
||||
plan, err := m.planRemovals(ctx, base, env)
|
||||
if err != nil {
|
||||
res.Outcome, res.Reason = "error", err.Error()
|
||||
m.logger.Printf("[WARN] [offbox] clean-up window %d: %v", w.ID, err)
|
||||
return
|
||||
}
|
||||
ids, refuse := offsiteGuard(snaps, plan, time.Now(), w.NewestAllowed, w.MaxRemove)
|
||||
if refuse != "" {
|
||||
res.Outcome, res.Reason = "guard-refused", refuse
|
||||
m.logger.Printf("[ERROR] [offbox] clean-up window %d: the fake-snapshot guard REFUSED — nothing deleted: %s (R-822)", w.ID, refuse)
|
||||
return
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
res.Outcome = "nothing"
|
||||
m.logger.Printf("[INFO] [offbox] clean-up window %d: the policy removes nothing", w.ID)
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
args := append(append([]string{"forget"}, ids...), "--prune")
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", args...); ferr != nil {
|
||||
res.Outcome, res.Reason = "error", truncate(out)
|
||||
m.logger.Printf("[WARN] [offbox] clean-up window %d: forget --prune failed (backups are safe): %v: %s", w.ID, ferr, truncate(out))
|
||||
} else {
|
||||
res.Outcome = "pruned"
|
||||
}
|
||||
if after, lerr := m.listGuardSnaps(ctx, base, env); lerr == nil {
|
||||
res.CountAfter = len(after)
|
||||
}
|
||||
res.Removed = res.CountBefore - res.CountAfter
|
||||
m.logger.Printf("[INFO] [offbox] clean-up window %d (%s): %d of %d planned snapshot(s) removed, %d -> %d, in %s",
|
||||
w.ID, why, res.Removed, len(plan), res.CountBefore, res.CountAfter, time.Since(start).Round(time.Second))
|
||||
}
|
||||
@@ -0,0 +1,237 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/settings"
|
||||
)
|
||||
|
||||
// ── decision 68/69 (v0.289.0): the append-only tier ─────────────────────────────────────────────────
|
||||
|
||||
func pinTarget(t *testing.T, sett *settings.Settings) {
|
||||
t.Helper()
|
||||
if err := sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||||
o.Transport = settings.TransportRclonePinned
|
||||
o.Port = 23
|
||||
o.Host, o.User, o.RepoPath = "u1-sub4.example", "u1-sub4", "/home/felhom-repo"
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func snapJSON(s []guardSnap) []byte { b, _ := json.Marshal(s); return b }
|
||||
|
||||
func planJSON(remove []guardSnap) []byte {
|
||||
b, _ := json.Marshal([]map[string]any{{"tags": []string{"app1"}, "remove": remove}})
|
||||
return b
|
||||
}
|
||||
|
||||
// windowRunner fakes restic for the retention step: snapshots, the dry-run plan, and records every
|
||||
// non-dry-run forget (the only call that deletes).
|
||||
type windowRunner struct {
|
||||
snaps []guardSnap
|
||||
plan []guardSnap
|
||||
forgets [][]string
|
||||
}
|
||||
|
||||
func (w *windowRunner) run(_ context.Context, _ []string, args ...string) ([]byte, error) {
|
||||
switch {
|
||||
case contains(args, "forget") && contains(args, "--dry-run"):
|
||||
return planJSON(w.plan), nil
|
||||
case contains(args, "forget"):
|
||||
w.forgets = append(w.forgets, append([]string{}, args...))
|
||||
return nil, nil
|
||||
case contains(args, "snapshots"):
|
||||
return snapJSON(w.snaps), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakeWindow struct {
|
||||
grant OffsiteWindow
|
||||
opened int
|
||||
closed []OffsiteWindowResult
|
||||
}
|
||||
|
||||
func (f *fakeWindow) Open(context.Context, int) (OffsiteWindow, error) {
|
||||
f.opened++
|
||||
return f.grant, nil
|
||||
}
|
||||
func (f *fakeWindow) Close(_ context.Context, r OffsiteWindowResult) error {
|
||||
f.closed = append(f.closed, r)
|
||||
return nil
|
||||
}
|
||||
|
||||
func snap(id string, at time.Time) guardSnap {
|
||||
return guardSnap{ID: id + "-full", ShortID: id, Time: at}
|
||||
}
|
||||
|
||||
// The pinned transport: rclone over port 23, the pinned key; never sftp.
|
||||
func TestOffboxBaseArgs_PinnedUsesRcloneOnPort23(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.Port = 0 })
|
||||
args, _ := m.offboxBaseArgs(sett.GetOffboxTarget())
|
||||
j := strings.Join(args, " ")
|
||||
if !strings.Contains(j, "-r rclone:/home/felhom-repo") || !strings.Contains(j, "rclone.program=ssh -p 23 ") || strings.Contains(j, "sftp") {
|
||||
t.Fatalf("pinned args = %q", j)
|
||||
}
|
||||
if !argsContainTimeout(args) {
|
||||
t.Fatal("ConnectTimeout lost on the pinned transport")
|
||||
}
|
||||
// The household's NAS stays SFTP.
|
||||
m2, sett2 := newOffboxManager(t)
|
||||
if j2 := strings.Join(func() []string { a, _ := m2.offboxBaseArgs(sett2.GetOffboxTarget()); return a }(), " "); !strings.Contains(j2, "sftp:") {
|
||||
t.Fatalf("NAS args = %q", j2)
|
||||
}
|
||||
}
|
||||
|
||||
// THE CONSEQUENCE: on the pinned tier, a run with no window deletes NOTHING — no forget reaches restic.
|
||||
// RED-PROOF: the pre-v0.289.0 retention ran `forget … --prune` unconditionally after every run.
|
||||
func TestRetention_PinnedWithoutWindowDeletesNothing(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
wr := &windowRunner{snaps: []guardSnap{snap("a", time.Now().Add(-40*24*time.Hour))}, plan: []guardSnap{snap("a", time.Now().Add(-40*24*time.Hour))}}
|
||||
m.SetOffboxRunner(wr.run)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run") // no client wired
|
||||
fw := &fakeWindow{grant: OffsiteWindow{Granted: false, Reason: "not due"}}
|
||||
m.SetOffsiteWindowClient(fw)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(wr.forgets) != 0 {
|
||||
t.Fatalf("a forget ran without a window: %v", wr.forgets)
|
||||
}
|
||||
if fw.opened != 1 || len(fw.closed) != 0 {
|
||||
t.Fatalf("opened=%d closed=%d", fw.opened, len(fw.closed))
|
||||
}
|
||||
}
|
||||
|
||||
// The full run path: RunOffboxBackup on a pinned target with no window never calls forget.
|
||||
func TestRunOffboxBackup_PinnedNeverForgetsWithoutWindow(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
var forgets int
|
||||
m.SetOffboxRunner(func(ctx context.Context, env []string, args ...string) ([]byte, error) {
|
||||
if contains(args, "forget") {
|
||||
forgets++
|
||||
}
|
||||
rr := &recordingOffboxRunner{}
|
||||
return rr.run(ctx, env, args...)
|
||||
})
|
||||
_ = m.RunOffboxBackup(context.Background())
|
||||
if forgets != 0 {
|
||||
t.Fatalf("the pinned run reached forget %d time(s)", forgets)
|
||||
}
|
||||
}
|
||||
|
||||
// R-822, the lab's shape: 13 future-dated fakes. The guard REFUSES and nothing is deleted; the hub
|
||||
// is told why (window closed with outcome guard-refused).
|
||||
func TestOffsiteGuard_LabThirteenFutureFakes_Refused(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
now := time.Now()
|
||||
real := []guardSnap{snap("r1", now.Add(-2*time.Hour)), snap("r2", now.Add(-26*time.Hour)), snap("r3", now.Add(-50*time.Hour))}
|
||||
all := append([]guardSnap{}, real...)
|
||||
for d := 1; d <= 7; d++ {
|
||||
all = append(all, snap(fmt.Sprintf("f%d", d), time.Date(2027, 1, d, 3, 0, 0, 0, time.UTC)))
|
||||
}
|
||||
for mth := 2; mth <= 7; mth++ {
|
||||
all = append(all, snap(fmt.Sprintf("m%d", mth), time.Date(2027, time.Month(mth), 15, 3, 0, 0, 0, time.UTC)))
|
||||
}
|
||||
wr := &windowRunner{snaps: all, plan: real} // the poisoned policy selects every REAL snapshot
|
||||
m.SetOffboxRunner(wr.run)
|
||||
fw := &fakeWindow{grant: OffsiteWindow{Granted: true, ID: 7, NewestAllowed: now, MaxRemove: 50}}
|
||||
m.SetOffsiteWindowClient(fw)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(wr.forgets) != 0 {
|
||||
t.Fatalf("the guard let a poisoned plan delete: %v", wr.forgets)
|
||||
}
|
||||
if len(fw.closed) != 1 || fw.closed[0].Outcome != "guard-refused" || !strings.Contains(fw.closed[0].Reason, "future") {
|
||||
t.Fatalf("window close = %+v", fw.closed)
|
||||
}
|
||||
}
|
||||
|
||||
// Past-dated fakes that make the policy drop a RECENT real snapshot: refused too.
|
||||
func TestOffsiteGuard_RecentRemovalRefused(t *testing.T) {
|
||||
now := time.Now()
|
||||
all := []guardSnap{snap("old", now.Add(-60*24*time.Hour)), snap("recent", now.Add(-3*24*time.Hour))}
|
||||
_, why := offsiteGuard(all, []guardSnap{snap("recent", now.Add(-3*24*time.Hour))}, now, now, 50)
|
||||
if !strings.Contains(why, "younger than 8 days") {
|
||||
t.Fatalf("why = %q", why)
|
||||
}
|
||||
_, why = offsiteGuard(all, nil, now, now.Add(-10*24*time.Hour), 50)
|
||||
if !strings.Contains(why, "newer than the hub allows") {
|
||||
t.Fatalf("newest-allowed bound not enforced: %q", why)
|
||||
}
|
||||
}
|
||||
|
||||
// Honest retention inside a window: the oldest first, at most MaxRemove, and the forget names ids.
|
||||
func TestOffsiteGuard_HonestPlanPrunesOldestFirstCapped(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
now := time.Now()
|
||||
plan := []guardSnap{snap("c", now.Add(-20*24*time.Hour)), snap("a", now.Add(-90*24*time.Hour)), snap("b", now.Add(-60*24*time.Hour))}
|
||||
all := append([]guardSnap{snap("keep", now.Add(-time.Hour))}, plan...)
|
||||
wr := &windowRunner{snaps: all, plan: plan}
|
||||
m.SetOffboxRunner(wr.run)
|
||||
fw := &fakeWindow{grant: OffsiteWindow{Granted: true, ID: 9, NewestAllowed: now, MaxRemove: 2}}
|
||||
m.SetOffsiteWindowClient(fw)
|
||||
m.offsiteWindowRetention(context.Background(), nil, nil, "after-run")
|
||||
if len(wr.forgets) != 1 {
|
||||
t.Fatalf("forgets = %v", wr.forgets)
|
||||
}
|
||||
got := strings.Join(wr.forgets[0], " ")
|
||||
if !strings.Contains(got, "forget a-full b-full --prune") || strings.Contains(got, "c-full") {
|
||||
t.Fatalf("forget = %q (want the two OLDEST, capped)", got)
|
||||
}
|
||||
if len(fw.closed) != 1 || fw.closed[0].Outcome != "pruned" || fw.closed[0].ID != 9 {
|
||||
t.Fatalf("close = %+v", fw.closed)
|
||||
}
|
||||
}
|
||||
|
||||
// The orphan reset on the pinned tier asks the HUB; no ssh `mv` from the box.
|
||||
func TestResetOrphaned_PinnedAsksTheHub(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
m.SetOffboxSSH(func(context.Context, string, string, int, string, string, string) ([]byte, error) {
|
||||
t.Fatal("the box issued a raw ssh command on the pinned tier")
|
||||
return nil, nil
|
||||
})
|
||||
called := 0
|
||||
m.SetOffsiteMoveAside(func(context.Context) (string, error) { called++; return "/home/felhom-repo.orphaned-20261003", nil })
|
||||
m.SetOffboxRunner(func(context.Context, []string, ...string) ([]byte, error) { return nil, nil })
|
||||
if err := m.resetOrphanedRepo(context.Background(), nil, nil, "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if called != 1 || sett.GetOffboxTarget().OrphanedRenamedTo != "/home/felhom-repo.orphaned-20261003" {
|
||||
t.Fatalf("hub move-aside called=%d recorded=%q", called, sett.GetOffboxTarget().OrphanedRenamedTo)
|
||||
}
|
||||
}
|
||||
|
||||
// Abandonment on the pinned tier: due → nothing deleted, the operator is told, the sweep goes quiet.
|
||||
func TestAbandon_PinnedDefersToOperator(t *testing.T) {
|
||||
m, sett := newOffboxManager(t)
|
||||
pinTarget(t, sett)
|
||||
sett.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
|
||||
o.AbandonRepoPath = "/home/felhom-repo.orphaned-20260901"
|
||||
o.AbandonStartedAt = time.Now().Add(-20 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
||||
o.AbandonAt = time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)
|
||||
})
|
||||
m.SetOffboxSSH(func(context.Context, string, string, int, string, string, string) ([]byte, error) {
|
||||
t.Fatal("the box tried to delete on the pinned tier")
|
||||
return nil, nil
|
||||
})
|
||||
var evs []string
|
||||
m.SetOffboxOrphanEvent(func(e, _ string) { evs = append(evs, e) })
|
||||
deleted, err := m.AbandonSweep(context.Background())
|
||||
if deleted || err != nil || len(evs) != 1 || evs[0] != "offbox_abandon_deferred" {
|
||||
t.Fatalf("deleted=%v err=%v events=%v", deleted, err, evs)
|
||||
}
|
||||
if again, _ := m.AbandonSweep(context.Background()); again {
|
||||
t.Fatal("second sweep deleted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user