v0.289.0: off-site key cannot delete — append-only rclone transport, box sends only its public key (hub registrar), retention only inside a hub window behind the fake-snapshot guard (decisions 68-69, R-820, R-822)
gates / gates (push) Successful in 27s
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -69,6 +69,11 @@ func (m *Manager) SetOffboxOrphanEvent(fn func(eventType, renamedTo string)) {
|
||||
}
|
||||
|
||||
// SetOffboxSSH overrides the raw-ssh exec used for the orphaned-repo move-aside (tests).
|
||||
// SetOffsiteMoveAside wires the hub's move-aside (decision 69): the orphan reset asks the hub.
|
||||
func (m *Manager) SetOffsiteMoveAside(fn func(ctx context.Context) (string, error)) {
|
||||
m.offsiteMoveAside = fn
|
||||
}
|
||||
|
||||
func (m *Manager) SetOffboxSSH(fn func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)) {
|
||||
m.offboxSSH = fn
|
||||
}
|
||||
@@ -208,7 +213,8 @@ func ClassifyOffsiteFailure(err error) OffsiteFailureClass {
|
||||
strings.Contains(s, "no route to host"), strings.Contains(s, "i/o timeout"),
|
||||
strings.Contains(s, "timed out"), strings.Contains(s, "permission denied"),
|
||||
strings.Contains(s, "host key"), strings.Contains(s, "handshake"),
|
||||
strings.Contains(s, "could not resolve"), strings.Contains(s, "network is unreachable"):
|
||||
strings.Contains(s, "could not resolve"), strings.Contains(s, "network is unreachable"),
|
||||
strings.Contains(s, "error talking http to rclone"): // measured: the rclone: backend's dead-ssh shape
|
||||
return OffsiteFailTransport
|
||||
default:
|
||||
return OffsiteFailUnknown
|
||||
@@ -311,27 +317,44 @@ func (m *Manager) resetOrphanedRepo(ctx context.Context, base, env []string, rea
|
||||
if t == nil {
|
||||
return fmt.Errorf("no offsite target configured")
|
||||
}
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
|
||||
date := time.Now().UTC().Format("20060102")
|
||||
base1 := t.RepoPath + ".orphaned-" + date
|
||||
newPath := base1
|
||||
for i := 2; i <= 20; i++ {
|
||||
// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
|
||||
// error also lands here; we then just try the mv and let it fail loudly rather than loop.
|
||||
out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
|
||||
if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
|
||||
break // absent (test -e exit 1) → free name
|
||||
var newPath string
|
||||
if t.Pinned() {
|
||||
// v0.289.0 (decision 69): the HUB sets the repository aside — the box's append-only key reaches only
|
||||
// the pinned rclone server and cannot rename a directory. The hub renames, never deletes, and picks a
|
||||
// name that never overwrites an earlier set-aside copy (`<repo>.orphaned-<date>[-n]`).
|
||||
if m.offsiteMoveAside == nil {
|
||||
return fmt.Errorf("offbox move-aside: the hub's key registrar is not configured on this box")
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): asking the hub to set %s aside, then re-init", reason, t.RepoPath)
|
||||
np, err := m.offsiteMoveAside(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w", err)
|
||||
}
|
||||
m.logger.Printf("[INFO] [offbox] the hub set the orphaned repo aside: %s -> %s (nothing deleted)", t.RepoPath, newPath)
|
||||
newPath = np
|
||||
} else {
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
|
||||
date := time.Now().UTC().Format("20060102")
|
||||
base1 := t.RepoPath + ".orphaned-" + date
|
||||
newPath = base1
|
||||
for i := 2; i <= 20; i++ {
|
||||
// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
|
||||
// error also lands here; we then just try the mv and let it fail loudly rather than loop.
|
||||
out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
|
||||
if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
|
||||
break // absent (test -e exit 1) → free name
|
||||
}
|
||||
newPath = fmt.Sprintf("%s-%d", base1, i)
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
|
||||
if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
|
||||
fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
|
||||
}
|
||||
newPath = fmt.Sprintf("%s-%d", base1, i)
|
||||
}
|
||||
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
|
||||
if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
|
||||
fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
|
||||
return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
|
||||
}
|
||||
// Fresh init under the current passphrase.
|
||||
ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
|
||||
@@ -628,6 +651,10 @@ func (m *Manager) ApplyOffsiteTarget(ctx context.Context, tgt *settings.OffboxTa
|
||||
if tgt.EscrowState != "escrowed" {
|
||||
tgt.EscrowState = "pending"
|
||||
}
|
||||
// Decision 69 (v0.289.0): a HUB-provisioned target is reached only through the append-only key the
|
||||
// registrar pinned — the apply-bridge proved that before calling here. The household's own NAS
|
||||
// (offboxConfigHandler) never comes through this function and stays Transport "".
|
||||
tgt.Transport = settings.TransportRclonePinned
|
||||
if err := m.settings.SetOffboxTarget(tgt); err != nil {
|
||||
return fmt.Errorf("apply offsite target: %w", err)
|
||||
}
|
||||
@@ -733,13 +760,32 @@ func (m *Manager) OffboxEscrowState() string {
|
||||
return t.EscrowState
|
||||
}
|
||||
|
||||
// offboxBaseArgs builds the restic global args (repo + sftp.args carrying the ConnectTimeout, key, pinned
|
||||
// known_hosts, port) and the env (RESTIC_PASSWORD_FILE). The ConnectTimeout is MANDATORY (fail-fast).
|
||||
// offboxBaseArgs builds the restic global args (the rclone: repo + rclone.program carrying the
|
||||
// ConnectTimeout, key, pinned known_hosts, port) and the env (RESTIC_PASSWORD_FILE).
|
||||
func (m *Manager) offboxBaseArgs(t *settings.OffboxTarget) ([]string, []string) {
|
||||
port := t.Port
|
||||
if port == 0 {
|
||||
port = 22
|
||||
}
|
||||
// v0.289.0 (decision 69, R-820): the `rclone:` backend over the box's APPEND-ONLY key. The hub's key
|
||||
// registrar pins this key in the sub-account's authorized_keys to
|
||||
// `command="rclone serve restic --stdio --append-only <repo>",restrict`, so whatever we ask for, the
|
||||
// provider runs that server: backups, restores, `check` and lock removal work; every delete is
|
||||
// refused (403) — measured on the provider 2026-10-03 (audits/offsite-append-only-2026-10-03/,
|
||||
// audits/offsite-lock-build-2026-10-03/partA: an sftp-written repo reads, extends, restores and
|
||||
// `check --read-data`s through it). restic 0.14.0 is enough; rclone is NOT needed in the image (it
|
||||
// runs on the provider). The program is split on whitespace by restic; the trailing "rclone" is the
|
||||
// remote command an UNPINNED key would run (and the pin ignores). ConnectTimeout stays MANDATORY.
|
||||
if t.Pinned() {
|
||||
if t.Port == 0 {
|
||||
port = 23 // the provider accepts OpenSSH keys on 23 only (measured: 22 refuses them)
|
||||
}
|
||||
rcloneProg := fmt.Sprintf("ssh -p %d -oBatchMode=yes -oConnectTimeout=%d -oStrictHostKeyChecking=yes -oUserKnownHostsFile=%s -oIdentitiesOnly=yes -i %s %s@%s rclone",
|
||||
port, offboxConnectTimeoutSec, m.offboxKnownHosts(), m.offboxKeyPath(), t.User, t.Host)
|
||||
return []string{"-r", "rclone:" + t.RepoPath, "-o", "rclone.program=" + rcloneProg},
|
||||
[]string{"RESTIC_PASSWORD_FILE=" + m.offboxPwPath()}
|
||||
}
|
||||
// The household's own SFTP NAS target (Transport ""): unchanged since v0.142.0.
|
||||
// restic's sftp backend connects via the `-o sftp.command` SSH invocation (the portable form across
|
||||
// restic versions — `sftp.args` is not recognized by restic 0.14). The ConnectTimeout makes a dead NAS
|
||||
// fail in ~N s (the load-bearing spike Q8 knob); StrictHostKeyChecking + a pinned known_hosts avoid
|
||||
@@ -1412,17 +1458,12 @@ func (m *Manager) runOffboxInternal(ctx context.Context, apps, base, env []strin
|
||||
if firstErr != nil {
|
||||
return res, firstErr
|
||||
}
|
||||
// Retention: keep a sane window, prune the rest. SP-2: `--group-by host,tags` so an app's OLD
|
||||
// unit-only-shape snapshots share a group with its NEW enlarged shape (same <stack> tag) and age
|
||||
// out naturally — the default host,paths grouping would strand old-shape snapshots in their own
|
||||
// permanently-retained group. prune takes an EXCLUSIVE lock (the C2 stale-lock step) → resticStep.
|
||||
// Retention (decision 68, v0.289.0): the box's key is append-only, so it cannot prune on its own. It
|
||||
// asks the hub for a clean-up window; only inside one, and only past the fake-snapshot guard (R-822),
|
||||
// does it forget/prune. No window → nothing is deleted (the interim, option 3). SP-2's
|
||||
// `--group-by host,tags` policy is unchanged — it lives in offsiteWindowRetention now.
|
||||
m.offboxProgress.setPhase(OffboxPhaseRetention)
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
if out, ferr := m.resticStep(fctx, env, base, "prune", "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune"); ferr != nil {
|
||||
// A prune failure is non-fatal to the backup itself (data is safe) — log, don't fail the run.
|
||||
m.logger.Printf("[WARN] [offbox] forget --prune failed (backups are safe): %v: %s", ferr, truncate(out))
|
||||
}
|
||||
m.offsiteWindowRetention(ctx, base, env, "after-run")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -1778,24 +1819,16 @@ func OffboxQuotaPercent(t *settings.OffboxTarget) int {
|
||||
return pct
|
||||
}
|
||||
|
||||
// offboxPruneOnly runs ONLY the retention/prune step (the over-quota path: new backups are refused but
|
||||
// pruning must stay available — it is the only way back under the quota). Repo-ensure first so a fresh
|
||||
// target still fails loudly; errors are non-fatal (same as the regular run's prune).
|
||||
// offboxPruneOnly is the over-quota path: new backups are refused. Pruning is the only way back under the
|
||||
// quota — and since decision 68 it happens ONLY inside a hub-opened window, behind the R-822 guard. When
|
||||
// the hub grants none, nothing is deleted and the household's quota sentence stays (no delete attempt —
|
||||
// a refused delete costs ~48 s of restic retries per file and writes an index, measured).
|
||||
func (m *Manager) offboxPruneOnly(ctx context.Context, base, env []string) {
|
||||
if rerr := m.ensureOffboxRepo(ctx, base, env); rerr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune: repo unreachable: %v", rerr)
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(ctx, offboxBackupTimeout)
|
||||
defer cancel()
|
||||
// SP-2: `--group-by host,tags` (mirrors runOffboxInternal's forget) so old unit-only-shape snapshots
|
||||
// age out with the enlarged shape instead of stranding in a permanently-retained host,paths group.
|
||||
fargs := append(append([]string{}, base...), "forget", "--group-by", "host,tags", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "6", "--prune")
|
||||
if out, ferr := m.runner()(fctx, env, fargs...); ferr != nil {
|
||||
m.logger.Printf("[WARN] [offbox] over-quota prune failed: %v: %s", ferr, truncate(out))
|
||||
} else {
|
||||
m.logger.Printf("[INFO] [offbox] over-quota: prune executed (new backups refused until under quota)")
|
||||
}
|
||||
m.offsiteWindowRetention(ctx, base, env, "over-quota")
|
||||
}
|
||||
|
||||
// offboxRecordStats reads the snapshot count (best-effort) for the UI; also fills repo size when stats works.
|
||||
|
||||
Reference in New Issue
Block a user