v0.142.0: offsite repo continuity — orphaned-repo guard (A) + run-status auto-refresh (C)

- Part A: classify restic cat-config failure (wrong-password=orphaned vs no-repo vs other); ORPHANED state + Hungarian card + offbox_repo_orphaned/reset events (once, not nightly); reset = move-aside (never delete) + init, unclaimed auto / claimed confirm. Red-proofs TestOffbox_OrphanDetection_* + ConfirmedReset.
- Part C: GET /backup/offbox/status + poll on backups_remote → flips Fut→Rendben/Hiba without manual reload.
This commit is contained in:
2026-07-17 10:47:30 +02:00
parent 1452dd2b17
commit 596505ed64
11 changed files with 587 additions and 18 deletions
+7
View File
@@ -35,6 +35,13 @@ type Manager struct {
// offbox (Part B): the restic-SFTP exec seam (nil → real restic) + the failure→operator-alert hook.
offboxRunner offboxRunner
offboxNotify func(dur time.Duration, snapshots int, err error)
// offboxOrphanEvent (v0.142.0), if set, pushes a hub event on offsite-repo continuity transitions
// ("offbox_repo_orphaned" / "offbox_repo_reset"); renamedTo names the move-aside path (reset only).
// Wired in main.go to the notifier. Nil-safe.
offboxOrphanEvent func(eventType, renamedTo string)
// offboxSSH (v0.142.0) is the raw-ssh exec seam for the orphaned-repo move-aside (restic has no
// rename); tests inject a fake. Nil → the real ssh invocation (defaultOffboxSSH).
offboxSSH func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)
// offboxSizer (3a) — the mandatory-set byte estimator for the pre-push enlargement gate, overridable
// in tests so the gate is unit-testable without a real du. Nil → the real dirSizeBytes (du -sb).
+215 -17
View File
@@ -6,6 +6,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
@@ -53,6 +54,162 @@ func (m *Manager) SetOffboxNotify(fn func(dur time.Duration, snapshots int, err
m.offboxNotify = fn
}
// SetOffboxOrphanEvent wires the offsite-repo continuity event push (main.go → notifier).
func (m *Manager) SetOffboxOrphanEvent(fn func(eventType, renamedTo string)) { m.offboxOrphanEvent = fn }
// SetOffboxSSH overrides the raw-ssh exec used for the orphaned-repo move-aside (tests).
func (m *Manager) SetOffboxSSH(fn func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error)) {
m.offboxSSH = fn
}
// ErrOffboxOrphaned is the sentinel returned when the offsite repo exists but is keyed under a
// passphrase this controller no longer has (the reinstall shape) — the run skips and the UI shows
// the orphan card instead of the raw restic error.
var ErrOffboxOrphaned = fmt.Errorf("offbox repo orphaned: exists but keyed under a previous, no-longer-available passphrase")
// classifyResticProbe maps a `restic cat config` failure to a repo class. The signatures are the exact
// restic stderr matched in the 2026-07-17 diagnosis + restic's no-repo message:
// - "orphaned": repo present, wrong key ("wrong password or no key found") — the definitive signal
// - "norepo": no repo at the location (init is the correct path)
// - "other": network/SFTP-auth/unknown — NOT orphaned; existing error handling
func classifyResticProbe(out []byte, err error) string {
if err == nil {
return "" // success — repo good
}
s := strings.ToLower(string(out))
switch {
case strings.Contains(s, "wrong password or no key found"):
return "orphaned"
case strings.Contains(s, "unable to open config file"),
strings.Contains(s, "is there a repository at the following location"),
strings.Contains(s, "no such file"),
strings.Contains(s, "does not exist"):
return "norepo"
default:
return "other"
}
}
func defaultOffboxSSH(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) {
if port == 0 {
port = 22
}
args := []string{
"-p", fmt.Sprint(port), "-oBatchMode=yes", fmt.Sprintf("-oConnectTimeout=%d", offboxConnectTimeoutSec),
"-oStrictHostKeyChecking=yes", "-oUserKnownHostsFile=" + knownHosts, "-i", keyPath,
user + "@" + host, remoteCmd,
}
cmd := exec.CommandContext(ctx, "ssh", args...)
return cmd.CombinedOutput()
}
func (m *Manager) sshRunner() func(ctx context.Context, host, user string, port int, keyPath, knownHosts, remoteCmd string) ([]byte, error) {
if m.offboxSSH != nil {
return m.offboxSSH
}
return defaultOffboxSSH
}
// OffboxOrphaned reports whether the offsite repo is in the ORPHANED state (persisted).
func (m *Manager) OffboxOrphaned() bool {
t := m.settings.GetOffboxTarget()
return t != nil && t.RepoState == "orphaned"
}
// OffboxOrphanedRenamedTo returns the last move-aside path (for the card copy; "" if none).
func (m *Manager) OffboxOrphanedRenamedTo() string {
t := m.settings.GetOffboxTarget()
if t == nil {
return ""
}
return t.OrphanedRenamedTo
}
// markOrphaned sets the persistent ORPHANED state and, ONLY on the transition into it (not already
// orphaned), pushes the offbox_repo_orphaned event — so scheduled runs never nightly-spam.
func (m *Manager) markOrphaned() {
already := m.OffboxOrphaned()
if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
o.RepoState = "orphaned"
if o.OrphanedAt == "" || !already {
o.OrphanedAt = time.Now().UTC().Format(time.RFC3339)
}
}); err != nil {
m.logger.Printf("[WARN] [offbox] persist orphaned state failed: %v", err)
}
if !already {
m.logger.Printf("[WARN] [offbox] offsite repo ORPHANED — remote holds backups written under a previous, no-longer-available key; runs will skip until reset")
if m.offboxOrphanEvent != nil {
m.offboxOrphanEvent("offbox_repo_orphaned", "")
}
}
}
// resetOrphanedRepo moves the orphaned repo aside (never deletes) and re-inits a fresh repo under the
// CURRENT passphrase. Reversible. Used by the unclaimed auto-reset (Scenario B) and the claimed
// confirmed reset (Scenario C). Caller holds the single-flight guarantee (run mutex) OR is the handler.
func (m *Manager) resetOrphanedRepo(ctx context.Context, base, env []string, reason string) error {
t := m.settings.GetOffboxTarget()
if t == nil {
return fmt.Errorf("no offsite target configured")
}
port := t.Port
if port == 0 {
port = 22
}
// Choose a move-aside name that never overwrites an earlier orphaned copy (edge rule: -2, -3).
date := time.Now().UTC().Format("20060102")
base1 := t.RepoPath + ".orphaned-" + date
newPath := base1
for i := 2; i <= 20; i++ {
// `test -e <p>` returns non-zero (exit 1) when absent — that is the name we want. A transport
// error also lands here; we then just try the mv and let it fail loudly rather than loop.
out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(), "test -e "+shellQuote(newPath))
if err != nil && !strings.Contains(strings.ToLower(string(out)), "denied") {
break // absent (test -e exit 1) → free name
}
newPath = fmt.Sprintf("%s-%d", base1, i)
}
m.logger.Printf("[WARN] [offbox] resetting orphaned repo (%s): move-aside %s -> %s, then re-init", reason, t.RepoPath, newPath)
if out, err := m.sshRunner()(ctx, t.Host, t.User, port, m.offboxKeyPath(), m.offboxKnownHosts(),
fmt.Sprintf("mv %s %s", shellQuote(t.RepoPath), shellQuote(newPath))); err != nil {
return fmt.Errorf("offbox move-aside failed: %w: %s", err, truncate(out))
}
// Fresh init under the current passphrase.
ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer icancel()
if out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...); err != nil {
return fmt.Errorf("offbox re-init after move-aside failed: %w: %s", err, truncate(out))
}
if err := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
o.RepoState = ""
o.OrphanedAt = ""
o.OrphanedRenamedTo = newPath
o.LastError = ""
}); err != nil {
m.logger.Printf("[WARN] [offbox] clear orphaned state failed: %v", err)
}
m.logger.Printf("[INFO] [offbox] orphaned repo reset complete — old history set aside at %s (move-aside, not deleted); fresh repo initialized", newPath)
if m.offboxOrphanEvent != nil {
m.offboxOrphanEvent("offbox_repo_reset", newPath)
}
return nil
}
// ResetOrphanedRepo is the handler entry point for the CLAIMED confirmed reset (Scenario C). It refuses
// unless the repo is currently orphaned. It builds the base/env and runs the move-aside + re-init.
func (m *Manager) ResetOrphanedRepo(ctx context.Context) error {
if !m.OffboxOrphaned() {
return fmt.Errorf("az offsite tároló nincs elárvult állapotban")
}
t := m.settings.GetOffboxTarget()
base, env := m.offboxBaseArgs(t)
return m.resetOrphanedRepo(ctx, base, env, "operator-confirmed (claimed)")
}
// shellQuote single-quotes a path for the remote shell (our repo paths have no single quotes).
func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" }
// SetOffboxSizer overrides the mandatory-set byte estimator (tests). SetOffboxEnlargeBlockedNotifier
// wires the edge-triggered enlargement-blocked notification (main.go). SetOffboxPlaceCopier overrides
// the place-to-live missing-only merge (tests).
@@ -368,23 +525,47 @@ func (m *Manager) resticStep(ctx context.Context, env, base []string, label stri
func (m *Manager) ensureOffboxRepo(ctx context.Context, base, env []string) error {
pctx, cancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer cancel()
if _, err := m.runner()(pctx, env, append(append([]string{}, base...), "cat", "config")...); err == nil {
return nil // repo exists
}
// Repo (probably) absent OR unreachable. Try init; if init succeeds the repo was absent. If init
// fails because it already exists (a race), treat as success; otherwise the error is real (e.g. a
// dead NAS — fail fast).
ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer icancel()
out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...)
if err == nil {
m.logger.Printf("[INFO] [offbox] initialized restic repo")
pout, perr := m.runner()(pctx, env, append(append([]string{}, base...), "cat", "config")...)
switch classifyResticProbe(pout, perr) {
case "": // success — repo good (and clear any stale orphaned flag)
if m.OffboxOrphaned() {
_ = m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) { o.RepoState = ""; o.OrphanedAt = "" })
}
return nil
case "orphaned":
// The repo EXISTS but is keyed under a passphrase we no longer have (the reinstall shape). An
// UNCLAIMED (as-delivered) box auto-resets (Scenario B); a CLAIMED box surfaces the orphan card
// and skips until the customer confirms a reset (Scenario C). Move-aside, never delete.
if !m.settings.GetClaimed() {
m.logger.Printf("[INFO] [offbox] orphaned repo on an UNCLAIMED box — auto-resetting (move-aside + re-init)")
if m.offboxOrphanEvent != nil {
m.offboxOrphanEvent("offbox_repo_orphaned", "")
}
if rerr := m.resetOrphanedRepo(ctx, base, env, "auto (unclaimed)"); rerr != nil {
m.markOrphaned() // auto-reset failed → fall back to the orphan card so it isn't silent
return ErrOffboxOrphaned
}
return nil // repo is fresh under the current passphrase → the run proceeds
}
m.markOrphaned()
return ErrOffboxOrphaned
case "norepo":
// No repo at the location → init (the normal first-run path). A race where it already exists is
// treated as success; any other init error is real (e.g. a dead NAS — fail fast).
ictx, icancel := context.WithTimeout(ctx, offboxProbeTimeout)
defer icancel()
out, err := m.runner()(ictx, env, append(append([]string{}, base...), "init")...)
if err == nil {
m.logger.Printf("[INFO] [offbox] initialized restic repo")
return nil
}
if strings.Contains(string(out), "already initialized") || strings.Contains(string(out), "already exists") {
return nil
}
return fmt.Errorf("offbox repo unreachable / init failed: %w: %s", err, truncate(out))
default: // "other" — network/SFTP-auth/unknown; NOT orphaned. Surface as before (fail fast).
return fmt.Errorf("offbox repo unreachable: %w: %s", perr, truncate(pout))
}
if strings.Contains(string(out), "already initialized") || strings.Contains(string(out), "already exists") {
return nil
}
return fmt.Errorf("offbox repo unreachable / init failed: %w: %s", err, truncate(out))
}
// RunOffboxBackup backs up every off-box-toggled app's recovery unit (recovery unit + DB dumps + volume
@@ -400,6 +581,12 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
m.logger.Printf("[INFO] [offbox] skipped — pending key escrow (no offsite run until the repo password is escrowed under R)")
return nil
}
// Offsite-repo continuity (v0.142.0): once ORPHANED, scheduled runs SKIP (the event fired on the
// detection transition — no nightly spam) until a reset clears it. The remote page shows the card.
if m.OffboxOrphaned() {
m.logger.Printf("[INFO] [offbox] skipped — offsite repo orphaned (awaiting reset)")
return nil
}
if m.migrationActive() {
m.logger.Printf("[INFO] [offbox] skipped — migration in progress")
return nil
@@ -467,7 +654,13 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
if perr := m.settings.UpdateOffboxStatus(func(o *settings.OffboxTarget) {
o.LastRun = time.Now().UTC().Format(time.RFC3339)
o.LastDuration = dur.Round(time.Second).String()
if runErr != nil {
if errors.Is(runErr, ErrOffboxOrphaned) {
// First-detection of the orphaned repo: RepoState (set by markOrphaned) drives the orphan
// card — do NOT surface the raw restic/sentinel text as the last-error banner.
o.LastStatus = "error"
o.LastError = ""
o.LastWarning = ""
} else if runErr != nil {
o.LastStatus = "error"
o.LastError = runErr.Error()
o.LastWarning = ""
@@ -503,7 +696,9 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
}); perr != nil {
m.logger.Printf("[WARN] [offbox] status persist (final) failed: %v", perr)
}
if m.offboxNotify != nil {
// The orphaned case has its OWN dedicated event (offbox_repo_orphaned) — do NOT also fire the
// generic backup-failed notification (no double/raw alert; the orphan card is the customer surface).
if m.offboxNotify != nil && !errors.Is(runErr, ErrOffboxOrphaned) {
m.offboxNotify(dur, snapshots, runErr)
}
// Edge-triggered enlarge-blocked notification: only apps that NEWLY crossed into the blocked state
@@ -518,6 +713,9 @@ func (m *Manager) RunOffboxBackup(ctx context.Context) error {
}
}
switch {
case errors.Is(runErr, ErrOffboxOrphaned):
m.logger.Printf("[WARN] [offbox] run skipped — offsite repo orphaned (card shown; awaiting reset)")
return nil // the orphaned STATE + event are the signal; not a hard run error for the scheduler
case runErr != nil:
m.logger.Printf("[ERROR] [offbox] backup failed after %s: %v", dur.Round(time.Second), runErr)
case len(missing) > 0:
@@ -0,0 +1,172 @@
package backup
import (
"context"
"fmt"
"strings"
"testing"
)
// classifyResticProbe maps the exact restic stderr to a repo class (the 2026-07-17 diagnosis
// signatures). ORPHANED only on the definitive wrong-password line; ambiguous errors are NOT orphaned.
func TestClassifyResticProbe(t *testing.T) {
cases := []struct {
out string
err error
want string
}{
{"", nil, ""}, // success
{"Fatal: wrong password or no key found", fmt.Errorf("exit status 1"), "orphaned"},
{"Fatal: unable to open config file: <sftp:...> does not exist\nIs there a repository at the following location?", fmt.Errorf("exit status 1"), "norepo"},
{"ssh: connect to host nas.local port 22: Connection timed out", fmt.Errorf("exit status 255"), "other"},
{"Load(<lock/...>): permission denied", fmt.Errorf("exit status 1"), "other"},
}
for _, c := range cases {
if got := classifyResticProbe([]byte(c.out), c.err); got != c.want {
t.Errorf("classify(%q) = %q, want %q", c.out, got, c.want)
}
}
}
// wrongPwRunner: `cat config` returns the wrong-password signature; other restic steps succeed (so a
// post-reset run can proceed). Records the subcommands seen.
func wrongPwRunner(seen *[]string) offboxRunner {
return func(_ context.Context, _ []string, args ...string) ([]byte, error) {
sub := ""
for i, a := range args {
if a == "cat" && i+1 < len(args) && args[i+1] == "config" {
sub = "cat-config"
} else if a == "init" {
sub = "init"
}
}
if sub == "" && len(args) > 0 {
sub = args[len(args)-1]
}
if seen != nil {
*seen = append(*seen, sub)
}
if sub == "cat-config" {
return []byte("Fatal: wrong password or no key found"), fmt.Errorf("exit status 1")
}
return nil, nil // init / unlock / backup / stats succeed
}
}
// Scenario A (RED-PROOF = the incident): a CLAIMED box whose repo is wrong-keyed enters the explicit
// ORPHANED state — the run skips cleanly (no raw restic banner, ONE event, no nightly re-fire) instead
// of erroring nightly with "exit status 1". Pre-fix (no classification) surfaced the raw error and set
// no state → these assertions FAIL.
func TestOffbox_OrphanDetection_Claimed(t *testing.T) {
m, sett := newOffboxManager(t)
if err := sett.SetClaimed(); err != nil { // claimed → orphan card, NEVER auto-reset
t.Fatal(err)
}
var events []string
m.SetOffboxOrphanEvent(func(evt, _ string) { events = append(events, evt) })
m.SetOffboxRunner(wrongPwRunner(nil))
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("run should skip cleanly on an orphaned repo, got %v", err)
}
if !m.OffboxOrphaned() {
t.Fatal("repo was not classified/persisted as ORPHANED")
}
if len(events) != 1 || events[0] != "offbox_repo_orphaned" {
t.Fatalf("expected exactly one offbox_repo_orphaned event, got %v", events)
}
got := sett.GetOffboxTarget()
if got.RepoState != "orphaned" || got.OrphanedAt == "" {
t.Fatalf("RepoState=%q OrphanedAt=%q, want orphaned + a stamp", got.RepoState, got.OrphanedAt)
}
// The raw restic error must NOT be surfaced as the last-error banner (the card explains instead).
if strings.Contains(got.LastError, "wrong password") || strings.Contains(got.LastError, "exit status") {
t.Fatalf("raw restic error leaked into LastError: %q", got.LastError)
}
// A second scheduled run SKIPS (no nightly spam) — no new event.
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("second run: %v", err)
}
if len(events) != 1 {
t.Fatalf("nightly re-fire — events=%v, want the single transition event only", events)
}
}
// Scenario B: an UNCLAIMED box auto-resets on detection — move-aside (never delete) + re-init; both
// events fire and the box ends un-orphaned (next run green).
func TestOffbox_OrphanDetection_UnclaimedAutoReset(t *testing.T) {
m, sett := newOffboxManager(t) // unclaimed by default
var events []string
m.SetOffboxOrphanEvent(func(evt, _ string) { events = append(events, evt) })
var sshCmds []string
m.SetOffboxSSH(func(_ context.Context, _, _ string, _ int, _, _, remoteCmd string) ([]byte, error) {
sshCmds = append(sshCmds, remoteCmd)
if strings.HasPrefix(remoteCmd, "test -e") {
return nil, fmt.Errorf("exit status 1") // absent → free name
}
return nil, nil // mv OK
})
m.SetOffboxRunner(wrongPwRunner(nil))
if err := m.RunOffboxBackup(context.Background()); err != nil {
t.Fatalf("unclaimed run should auto-reset + succeed, got %v", err)
}
if m.OffboxOrphaned() {
t.Fatal("unclaimed box stayed orphaned — auto-reset did not clear the state")
}
got := sett.GetOffboxTarget()
if got.OrphanedRenamedTo == "" || !strings.Contains(got.OrphanedRenamedTo, ".orphaned-") {
t.Fatalf("move-aside path not recorded: %q", got.OrphanedRenamedTo)
}
var mvSeen bool
for _, c := range sshCmds {
if strings.HasPrefix(c, "mv ") {
mvSeen = true
}
}
if !mvSeen {
t.Fatalf("no move-aside mv issued: %v", sshCmds)
}
// Both transition events fired (orphaned → reset). No delete anywhere.
if len(events) != 2 || events[0] != "offbox_repo_orphaned" || events[1] != "offbox_repo_reset" {
t.Fatalf("events = %v, want [orphaned reset]", events)
}
}
// Scenario C: the claimed confirmed reset (ResetOrphanedRepo) refuses unless orphaned, then move-aside +
// re-init + clear state.
func TestOffbox_ConfirmedReset(t *testing.T) {
m, sett := newOffboxManager(t)
if err := sett.SetClaimed(); err != nil {
t.Fatal(err)
}
// refuse when not orphaned
if err := m.ResetOrphanedRepo(context.Background()); err == nil {
t.Fatal("reset must refuse when the repo is not orphaned")
}
// mark orphaned, then confirm reset
m.SetOffboxRunner(wrongPwRunner(nil))
_ = m.RunOffboxBackup(context.Background())
if !m.OffboxOrphaned() {
t.Fatal("precondition: not orphaned")
}
var mv bool
m.SetOffboxSSH(func(_ context.Context, _, _ string, _ int, _, _, cmd string) ([]byte, error) {
if strings.HasPrefix(cmd, "test -e") {
return nil, fmt.Errorf("exit 1")
}
if strings.HasPrefix(cmd, "mv ") {
mv = true
}
return nil, nil
})
if err := m.ResetOrphanedRepo(context.Background()); err != nil {
t.Fatalf("confirmed reset: %v", err)
}
if !mv {
t.Fatal("confirmed reset did not move the old repo aside")
}
if m.OffboxOrphaned() {
t.Fatal("state not cleared after confirmed reset")
}
}