package backup import ( "context" "crypto/sha256" "encoding/hex" "fmt" "strings" "sync" "time" "gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec" ) // The stray-copy fix (night brief 2026-10-23, Part 1). The capped second-drive copy (R-1214) runs `rsync -a --delete` // in its own container. Killing the `docker` CLI — the copy's timeout, a controller restart, a self-update — does NOT // stop that container: the copy goes on with `--delete`, outside every guard the controller holds in memory (R-1180: // no mirror during a restore of that app). So: // // - every copy container has a fixed name (one per destination) and the label tier2CopyLabel; // - a leftover of the same copy is removed before a new one starts, and the container is removed by name when the // copy ends with an error (the timeout included); // - at controller start, before the scheduler or the web server can start a backup or a restore, every labelled // container is removed (ClearLeftoverTier2Copies, each one logged); // - a restore is refused while a copy runs or while a leftover may still run (Tier2CopyBlocksRestore). // // Pinned by TestStrayCopy_* (tier2_copy_lifecycle_test.go) and the web test TestStrayCopy_RestoreRefused*. // tier2CopyLabel marks every second-drive copy container; its value is "1". tier2CopyAppLabel names the app. const ( tier2CopyLabel = "felhom.tier2-copy" tier2CopyAppLabel = "felhom.tier2-copy.app" tier2CopyPrefix = "felhom-tier2-copy-" ) // tier2CopyTimeout bounds one copy. tier2CopyTimeoutOverride is set ONLY by a test build's ldflags // (-X …/internal/backup.tier2CopyTimeoutOverride=2m) to prove the timeout path on a scratch box; a release leaves it // empty and an unparseable value is ignored. var ( tier2CopyTimeout = 60 * time.Minute tier2CopyTimeoutOverride = "" ) func tier2CopyDeadline() time.Duration { if tier2CopyTimeoutOverride != "" { if d, err := time.ParseDuration(tier2CopyTimeoutOverride); err == nil && d > 0 { return d } } return tier2CopyTimeout } // tier2CopyName returns the fixed container name for a copy into dst and the app it belongs to. The app is the path // segment after backups/secondary/ or backups/primary/ ("" when the path has neither); the name carries 8 hex of the // destination's hash, so two legs of one app never share a name. func tier2CopyName(dst string) (name, app string) { for _, marker := range []string{"/backups/secondary/", "/backups/primary/"} { if i := strings.Index(dst, marker); i >= 0 { app = strings.SplitN(dst[i+len(marker):], "/", 2)[0] break } } sum := sha256.Sum256([]byte(dst)) return tier2CopyPrefix + tier2DockerSafe(app) + "-" + hex.EncodeToString(sum[:4]), app } // tier2DockerSafe keeps [a-z0-9_.-] (a Docker name/label value), max 40 characters; "x" when nothing is left. func tier2DockerSafe(s string) string { var b strings.Builder for _, r := range strings.ToLower(s) { if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' { b.WriteRune(r) } if b.Len() >= 40 { break } } if b.Len() == 0 { return "x" } return b.String() } // tier2Docker runs one docker command for the copy's lifecycle (rm, ps). A seam: tests never reach Docker. var tier2Docker = func(ctx context.Context, args ...string) ([]byte, error) { return dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput() } // The controller's own record of copies (package-level: one controller, one backup manager per process). // - inFlight: app → copies running now (capped or in-process); // - running: the container names THIS process runs now — the start-up check never removes one of them; // - strays: names whose removal failed — cleared when a later removal of that name works; // - startupPending: the start-up check has not succeeded yet. // // Restores are refused while any of these is non-empty (review 2026-10-23 night, N2/N3: a stray used to block every // restore until a restart, and the start-up retry used to remove this process's own running copies). var tier2Copies = struct { sync.Mutex inFlight map[string]int running map[string]bool strays map[string]bool startupPending bool retrying bool }{inFlight: map[string]int{}, running: map[string]bool{}, strays: map[string]bool{}} // tier2CopyLocks serializes copies into the SAME destination within this process (review N4): a second run waits for // the first instead of removing its container by name (which killed it mid-copy and mailed a false failure). var tier2CopyLocks = struct { sync.Mutex m map[string]*sync.Mutex }{m: map[string]*sync.Mutex{}} func tier2CopyLock(name string) *sync.Mutex { tier2CopyLocks.Lock() defer tier2CopyLocks.Unlock() l := tier2CopyLocks.m[name] if l == nil { l = &sync.Mutex{} tier2CopyLocks.m[name] = l } return l } func tier2CopyBegin(app string) { tier2Copies.Lock() tier2Copies.inFlight[app]++ tier2Copies.Unlock() } func tier2CopyEnd(app string) { tier2Copies.Lock() if tier2Copies.inFlight[app]--; tier2Copies.inFlight[app] <= 0 { delete(tier2Copies.inFlight, app) } tier2Copies.Unlock() } func tier2MarkRunning(name string, on bool) { tier2Copies.Lock() if on { tier2Copies.running[name] = true } else { delete(tier2Copies.running, name) } tier2Copies.Unlock() } // tier2RemoveCopy removes the copy container `name` (no error when it does not exist). It reports whether the // container is surely gone; when not, the name is recorded as a stray (restores refused) and retried every minute. func tier2RemoveCopy(name, why string) bool { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() out, err := tier2Docker(ctx, "rm", "-f", name) if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") { tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed (retried every minute)", name, why, err, strings.TrimSpace(string(out))) tier2Copies.Lock() tier2Copies.strays[name] = true tier2Copies.Unlock() tier2StartRetry(context.Background()) return false } tier2Copies.Lock() delete(tier2Copies.strays, name) tier2Copies.Unlock() return true } // tier2RetryEvery is the retry period for strays and a failed start-up check; tier2RetryOff (tests only) keeps the // background retry from starting at all — a timer that outlives a test would reach the real Docker of the host. var ( tier2RetryEvery = time.Minute tier2RetryOff = false ) // tier2StartRetry starts ONE background retry (if none runs): every tier2RetryEvery it redoes a failed start-up check // and the removal of every stray, and ends when nothing is left or ctx ends. func tier2StartRetry(ctx context.Context) { tier2Copies.Lock() if tier2Copies.retrying || tier2RetryOff { tier2Copies.Unlock() return } tier2Copies.retrying = true tier2Copies.Unlock() every := tier2RetryEvery go func() { t := time.NewTicker(every) defer t.Stop() for { select { case <-ctx.Done(): tier2Copies.Lock() tier2Copies.retrying = false tier2Copies.Unlock() return case <-t.C: } if tier2RetryOnce(ctx) { tier2Logf("[INFO] [tier2] no leftover or stray second-drive copy container is left — restores allowed again") return } } }() } // tier2RetryOnce does one retry round; it reports (and clears `retrying`) when nothing is left to retry. func tier2RetryOnce(ctx context.Context) bool { tier2Copies.Lock() pending := tier2Copies.startupPending var names []string for n := range tier2Copies.strays { names = append(names, n) } tier2Copies.Unlock() if pending && clearLeftoverTier2CopiesOnce(ctx) == nil { tier2Copies.Lock() tier2Copies.startupPending = false tier2Copies.Unlock() } for _, n := range names { tier2RemoveCopyQuiet(ctx, n) } tier2Copies.Lock() defer tier2Copies.Unlock() if !tier2Copies.startupPending && len(tier2Copies.strays) == 0 { tier2Copies.retrying = false return true } return false } // tier2RemoveCopyQuiet is the retry's removal: a failure keeps the stray without starting another retry. func tier2RemoveCopyQuiet(ctx context.Context, name string) { rctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() out, err := tier2Docker(rctx, "rm", "-f", name) if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") { return } tier2Copies.Lock() delete(tier2Copies.strays, name) tier2Copies.Unlock() tier2Logf("[INFO] [tier2] the stray copy container %s is removed", name) } // ClearLeftoverTier2Copies removes every labelled copy container a previous controller process started and lost // (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each removal is // logged. Until it succeeds, restores are refused; on failure it is retried every minute in the background. func ClearLeftoverTier2Copies(ctx context.Context) error { tier2Copies.Lock() tier2Copies.startupPending = true tier2Copies.Unlock() err := clearLeftoverTier2CopiesOnce(ctx) if err == nil { tier2Copies.Lock() tier2Copies.startupPending = false tier2Copies.Unlock() return nil } tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err) tier2StartRetry(ctx) return err } func clearLeftoverTier2CopiesOnce(ctx context.Context) error { lctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() out, err := tier2Docker(lctx, "ps", "-a", "--filter", "label="+tier2CopyLabel+"=1", "--format", "{{.Names}}") if err != nil { return fmt.Errorf("docker ps: %v %s", err, strings.TrimSpace(string(out))) } var names []string tier2Copies.Lock() for _, n := range strings.Fields(string(out)) { if !tier2Copies.running[n] { // never a copy THIS process runs (review N3) names = append(names, n) } } tier2Copies.Unlock() if len(names) == 0 { tier2Logf("[INFO] [tier2] start-up: no leftover second-drive copy container") return nil } failed := 0 for _, n := range names { rctx, rcancel := context.WithTimeout(ctx, 60*time.Second) rout, rerr := tier2Docker(rctx, "rm", "-f", n) rcancel() if rerr != nil { failed++ tier2Logf("[ERROR] [tier2] start-up: could not remove the leftover copy container %s: %v %s", n, rerr, strings.TrimSpace(string(rout))) continue } tier2Logf("[WARN] [tier2] start-up: removed a leftover second-drive copy container %s — a copy a previous controller process lost; the copy is redone at the next run", n) } if failed > 0 { return fmt.Errorf("%d of %d leftover copy container(s) not removed", failed, len(names)) } return nil } // Tier2CopyBlocksRestore reports whether a restore must wait: a second-drive copy runs now (its `--delete` would remove // files the restore reads or writes), or a copy container may still run that this process could not remove. The // returned app is "" when the cause is not one app. func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) { tier2Copies.Lock() defer tier2Copies.Unlock() if tier2Copies.startupPending || len(tier2Copies.strays) > 0 { return "", true } for a := range tier2Copies.inFlight { return a, true } return "", false } // tier2ResetForTest empties the record (tests only). func tier2ResetForTest() { tier2Copies.Lock() tier2Copies.inFlight, tier2Copies.running, tier2Copies.strays = map[string]int{}, map[string]bool{}, map[string]bool{} tier2Copies.startupPending, tier2Copies.retrying = false, false tier2Copies.Unlock() } // MarkTier2CopyForTest records a copy of app as running until the returned func is called — a test-only seam (another // package's test exercises the restore refusal without a real copy), in the spirit of AcquireRunningForTest. func MarkTier2CopyForTest(app string) func() { tier2CopyBegin(app) return func() { tier2CopyEnd(app) } }