Stray-copy fix: the capped second-drive copy container never outlives the copy — fixed name + label, removed by name on any error (the timeout included), every leftover removed at start before the scheduler and web server; a restore is refused while a copy runs or may still run; --mount (refuses a path the daemon does not see) and the image ID with --pull never (review); red-proved; unreleased
gates / gates (push) Successful in 1m53s
gates / gates (push) Successful in 1m53s
This commit is contained in:
@@ -0,0 +1,212 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
|
||||
)
|
||||
|
||||
// The stray-copy fix (night brief 2026-10-23, Part 1). The capped second-drive copy (R-1214) runs `rsync -a --delete`
|
||||
// in its own container. Killing the `docker` CLI — the copy's timeout, a controller restart, a self-update — does NOT
|
||||
// stop that container: the copy goes on with `--delete`, outside every guard the controller holds in memory (R-1180:
|
||||
// no mirror during a restore of that app). So:
|
||||
//
|
||||
// - every copy container has a fixed name (one per destination) and the label tier2CopyLabel;
|
||||
// - a leftover of the same copy is removed before a new one starts, and the container is removed by name when the
|
||||
// copy ends with an error (the timeout included);
|
||||
// - at controller start, before the scheduler or the web server can start a backup or a restore, every labelled
|
||||
// container is removed (ClearLeftoverTier2Copies, each one logged);
|
||||
// - a restore is refused while a copy runs or while a leftover may still run (Tier2CopyBlocksRestore).
|
||||
//
|
||||
// Pinned by TestStrayCopy_* (tier2_copy_lifecycle_test.go) and the web test TestStrayCopy_RestoreRefused*.
|
||||
|
||||
// tier2CopyLabel marks every second-drive copy container; its value is "1". tier2CopyAppLabel names the app.
|
||||
const (
|
||||
tier2CopyLabel = "felhom.tier2-copy"
|
||||
tier2CopyAppLabel = "felhom.tier2-copy.app"
|
||||
tier2CopyPrefix = "felhom-tier2-copy-"
|
||||
)
|
||||
|
||||
// tier2CopyTimeout bounds one copy. tier2CopyTimeoutOverride is set ONLY by a test build's ldflags
|
||||
// (-X …/internal/backup.tier2CopyTimeoutOverride=2m) to prove the timeout path on a scratch box; a release leaves it
|
||||
// empty and an unparseable value is ignored.
|
||||
var (
|
||||
tier2CopyTimeout = 60 * time.Minute
|
||||
tier2CopyTimeoutOverride = ""
|
||||
)
|
||||
|
||||
func tier2CopyDeadline() time.Duration {
|
||||
if tier2CopyTimeoutOverride != "" {
|
||||
if d, err := time.ParseDuration(tier2CopyTimeoutOverride); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return tier2CopyTimeout
|
||||
}
|
||||
|
||||
// tier2CopyName returns the fixed container name for a copy into dst and the app it belongs to. The app is the path
|
||||
// segment after backups/secondary/ or backups/primary/ ("" when the path has neither); the name carries 8 hex of the
|
||||
// destination's hash, so two legs of one app never share a name.
|
||||
func tier2CopyName(dst string) (name, app string) {
|
||||
for _, marker := range []string{"/backups/secondary/", "/backups/primary/"} {
|
||||
if i := strings.Index(dst, marker); i >= 0 {
|
||||
app = strings.SplitN(dst[i+len(marker):], "/", 2)[0]
|
||||
break
|
||||
}
|
||||
}
|
||||
sum := sha256.Sum256([]byte(dst))
|
||||
return tier2CopyPrefix + tier2DockerSafe(app) + "-" + hex.EncodeToString(sum[:4]), app
|
||||
}
|
||||
|
||||
// tier2DockerSafe keeps [a-z0-9_.-] (a Docker name/label value), max 40 characters; "x" when nothing is left.
|
||||
func tier2DockerSafe(s string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(s) {
|
||||
if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' {
|
||||
b.WriteRune(r)
|
||||
}
|
||||
if b.Len() >= 40 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
return "x"
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// tier2Docker runs one docker command for the copy's lifecycle (rm, ps). A seam: tests never reach Docker.
|
||||
var tier2Docker = func(ctx context.Context, args ...string) ([]byte, error) {
|
||||
return dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput()
|
||||
}
|
||||
|
||||
// The controller's own record of copies (package-level: one controller, one backup manager per process).
|
||||
var tier2Copies = struct {
|
||||
sync.Mutex
|
||||
inFlight map[string]int // app → copies running now (capped or in-process)
|
||||
stray bool // a container this process could not remove may still run, or the start-up removal has not succeeded
|
||||
}{inFlight: map[string]int{}}
|
||||
|
||||
func tier2CopyBegin(app string) {
|
||||
tier2Copies.Lock()
|
||||
tier2Copies.inFlight[app]++
|
||||
tier2Copies.Unlock()
|
||||
}
|
||||
|
||||
func tier2CopyEnd(app string) {
|
||||
tier2Copies.Lock()
|
||||
if tier2Copies.inFlight[app]--; tier2Copies.inFlight[app] <= 0 {
|
||||
delete(tier2Copies.inFlight, app)
|
||||
}
|
||||
tier2Copies.Unlock()
|
||||
}
|
||||
|
||||
func tier2SetStray(v bool) {
|
||||
tier2Copies.Lock()
|
||||
tier2Copies.stray = v
|
||||
tier2Copies.Unlock()
|
||||
}
|
||||
|
||||
// tier2RemoveCopy removes the copy container `name` (no error when it does not exist). It reports whether the
|
||||
// container is surely gone; when not, the stray flag is set, so restores stay refused until a removal succeeds.
|
||||
func tier2RemoveCopy(name, why string) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
out, err := tier2Docker(ctx, "rm", "-f", name)
|
||||
if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
|
||||
tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed",
|
||||
name, why, err, strings.TrimSpace(string(out)))
|
||||
tier2SetStray(true)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ClearLeftoverTier2Copies removes every labelled copy container — a copy a previous controller process started and
|
||||
// lost (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each
|
||||
// removal is logged. Until it succeeds, restores are refused; on failure it retries every minute in the background
|
||||
// until it succeeds or ctx ends.
|
||||
func ClearLeftoverTier2Copies(ctx context.Context) error {
|
||||
tier2SetStray(true)
|
||||
err := clearLeftoverTier2CopiesOnce(ctx)
|
||||
if err == nil {
|
||||
tier2SetStray(false)
|
||||
return nil
|
||||
}
|
||||
tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err)
|
||||
go func() {
|
||||
t := time.NewTicker(time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if clearLeftoverTier2CopiesOnce(ctx) == nil {
|
||||
tier2SetStray(false)
|
||||
tier2Logf("[INFO] [tier2] the leftover second-drive copy check succeeded — restores allowed again")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return err
|
||||
}
|
||||
|
||||
func clearLeftoverTier2CopiesOnce(ctx context.Context) error {
|
||||
lctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
out, err := tier2Docker(lctx, "ps", "-a", "--filter", "label="+tier2CopyLabel+"=1", "--format", "{{.Names}}")
|
||||
if err != nil {
|
||||
return fmt.Errorf("docker ps: %v %s", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
names := strings.Fields(string(out))
|
||||
if len(names) == 0 {
|
||||
tier2Logf("[INFO] [tier2] start-up: no leftover second-drive copy container")
|
||||
return nil
|
||||
}
|
||||
failed := 0
|
||||
for _, n := range names {
|
||||
rctx, rcancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
rout, rerr := tier2Docker(rctx, "rm", "-f", n)
|
||||
rcancel()
|
||||
if rerr != nil {
|
||||
failed++
|
||||
tier2Logf("[ERROR] [tier2] start-up: could not remove the leftover copy container %s: %v %s", n, rerr, strings.TrimSpace(string(rout)))
|
||||
continue
|
||||
}
|
||||
tier2Logf("[WARN] [tier2] start-up: removed a leftover second-drive copy container %s — a copy a previous controller process lost; the copy is redone at the next run", n)
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d leftover copy container(s) not removed", failed, len(names))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Tier2CopyBlocksRestore reports whether a restore must wait: a second-drive copy runs now (its `--delete` would remove
|
||||
// files the restore reads or writes), or a copy container may still run that this process could not remove. The
|
||||
// returned app is "" when the cause is not one app.
|
||||
func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) {
|
||||
tier2Copies.Lock()
|
||||
defer tier2Copies.Unlock()
|
||||
if tier2Copies.stray {
|
||||
return "", true
|
||||
}
|
||||
for a := range tier2Copies.inFlight {
|
||||
return a, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// MarkTier2CopyForTest records a copy of app as running until the returned func is called — a test-only seam (another
|
||||
// package's test exercises the restore refusal without a real copy), in the spirit of AcquireRunningForTest.
|
||||
func MarkTier2CopyForTest(app string) func() {
|
||||
tier2CopyBegin(app)
|
||||
return func() { tier2CopyEnd(app) }
|
||||
}
|
||||
Reference in New Issue
Block a user