felhom-controller/controller/internal/backup/tier2_copy_lifecycle.go

package backup

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
)

// The stray-copy fix (night brief 2026-10-23, Part 1). The capped second-drive copy (R-1214) runs `rsync -a --delete`
// in its own container. Killing the `docker` CLI — the copy's timeout, a controller restart, a self-update — does NOT
// stop that container: the copy goes on with `--delete`, outside every guard the controller holds in memory (R-1180:
// no mirror during a restore of that app). So:
//
//   - every copy container has a fixed name (one per destination) and the label tier2CopyLabel;
//   - a leftover of the same copy is removed before a new one starts, and the container is removed by name when the
//     copy ends with an error (the timeout included);
//   - at controller start, before the scheduler or the web server can start a backup or a restore, every labelled
//     container is removed (ClearLeftoverTier2Copies, each one logged);
//   - a restore is refused while a copy runs or while a leftover may still run (Tier2CopyBlocksRestore).
//
// Pinned by TestStrayCopy_* (tier2_copy_lifecycle_test.go) and the web test TestStrayCopy_RestoreRefused*.

// tier2CopyLabel marks every second-drive copy container; its value is "1". tier2CopyAppLabel names the app.
const (
	tier2CopyLabel    = "felhom.tier2-copy"
	tier2CopyAppLabel = "felhom.tier2-copy.app"
	tier2CopyPrefix   = "felhom-tier2-copy-"
)

// tier2CopyTimeout bounds one copy. tier2CopyTimeoutOverride is set ONLY by a test build's ldflags
// (-X …/internal/backup.tier2CopyTimeoutOverride=2m) to prove the timeout path on a scratch box; a release leaves it
// empty and an unparseable value is ignored.
var (
	tier2CopyTimeout         = 60 * time.Minute
	tier2CopyTimeoutOverride = ""
)

func tier2CopyDeadline() time.Duration {
	if tier2CopyTimeoutOverride != "" {
		if d, err := time.ParseDuration(tier2CopyTimeoutOverride); err == nil && d > 0 {
			return d
		}
	}
	return tier2CopyTimeout
}

// tier2CopyName returns the fixed container name for a copy into dst and the app it belongs to. The app is the path
// segment after backups/secondary/ or backups/primary/ ("" when the path has neither); the name carries 8 hex of the
// destination's hash, so two legs of one app never share a name.
func tier2CopyName(dst string) (name, app string) {
	for _, marker := range []string{"/backups/secondary/", "/backups/primary/"} {
		if i := strings.Index(dst, marker); i >= 0 {
			app = strings.SplitN(dst[i+len(marker):], "/", 2)[0]
			break
		}
	}
	sum := sha256.Sum256([]byte(dst))
	return tier2CopyPrefix + tier2DockerSafe(app) + "-" + hex.EncodeToString(sum[:4]), app
}

// tier2DockerSafe keeps [a-z0-9_.-] (a Docker name/label value), max 40 characters; "x" when nothing is left.
func tier2DockerSafe(s string) string {
	var b strings.Builder
	for _, r := range strings.ToLower(s) {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' {
			b.WriteRune(r)
		}
		if b.Len() >= 40 {
			break
		}
	}
	if b.Len() == 0 {
		return "x"
	}
	return b.String()
}

// tier2Docker runs one docker command for the copy's lifecycle (rm, ps). A seam: tests never reach Docker.
var tier2Docker = func(ctx context.Context, args ...string) ([]byte, error) {
	return dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput()
}

// The controller's own record of copies (package-level: one controller, one backup manager per process).
//   - inFlight: app → copies running now (capped or in-process);
//   - running: the container names THIS process runs now — the start-up check never removes one of them;
//   - strays: names whose removal failed — cleared when a later removal of that name works;
//   - startupPending: the start-up check has not succeeded yet.
//
// Restores are refused while any of these is non-empty (review 2026-10-23 night, N2/N3: a stray used to block every
// restore until a restart, and the start-up retry used to remove this process's own running copies).
var tier2Copies = struct {
	sync.Mutex
	inFlight       map[string]int
	running        map[string]bool
	strays         map[string]bool
	startupPending bool
	retrying       bool
}{inFlight: map[string]int{}, running: map[string]bool{}, strays: map[string]bool{}}

// tier2CopyLocks serializes copies into the SAME destination within this process (review N4): a second run waits for
// the first instead of removing its container by name (which killed it mid-copy and mailed a false failure).
var tier2CopyLocks = struct {
	sync.Mutex
	m map[string]*sync.Mutex
}{m: map[string]*sync.Mutex{}}

func tier2CopyLock(name string) *sync.Mutex {
	tier2CopyLocks.Lock()
	defer tier2CopyLocks.Unlock()
	l := tier2CopyLocks.m[name]
	if l == nil {
		l = &sync.Mutex{}
		tier2CopyLocks.m[name] = l
	}
	return l
}

func tier2CopyBegin(app string) {
	tier2Copies.Lock()
	tier2Copies.inFlight[app]++
	tier2Copies.Unlock()
}

func tier2CopyEnd(app string) {
	tier2Copies.Lock()
	if tier2Copies.inFlight[app]--; tier2Copies.inFlight[app] <= 0 {
		delete(tier2Copies.inFlight, app)
	}
	tier2Copies.Unlock()
}

func tier2MarkRunning(name string, on bool) {
	tier2Copies.Lock()
	if on {
		tier2Copies.running[name] = true
	} else {
		delete(tier2Copies.running, name)
	}
	tier2Copies.Unlock()
}

// tier2RemoveCopy removes the copy container `name` (no error when it does not exist). It reports whether the
// container is surely gone; when not, the name is recorded as a stray (restores refused) and retried every minute.
func tier2RemoveCopy(name, why string) bool {
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()
	out, err := tier2Docker(ctx, "rm", "-f", name)
	if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
		tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed (retried every minute)",
			name, why, err, strings.TrimSpace(string(out)))
		tier2Copies.Lock()
		tier2Copies.strays[name] = true
		tier2Copies.Unlock()
		tier2StartRetry(context.Background())
		return false
	}
	tier2Copies.Lock()
	delete(tier2Copies.strays, name)
	tier2Copies.Unlock()
	return true
}

// tier2RetryEvery is the retry period for strays and a failed start-up check; tier2RetryOff (tests only) keeps the
// background retry from starting at all — a timer that outlives a test would reach the real Docker of the host.
var (
	tier2RetryEvery = time.Minute
	tier2RetryOff   = false
)

// tier2StartRetry starts ONE background retry (if none runs): every tier2RetryEvery it redoes a failed start-up check
// and the removal of every stray, and ends when nothing is left or ctx ends.
func tier2StartRetry(ctx context.Context) {
	tier2Copies.Lock()
	if tier2Copies.retrying || tier2RetryOff {
		tier2Copies.Unlock()
		return
	}
	tier2Copies.retrying = true
	tier2Copies.Unlock()
	every := tier2RetryEvery
	go func() {
		t := time.NewTicker(every)
		defer t.Stop()
		for {
			select {
			case <-ctx.Done():
				tier2Copies.Lock()
				tier2Copies.retrying = false
				tier2Copies.Unlock()
				return
			case <-t.C:
			}
			if tier2RetryOnce(ctx) {
				tier2Logf("[INFO] [tier2] no leftover or stray second-drive copy container is left — restores allowed again")
				return
			}
		}
	}()
}

// tier2RetryOnce does one retry round; it reports (and clears `retrying`) when nothing is left to retry.
func tier2RetryOnce(ctx context.Context) bool {
	tier2Copies.Lock()
	pending := tier2Copies.startupPending
	var names []string
	for n := range tier2Copies.strays {
		names = append(names, n)
	}
	tier2Copies.Unlock()
	if pending && clearLeftoverTier2CopiesOnce(ctx) == nil {
		tier2Copies.Lock()
		tier2Copies.startupPending = false
		tier2Copies.Unlock()
	}
	for _, n := range names {
		tier2RemoveCopyQuiet(ctx, n)
	}
	tier2Copies.Lock()
	defer tier2Copies.Unlock()
	if !tier2Copies.startupPending && len(tier2Copies.strays) == 0 {
		tier2Copies.retrying = false
		return true
	}
	return false
}

// tier2RemoveCopyQuiet is the retry's removal: a failure keeps the stray without starting another retry.
func tier2RemoveCopyQuiet(ctx context.Context, name string) {
	rctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()
	out, err := tier2Docker(rctx, "rm", "-f", name)
	if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
		return
	}
	tier2Copies.Lock()
	delete(tier2Copies.strays, name)
	tier2Copies.Unlock()
	tier2Logf("[INFO] [tier2] the stray copy container %s is removed", name)
}

// ClearLeftoverTier2Copies removes every labelled copy container a previous controller process started and lost
// (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each removal is
// logged. Until it succeeds, restores are refused; on failure it is retried every minute in the background.
func ClearLeftoverTier2Copies(ctx context.Context) error {
	tier2Copies.Lock()
	tier2Copies.startupPending = true
	tier2Copies.Unlock()
	err := clearLeftoverTier2CopiesOnce(ctx)
	if err == nil {
		tier2Copies.Lock()
		tier2Copies.startupPending = false
		tier2Copies.Unlock()
		return nil
	}
	tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err)
	tier2StartRetry(ctx)
	return err
}

func clearLeftoverTier2CopiesOnce(ctx context.Context) error {
	lctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()
	out, err := tier2Docker(lctx, "ps", "-a", "--filter", "label="+tier2CopyLabel+"=1", "--format", "{{.Names}}")
	if err != nil {
		return fmt.Errorf("docker ps: %v %s", err, strings.TrimSpace(string(out)))
	}
	var names []string
	tier2Copies.Lock()
	for _, n := range strings.Fields(string(out)) {
		if !tier2Copies.running[n] { // never a copy THIS process runs (review N3)
			names = append(names, n)
		}
	}
	tier2Copies.Unlock()
	if len(names) == 0 {
		tier2Logf("[INFO] [tier2] start-up: no leftover second-drive copy container")
		return nil
	}
	failed := 0
	for _, n := range names {
		rctx, rcancel := context.WithTimeout(ctx, 60*time.Second)
		rout, rerr := tier2Docker(rctx, "rm", "-f", n)
		rcancel()
		if rerr != nil {
			failed++
			tier2Logf("[ERROR] [tier2] start-up: could not remove the leftover copy container %s: %v %s", n, rerr, strings.TrimSpace(string(rout)))
			continue
		}
		tier2Logf("[WARN] [tier2] start-up: removed a leftover second-drive copy container %s — a copy a previous controller process lost; the copy is redone at the next run", n)
	}
	if failed > 0 {
		return fmt.Errorf("%d of %d leftover copy container(s) not removed", failed, len(names))
	}
	return nil
}

// Tier2CopyBlocksRestore reports whether a restore must wait: a second-drive copy runs now (its `--delete` would remove
// files the restore reads or writes), or a copy container may still run that this process could not remove. The
// returned app is "" when the cause is not one app.
func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) {
	tier2Copies.Lock()
	defer tier2Copies.Unlock()
	if tier2Copies.startupPending || len(tier2Copies.strays) > 0 {
		return "", true
	}
	for a := range tier2Copies.inFlight {
		return a, true
	}
	return "", false
}

// tier2ResetForTest empties the record (tests only).
func tier2ResetForTest() {
	tier2Copies.Lock()
	tier2Copies.inFlight, tier2Copies.running, tier2Copies.strays = map[string]int{}, map[string]bool{}, map[string]bool{}
	tier2Copies.startupPending, tier2Copies.retrying = false, false
	tier2Copies.Unlock()
}

// MarkTier2CopyForTest records a copy of app as running until the returned func is called — a test-only seam (another
// package's test exercises the restore refusal without a real copy), in the spirit of AcquireRunningForTest.
func MarkTier2CopyForTest(app string) func() {
	tier2CopyBegin(app)
	return func() { tier2CopyEnd(app) }
}