felhom-controller/controller/internal/backup/tier2_copy_lifecycle.go

package backup

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"strings"
	"sync"
	"time"

	"gitea.dooplex.hu/admin/felhom-controller/internal/dockerexec"
)

// The stray-copy fix (night brief 2026-10-23, Part 1). The capped second-drive copy (R-1214) runs `rsync -a --delete`
// in its own container. Killing the `docker` CLI — the copy's timeout, a controller restart, a self-update — does NOT
// stop that container: the copy goes on with `--delete`, outside every guard the controller holds in memory (R-1180:
// no mirror during a restore of that app). So:
//
//   - every copy container has a fixed name (one per destination) and the label tier2CopyLabel;
//   - a leftover of the same copy is removed before a new one starts, and the container is removed by name when the
//     copy ends with an error (the timeout included);
//   - at controller start, before the scheduler or the web server can start a backup or a restore, every labelled
//     container is removed (ClearLeftoverTier2Copies, each one logged);
//   - a restore is refused while a copy runs or while a leftover may still run (Tier2CopyBlocksRestore).
//
// Pinned by TestStrayCopy_* (tier2_copy_lifecycle_test.go) and the web test TestStrayCopy_RestoreRefused*.

// tier2CopyLabel marks every second-drive copy container; its value is "1". tier2CopyAppLabel names the app.
const (
	tier2CopyLabel    = "felhom.tier2-copy"
	tier2CopyAppLabel = "felhom.tier2-copy.app"
	tier2CopyPrefix   = "felhom-tier2-copy-"
)

// tier2CopyTimeout bounds one copy. tier2CopyTimeoutOverride is set ONLY by a test build's ldflags
// (-X …/internal/backup.tier2CopyTimeoutOverride=2m) to prove the timeout path on a scratch box; a release leaves it
// empty and an unparseable value is ignored.
var (
	tier2CopyTimeout         = 60 * time.Minute
	tier2CopyTimeoutOverride = ""
)

func tier2CopyDeadline() time.Duration {
	if tier2CopyTimeoutOverride != "" {
		if d, err := time.ParseDuration(tier2CopyTimeoutOverride); err == nil && d > 0 {
			return d
		}
	}
	return tier2CopyTimeout
}

// tier2CopyName returns the fixed container name for a copy into dst and the app it belongs to. The app is the path
// segment after backups/secondary/ or backups/primary/ ("" when the path has neither); the name carries 8 hex of the
// destination's hash, so two legs of one app never share a name.
func tier2CopyName(dst string) (name, app string) {
	for _, marker := range []string{"/backups/secondary/", "/backups/primary/"} {
		if i := strings.Index(dst, marker); i >= 0 {
			app = strings.SplitN(dst[i+len(marker):], "/", 2)[0]
			break
		}
	}
	sum := sha256.Sum256([]byte(dst))
	return tier2CopyPrefix + tier2DockerSafe(app) + "-" + hex.EncodeToString(sum[:4]), app
}

// tier2DockerSafe keeps [a-z0-9_.-] (a Docker name/label value), max 40 characters; "x" when nothing is left.
func tier2DockerSafe(s string) string {
	var b strings.Builder
	for _, r := range strings.ToLower(s) {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' {
			b.WriteRune(r)
		}
		if b.Len() >= 40 {
			break
		}
	}
	if b.Len() == 0 {
		return "x"
	}
	return b.String()
}

// tier2Docker runs one docker command for the copy's lifecycle (rm, ps). A seam: tests never reach Docker.
var tier2Docker = func(ctx context.Context, args ...string) ([]byte, error) {
	return dockerexec.CommandContext(ctx, "docker", args...).CombinedOutput()
}

// The controller's own record of copies (package-level: one controller, one backup manager per process).
var tier2Copies = struct {
	sync.Mutex
	inFlight map[string]int // app → copies running now (capped or in-process)
	stray    bool           // a container this process could not remove may still run, or the start-up removal has not succeeded
}{inFlight: map[string]int{}}

func tier2CopyBegin(app string) {
	tier2Copies.Lock()
	tier2Copies.inFlight[app]++
	tier2Copies.Unlock()
}

func tier2CopyEnd(app string) {
	tier2Copies.Lock()
	if tier2Copies.inFlight[app]--; tier2Copies.inFlight[app] <= 0 {
		delete(tier2Copies.inFlight, app)
	}
	tier2Copies.Unlock()
}

func tier2SetStray(v bool) {
	tier2Copies.Lock()
	tier2Copies.stray = v
	tier2Copies.Unlock()
}

// tier2RemoveCopy removes the copy container `name` (no error when it does not exist). It reports whether the
// container is surely gone; when not, the stray flag is set, so restores stay refused until a removal succeeds.
func tier2RemoveCopy(name, why string) bool {
	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
	defer cancel()
	out, err := tier2Docker(ctx, "rm", "-f", name)
	if err != nil && !strings.Contains(strings.ToLower(string(out)), "no such container") {
		tier2Logf("[ERROR] [tier2] could not remove the copy container %s (%s): %v %s — restores stay refused until it is removed",
			name, why, err, strings.TrimSpace(string(out)))
		tier2SetStray(true)
		return false
	}
	return true
}

// ClearLeftoverTier2Copies removes every labelled copy container — a copy a previous controller process started and
// lost (restart, self-update, crash). It runs at controller start, BEFORE the scheduler and the web server; each
// removal is logged. Until it succeeds, restores are refused; on failure it retries every minute in the background
// until it succeeds or ctx ends.
func ClearLeftoverTier2Copies(ctx context.Context) error {
	tier2SetStray(true)
	err := clearLeftoverTier2CopiesOnce(ctx)
	if err == nil {
		tier2SetStray(false)
		return nil
	}
	tier2Logf("[ERROR] [tier2] the start-up check for a leftover second-drive copy failed: %v — restores are refused until it succeeds; retrying every minute", err)
	go func() {
		t := time.NewTicker(time.Minute)
		defer t.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case <-t.C:
				if clearLeftoverTier2CopiesOnce(ctx) == nil {
					tier2SetStray(false)
					tier2Logf("[INFO] [tier2] the leftover second-drive copy check succeeded — restores allowed again")
					return
				}
			}
		}
	}()
	return err
}

func clearLeftoverTier2CopiesOnce(ctx context.Context) error {
	lctx, cancel := context.WithTimeout(ctx, 60*time.Second)
	defer cancel()
	out, err := tier2Docker(lctx, "ps", "-a", "--filter", "label="+tier2CopyLabel+"=1", "--format", "{{.Names}}")
	if err != nil {
		return fmt.Errorf("docker ps: %v %s", err, strings.TrimSpace(string(out)))
	}
	names := strings.Fields(string(out))
	if len(names) == 0 {
		tier2Logf("[INFO] [tier2] start-up: no leftover second-drive copy container")
		return nil
	}
	failed := 0
	for _, n := range names {
		rctx, rcancel := context.WithTimeout(ctx, 60*time.Second)
		rout, rerr := tier2Docker(rctx, "rm", "-f", n)
		rcancel()
		if rerr != nil {
			failed++
			tier2Logf("[ERROR] [tier2] start-up: could not remove the leftover copy container %s: %v %s", n, rerr, strings.TrimSpace(string(rout)))
			continue
		}
		tier2Logf("[WARN] [tier2] start-up: removed a leftover second-drive copy container %s — a copy a previous controller process lost; the copy is redone at the next run", n)
	}
	if failed > 0 {
		return fmt.Errorf("%d of %d leftover copy container(s) not removed", failed, len(names))
	}
	return nil
}

// Tier2CopyBlocksRestore reports whether a restore must wait: a second-drive copy runs now (its `--delete` would remove
// files the restore reads or writes), or a copy container may still run that this process could not remove. The
// returned app is "" when the cause is not one app.
func (m *Manager) Tier2CopyBlocksRestore() (app string, blocked bool) {
	tier2Copies.Lock()
	defer tier2Copies.Unlock()
	if tier2Copies.stray {
		return "", true
	}
	for a := range tier2Copies.inFlight {
		return a, true
	}
	return "", false
}

// MarkTier2CopyForTest records a copy of app as running until the returned func is called — a test-only seam (another
// package's test exercises the restore refusal without a real copy), in the spirit of AcquireRunningForTest.
func MarkTier2CopyForTest(app string) func() {
	tier2CopyBegin(app)
	return func() { tier2CopyEnd(app) }
}