felhom-controller/controller/internal/backup/appstop_marker.go

package backup

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"sync"
	"time"
)

// ── The app-stop marker (R-166 part 2, decision D-b "in-flight operations") ───────────────────────
//
// Several operations stop a customer's app, do something to its data, and start it again. Between
// the stop and the start, NOTHING ON DISK RECORDED THAT AN APP WAS OWED A RESTART. A controller that
// died in that window left the app down with no explanation anywhere — and because a stopped app has
// zero containers, the boot reconciler read it as a deliberate customer stop and deliberately left
// it alone. Silently, indefinitely.
//
// A `defer` is NOT the fix and must never be described as one. Campaign 8 fault 10 established this
// on live hardware: a SIGKILL runs no deferred function, and what brought the quiesce loop's stacks
// back was its persisted marker read by Recover() one second after restart. The defer covers the
// graceful exits; the marker covers the hard crash and the power cut. This file is that marker for
// the app-data path, modelled directly on internal/quiesce's.
//
// WHY ITS OWN FILE, not quiesce's: one file, one writer. Quiesce's marker records a whole-guest
// backup window and is written by the quiesce loop; this one records an app-data operation and is
// written by the backup manager and the exporter. Sharing the file would give it two writers with
// two lifetimes, and one clearing the other's record is a stranded app by a different route.
//
// SAFETY (D-b's binding rule): losing this file must never be worse than not having it. A lost or
// corrupt marker means the app is not auto-restarted by THIS mechanism — which is precisely the
// pre-v0.189.0 position, not a new hazard. It never deletes, restores, or touches a backup artifact.

// AppStopReason names WHY an app was stopped, so the recovery log tells an operator which operation
// was interrupted rather than merely that something was.
type AppStopReason string

const (
	// ReasonVolumeDump — DumpAppVolumesSafe: stop, tar the volumes consistently, start.
	ReasonVolumeDump AppStopReason = "volume_dump"
	// ReasonOffboxReconstitute — a full offsite restore overwriting the app's files.
	ReasonOffboxReconstitute AppStopReason = "offbox_reconstitute"
	// ReasonAppExport — a .fab export taken with "stop the app first".
	ReasonAppExport AppStopReason = "app_export"
)

// humanReason is the operator-facing phrasing for each reason.
func (r AppStopReason) humanReason() string {
	switch r {
	case ReasonVolumeDump:
		return "an app-data backup (volume dump)"
	case ReasonOffboxReconstitute:
		return "an off-site restore"
	case ReasonAppExport:
		return "an app export"
	default:
		return string(r)
	}
}

// AppStopMarker is the persisted "these apps were stopped by an operation that has not reported
// finishing — they are owed a restart" note.
type AppStopMarker struct {
	Active    bool          `json:"active"`
	OpID      string        `json:"op_id"`
	Reason    AppStopReason `json:"reason"`
	Stacks    []string      `json:"stacks"`
	StartedAt time.Time     `json:"started_at"`
}

// AppStopStarter is the one thing recovery needs: the ability to start a stack. StartStack must be
// idempotent (it is — `compose up -d` on a running stack is a no-op).
//
// R-174: production MUST pass a GATED starter, never the raw stack manager. Recover runs at STARTUP —
// exactly when an external drive may not have come back — and `Manager.StartStack` has no drive gate
// of its own. See `gatedAppStopStarter` in cmd/controller/main.go.
type AppStopStarter interface {
	StartStack(name string) error
}

// ErrStartRefused is what a gated starter returns when a DELIBERATE HOLDER — today the drive gate —
// says an app must not be started. Wrap it (`fmt.Errorf("%w: …", ErrStartRefused)`) so the reason
// survives; Recover matches with errors.Is.
//
// IT IS NOT A FAILURE, AND THE DISTINCTION IS THE WHOLE POINT OF THE TYPE. A refusal means the
// holder is doing its job and owns the restart; a failure means the restart was attempted and broke.
// Collapsing the two would put a deliberately-held app into `Failed`, which main.go reports through
// `NotifyBackupFailed` — a type that is customer-enabled by default (`settings.DefaultEnabledEvents`)
// and carries the Hungarian "A biztonsági mentés sikertelen!". That is R-171's defect one path over:
// a false alarm about an app the drive gate is deliberately holding. Both buckets keep the marker;
// only `Failed` alarms.
var ErrStartRefused = errors.New("start refused by a deliberate holder")

// AppStopGuard owns one marker file. Construct with NewAppStopGuard; the zero value is inert (every
// method is a no-op on a nil guard), so a caller that was never wired degrades to pre-v0.189.0
// behaviour instead of panicking.
type AppStopGuard struct {
	path   string
	logger *log.Logger
	now    func() time.Time
	// starter is only needed by Recover; Begin/End work without one.
	starter AppStopStarter

	// suppressed (R-330) is the in-memory "the alarm must not fire for these, we are holding them"
	// set. It is deliberately NOT persisted: after a restart the guard no longer holds anything —
	// Recover() either brings the apps back or leaves them genuinely down, and a down app must alarm.
	// Reviving a suppression across a crash would silence exactly the case the alarm exists for.
	// See appstop_suppress.go for the whole design.
	suppressMu sync.Mutex
	suppressed map[string]appStopHold
}

// AppStopRecovery is what Recover found and did. Returned rather than pushed through a notifier
// seam, because of a hard ordering constraint: Recover must COMPLETE before the boot reconciler is
// launched (§8.4, main.go:236) and the hub notifier is not constructed until main.go:307. A seam
// wired after the fact would be a seam that never fires — the "built but never wired" shape this
// project has now hit four times. Returning the outcome lets main.go report it the moment the
// notifier exists, and makes the reporting decision visible at the call site instead of buried here.
type AppStopRecovery struct {
	Reason    AppStopReason
	OpID      string
	StartedAt time.Time
	Restarted []string // apps started again by this recovery
	Failed    []string // apps whose restart was ATTEMPTED and broke (the marker was kept for these)
	// Refused are apps a deliberate holder said must not start — today, an absent data drive
	// (R-174). The marker is kept for these too, but they are NOT a fault and MUST NOT alarm: the
	// holder owns the restart. Separate from Failed for the reason recorded on ErrStartRefused.
	Refused []string
}

// Alarming reports whether this recovery is worth paging an operator about. A recovery that only
// REFUSED starts is the drive gate working as designed, and reporting it through the customer-enabled
// `backup_failed` type would be the R-171 false alarm one path over.
func (r *AppStopRecovery) Alarming() bool {
	if r == nil {
		return false
	}
	return len(r.Failed) > 0 || len(r.Restarted) > 0
}

// Message is the operator-facing headline for an interrupted operation.
func (r *AppStopRecovery) Message() string {
	if r == nil {
		return ""
	}
	if len(r.Failed) > 0 {
		m := fmt.Sprintf("%s was interrupted by a controller restart and %d of %d app(s) could NOT be restarted",
			r.Reason.humanReason(), len(r.Failed), len(r.Restarted)+len(r.Failed))
		if len(r.Refused) > 0 {
			m += fmt.Sprintf(" (a further %d are held by an absent drive and are not counted as failures)", len(r.Refused))
		}
		return m
	}
	if len(r.Refused) > 0 && len(r.Restarted) == 0 {
		return fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) are left stopped and HELD: their data drive is not available, so the drive gate restarts them when it returns",
			r.Reason.humanReason(), len(r.Refused))
	}
	m := fmt.Sprintf("%s was interrupted by a controller restart — %d app(s) were left stopped and have been restarted",
		r.Reason.humanReason(), len(r.Restarted))
	if len(r.Refused) > 0 {
		m += fmt.Sprintf("; %d more are held by an absent drive", len(r.Refused))
	}
	return m
}

// Detail is the machine-readable tail. App/stack NAMES only — never env values (§9.5).
func (r *AppStopRecovery) Detail() string {
	if r == nil {
		return ""
	}
	d := fmt.Sprintf("op=%s reason=%s started_at=%s restarted=%v", r.OpID, r.Reason,
		r.StartedAt.UTC().Format(time.RFC3339), r.Restarted)
	if len(r.Failed) > 0 {
		d += fmt.Sprintf(" restart_failed=%v", r.Failed)
	}
	if len(r.Refused) > 0 {
		d += fmt.Sprintf(" held_by_drive=%v", r.Refused)
	}
	return d
}

// NewAppStopGuard builds a guard over the given marker path.
func NewAppStopGuard(path string, logger *log.Logger) *AppStopGuard {
	if logger == nil {
		logger = log.Default()
	}
	return &AppStopGuard{path: path, logger: logger, now: time.Now}
}

// SetStarter wires the stack-start seam used by Recover. INIT-ONLY — call once at startup, before
// Recover. Separate from the constructor because the guard is built alongside the backup manager,
// which learns its stack provider later (the same shape as SetStackProvider).
func (g *AppStopGuard) SetStarter(s AppStopStarter) {
	if g == nil {
		return
	}
	g.starter = s
}

// Begin records that `stacks` are about to be stopped by `reason`. It MUST be called BEFORE the
// first stop — an error here means the marker could not be written, and the caller must not proceed
// to stop an app it cannot promise to restart.
func (g *AppStopGuard) Begin(opID string, reason AppStopReason, stackNames []string) error {
	if g == nil || g.path == "" {
		return nil // not wired — pre-v0.189.0 behaviour, never a hard failure
	}
	if len(stackNames) == 0 {
		return nil
	}
	if err := g.write(AppStopMarker{
		Active:    true,
		OpID:      opID,
		Reason:    reason,
		Stacks:    append([]string(nil), stackNames...),
		StartedAt: g.now(),
	}); err != nil {
		return err
	}
	// R-330: the marker is on disk, so the caller is now cleared to stop these apps — which is
	// exactly the moment the app-down alarm must stop counting them. Marked AFTER the write, so a
	// Begin that failed (and therefore stopped nothing) suppresses nothing either.
	g.markStopped(stackNames)
	return nil
}

// End clears the marker after a successful restart. Best-effort by contract: a failure to clear is
// logged, never returned as the operation's error — a stale marker costs one idempotent StartStack
// on the next boot, which is exactly D-b's "worst acceptable outcome" and far cheaper than failing
// a backup that actually succeeded.
func (g *AppStopGuard) End() {
	if g == nil {
		return
	}
	// R-330: start the post-restart grace BEFORE the early return below, and unconditionally. End()
	// is the one "we gave the app back" signal on all three paths, and a guard with no marker path
	// still owes its suppressed stacks a release — otherwise an unwired-path guard would hold them
	// until appStopMaxHold.
	g.releaseStarted()
	if g.path == "" {
		return
	}
	if err := os.Remove(g.path); err != nil && !os.IsNotExist(err) {
		g.logger.Printf("[ERROR] [appstop] could not clear the app-stop marker at %s: %v (a stale marker costs one idempotent restart at next startup)", g.path, err)
	}
}

// Recover restarts any apps left stopped by an operation that died before restarting them, then
// clears the marker. Call ONCE at startup, and — critically — call it to COMPLETION before the boot
// reconciler is launched, so an app this marker explains is not also reported as an unexplained boot
// orphan (§8.4).
//
// Idempotent: StartStack on a running stack is tolerated, and an absent or inactive marker is a
// no-op. On a restart FAILURE the marker is deliberately LEFT IN PLACE — the next startup retries,
// and in the meantime the app is down with desired_state:running, so the boot reconciler sees it as
// an orphan and the dead-app alarm owns it. Clearing a marker whose restart failed would erase the
// only durable record that an app is owed one.
//
// Returns nil when there was nothing to recover — so "no interrupted operation" and "the recovery
// never ran" are distinguishable to the caller, not only in a log (standing rule 3).
func (g *AppStopGuard) Recover() *AppStopRecovery {
	if g == nil || g.path == "" {
		return nil
	}
	m, ok := g.read()
	if !ok || !m.Active || len(m.Stacks) == 0 {
		return nil
	}
	if g.starter == nil {
		g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) were stopped by %s and are owed a restart, but no stack starter is wired — leaving the marker for the next startup: %v",
			len(m.Stacks), m.Reason.humanReason(), m.Stacks)
		return nil
	}

	g.logger.Printf("[WARN] [appstop] crash recovery: %s (op %q) was interrupted and left %d app(s) stopped — restarting them: %v",
		m.Reason.humanReason(), m.OpID, len(m.Stacks), m.Stacks)

	res := &AppStopRecovery{Reason: m.Reason, OpID: m.OpID, StartedAt: m.StartedAt}
	for _, name := range m.Stacks {
		if err := g.starter.StartStack(name); err != nil {
			// R-174: a REFUSAL is not a failure. The starter's gate has said this app must not be
			// started (an absent data drive), so the app is left down deliberately and the holder
			// owns the restart. Logged at WARN with the reason, and kept out of Failed so it never
			// reaches the customer-enabled backup_failed alarm — see ErrStartRefused.
			if errors.Is(err, ErrStartRefused) {
				g.logger.Printf("[WARN] [appstop] crash recovery: NOT restarting %s — %v; the marker is KEPT and the holder owns the restart", name, err)
				res.Refused = append(res.Refused, name)
				continue
			}
			g.logger.Printf("[ERROR] [appstop] crash recovery: restart %s failed: %v", name, err)
			res.Failed = append(res.Failed, name)
			continue
		}
		g.logger.Printf("[INFO] [appstop] crash recovery: restarted %s after the interrupted %s", name, m.Reason.humanReason())
		res.Restarted = append(res.Restarted, name)
	}
	sort.Strings(res.Failed)
	sort.Strings(res.Refused)
	sort.Strings(res.Restarted)

	// The marker is kept for BOTH unfinished outcomes, for the same reason and with different
	// urgency: a failed restart is retried next startup, and a refused one is genuinely unfinished
	// until its drive returns. Clearing it in either case would erase the only durable record that
	// an app is owed a restart.
	if len(res.Failed) > 0 {
		g.logger.Printf("[ERROR] [appstop] crash recovery: %d app(s) could not be restarted — KEEPING the marker so the next startup retries; the dead-app alarm owns them meanwhile: %v",
			len(res.Failed), res.Failed)
		return res
	}
	if len(res.Refused) > 0 {
		g.logger.Printf("[WARN] [appstop] crash recovery: %d app(s) were deliberately NOT restarted (drive absent) — KEEPING the marker; this is the gate working, not a fault: %v",
			len(res.Refused), res.Refused)
		return res
	}
	g.End()
	return res
}

// HeldStacks returns the stacks an app-data operation is CURRENTLY holding down, or nil.
//
// Read-only and nil-safe. It exists for the boot reconciler (§8.2): once R-157 mechanism A widened
// the boot window, the sweep could overlap a running volume dump or export and "recover" an app that
// is deliberately stopped mid-operation — restarting it under a tar, which is the inconsistency the
// stop was taken to avoid. Recover() has already run to completion by then, so a marker seen through
// this method belongs to an operation running NOW, not to a crashed one.
func (g *AppStopGuard) HeldStacks() []string {
	if g == nil || g.path == "" {
		return nil
	}
	m, ok := g.read()
	if !ok || !m.Active {
		return nil
	}
	return append([]string(nil), m.Stacks...)
}

// ---- marker persistence (atomic, 0600) — the quiesce shape ------------------------------------

func (g *AppStopGuard) write(m AppStopMarker) error {
	data, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return err
	}
	if err := os.MkdirAll(filepath.Dir(g.path), 0o755); err != nil {
		return err
	}
	tmp := g.path + ".tmp"
	f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
	if err != nil {
		return err
	}
	if _, err := f.Write(data); err != nil {
		f.Close()
		os.Remove(tmp)
		return err
	}
	// fsync before rename: the whole point is surviving a power cut, and a rename that lands ahead
	// of the bytes it points at is a marker that reads as corrupt at exactly the wrong moment.
	if err := f.Sync(); err != nil {
		f.Close()
		os.Remove(tmp)
		return err
	}
	if err := f.Close(); err != nil {
		os.Remove(tmp)
		return err
	}
	return os.Rename(tmp, g.path)
}

func (g *AppStopGuard) read() (AppStopMarker, bool) {
	data, err := os.ReadFile(g.path)
	if err != nil {
		return AppStopMarker{}, false
	}
	var m AppStopMarker
	if err := json.Unmarshal(data, &m); err != nil {
		// Never a silent skip (§9.4): a corrupt marker is LOUD and the bad file is quarantined, so a
		// genuinely interrupted operation leaves a trace instead of vanishing. Still returns false —
		// "no usable marker ⇒ no recovery" is the correct contract, and matches quiesce's.
		g.logger.Printf("[WARN] [appstop] the app-stop marker at %s is corrupt (%v) — quarantining; apps are NOT auto-restarted from it", g.path, err)
		_ = os.Rename(g.path, fmt.Sprintf("%s.corrupt-%d", g.path, g.now().Unix()))
		return AppStopMarker{}, false
	}
	return m, true
}