felhom-agent/internal/reconcile/recover.go

package reconcile

import (
	"context"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
)

// Recover consumes the journal's in-flight set at startup: resume-or-rollback for any
// op that was mid-execution when the agent crashed (doc 03 §10). This MUST run before
// the engine begins issuing new mutations.
//
// Why it is load-bearing for signed destructive ops (and why it lands with the gate):
// the idempotency-key store dedupes a COMPLETED op, but an op that crashed AFTER the
// Proxmox POST and BEFORE its terminal record (OpTaskRunning) is not covered by that —
// its nonce is already consumed, so a redelivery is rejected as a replay, yet it never
// reached a terminal state. Only this startup consumer can resolve it: re-check the
// Proxmox task and record the real outcome.
//
// Resolution per in-flight entry:
//   - has a task id (OpTaskRunning): re-read the task status once. Stopped → record the
//     real terminal state (OK → succeeded, else failed). Still running → leave it
//     in-flight (a later Recover or the task's own completion resolves it). Unreadable →
//     leave it (cannot safely decide).
//   - no task id (OpStarted only): the Proxmox POST was never confirmed, so the op
//     never took effect — record failed (fail-safe, the documented FileNonceStore
//     direction). A convergent reconcile op is simply re-issued next pass; a one-shot
//     op did NOT mark its idempotency key applied, so it is not falsely deduped.
func (e *Engine) Recover(ctx context.Context) RecoverResult {
	var res RecoverResult
	if e.journal == nil {
		return res
	}
	for _, entry := range e.journal.InFlight() {
		res.Examined++

		// Scratch entries (slice-6 restore-test) are resolved by TEARDOWN, not by
		// re-checking a sub-task UPID — a leaked scratch guest is the failure mode that
		// matters. Handle them BEFORE the generic UPID path (else the restore sub-task's OK
		// status would mark the entry succeeded while the guest still exists → leak).
		if entry.Scratch {
			e.recoverScratch(ctx, entry, &res)
			continue
		}

		// Rollback entries (slice-7 bring-up) own a guest the agent was CREATING. An in-flight
		// one means "VMID may be a half-built guest → destroy it" (compensating rollback) — same
		// reason the Scratch path runs before the generic UPID path: the restore sub-task's OK
		// status would otherwise mark the entry succeeded and leave a half-provisioned guest.
		if entry.Rollback {
			e.recoverBringUp(ctx, entry, &res)
			continue
		}

		if entry.UPID == "" {
			// POST never confirmed → abandon (fail-safe).
			e.append(terminal(entry, OpFailed))
			res.RolledBack++
			e.logger.Warn("recover: in-flight op had no task id; marked failed (fail-safe)",
				"op_id", entry.OpID, "vmid", entry.VMID, "kind", entry.Kind)
			continue
		}
		st, err := e.api.TaskStatusOnce(ctx, entry.UPID)
		if err != nil {
			res.Unresolved++
			e.logger.Warn("recover: cannot read in-flight task status; left in-flight",
				"op_id", entry.OpID, "upid", entry.UPID, "err", err)
			continue
		}
		if st.Running() {
			res.StillRunning++
			e.logger.Info("recover: in-flight task still running; left in-flight",
				"op_id", entry.OpID, "upid", entry.UPID)
			continue
		}
		// Stopped: record the real outcome.
		if st.OK() {
			e.append(terminal(entry, OpSucceeded))
			res.Resumed++
			e.logger.Info("recover: in-flight task completed OK; marked succeeded",
				"op_id", entry.OpID, "upid", entry.UPID)
		} else {
			e.append(terminal(entry, OpFailed))
			res.Failed++
			e.logger.Warn("recover: in-flight task ended non-OK; marked failed",
				"op_id", entry.OpID, "upid", entry.UPID, "exitstatus", st.ExitStatus)
		}
	}
	if res.Examined > 0 {
		e.logger.Info("recover: in-flight journal reconciled", "result", res)
	}
	return res
}

// recoverScratch resolves a leaked restore-test scratch guest (slice 6, doc 03 §8/§10).
// The invariant: a Scratch entry in-flight at startup means "scratch guest VMID may exist
// and must be destroyed." It is idempotent — if the guest is already gone (crash after the
// destroy task but before the terminal record), it records terminal-clean. The teardown
// routes through the gate as a benign ClassGuestDestroy (agent-tagged scratch provenance) —
// the same audit-bearing path the normal teardown uses.
func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *RecoverResult) {
	lxc, err := e.api.ListLXC(ctx)
	if err != nil {
		// Can't tell whether the guest exists → leave in-flight; a later Recover retries.
		res.Unresolved++
		e.logger.Warn("recover: cannot list guests to resolve leaked scratch; left in-flight",
			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
		return
	}
	exists := false
	for _, g := range lxc {
		if g.VMID == entry.VMID {
			exists = true
			break
		}
	}
	if !exists {
		// Already gone (idempotent) → the scratch left no leak.
		e.append(terminal(entry, OpSucceeded))
		res.ScratchClean++
		e.logger.Info("recover: leaked-scratch entry resolved; guest already gone",
			"op_id", entry.OpID, "vmid", entry.VMID)
		return
	}

	dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, entry.VMID), nil)
	if !dec.Allowed {
		// Should be benign; if not, fail-safe (leave in-flight, do NOT force a destroy).
		res.Unresolved++
		e.logger.Error("recover: scratch teardown refused by gate (unexpected); left in-flight",
			"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
		return
	}
	upid, err := e.api.DestroyLXC(ctx, entry.VMID)
	if err != nil {
		res.Unresolved++
		e.logger.Warn("recover: destroying leaked scratch failed; left in-flight (will retry)",
			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
		return
	}
	if upid != "" {
		if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
			res.Unresolved++
			e.logger.Warn("recover: leaked-scratch destroy task failed; left in-flight (will retry)",
				"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
			return
		}
	}
	e.append(terminal(entry, OpSucceeded))
	res.ScratchDestroyed++
	e.logger.Warn("recover: destroyed leaked restore-test scratch guest",
		"op_id", entry.OpID, "vmid", entry.VMID)
}

// recoverBringUp rolls back a half-built bring-up guest left in-flight by a mid-job crash
// (slice 7, doc 03 §9). Invariant: a Rollback entry in-flight at startup means "VMID may be a
// half-provisioned guest and MUST be destroyed" (compensating rollback — the guest is only kept
// when the bring-up reached its terminal OpSucceeded). Idempotent: already-gone records
// terminal-clean. Routes the destroy through the gate as benign ClassGuestDestroy (SameTxnCreated
// provenance) — the same audit-bearing path the in-job rollback uses.
func (e *Engine) recoverBringUp(ctx context.Context, entry JournalEntry, res *RecoverResult) {
	lxc, err := e.api.ListLXC(ctx)
	if err != nil {
		res.Unresolved++
		e.logger.Warn("recover: cannot list guests to resolve half-built bring-up; left in-flight",
			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
		return
	}
	exists := false
	for _, g := range lxc {
		if g.VMID == entry.VMID {
			exists = true
			break
		}
	}
	if !exists {
		// Already gone (e.g. the restore POST never created it) → no leak.
		e.append(terminal(entry, OpSucceeded))
		res.BringUpClean++
		e.logger.Info("recover: half-built bring-up resolved; guest already gone",
			"op_id", entry.OpID, "vmid", entry.VMID)
		return
	}

	dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, entry.VMID), nil)
	if !dec.Allowed {
		// Should be benign (SameTxnCreated); if not, fail-safe — do NOT force a destroy.
		res.Unresolved++
		e.logger.Error("recover: bring-up rollback refused by gate (unexpected); left in-flight",
			"op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason)
		return
	}
	upid, err := e.api.DestroyLXC(ctx, entry.VMID)
	if err != nil {
		res.Unresolved++
		e.logger.Warn("recover: destroying half-built bring-up failed; left in-flight (will retry)",
			"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
		return
	}
	if upid != "" {
		if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil {
			res.Unresolved++
			e.logger.Warn("recover: half-built bring-up destroy task failed; left in-flight (will retry)",
				"op_id", entry.OpID, "vmid", entry.VMID, "err", err)
			return
		}
	}
	e.append(terminal(entry, OpSucceeded))
	res.BringUpRolledBack++
	e.logger.Warn("recover: rolled back half-built bring-up guest",
		"op_id", entry.OpID, "vmid", entry.VMID)
}

// RecoverResult summarizes a startup recovery pass.
type RecoverResult struct {
	Examined          int
	Resumed           int // task found completed OK and recorded succeeded
	Failed            int // task found ended non-OK and recorded failed
	RolledBack        int // no task id → abandoned (fail-safe)
	StillRunning      int // task still executing → left in-flight
	Unresolved        int // task status unreadable → left in-flight
	ScratchClean      int // scratch entry resolved: guest already gone (no leak)
	ScratchDestroyed  int // scratch entry resolved: leaked guest destroyed
	BringUpClean      int // bring-up rollback entry resolved: guest already gone (no leak)
	BringUpRolledBack int // bring-up rollback entry resolved: half-built guest destroyed
}

// terminal builds a terminal journal record preserving the op's identity, with the
// idempotency key + scratch flag carried through.
func terminal(e JournalEntry, state OpState) JournalEntry {
	return JournalEntry{
		OpID:     e.OpID,
		VMID:     e.VMID,
		Kind:     e.Kind,
		UPID:     e.UPID,
		State:    state,
		IdempKey: e.IdempKey,
		Scratch:  e.Scratch,
		Rollback: e.Rollback,
		At:       time.Now().UTC(),
	}
}