package reconcile import ( "context" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" ) // Recover consumes the journal's in-flight set at startup: resume-or-rollback for any // op that was mid-execution when the agent crashed (doc 03 §10). This MUST run before // the engine begins issuing new mutations. // // Why it is load-bearing for signed destructive ops (and why it lands with the gate): // the idempotency-key store dedupes a COMPLETED op, but an op that crashed AFTER the // Proxmox POST and BEFORE its terminal record (OpTaskRunning) is not covered by that — // its nonce is already consumed, so a redelivery is rejected as a replay, yet it never // reached a terminal state. Only this startup consumer can resolve it: re-check the // Proxmox task and record the real outcome. // // Resolution per in-flight entry: // - has a task id (OpTaskRunning): re-read the task status once. Stopped → record the // real terminal state (OK → succeeded, else failed). Still running → leave it // in-flight (a later Recover or the task's own completion resolves it). Unreadable → // leave it (cannot safely decide). // - no task id (OpStarted only): the Proxmox POST was never confirmed, so the op // never took effect — record failed (fail-safe, the documented FileNonceStore // direction). A convergent reconcile op is simply re-issued next pass; a one-shot // op did NOT mark its idempotency key applied, so it is not falsely deduped. func (e *Engine) Recover(ctx context.Context) RecoverResult { var res RecoverResult if e.journal == nil { return res } for _, entry := range e.journal.InFlight() { res.Examined++ // PROOF-OF-LAUNCH gate, checked FIRST (campaign pool-effects F1c): an entry with no // journaled UPID means the restore/POST was never confirmed → this transaction created // NOTHING → NEVER destroy the vmid. For Scratch/Rollback (guest-creating) entries this // is load-bearing: a pre-existing guest — possibly another customer's, invisible to the // pool-blind ListLXC existence check — may sit at that vmid, and the old destroy-if- // exists path would have destroyed it under a broad token. Abandon fail-safe instead. // Accepted residual: a crash between obtaining and journaling the UPID may leak a // half-built guest (cleanable) — preferable to destroying an innocent one. if entry.UPID == "" { e.append(terminal(entry, OpFailed)) res.RolledBack++ e.logger.Warn("recover: in-flight op had no task id; marked failed (fail-safe, no destroy)", "op_id", entry.OpID, "vmid", entry.VMID, "kind", entry.Kind) continue } // Scratch entries (slice-6 restore-test) are resolved by TEARDOWN, not by // re-checking a sub-task UPID — a leaked scratch guest is the failure mode that // matters. Handle them BEFORE the generic UPID path (else the restore sub-task's OK // status would mark the entry succeeded while the guest still exists → leak). The // UPID they carry is the launch proof authorizing the destroy-by-existence below. if entry.Scratch { e.recoverScratch(ctx, entry, &res) continue } // Rollback entries (slice-7 bring-up) own a guest the agent was CREATING. An in-flight // launch-proven one means "VMID may be a half-built guest → destroy it" (compensating // rollback) — same reason the Scratch path runs before the generic UPID path: the // restore sub-task's OK status would otherwise mark the entry succeeded and leave a // half-provisioned guest. if entry.Rollback { e.recoverBringUp(ctx, entry, &res) continue } st, err := e.api.TaskStatusOnce(ctx, entry.UPID) if err != nil { res.Unresolved++ e.logger.Warn("recover: cannot read in-flight task status; left in-flight", "op_id", entry.OpID, "upid", entry.UPID, "err", err) continue } if st.Running() { res.StillRunning++ e.logger.Info("recover: in-flight task still running; left in-flight", "op_id", entry.OpID, "upid", entry.UPID) continue } // Stopped: record the real outcome. if st.OK() { e.append(terminal(entry, OpSucceeded)) res.Resumed++ e.logger.Info("recover: in-flight task completed OK; marked succeeded", "op_id", entry.OpID, "upid", entry.UPID) } else { e.append(terminal(entry, OpFailed)) res.Failed++ e.logger.Warn("recover: in-flight task ended non-OK; marked failed", "op_id", entry.OpID, "upid", entry.UPID, "exitstatus", st.ExitStatus) } } if res.Examined > 0 { e.logger.Info("recover: in-flight journal reconciled", "result", res) } return res } // recoverScratch resolves a leaked restore-test scratch guest (slice 6, doc 03 §8/§10). // The invariant: a Scratch entry in-flight at startup means "scratch guest VMID may exist // and must be destroyed." It is idempotent — if the guest is already gone (crash after the // destroy task but before the terminal record), it records terminal-clean. The teardown // routes through the gate as a benign ClassGuestDestroy (agent-tagged scratch provenance) — // the same audit-bearing path the normal teardown uses. func (e *Engine) recoverScratch(ctx context.Context, entry JournalEntry, res *RecoverResult) { lxc, err := e.api.ListLXC(ctx) if err != nil { // Can't tell whether the guest exists → leave in-flight; a later Recover retries. res.Unresolved++ e.logger.Warn("recover: cannot list guests to resolve leaked scratch; left in-flight", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } exists := false for _, g := range lxc { if g.VMID == entry.VMID { exists = true break } } if !exists { // Already gone (idempotent) → the scratch left no leak. e.append(terminal(entry, OpSucceeded)) res.ScratchClean++ e.logger.Info("recover: leaked-scratch entry resolved; guest already gone", "op_id", entry.OpID, "vmid", entry.VMID) return } dec := e.gate.Authorize(IntentForScratchDestroy(e.hostID, entry.VMID), nil) if !dec.Allowed { // Should be benign; if not, fail-safe (leave in-flight, do NOT force a destroy). res.Unresolved++ e.logger.Error("recover: scratch teardown refused by gate (unexpected); left in-flight", "op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason) return } upid, err := e.api.DestroyLXC(ctx, entry.VMID) if err != nil { res.Unresolved++ e.logger.Warn("recover: destroying leaked scratch failed; left in-flight (will retry)", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } if upid != "" { if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil { res.Unresolved++ e.logger.Warn("recover: leaked-scratch destroy task failed; left in-flight (will retry)", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } } e.append(terminal(entry, OpSucceeded)) res.ScratchDestroyed++ e.logger.Warn("recover: destroyed leaked restore-test scratch guest", "op_id", entry.OpID, "vmid", entry.VMID) } // recoverBringUp rolls back a half-built bring-up guest left in-flight by a mid-job crash // (slice 7, doc 03 §9). Invariant: a Rollback entry in-flight at startup means "VMID may be a // half-provisioned guest and MUST be destroyed" (compensating rollback — the guest is only kept // when the bring-up reached its terminal OpSucceeded). Idempotent: already-gone records // terminal-clean. Routes the destroy through the gate as benign ClassGuestDestroy (SameTxnCreated // provenance) — the same audit-bearing path the in-job rollback uses. func (e *Engine) recoverBringUp(ctx context.Context, entry JournalEntry, res *RecoverResult) { lxc, err := e.api.ListLXC(ctx) if err != nil { res.Unresolved++ e.logger.Warn("recover: cannot list guests to resolve half-built bring-up; left in-flight", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } exists := false for _, g := range lxc { if g.VMID == entry.VMID { exists = true break } } if !exists { // Already gone (e.g. the restore POST never created it) → no leak. e.append(terminal(entry, OpSucceeded)) res.BringUpClean++ e.logger.Info("recover: half-built bring-up resolved; guest already gone", "op_id", entry.OpID, "vmid", entry.VMID) return } dec := e.gate.Authorize(IntentForRollbackDestroy(e.hostID, entry.VMID), nil) if !dec.Allowed { // Should be benign (SameTxnCreated); if not, fail-safe — do NOT force a destroy. res.Unresolved++ e.logger.Error("recover: bring-up rollback refused by gate (unexpected); left in-flight", "op_id", entry.OpID, "vmid", entry.VMID, "reason", dec.Reason) return } upid, err := e.api.DestroyLXC(ctx, entry.VMID) if err != nil { res.Unresolved++ e.logger.Warn("recover: destroying half-built bring-up failed; left in-flight (will retry)", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } if upid != "" { if _, err := e.api.WaitTask(ctx, upid, proxmox.WaitOptions{}); err != nil { res.Unresolved++ e.logger.Warn("recover: half-built bring-up destroy task failed; left in-flight (will retry)", "op_id", entry.OpID, "vmid", entry.VMID, "err", err) return } } e.append(terminal(entry, OpSucceeded)) res.BringUpRolledBack++ e.logger.Warn("recover: rolled back half-built bring-up guest", "op_id", entry.OpID, "vmid", entry.VMID) } // RecoverResult summarizes a startup recovery pass. type RecoverResult struct { Examined int Resumed int // task found completed OK and recorded succeeded Failed int // task found ended non-OK and recorded failed RolledBack int // no task id → abandoned (fail-safe) StillRunning int // task still executing → left in-flight Unresolved int // task status unreadable → left in-flight ScratchClean int // scratch entry resolved: guest already gone (no leak) ScratchDestroyed int // scratch entry resolved: leaked guest destroyed BringUpClean int // bring-up rollback entry resolved: guest already gone (no leak) BringUpRolledBack int // bring-up rollback entry resolved: half-built guest destroyed } // terminal builds a terminal journal record preserving the op's identity, with the // idempotency key + scratch flag carried through. func terminal(e JournalEntry, state OpState) JournalEntry { return JournalEntry{ OpID: e.OpID, VMID: e.VMID, Kind: e.Kind, UPID: e.UPID, State: state, IdempKey: e.IdempKey, Scratch: e.Scratch, Rollback: e.Rollback, At: time.Now().UTC(), } }