felhom-controller/controller/internal/stacks/remove_interrupted.go

package stacks

import (
	"encoding/json"
	"os"
	"path/filepath"
	"time"
)

// ── A Remove cut off by a controller restart is finished at boot (R-682) ─────────────────────────────
//
// MEASURED 2026-09-24 on 9202 (chaos round 9): a kill 2 s after the Remove press answered the household
// 502; after the restart the app read deployed and stopped with NO container left — half-removed, and
// nothing told the household to press Remove again. An install (R-681) and an update journal themselves
// and are finished at boot; a remove did not.
//
// THE MECHANISM: RemoveStack writes removePendingFile after every refusal and BEFORE `compose down`, and
// removes it on every return (success or a reported failure — the household saw that answer). So a
// marker found at start means the process died mid-remove, and RecoverInterruptedRemoves finishes it:
//   - app.yaml already gone → the remove had reached its last step; leftovers of the applied record and
//     the marker go;
//   - otherwise → the SAME RemoveStack a second press runs, but KEEPING the drive data and the backups
//     even when the household had asked to delete them. Deleting customer data unattended at boot, while
//     drives may still be settling, is the one step this recovery does not take (R-442's fail-closed
//     direction: when unsure, the data stays). The kept request is logged; the app reads not installed,
//     and its leftover data stays reachable through the not-installed app's own delete action.
// Pinned by r682_remove_interrupted_test.go.

const removePendingFile = ".felhom-remove-pending"

type removeJournal struct {
	At               string `json:"at"`
	RemoveHDDData    bool   `json:"remove_hdd_data"`
	BackupPathsAsked int    `json:"backup_paths_asked"`
}

func markRemovePending(stackDir string, removeHDDData bool, backupPaths int) error {
	b, _ := json.Marshal(removeJournal{At: time.Now().UTC().Format(time.RFC3339), RemoveHDDData: removeHDDData, BackupPathsAsked: backupPaths})
	return os.WriteFile(filepath.Join(stackDir, removePendingFile), append(b, '\n'), 0o644)
}

func clearRemovePending(stackDir string) {
	_ = os.Remove(filepath.Join(stackDir, removePendingFile))
}

// RecoverInterruptedRemoves runs once at start, after the initial scan and BEFORE the boot reconciler
// (which would otherwise start a half-removed app). It returns the names it finished.
func (m *Manager) RecoverInterruptedRemoves() []string {
	m.mu.RLock()
	type cand struct{ name, dir string }
	var cands []cand
	for name, st := range m.stacks {
		if st.ComposePath == "" {
			continue
		}
		dir := filepath.Dir(st.ComposePath)
		if _, err := os.Stat(filepath.Join(dir, removePendingFile)); err == nil {
			cands = append(cands, cand{name, dir})
		}
	}
	m.mu.RUnlock()

	var out []string
	for _, c := range cands {
		var j removeJournal
		if b, err := os.ReadFile(filepath.Join(c.dir, removePendingFile)); err == nil {
			_ = json.Unmarshal(b, &j) // an unreadable journal finishes the safe way (data kept)
		}
		if _, err := os.Stat(filepath.Join(c.dir, "app.yaml")); os.IsNotExist(err) {
			for _, p := range []string{AppliedComposePath(c.dir), filepath.Join(c.dir, appliedMetaDir)} {
				_ = os.RemoveAll(p)
			}
			clearRemovePending(c.dir)
			m.logger.Printf("[INFO] [stacks] remove %s: had reached its last step before the restart; marker cleared (R-682)", c.name)
			out = append(out, c.name)
			continue
		}
		m.logger.Printf("[WARN] [stacks] remove %s was INTERRUPTED by a controller restart (pressed %s) — finishing it now, keeping drive data and backups (R-682)", c.name, j.At)
		if j.RemoveHDDData || j.BackupPathsAsked > 0 {
			m.logger.Printf("[WARN] [stacks] remove %s: the household had asked to delete drive data=%v and %d backup path(s) — NOT done unattended; the data stays (R-682)", c.name, j.RemoveHDDData, j.BackupPathsAsked)
		}
		if _, err := m.RemoveStack(c.name, false, nil); err != nil {
			// ONE attempt, never a retry at a later start: a refusal here (the app came back running, or
			// is busy) leaves it as the household sees it, and a marker kept for later would turn a
			// deliberate stop next week into a removal nobody asked for. A refusal returns before the
			// journal is written, so the marker is cleared here explicitly.
			clearRemovePending(c.dir)
			m.logger.Printf("[ERROR] [stacks] remove %s: finishing the interrupted remove failed: %v — the Remove button finishes it", c.name, err)
			continue
		}
		out = append(out, c.name)
	}
	return out
}