felhom-controller/controller/internal/backup/hollow_watch.go

package backup

import (
	"sort"
	"sync"
	"time"
)

// ── Part D (v0.279.0): a RUNNING app whose newest copy holds no data is an alarm ─────────────────────
//
// Measured 2026-09-28 on demo-hp (controller 0.277.0): a freshly installed nextcloud carried a leftover
// hold (R-704), so the dump leg skipped its volumes and its unit was never captured; the off-site run then
// pushed a snapshot that "carried NO database dump and NO volume tar". The ONLY trace was one WARN line in
// the container log (R-412 leg 1 made it honest wording, nothing more): no event, no page sentence. The app
// was running and unprotected, and nobody could know.
//
// R-704 closed that cause. This closes the CLASS: at the end of each data leg, every installed app that is
// RUNNING and has named volumes (the data a unit must carry — a database lives in a volume) must have a
// copy that carries data (unitCarriesData — the manifest lists a dump or a tar). If not:
//   - the operator hears it once per app, per tier, per day (the existing operator-only backup_run_failures
//     digest, wired in main.go — no new event type);
//   - the household sees one sentence per app on the backup page, until a later check finds data.
// A HELD app that is really stopped is not flagged — a hold is a deliberate stop, and its copy is the one
// the hold names. A held app that RUNS (yesterday's shape) is flagged.
// Pinned by internal/backup/r_partd_hollow_watch_test.go.

// Hollow-copy tiers.
const (
	HollowTierLocal   = "local"   // the app's own recovery unit (Tier 1), checked at the end of the dump leg
	HollowTierOffsite = "offsite" // the unit the off-site run just pushed (Tier 3)
)

// HollowCopy is one running app whose newest copy on a tier holds no data.
type HollowCopy struct {
	App  string
	Tier string
	At   time.Time // when the check found it
}

type hollowWatch struct {
	mu       sync.Mutex
	current  map[string]HollowCopy // key app|tier
	notified map[string]string     // key app|tier → the day (YYYY-MM-DD) the operator was told
	notify   func(app, tier string)
	now      func() time.Time
}

// SetHollowCopyNotify wires the operator signal (main.go → notifier). INIT-ONLY.
func (m *Manager) SetHollowCopyNotify(fn func(app, tier string)) {
	m.hollow.mu.Lock()
	m.hollow.notify = fn
	m.hollow.mu.Unlock()
}

// HollowCopies returns the flagged apps, sorted, for the backup page.
func (m *Manager) HollowCopies() []HollowCopy {
	m.hollow.mu.Lock()
	defer m.hollow.mu.Unlock()
	out := make([]HollowCopy, 0, len(m.hollow.current))
	for _, h := range m.hollow.current {
		out = append(out, h)
	}
	sort.Slice(out, func(i, j int) bool {
		if out[i].App != out[j].App {
			return out[i].App < out[j].App
		}
		return out[i].Tier < out[j].Tier
	})
	return out
}

// watchesForData: a deployed app that RUNS and has named volumes. A held app that is stopped is skipped.
func (m *Manager) watchesForData(app string) bool {
	if m.stackProvider == nil || m.cfg != nil && m.cfg.IsProtectedStack(app) {
		return false
	}
	if len(m.stackProvider.GetDockerVolumes(app)) == 0 {
		return false
	}
	return m.stackProvider.RefreshAndIsRunning(app)
}

// judgeCopy records the verdict for one app's copy on a tier: flags (and tells the operator, once a day) a
// running app's copy with no data, clears the flag when the copy carries data.
func (m *Manager) judgeCopy(app, tier, unitDir string) {
	key := app + "|" + tier
	hollow := m.watchesForData(app) && !unitCarriesData(unitDir)
	m.hollow.mu.Lock()
	if m.hollow.current == nil {
		m.hollow.current = map[string]HollowCopy{}
		m.hollow.notified = map[string]string{}
	}
	now := time.Now
	if m.hollow.now != nil {
		now = m.hollow.now
	}
	if !hollow {
		delete(m.hollow.current, key)
		m.hollow.mu.Unlock()
		return
	}
	t := now()
	m.hollow.current[key] = HollowCopy{App: app, Tier: tier, At: t}
	day := t.Format("2006-01-02")
	tell := m.hollow.notify != nil && m.hollow.notified[key] != day
	if tell {
		m.hollow.notified[key] = day
	}
	fn := m.hollow.notify
	m.hollow.mu.Unlock()
	m.logger.Printf("[ERROR] [backup] %s is RUNNING but its newest %s copy (%s) holds NO database dump and NO volume tar — it is not protected (Part D)", app, tier, unitDir)
	if tell {
		fn(app, tier)
	}
}

// checkLocalCopies judges every deployed app's own unit at the end of the dump leg.
func (m *Manager) checkLocalCopies() {
	if m.stackProvider == nil {
		return
	}
	for _, s := range m.stackProvider.ListDeployedStacks() {
		m.judgeCopy(s.Name, HollowTierLocal, m.primaryUnitDirFor(s.Name))
	}
}

// FlagHollowCopyForTest records a flagged copy without a backup run (the web package's render test).
// Test-only by name: only judgeCopy may decide a copy is hollow.
func (m *Manager) FlagHollowCopyForTest(app, tier string) {
	m.hollow.mu.Lock()
	defer m.hollow.mu.Unlock()
	if m.hollow.current == nil {
		m.hollow.current = map[string]HollowCopy{}
		m.hollow.notified = map[string]string{}
	}
	m.hollow.current[app+"|"+tier] = HollowCopy{App: app, Tier: tier, At: time.Now()}
}