felhom-agent/internal/pbs/verify.go

package pbs

import (
	"context"
	"log/slog"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)

// DefaultVerifyCadence is the verify maintenance interval — more frequent than the full
// self-restore-test, because it's the cheap, key-free, ciphertext-level integrity check (§8).
const DefaultVerifyCadence = 6 * time.Hour

// Target is one PBS datastore to verify, with its client.
type Target struct {
	Datastore string
	Client    *Client
	// StorageID is the PVE storage-entry id this target came from. Carried so an auth failure can
	// NAME the storage the operator has to fix, and so the pbsdr bridge can match the failure to its
	// own descriptor (a host may hold several PBS storages).
	StorageID string
}

// Targets resolves the current set of PBS datastores to verify (re-derived each cycle from
// the PVE storage config — wired in main.go so pbs stays decoupled from how clients are built).
type Targets func(ctx context.Context) ([]Target, error)

// VerifyLoop is the verify maintenance loop (slice 6 Phase B). It runs on its OWN cadence and
// is a reporting/maintenance task like the slice-5 watchdog — it does NOT go through the
// reconcile gate/journal (it mutates no guest). Each cycle, per datastore: trigger a verify →
// poll the task → re-list snapshots → record the per-snapshot verify-state for the report.
type VerifyLoop struct {
	targets Targets
	store   *SnapshotStore
	cadence time.Duration
	logger  *slog.Logger
}

// VerifyLoopOptions configures a VerifyLoop.
type VerifyLoopOptions struct {
	Targets Targets
	Store   *SnapshotStore
	Cadence time.Duration // 0 → default 6h; negative → disabled
	Logger  *slog.Logger
}

// NewVerifyLoop builds a VerifyLoop.
func NewVerifyLoop(opts VerifyLoopOptions) *VerifyLoop {
	logger := opts.Logger
	if logger == nil {
		logger = slog.Default()
	}
	cadence := opts.Cadence
	if cadence == 0 {
		cadence = DefaultVerifyCadence
	}
	return &VerifyLoop{targets: opts.Targets, store: opts.Store, cadence: cadence, logger: logger}
}

// Run verifies on the cadence until ctx is cancelled. It does an immediate first pass (so a
// freshly-started agent reports snapshot inventory + verify-state promptly), then on each
// tick. A negative cadence (or nil targets/store) disables it. Returns nil on cancellation.
func (l *VerifyLoop) Run(ctx context.Context) error {
	if l.cadence < 0 || l.targets == nil || l.store == nil {
		l.logger.Info("pbs: verify loop disabled")
		<-ctx.Done()
		return nil
	}
	l.logger.Info("pbs: verify loop starting", "cadence", l.cadence)
	l.tick(ctx) // immediate inventory + verify
	t := time.NewTicker(l.cadence)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			l.logger.Info("pbs: verify loop shutting down", "reason", ctx.Err())
			return nil
		case <-t.C:
			l.tick(ctx)
		}
	}
}

// RunOnce performs a single synchronous verify+list pass over all targets (used by the
// selftest harness and the live runbook). Same work as one cadence tick.
func (l *VerifyLoop) RunOnce(ctx context.Context) { l.tick(ctx) }

// tick verifies + re-lists each target datastore once. Deterministic enough to drive directly
// in tests. A per-target error is logged and skipped (other datastores still report).
func (l *VerifyLoop) tick(ctx context.Context) {
	targets, err := l.targets(ctx)
	if err != nil {
		l.logger.Warn("pbs: verify loop could not resolve targets; skipping", "err", err)
		return
	}
	for _, t := range targets {
		l.verifyOne(ctx, t)
	}
}

// verifyOne triggers a verify, waits for it, then re-lists + records the snapshots' state.
func (l *VerifyLoop) verifyOne(ctx context.Context, t Target) {
	if upid, err := t.Client.Verify(ctx, t.Datastore); err != nil {
		// Verify-trigger failure is non-fatal: still re-list so we report current state.
		l.logger.Warn("pbs: verify trigger failed; reporting current snapshot state", "datastore", t.Datastore, "err", err)
	} else if err := t.Client.WaitVerify(ctx, upid, 2*time.Second, 30*time.Minute); err != nil {
		l.logger.Warn("pbs: verify task wait failed; reporting current snapshot state", "datastore", t.Datastore, "err", err)
	}

	snaps, err := t.Client.Snapshots(ctx, t.Datastore)
	if err != nil {
		l.logger.Warn("pbs: snapshot list failed", "datastore", t.Datastore, "err", err)
		return
	}
	out := make([]hub.PBSSnapshot, 0, len(snaps))
	failed := 0
	for _, s := range snaps {
		h := s.ToHub()
		if h.VerifyState == VerifyFailed {
			failed++
		}
		out = append(out, h)
	}
	l.store.Record(t.Datastore, out)
	if failed > 0 {
		l.logger.Error("pbs: datastore has FAILED-verify snapshots", "datastore", t.Datastore, "failed", failed, "total", len(out))
	} else {
		l.logger.Info("pbs: verify cycle complete", "datastore", t.Datastore, "snapshots", len(out))
	}
}