Old kernels removed by the box (R-1368, wrapper mode kernel-cleanup); security profiles refreshed after a kernel change (R-1388, mode profile-refresh); Go 1.27.4 (R-1391)

internal/osupdate/kernel.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"log/slog"
 	"regexp"
+	"sort"
+	"strings"
 	"time"
 
 	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
@@ -444,3 +446,129 @@ func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParam
 	}
 	return l.finish(ctx, lg, rep)
 }
+
+// ---- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ----
+//
+// The rule, its refusals and "may this box do it" live in the root wrapper (mode kernel-cleanup); the plan names NO
+// package. This file decides only WHEN: in the night leg, after a healthy host step, never in a night with a kernel
+// step. The wrapper adds its own 20-hour window after a step.
+
+// KernelCleanupResult is the wrapper's kernel_cleanup object.
+type KernelCleanupResult struct {
+	Outcome string `json:"outcome"` // removed | nothing | not_run | dry_run | failed
+	DryRun  bool   `json:"dry_run"`
+	Would   string `json:"would"` // dry run only: remove | nothing | refuse | not_run
+	NotRun  []struct {
+		Code   string `json:"code"`
+		Reason string `json:"reason"`
+	} `json:"not_run"`
+	Refusals []struct {
+		Code   string `json:"code"`
+		Reason string `json:"reason"`
+	} `json:"refusals"`
+	Kept        map[string][]string `json:"kept"`
+	Removal     []string            `json:"removal"`
+	Removed     []string            `json:"removed"`
+	BytesGained *int64              `json:"bytes_gained"`
+	Leftovers   []string            `json:"leftovers"`
+}
+
+// kernelCleanupSkip says why tonight's pass must not call the clean-up ("" = call it).
+func kernelCleanupSkip(appliance bool, trigger string, host, pve, kernel Report, blk hub.WireOSUpdate, okStep func(Report) bool) string {
+	switch {
+	case trigger != "night":
+		return "it runs only in the night leg (never a debug pass)"
+	case !appliance:
+		return "not an appliance install (a BYO host belongs to its owner)"
+	case !blk.Enabled:
+		return "OS updates are switched off for this box"
+	case host.Layer == "" || !okStep(host):
+		return "no healthy host step this pass"
+	case pve.Layer != "" && !okStep(pve):
+		return "the Proxmox step did not end healthy"
+	case kernel.Layer != "":
+		return "a kernel step ran in this night (" + kernel.Outcome + ") — never in the same night"
+	}
+	if due, _ := kernelDue(blk, trigger); due {
+		return "a kernel step is due tonight — never in the same night"
+	}
+	return ""
+}
+
+// runKernelCleanup calls the wrapper once. The hub hears about it only when there is something to tell: kernels were
+// removed (outcome "cleaned"), the wrapper refused, or the purge failed. "Nothing to remove" and "not run" are log
+// lines — one report per box per night would bury the kernel step's own line on the hub's System page.
+func (l *Leg) runKernelCleanup(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
+	lg := l.log().With("run", runID, "layer", LayerKernel, "mode", "kernel-cleanup", "vmid", vmid, "trigger", trigger)
+	start := l.now()
+	wr, err := l.call(ctx, runID, kernelPlan("kernel-cleanup", vmid, map[string]any{"release_id": "kernel-cleanup",
+		"run_id": runID, "trigger": trigger, "ring": blk.Ring}))
+	// NO kernel view and no kernel version in this report: the hub reads a kernel-layer report's `kernel.to` /
+	// `release_id` as "the kernel this STEP was for" (kernelOf), and a clean-up is not a step for any kernel.
+	rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "kernel-cleanup",
+		ReleaseID: "kernel-cleanup", PassSeconds: wr.PassSeconds}
+	var res KernelCleanupResult
+	_ = json.Unmarshal(wr.KernelCleanup, &res)
+	took := l.now().Sub(start).Round(time.Second).String()
+	switch {
+	case err != nil:
+		lg.Warn("osupdate: kernel clean-up — the wrapper gave no answer; not retried tonight", "err", err, "took", took)
+		rep.Outcome, rep.HealthReason = "failed", "kernel clean-up: "+err.Error()
+		return l.finish(ctx, lg, rep)
+	case wr.refused():
+		lg.Warn("osupdate: kernel clean-up REFUSED — nothing was removed; not retried tonight", "refused", string(wr.Refused), "took", took)
+		rep.Outcome, rep.Refused, rep.HealthReason = "refused", wr.Refused, "the clean-up of old kernels was refused; nothing was removed"
+		return l.finish(ctx, lg, rep)
+	case wr.failed():
+		lg.Error("osupdate: kernel clean-up FAILED — look at the box; not retried tonight", "failed", string(wr.Failed), "took", took)
+		rep.Outcome, rep.Refused, rep.HealthReason = "failed", wr.Failed, "the clean-up of old kernels failed"
+		return l.finish(ctx, lg, rep)
+	}
+	switch res.Outcome {
+	case "removed":
+		gained := "unknown"
+		if res.BytesGained != nil {
+			gained = fmt.Sprintf("%.1f GiB", float64(*res.BytesGained)/(1<<30))
+		}
+		rep.Outcome, rep.Healthy = "cleaned", true
+		rep.HealthReason = truncate(fmt.Sprintf("removed %d old kernel package(s): %s; kept %s; %s gained; %d left-over path(s)",
+			len(res.Removed), strings.Join(res.Removed, ", "), strings.Join(sortedKeys(res.Kept), ", "), gained, len(res.Leftovers)), 600)
+		lg.Info("osupdate: kernel clean-up — old kernels removed", "removed", len(res.Removed), "names", strings.Join(res.Removed, ","),
+			"kept", strings.Join(sortedKeys(res.Kept), ","), "bytes_gained", res.BytesGained, "leftovers", len(res.Leftovers), "took", took)
+		return l.finish(ctx, lg, rep)
+	case "nothing":
+		lg.Info("osupdate: kernel clean-up — nothing to remove", "kept", strings.Join(sortedKeys(res.Kept), ","), "took", took)
+	case "not_run":
+		code, why := "", ""
+		if len(res.NotRun) > 0 {
+			code, why = res.NotRun[0].Code, res.NotRun[0].Reason
+		}
+		lg.Info("osupdate: kernel clean-up — not run on this box tonight", "code", code, "reason", why, "took", took)
+	default:
+		lg.Warn("osupdate: kernel clean-up — the wrapper answered with no outcome this agent knows (an older wrapper?)", "outcome", res.Outcome, "took", took)
+	}
+	return Report{}
+}
+
+// KernelCleanupDryRun asks the wrapper what the clean-up WOULD do, and changes nothing (the debug tool: `--selftest
+// kernel-cleanup-dry-run`). The raw kernel_cleanup object, or the refusal.
+func (l *Leg) KernelCleanupDryRun(ctx context.Context, vmid int) (json.RawMessage, error) {
+	wr, err := l.call(ctx, "kcdry"+l.now().UTC().Format("150405"), kernelPlan("kernel-cleanup", vmid,
+		map[string]any{"release_id": "kernel-cleanup", "dry_run": true}))
+	if err != nil {
+		return nil, err
+	}
+	if wr.refused() {
+		return nil, fmt.Errorf("kernel-cleanup dry run refused: %s", wr.Refused)
+	}
+	return wr.KernelCleanup, nil
+}
+
+func sortedKeys(m map[string][]string) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}