diff --git a/CHANGELOG.md b/CHANGELOG.md index 48d4262..8055d68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ Commit only. Each part can be released alone; none needs release 2b. No `configs/` change in this section. +- **The hub is told what the kernel clean-up did and what the profile refresh did (R-1383; felhom.eu `11` §5.11.1, + §5.12).** (1) The clean-up's report carries `kernel_cleanup` (`osupdate.CleanupDetail`: removed, kept, bytes_gained, + leftovers). It still has no `kernel` view and `release_id` stays `kernel-cleanup` — the hub reads only those two as + „the kernel this step was for” (the test asserts the wire form). (2) The host report carries `profile_refresh` + (`osupdate.ProfileRefreshStatus`, from `Leg.ProfileRefreshStanza`, wired in `main.go`): the root wrapper's own record + of the last refresh — so it outlives an agent restart — and how this process's last ask ended. One small file read, + no wrapper call, names checked and capped. No OS report is sent for the refresh (an unknown layer reads as a guest + step on the hub). Tests `TestKernelCleanup_NightOrderOnceAndNoNames` (extended), `TestProfileRefreshStanza_RecordAndLastAsk`, + `TestMainWiresPVETrust`; 9 mutations seen red (one mutant did not compile at first and was done again). - **A boot token that a clock step cannot move (R-1395).** `/disks` answers `guest_boot_key` = `-` beside the unchanged `guest_boot_id` (`-`). The kernel computes the boot time from the wall clock, so a step of the host's clock moved the old token and the diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 4a51a12..18dea85 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -1318,6 +1318,8 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int }} // R-1388 / R-1301 (`11` §5.12): after a start, load the security profiles from a fresh compile when the box runs // another kernel than at the last refresh (the wrapper keeps that record and answers "not needed" otherwise). + // R-1383 (release 3): the last refresh, and how this process's last ask ended, ride the host report. + collector.SetProfileRefreshSource(osLeg.ProfileRefreshStanza) go osLeg.ProfileRefresh(daemonCtx, "agent start", profileGate, 12, 5*time.Minute) // R-1285: the NEW agent runs the bundle policy pass once after its self-update commits — in the normal two-job // delivery the bundle was installed by the OLD process, which has no such pass (measured 2026-10-26). diff --git a/cmd/felhom-agent/pvetrust_wiring_test.go b/cmd/felhom-agent/pvetrust_wiring_test.go index 94ac95a..858644d 100644 --- a/cmd/felhom-agent/pvetrust_wiring_test.go +++ b/cmd/felhom-agent/pvetrust_wiring_test.go @@ -31,6 +31,10 @@ func TestMainWiresPVETrust(t *testing.T) { seen["ensure"]++ case sel.Sel.Name == "SetProxmoxTrustSource": seen["stanza"]++ + case sel.Sel.Name == "SetProfileRefreshSource" && len(node.Args) == 1: + if a, ok := node.Args[0].(*ast.SelectorExpr); ok && a.Sel.Name == "ProfileRefreshStanza" { + seen["profile refresh → report"]++ // R-1383 + } case sel.Sel.Name == "SetGuestPowerGaveUpSource" && len(node.Args) == 1: if a, ok := node.Args[0].(*ast.SelectorExpr); ok && a.Sel.Name == "GuestPowerGaveUpStanza" { seen["guest-power give-up → report"]++ // R-1383 @@ -58,7 +62,7 @@ func TestMainWiresPVETrust(t *testing.T) { return true }) want := map[string]int{"ensure": 2, "stanza": 1, "hook": 1, "timer": 1, "apply=SetTLS": 1, "profile-refresh": 2, - "guest-power give-up → report": 1} + "guest-power give-up → report": 1, "profile refresh → report": 1} for k, n := range want { if seen[k] < n { t.Errorf("main.go wires %q %d time(s), want at least %d (the two-call ones: at start and after a bundle install)", k, seen[k], n) diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 3a66623..5ee05d9 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -126,6 +126,7 @@ type Collector struct { leafFP string // v0.48.0: served local-API leaf fp (static per process; "" when local API disabled) pveTrust func() any // R-1380: the proxmox_trust stanza (nil = omitted) guestPowerGaveUp func() any // R-1383: the guest_power_gave_up stanza (nil = omitted) + profileRefresh func() any // R-1383: the profile_refresh stanza (nil = omitted) addrEnum AddressEnumerator // v0.119.0: host interface enumeration; nil => the REAL one (see collectAddresses) wg WireguardReporter // S3: offsite-tunnel status (nil → stanza omitted) pbsdr PBSDRReporter // slice 2: PBS DR tier bridge state (nil → stanza omitted) @@ -217,6 +218,12 @@ func (c *Collector) SetProxmoxTrustSource(f func() any) *Collector { return c } +// SetProfileRefreshSource wires the `profile_refresh` stanza (R-1383). Optional; nil, or a nil result, omits it. +func (c *Collector) SetProfileRefreshSource(f func() any) *Collector { + c.profileRefresh = f + return c +} + // SetGuestPowerGaveUpSource wires the `guest_power_gave_up` stanza (R-1383). Optional; nil, or a nil result, omits it. func (c *Collector) SetGuestPowerGaveUpSource(f func() any) *Collector { c.guestPowerGaveUp = f @@ -405,6 +412,13 @@ func (c *Collector) SetOOBReporter(o OOBReporter) *Collector { // error (no useful report — the cycle skips the POST); a failed per-guest // GuestConfig degrades that guest to status="unknown" without spec but still sends; // a cloudflared probe failure yields status="unknown" and is never fatal. +func (c *Collector) profileRefreshStanza() any { + if c.profileRefresh == nil { + return nil + } + return c.profileRefresh() +} + func (c *Collector) guestPowerGaveUpStanza() any { if c.guestPowerGaveUp == nil { return nil @@ -450,6 +464,7 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { LeafFingerprint: c.leafFP, ProxmoxTrust: c.proxmoxTrust(), GuestPowerGaveUp: c.guestPowerGaveUpStanza(), + ProfileRefresh: c.profileRefreshStanza(), Addresses: c.collectAddresses(), System: c.systemInfo(ctx, ns), } diff --git a/internal/hub/report.go b/internal/hub/report.go index b088c6b..2105510 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -182,6 +182,10 @@ type HostReport struct { // byte-stable. The hub parser is pinned by hub/internal/monitor/controller_supervisor_test.go // against the JSON TestControllerSupervisorStanza_WireShape pins here. ControllerSupervisor *ControllerSupervisorStatus `json:"controller_supervisor,omitempty"` + // ProfileRefresh (R-1383, release 3): the box's last security-profile refresh and how the agent's last ask ended + // (osupdate.ProfileRefreshStatus, field-exact; hub sysfacts.ProfileRefresh). `any` + omitempty: absent when there + // is nothing to say or the source is not wired. + ProfileRefresh any `json:"profile_refresh,omitempty"` // GuestPowerGaveUp (R-1383, D78): the guests the guest-power watchdog stopped trying to start (localapi // GuestPowerGiveUp, field-exact: vmid, since, attempts). The hub raises one operator alarm per guest and clears it // when the guest runs. `any` + omitempty: absent when there is none or the source is not wired. diff --git a/internal/osupdate/kernel.go b/internal/osupdate/kernel.go index af8f354..3f51d3c 100644 --- a/internal/osupdate/kernel.go +++ b/internal/osupdate/kernel.go @@ -531,6 +531,8 @@ func (l *Leg) runKernelCleanup(ctx context.Context, runID string, vmid int, trig gained = fmt.Sprintf("%.1f GiB", float64(*res.BytesGained)/(1<<30)) } rep.Outcome, rep.Healthy = "cleaned", true + rep.KernelCleanup = &CleanupDetail{Removed: capNames(res.Removed, 40), Kept: capNames(sortedKeys(res.Kept), 40), + BytesGained: res.BytesGained, Leftovers: len(res.Leftovers)} rep.HealthReason = truncate(fmt.Sprintf("removed %d old kernel package(s): %s; kept %s; %s gained; %d left-over path(s)", len(res.Removed), strings.Join(res.Removed, ", "), strings.Join(sortedKeys(res.Kept), ", "), gained, len(res.Leftovers)), 600) lg.Info("osupdate: kernel clean-up — old kernels removed", "removed", len(res.Removed), "names", strings.Join(res.Removed, ","), @@ -564,6 +566,22 @@ func (l *Leg) KernelCleanupDryRun(ctx context.Context, vmid int) (json.RawMessag return wr.KernelCleanup, nil } +// capNames bounds a list that goes to the hub: at most n names, each at most 120 bytes; never nil (the hub reads +// "removed: []" as "none", not as "unknown"). +func capNames(in []string, n int) []string { + out := make([]string, 0, len(in)) + for _, x := range in { + if len(out) == n { + break + } + if len(x) > 120 { + x = x[:120] + } + out = append(out, x) + } + return out +} + func sortedKeys(m map[string][]string) []string { out := make([]string, 0, len(m)) for k := range m { diff --git a/internal/osupdate/kernel_test.go b/internal/osupdate/kernel_test.go index d5d8c0a..5a46566 100644 --- a/internal/osupdate/kernel_test.go +++ b/internal/osupdate/kernel_test.go @@ -400,6 +400,25 @@ func TestKernelCleanup_NightOrderOnceAndNoNames(t *testing.T) { if len(lastRep.Kernel) != 0 || kverRE.MatchString(lastRep.ReleaseID) { t.Fatalf("the clean-up's report names a kernel (the hub reads that as a kernel step): %+v", lastRep) } + // R-1383 (release 3): the detail the hub's System page shows — removed, kept, gained, left-overs — rides the same + // report as its own object. On the wire it is `kernel_cleanup`, and the report STILL has no `kernel` key. + d := lastRep.KernelCleanup + if d == nil || len(d.Removed) != 1 || d.Removed[0] != "proxmox-kernel-7.0.14-20-pve-signed" || + strings.Join(d.Kept, ",") != "7.0.14-22-pve,7.0.2-6-pve" || d.BytesGained == nil || *d.BytesGained != 1073741824 || d.Leftovers != 0 { + t.Fatalf("the clean-up's detail did not reach the hub: %+v", d) + } + wire, _ := json.Marshal(lastRep) + var keys map[string]json.RawMessage + _ = json.Unmarshal(wire, &keys) + if _, has := keys["kernel"]; has { + t.Fatalf("the clean-up's report carries a `kernel` view on the wire: %s", wire) + } + if string(keys["kernel_cleanup"]) != `{"removed":["proxmox-kernel-7.0.14-20-pve-signed"],"kept":["7.0.14-22-pve","7.0.2-6-pve"],"bytes_gained":1073741824,"leftovers":0}` { + t.Fatalf("kernel_cleanup on the wire = %s", keys["kernel_cleanup"]) + } + if string(keys["release_id"]) != `"kernel-cleanup"` { + t.Fatalf("release_id on the wire = %s (a kernel version there is read as a step for it)", keys["release_id"]) + } // the same night again: the pass is skipped as a whole — never a second clean-up l.Run(context.Background(), 9201, "night") if len(cleanupCalls(w)) != 1 { diff --git a/internal/osupdate/leg.go b/internal/osupdate/leg.go index f96f863..483a9da 100644 --- a/internal/osupdate/leg.go +++ b/internal/osupdate/leg.go @@ -188,10 +188,23 @@ type Report struct { // outcomes of this layer: staged | applied (the new kernel is the default) | fell_back | health_failed (self-revert // started) | self_reverted | revert_failed | judging | nothing | refused | failed. Kernel json.RawMessage `json:"kernel,omitempty"` + // KernelCleanup (R-1383, release 3): ONLY on the mode kernel-cleanup report with outcome "cleaned" — what the + // clean-up removed, kept and gained, for the hub's System page. It is not the `kernel` view: this report still + // carries no kernel view and no kernel as its release_id, or the hub would read it as a kernel STEP for that kernel + // (the hub's kernelOf reads only those two). Pinned by TestKernelCleanup_NightOrderOnceAndNoNames. + KernelCleanup *CleanupDetail `json:"kernel_cleanup,omitempty"` unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it } +// CleanupDetail is the clean-up's result as the hub shows it (hub osupdates.CleanupDetail, field-exact). +type CleanupDetail struct { + Removed []string `json:"removed"` // package names + Kept []string `json:"kept"` // kernel versions that stay + BytesGained *int64 `json:"bytes_gained,omitempty"` // nil = the wrapper could not measure it + Leftovers int `json:"leftovers"` // paths of a removed kernel that are still there +} + // Reporter posts a report to the hub (*hub.Client). type Reporter interface { PostOSReport(ctx context.Context, body []byte) error @@ -211,9 +224,14 @@ type Leg struct { MinGap time.Duration // between night runs (default 20 h) Now func() time.Time Sleep func(context.Context, time.Duration) + // ProfileRecordPath is the root wrapper's record of the last security-profile refresh (0644, read-only for the + // agent); "" = DefaultProfileRecordPath. A seam for the tests. + ProfileRecordPath string - mu sync.Mutex - block *hub.WireOSUpdate + mu sync.Mutex + block *hub.WireOSUpdate + profileAsk string // how the last ask of the profile refresh ended in THIS process ("" = never asked) + profileAskAt time.Time } // planFile / reportFile: the plan the agent writes and the copy of the report the wrapper keeps beside it (R-868). diff --git a/internal/osupdate/profilerefresh.go b/internal/osupdate/profilerefresh.go index 814d219..9ed4822 100644 --- a/internal/osupdate/profilerefresh.go +++ b/internal/osupdate/profilerefresh.go @@ -5,11 +5,14 @@ package osupdate // profile-refresh loads every profile from a fresh compile, once per kernel the box starts on, and restarts only a // failed service whose own program's profile was stale. This file decides WHEN to ask: at every agent start and after // a config bundle was installed (a new wrapper may know the mode the old one refused) — the wrapper itself answers -// "not needed" when this kernel was already done. Nothing is sent to the hub (no hub field exists for it; R-1383). +// "not needed" when this kernel was already done. No OS report is sent for it (an unknown layer reads as a GUEST step on +// the hub); since release 3 the result rides the host report as the `profile_refresh` stanza (ProfileRefreshStanza). import ( "context" "encoding/json" + "os" + "regexp" "strings" "time" ) @@ -29,6 +32,76 @@ type ProfileRefreshResult struct { Seconds float64 `json:"seconds"` } +// DefaultProfileRecordPath is where the root wrapper keeps the record of its last refresh (PROFILE_STATE, 0644). +const DefaultProfileRecordPath = "/var/lib/felhom-os-apply/profile-refresh.json" + +// ProfileRefreshStatus is the host report's `profile_refresh` stanza (R-1383, release 3; hub sysfacts.ProfileRefresh, +// field-exact): the last refresh the box did — from the root wrapper's own record, so it outlives an agent restart — +// and how the last ask of THIS agent process ended. The refresh restarted a failed service on the N100 and nobody +// outside the box's journal could see it. +type ProfileRefreshStatus struct { + Kernel string `json:"kernel,omitempty"` // the kernel the last refresh ran on + At string `json:"at,omitempty"` // RFC3339, when + Rewritten int `json:"rewritten"` // how many compiled profiles changed + Restarted []string `json:"restarted,omitempty"` // units it restarted (failed, with a stale profile) + LastAsk string `json:"last_ask,omitempty"` // refreshed | not_needed | failed | refused | no_answer + LastAskAt string `json:"last_ask_at,omitempty"` +} + +var profileNameRe = regexp.MustCompile(`^[A-Za-z0-9@._:+-]{1,120}$`) + +// ProfileRefreshStanza builds the stanza, or an UNTYPED nil when there is nothing to say (no record and never asked: +// a BYO host, or an agent that did not ask yet) — the report then omits the field. It reads one small root-written +// file and never calls the wrapper. Names that are not plain unit / kernel names are dropped, the list is capped. +func (l *Leg) ProfileRefreshStanza() any { + path := l.ProfileRecordPath + if path == "" { + path = DefaultProfileRecordPath + } + var st ProfileRefreshStatus + have := false + if raw, err := os.ReadFile(path); err == nil && len(raw) < 64<<10 { + var rec struct { + Kernel string `json:"kernel"` + At string `json:"at"` + Rewritten []string `json:"rewritten"` + Restarted []string `json:"restarted"` + } + if json.Unmarshal(raw, &rec) == nil && (rec.Kernel != "" || rec.At != "") { + have = true + if profileNameRe.MatchString(rec.Kernel) { + st.Kernel = rec.Kernel + } + if _, err := time.Parse(time.RFC3339, rec.At); err == nil { + st.At = rec.At + } + st.Rewritten = len(rec.Rewritten) + for _, u := range rec.Restarted { + if len(st.Restarted) < 16 && profileNameRe.MatchString(u) { + st.Restarted = append(st.Restarted, u) + } + } + } + } + l.mu.Lock() + ask, at := l.profileAsk, l.profileAskAt + l.mu.Unlock() + if ask != "" { + have = true + st.LastAsk, st.LastAskAt = ask, at.UTC().Format(time.RFC3339) + } + if !have { + return nil + } + return st +} + +func (l *Leg) noteProfileAsk(outcome string) { + l.mu.Lock() + l.profileAsk, l.profileAskAt = outcome, l.now() + l.mu.Unlock() +} + // ProfileGate takes the heavy-op gate (a backup, a restore-test or an OS step must not run beside the refresh). type ProfileGate func() (release func(), busy string, ok bool) @@ -69,12 +142,15 @@ func (l *Leg) ProfileRefresh(ctx context.Context, why string, gate ProfileGate, switch { case err != nil: lg.Warn("osupdate: profile refresh — the wrapper gave no answer; the next agent start asks again", "err", err, "took", took) + l.noteProfileAsk("no_answer") return "" case wr.refused(): lg.Info("osupdate: profile refresh refused (an older wrapper that does not know the mode, or a BYO host)", "refused", string(wr.Refused)) + l.noteProfileAsk("refused") return "" case wr.failed(): lg.Error("osupdate: profile refresh FAILED — the profiles are as before; the next agent start asks again", "failed", string(wr.Failed), "took", took) + l.noteProfileAsk("failed") return "failed" } var res ProfileRefreshResult @@ -97,5 +173,8 @@ func (l *Leg) ProfileRefresh(ctx context.Context, why string, gate ProfileGate, default: lg.Warn("osupdate: profile refresh — the wrapper answered with no outcome this agent knows", "outcome", res.Outcome) } + if res.Outcome == "refreshed" || res.Outcome == "not_needed" { + l.noteProfileAsk(res.Outcome) + } return res.Outcome } diff --git a/internal/osupdate/profilerefresh_test.go b/internal/osupdate/profilerefresh_test.go index 1713462..10c53c3 100644 --- a/internal/osupdate/profilerefresh_test.go +++ b/internal/osupdate/profilerefresh_test.go @@ -3,6 +3,9 @@ package osupdate import ( "context" "encoding/json" + "os" + "path/filepath" + "strings" "testing" "time" @@ -95,3 +98,55 @@ func TestProfileRefresh_RefusedAndFailed(t *testing.T) { t.Fatalf("failed → %q", got) } } + +// R-1383 (release 3): the refresh restarted a failed service on the N100 and only the box's journal knew. The stanza +// that rides the host report: nothing before any refresh (an UNTYPED nil — the report omits the field); after a +// refresh, the root wrapper's own record (so it outlives an agent restart) and how this process's last ask ended. +// COMPANION RED-PROOFS (observed, felhom.eu audits/night-build-2026-11-14/B/redproof-agent-b3.txt). +func TestProfileRefreshStanza_RecordAndLastAsk(t *testing.T) { + w := &fakeWrapper{t: t, profileRep: refreshed()} + l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) + rec := filepath.Join(t.TempDir(), "profile-refresh.json") + l.ProfileRecordPath = rec + now := time.Date(2026, 11, 14, 11, 30, 11, 0, time.UTC) + l.Now = func() time.Time { return now } + if st := l.ProfileRefreshStanza(); st != nil { + t.Fatalf("before any refresh the stanza is %#v, want an untyped nil", st) + } + // the wrapper refreshes and writes ITS record (the fake wrapper does not, so the test writes what the real one does) + if got := l.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "refreshed" { + t.Fatalf("outcome = %q", got) + } + if err := os.WriteFile(rec, []byte(`{"at":"2026-11-14T11:30:11Z","kernel":"7.0.14-28-pve","restarted":["chrony.service","bad name; rm -rf"],`+ + `"rewritten":["f6da0254.0/usr.sbin.chronyd","f6da0254.0/lxc-containers"]}`), 0o644); err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(l.ProfileRefreshStanza()) + if string(b) != `{"kernel":"7.0.14-28-pve","at":"2026-11-14T11:30:11Z","rewritten":2,"restarted":["chrony.service"],"last_ask":"refreshed","last_ask_at":"2026-11-14T11:30:11Z"}` { + t.Fatalf("stanza after a refresh = %s", b) + } + // an agent restart: a new process, the same record — the box's last refresh is still told + l2, _ := newLeg(t, &fakeWrapper{t: t}, &hub.WireOSUpdate{Ring: 1, Enabled: true}) + l2.ProfileRecordPath = rec + b, _ = json.Marshal(l2.ProfileRefreshStanza()) + if string(b) != `{"kernel":"7.0.14-28-pve","at":"2026-11-14T11:30:11Z","rewritten":2,"restarted":["chrony.service"]}` { + t.Fatalf("stanza after an agent restart = %s", b) + } + // a failed ask is told, beside the older record + l2.Now = func() time.Time { return now.Add(time.Hour) } + l2.Runner = (&fakeWrapper{t: t, profileRep: &WrapperReport{Failed: json.RawMessage(`{"code":"F1","reason":"apparmor_parser failed"}`)}}) + if got := l2.ProfileRefresh(context.Background(), "agent start", nil, 1, time.Second); got != "failed" { + t.Fatalf("outcome = %q", got) + } + b, _ = json.Marshal(l2.ProfileRefreshStanza()) + if !strings.Contains(string(b), `"last_ask":"failed","last_ask_at":"2026-11-14T12:30:11Z"`) || !strings.Contains(string(b), `"kernel":"7.0.14-28-pve"`) { + t.Fatalf("stanza after a failed ask = %s", b) + } + // a record that is not JSON, or huge, says nothing and breaks nothing + _ = os.WriteFile(rec, []byte("not json"), 0o644) + l3, _ := newLeg(t, &fakeWrapper{t: t}, &hub.WireOSUpdate{Ring: 1, Enabled: true}) + l3.ProfileRecordPath = rec + if st := l3.ProfileRefreshStanza(); st != nil { + t.Fatalf("an unreadable record gave the stanza %#v", st) + } +}