diff --git a/documentation/audits/os-guest-lane-2026-10-04/partA/A1-snapshot.txt b/documentation/audits/os-guest-lane-2026-10-04/partA/A1-snapshot.txt new file mode 100644 index 00000000..74a7c46c --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partA/A1-snapshot.txt @@ -0,0 +1,30 @@ +node=demo-hp +libc6 2.41-12+deb13u3 +adventurelog Up 38 minutes (healthy) +adventurelog-frontend Up 38 minutes (healthy) +adventurelog-postgres Up 38 minutes (healthy) +bentopdf Up 38 minutes (healthy) +bookstack Up 38 minutes (healthy) +bookstack-db Up 38 minutes (healthy) +calibre-web Up 38 minutes (healthy) +cloudflared Up 38 minutes +docmost Up 38 minutes (healthy) +docmost-postgres Up 38 minutes (healthy) +docmost-redis Up 38 minutes (healthy) +felhom-controller Up 38 minutes (healthy) +filebrowser Up 37 minutes (healthy) +kimai Up 38 minutes (healthy) +kimai-db Up 38 minutes (healthy) +opengist Up 38 minutes (healthy) +paperless-postgres Up 38 minutes (healthy) +paperless-redis Up 38 minutes (healthy) +paperless-webserver Up 37 minutes (healthy) +privatebin Up 38 minutes (healthy) +romm Up 37 minutes (healthy) +romm-db Up 37 minutes (healthy) +romm-redis Up 37 minutes (healthy) +traefik Up 38 minutes +python3: can't open file '/root/pveapi.py': [Errno 13] Permission denied +POST /nodes/demo-hp/lxc/9201/snapshot: http 200 task exit=snapshot feature is not available seconds=0.1 +`-> current You are here! + data 61.90 2.69 diff --git a/documentation/audits/os-guest-lane-2026-10-04/partA/A2-after.txt b/documentation/audits/os-guest-lane-2026-10-04/partA/A2-after.txt new file mode 100644 index 00000000..d3460224 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partA/A2-after.txt @@ -0,0 +1,2 @@ + data 61.90 2.69 +`-> current You are here! diff --git a/documentation/audits/os-guest-lane-2026-10-04/partA/README.md b/documentation/audits/os-guest-lane-2026-10-04/partA/README.md new file mode 100644 index 00000000..6b032786 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partA/README.md @@ -0,0 +1,19 @@ +# Part A — the snapshot undo (R-837), demo-hp 9201, 2026-10-04 ~10:30 CEST + +**Result: a snapshot of a customer guest is NOT possible. The automatic undo is not built (the brief's stop rule).** + +- Thin pool before: data 61.90 %, metadata 2.69 % (`A1-snapshot.txt`). After: unchanged (`A2-after.txt`) — nothing was created. +- The agent's token has the rights: role `FelhomAgentGuest` on `/pool/felhom` holds `VM.Snapshot` and + `VM.Snapshot.Rollback`. Called AS THE TOKEN (`POST /nodes/demo-hp/lxc/9201/snapshot`, http 200): the task ends + `snapshot feature is not available` in 0.1 s. As root, `pct snapshot 9201 r837root`: the same. +- Why, from the PVE source: `PVE/AbstractConfig.pm:755-757` dies with that message when + `has_feature('snapshot', …, $snapname eq 'vzdump')` fails; `PVE/LXC/Config.pm:97-110` checks EVERY mount point and + skips non-backup ones ONLY when that last flag is set — i.e. only for the backup's own snapshot named `vzdump`. A + customer guest always carries two host-path binds (`mp8 /mnt/felhom-drives`, `mp9 …/bootstrap`), which have no + snapshot feature. So: rootfs and mp0 are LVM-thin and could snapshot; the guest cannot. +- The nightly whole-guest backup still works in snapshot mode (`create storage snapshot 'vzdump'`, 04:34:55). +- **Rejected, not tried:** naming a snapshot `vzdump` to pass the check — the name is the backup's own and a + collision would break the night's backup; and taking raw LVM thin snapshots of rootfs + mp0 behind PVE's back (a + new mechanism nobody has measured — a STATUS decision, not a session improvisation). +- Steps 2–3 (apply by hand, roll back) were not run: there is nothing to roll back to. 9201 is brought current by the + product's own OS leg in Part G. diff --git a/documentation/audits/os-guest-lane-2026-10-04/partB/redproof.txt b/documentation/audits/os-guest-lane-2026-10-04/partB/redproof.txt new file mode 100644 index 00000000..83a476e1 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partB/redproof.txt @@ -0,0 +1,14 @@ +# Red-proof of configs/felhom-os-apply, 2026-10-04: for each refusal code, every `raise Refused("Rx", …)` is replaced by `pass` in a COPY, and the suite is run against the copy. +R1: 7 raise(s) removed -> suite rc=1; failing tests: test_R1_not_owned_by_the_agent, test_R1_path_outside_the_plan_dir, test_R1_symlink; own test(s) failed: YES +R2: 2 raise(s) removed -> suite rc=1; failing tests: test_R2_non_debian_origin_in_the_plan, test_R2_non_debian_origin_in_the_simulation; own test(s) failed: YES +R3: 1 raise(s) removed -> suite rc=1; failing tests: test_R3_slow_lane; own test(s) failed: YES +R4: 1 raise(s) removed -> suite rc=1; failing tests: test_R4_removal; own test(s) failed: YES +R5: 1 raise(s) removed -> suite rc=1; failing tests: test_R5_downgrade_exact; own test(s) failed: YES +R6: 4 raise(s) removed -> suite rc=1; failing tests: test_R6_allow_new_is_slow_lane, test_R6_new_package, test_R6_unlisted_package; own test(s) failed: YES +R7: 7 raise(s) removed -> suite rc=1; failing tests: test_R7_not_downloadable_and_no_snapshot, test_snapshot_does_not_have_it_either; own test(s) failed: YES +R8: 1 raise(s) removed -> suite rc=1; failing tests: test_R8_free_space; own test(s) failed: YES +R9: 2 raise(s) removed -> suite rc=1; failing tests: test_R9_apt_lock_held, test_R9_guest_locked_by_a_backup; own test(s) failed: YES +R10: 4 raise(s) removed -> suite rc=1; failing tests: test_R10_bind_only_in_a_snapshot_section, test_R10_not_running, test_R10_not_the_boxs_own_guest, test_R10_reserved_vmid; own test(s) failed: YES +R11: 9 raise(s) removed -> suite rc=1; failing tests: test_R11_bad_name, test_R11_bad_version_string, test_R11_duplicate; own test(s) failed: YES +R12: 1 raise(s) removed -> suite rc=1; failing tests: test_R12_host_layer; own test(s) failed: YES +R13: 1 raise(s) removed -> suite rc=1; failing tests: test_R13_repair_does_not_fix_it; own test(s) failed: YES diff --git a/documentation/audits/os-guest-lane-2026-10-04/partC/agent-leg-redproofs.txt b/documentation/audits/os-guest-lane-2026-10-04/partC/agent-leg-redproofs.txt new file mode 100644 index 00000000..ceef95c0 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partC/agent-leg-redproofs.txt @@ -0,0 +1,35 @@ +# agent v0.140.0 OS leg red-proofs, 2026-10-04 (each mutation applied, COMPILES, tests run, reverted; the package result line is printed so a build failure cannot pass as a red-proof) +== mutation: fast-lane +--- FAIL: TestRing0_PlansTheFastLaneOnly (0.00s) + leg_test.go:114: ring-0 plan = [map[name:libc6 origin:Debian version:u4] map[name:openssl origin:Debian-Security version:u3] map[name:docker-ce origin:Debian version:29.8]], want libc6 + openssl only +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.199s +== mutation: switch +--- FAIL: TestSwitchOff_ReportsOnly (0.00s) + leg_test.go:174: rep={RunID:20261004T040000Z Trigger:night Mode:inventory Ring:0 ReleaseID:ring0-20261004T040000Z Outcome: Healthy:true HealthReason: VMID:9201 Upgraded:[] Installed:[{Name:libc6 Version:u3 Origin:Deb +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.216s +== mutation: health-containers +--- FAIL: TestHealth_FailsAfterTheWait (0.00s) + leg_test.go:187: rep = {RunID:20261004T040000Z Trigger:night Mode:apply Ring:0 ReleaseID:ring0-20261004T040000Z Outcome:health_failed Healthy:false HealthReason:app was healthy and is VMID:9201 Upgraded:[{Name:libc6 +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.206s +== mutation: health-controller +ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.197s +== mutation: once-per-night +--- FAIL: TestOncePerNight (0.00s) + leg_test.go:235: a second night run in the same night ran: {RunID:20261004T040000Z Trigger:night Mode:apply Ring:0 ReleaseID:ring0-20261004T040000Z Outcome:applied Healthy:true HealthReason: VMID:9201 Upgraded:[{Name +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.203s +== mutation: ring1-uses-release +--- FAIL: TestRing1_InstallsExactlyTheRelease (0.00s) + leg_test.go:142: ring-1 plan = map[lane:fast layer:guest mode:apply packages:[map[name:libc6 origin:Debian version:u4-approvedx]] release_id:os-1 snapshot:20261004T080000Z vmid:9201] +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.195s +== reverted +ok gitea.dooplex.hu/admin/felhom-agent/internal/osupdate (cached) +== mutation: health-controller (after adding the case "only the controller differs") +--- FAIL: TestHealthVerdict (0.00s) + leg_test.go:225: only the controller differs: got true (), want false +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/osupdate 0.197s +== RP (local API hook): run the leg AFTER the gate is released + afterbackup_test.go:55: the heavy-op gate was free while the leg ran — a restore-test could overlap it +FAIL gitea.dooplex.hu/admin/felhom-agent/internal/localapi 0.563s +== RP (local API hook): run the leg after a FAILED backup too + afterbackup_test.go:60: the leg ran after a FAILED backup: [8200] +FAIL (recorded from the run above) diff --git a/documentation/audits/os-guest-lane-2026-10-04/partD/hub-redproofs.txt b/documentation/audits/os-guest-lane-2026-10-04/partD/hub-redproofs.txt new file mode 100644 index 00000000..3760a9ba --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partD/hub-redproofs.txt @@ -0,0 +1,18 @@ +# hub v0.130.0 approval-rule red-proofs, 2026-10-04 (each mutation applied, tests run, reverted) +== mutation: age +--- FAIL: TestApproval_WaitHealthyAndOneNight (0.03s) + service_test.go:65: fresh set approved or wrong reason: {Fingerprint:8415f8a9ec63c815 FirstSeen:2026-10-04 12:00:00 +0000 UTC Packages:2 Waiting:hp has 0 of 1 night run(s) since the set was first seen} +== mutation: healthy +--- FAIL: TestApproval_UnhealthyRunBlocks (0.03s) + service_test.go:104: approved although a ring-0 run was not healthy: {Fingerprint:5088e82274a46ef8 FirstSeen:2026-10-04 12:00:00 +0000 UTC Packages:1 Waiting:} +== mutation: nights +--- FAIL: TestApproval_WaitHealthyAndOneNight (0.03s) + service_test.go:70: approved without a night run: {Fingerprint:8415f8a9ec63c815 FirstSeen:2026-10-04 12:00:00 +0000 UTC Packages:2 Waiting:} +== mutation: origin +--- FAIL: TestCandidate_OnlyDebianOrigins (0.03s) + service_test.go:140: a Docker package entered the candidate +== mutation: agree +--- FAIL: TestCandidate_DisagreementLeftOut (0.03s) + service_test.go:128: candidate = map[curl:{Name:curl Version:8.14.1-2+deb13u5 Origin:Debian} libc6:{Name:libc6 Version:2.41-12+deb13u4 Origin:Debian}] +== reverted +ok gitea.dooplex.hu/admin/felhom-hub/internal/osupdates (cached) diff --git a/documentation/audits/os-guest-lane-2026-10-04/partE/r726-redproof.txt b/documentation/audits/os-guest-lane-2026-10-04/partE/r726-redproof.txt new file mode 100644 index 00000000..7cd1d347 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partE/r726-redproof.txt @@ -0,0 +1,11 @@ +# controller R-726 (decision 78) red-proof, 2026-10-04 +== mutation: drop the first-night condition +--- FAIL: TestR726_ReturningHouseholdFirstNightSetsAside (0.00s) + offbox_orphan_test.go:204: the returning household's box stayed orphaned +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup 0.014s +== mutation: first night = any claimed box (ignore LastSuccess) +--- FAIL: TestOffbox_OrphanDetection_Claimed (10.10s) + offbox_orphan_test.go:81: expected exactly one offbox_repo_orphaned event, got [offbox_repo_orphaned offbox_repo_orphaned] +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/backup 10.113s +== reverted +ok gitea.dooplex.hu/admin/felhom-controller/internal/backup 0.015s diff --git a/documentation/audits/os-guest-lane-2026-10-04/partF/redproof.txt b/documentation/audits/os-guest-lane-2026-10-04/partF/redproof.txt new file mode 100644 index 00000000..767f82d5 --- /dev/null +++ b/documentation/audits/os-guest-lane-2026-10-04/partF/redproof.txt @@ -0,0 +1,6 @@ +# controller R-838 red-proof, 2026-10-04 +== mutation: a running file browser is left alone again (the pre-fix behaviour) + infra_tunnel_test.go:275: compose after the move: +FAIL gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.009s +== reverted +ok gitea.dooplex.hu/admin/felhom-controller/internal/stacks 0.011s diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 753579e3..cd826b14 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,23 @@ +## v0.130.0 — OS updates, guest fast lane: rings, the per-box switch, OS releases approved from ring 0 (`11` §8 step 2) + +- **The OS release.** Ring-0 boxes (the demo boxes) report every OS-leg run to `POST /api/v1/hosts/{id}/os-report` + (per-host key, self-scoped) with the FULL installed set (`11` C9). The CANDIDATE is every `Debian` / + `Debian-Security` package=version that all ring-0 boxes having it agree on (a disagreement is left out). It is + APPROVED when, since first seen, `OS_APPROVE_AFTER` (24 h) has passed with every ring-0 report healthy and every + ring-0 box has `OS_APPROVE_NIGHTS` (1) post-backup night runs. Either override is a TEST configuration, logged at + start. The approval time is the snapshot.debian.org timestamp (decision 79). Ring-1 boxes are nudged (desired + generation bump). +- **The box's block.** `os_update {ring, enabled, release}` is merged into every desired-state document at read time + (like the wireguard block). Ring 1 gets the newest release; ring 0 none (it installs everything pending). Golden: + `internal/api/testdata/desired-state-osupdate.golden.json`, byte-identical with the agent's. +- **Operator routes:** `POST /os/ring/` (`ring=0|1`), `POST /os/enabled/` (`on=0|1`, default ON), + `POST /os/approve-now` (an operator event), `GET /os/fleet` (one line per box: ring, switch, release, last outcome, + pending, not covered, restart-needed). +- **Events:** `os_update_applied` (info, the household's line — recorded, not mailed; hu/en in the bundle), + `os_update_failed` and `os_update_health_failed` (error, operator — mailed), `os_release_approved`, + `os_release_approved_now`, `os_update_settings_changed` (operator). +- Tests `internal/osupdates/service_test.go` (5 approval-rule red-proofs), `internal/api/os_updates_test.go`. + ## v0.129.0 — the operator can raise ONE clean-up window's cap after a long gap (R-833) - After weeks without windows the honest backlog exceeds half the snapshots, and the box's guard refuses every window diff --git a/hub/cmd/hub/main.go b/hub/cmd/hub/main.go index 56f8d04f..08cd7e81 100644 --- a/hub/cmd/hub/main.go +++ b/hub/cmd/hub/main.go @@ -25,6 +25,7 @@ import ( "gitea.dooplex.hu/admin/felhom-hub/internal/offsite" "gitea.dooplex.hu/admin/felhom-hub/internal/offsiteheal" "gitea.dooplex.hu/admin/felhom-hub/internal/offsitekeys" + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" "gitea.dooplex.hu/admin/felhom-hub/internal/pbsdrheal" "gitea.dooplex.hu/admin/felhom-hub/internal/poke" "gitea.dooplex.hu/admin/felhom-hub/internal/store" @@ -384,6 +385,57 @@ func main() { // unconfigured or the customer has no offsite tier. apiHandler.SetOffsiteReissuer(webServer.ReissueOffsiteForCustomer) + // OS updates, guest fast lane (hub v0.130.0, `11` §8 step 2). A release is approved when every ring-0 box runs the + // set, healthy, for OS_APPROVE_AFTER (default 24h) and through OS_APPROVE_NIGHTS night runs (default 1) — the + // ruled "1–2 day wait". Either override is a TEST configuration and is logged loudly. + osSvc := &osupdates.Service{Store: dataStore, Emit: dispatcher.ProcessEvent, Logger: logger, + ApproveAfter: 24 * time.Hour, NightsRequired: 1, + Bump: func(hostID string) { + if _, err := dataStore.BumpHostDesired(hostID); err != nil { + logger.Printf("[WARN] osupdates: bump desired for %s: %v", hostID, err) + } + }} + if v := os.Getenv("OS_APPROVE_AFTER"); v != "" { + if d, derr := time.ParseDuration(v); derr == nil && d >= 0 { + osSvc.ApproveAfter = d + logger.Printf("[WARN] OS_APPROVE_AFTER=%s — OS releases approve after %s instead of 24h (TEST CONFIGURATION)", v, d) + } else { + logger.Printf("[ERROR] OS_APPROVE_AFTER=%q invalid — keeping 24h", v) + } + } + if v := os.Getenv("OS_APPROVE_NIGHTS"); v != "" { + if n, nerr := strconv.Atoi(v); nerr == nil && n >= 0 { + osSvc.NightsRequired = n + logger.Printf("[WARN] OS_APPROVE_NIGHTS=%s — OS releases need %d night run(s) instead of 1 (TEST CONFIGURATION)", v, n) + } else { + logger.Printf("[ERROR] OS_APPROVE_NIGHTS=%q invalid — keeping 1", v) + } + } + logger.Printf("[INFO] osupdates: approval rule = every ring-0 box healthy for %s and %d night run(s)", osSvc.ApproveAfter, osSvc.NightsRequired) + apiHandler.SetOSUpdateService(osSvc) + webServer.SetOSUpdateAdmin(osSvc) + go func() { + tk := time.NewTicker(60 * time.Second) + defer tk.Stop() + last := "" + for { + select { + case <-ctx.Done(): + return + case <-tk.C: + st, err := osSvc.Evaluate() + if err != nil { + logger.Printf("[WARN] osupdates: evaluate: %v", err) + continue + } + if msg := st.Fingerprint + "|" + st.Waiting; msg != last { + last = msg + logger.Printf("[INFO] osupdates: candidate %s (%d packages, first seen %s): %s", st.Fingerprint, st.Packages, st.FirstSeen.UTC().Format(time.RFC3339), map[bool]string{true: "approved / nothing to wait for", false: "waiting — " + st.Waiting}[st.Waiting == ""]) + } + } + } + }() + // Decision 69 (v0.127.0): the off-site KEY REGISTRAR. The box sends its public key; the hub writes it // into the sub-account's authorized_keys pinned append-only; the daily check reads every file. if offsiteKeyReady { diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index d2338821..9ef39b02 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -50,6 +50,7 @@ type Poker interface { type Handler struct { store *store.Store offsiteKeys OffsiteKeyService // decision 69 key registrar (nil → 503) + osUpdates OSUpdateService // `11` §8 step 2, the guest fast lane (nil → 503, no block merged) apiKey string resendAPIKey string fromEmail string @@ -288,6 +289,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/pbs/consume-token"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/pbs/consume-token") h.handleConsumePBSToken(w, r, hostID) + // OS updates (hub v0.130.0): the agent's report after every OS-leg run — per-host key, self-scoped. + case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/os-report"): + hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/os-report") + h.handleOSReport(w, r, hostID) // Desired-state serving (slice 10A) — per-host-key, self-scoped (a host reads only its own). case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/desired-state"): hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/desired-state") @@ -1747,6 +1752,7 @@ func (h *Handler) handleGetDesiredState(w http.ResponseWriter, r *http.Request, // S2: merge the hub-OWNED wireguard block at read time (no peer → pass-through unchanged; // the stored operator blob is never modified). See api/wg.go mergeWireguard. desired = h.mergeWireguard(pathHostID, desired) + desired = h.mergeOSUpdate(pathHostID, desired) resp := map[string]interface{}{ "generation": host.DesiredGeneration, "desired_state": json.RawMessage(desired), // opaque to the hub — agent owns the schema diff --git a/hub/internal/api/os_updates.go b/hub/internal/api/os_updates.go new file mode 100644 index 00000000..a669c4e5 --- /dev/null +++ b/hub/internal/api/os_updates.go @@ -0,0 +1,71 @@ +package api + +import ( + "encoding/json" + "io" + "net/http" + + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" +) + +// OSUpdateService is the hub half of the guest fast lane (`11` §8 step 2; hub v0.130.0). +type OSUpdateService interface { + Ingest(hostID string, r osupdates.Report) error + DesiredBlock(hostID string) osupdates.Block +} + +// SetOSUpdateService wires the OS-update service. nil → the report endpoint answers 503 and no block is merged. +func (h *Handler) SetOSUpdateService(s OSUpdateService) { h.osUpdates = s } + +// handleOSReport: POST /api/v1/hosts/{id}/os-report — the agent's report after every OS-leg run. Per-host key, +// SELF-SCOPED (a host reports only for itself). +func (h *Handler) handleOSReport(w http.ResponseWriter, r *http.Request, pathHostID string) { + authHostID, _, isGlobal, ok := h.checkAuthHost(r) + if !ok { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if pathHostID == "" || (!isGlobal && authHostID != pathHostID) { + http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden) + return + } + if h.osUpdates == nil { + http.Error(w, "os updates not configured", http.StatusServiceUnavailable) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 4<<20)) + if err != nil { + http.Error(w, "read error", http.StatusBadRequest) + return + } + var rep osupdates.Report + if err := json.Unmarshal(body, &rep); err != nil || rep.RunID == "" { + http.Error(w, "body must be an os report with run_id", http.StatusBadRequest) + return + } + if err := h.osUpdates.Ingest(pathHostID, rep); err != nil { + h.logger.Printf("[WARN] os-report from %s: %v", pathHostID, err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + writeJSON(w, http.StatusOK, map[string]any{"ok": true}) +} + +// mergeOSUpdate adds the hub-OWNED `os_update` block to a host's desired state at read time (like +// mergeWireguard): the stored operator blob is never modified. No service → pass-through unchanged. +func (h *Handler) mergeOSUpdate(hostID, desired string) string { + if h.osUpdates == nil { + return desired + } + var doc map[string]interface{} + if err := json.Unmarshal([]byte(desired), &doc); err != nil { + h.logger.Printf("[ERROR] os_update merge %s: stored desired_json unparsable: %v (serving unmerged)", hostID, err) + return desired + } + doc["os_update"] = h.osUpdates.DesiredBlock(hostID) + out, err := json.Marshal(doc) + if err != nil { + return desired + } + return string(out) +} diff --git a/hub/internal/api/os_updates_test.go b/hub/internal/api/os_updates_test.go new file mode 100644 index 00000000..04aad426 --- /dev/null +++ b/hub/internal/api/os_updates_test.go @@ -0,0 +1,83 @@ +package api + +import ( + "encoding/json" + "log" + "net/http" + "os" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/osupdates" + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// The os_update block a box receives is a contract DUPLICATED with felhom-agent: +// testdata/desired-state-osupdate.golden.json MUST stay byte-identical with the agent's +// internal/hub/testdata copy (the agent's test decodes it). This test proves the hub SERVES exactly that shape. +func TestOSUpdate_DesiredBlockMatchesTheGolden(t *testing.T) { + h, st, _ := newTestHandler(t) + seedHost(t, st, "h1", "c1", "HKEY1") + svc := &osupdates.Service{Store: st, ApproveAfter: 0, NightsRequired: 0, Logger: log.New(os.Stderr, "", 0)} + h.SetOSUpdateService(svc) + rel := `[{"name":"libc6","version":"2.41-12+deb13u4","origin":"Debian"},{"name":"openssl","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]` + if err := st.SaveOSRelease(storeRelease("os-20261004-120000", rel)); err != nil { + t.Fatal(err) + } + rr := do(h, http.MethodGet, "/hosts/h1/desired-state", "HKEY1", "") + if rr.Code != 200 { + t.Fatalf("GET desired-state: %d", rr.Code) + } + var got struct { + DesiredState struct { + OSUpdate json.RawMessage `json:"os_update"` + } `json:"desired_state"` + } + json.Unmarshal(rr.Body.Bytes(), &got) + raw, err := os.ReadFile("testdata/desired-state-osupdate.golden.json") + if err != nil { + t.Fatal(err) + } + var golden struct { + DesiredState struct { + OSUpdate json.RawMessage `json:"os_update"` + } `json:"desired_state"` + } + if err := json.Unmarshal(raw, &golden); err != nil { + t.Fatal(err) + } + var a, b any + json.Unmarshal(got.DesiredState.OSUpdate, &a) + json.Unmarshal(golden.DesiredState.OSUpdate, &b) + ab, _ := json.Marshal(a) + bb, _ := json.Marshal(b) + if string(ab) != string(bb) { + t.Fatalf("served os_update diverged from the golden:\n served: %s\n golden: %s", ab, bb) + } +} + +// A box reports only for itself; another box's key is refused and nothing is stored. +func TestOSReport_SelfScoped(t *testing.T) { + h, st, _ := newTestHandler(t) + seedHost(t, st, "h1", "c1", "HKEY1") + seedHost(t, st, "h2", "c2", "HKEY2") + h.SetOSUpdateService(&osupdates.Service{Store: st}) + body := `{"run_id":"r1","trigger":"night","mode":"apply","outcome":"applied","healthy":true,"upgraded":[{"name":"libc6","version":"x","origin":"Debian"}]}` + if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY2", body); rr.Code != http.StatusForbidden { + t.Fatalf("cross-host report → %d, want 403", rr.Code) + } + if r, _ := st.LatestOSReport("h1"); r != nil { + t.Fatal("a refused report was stored") + } + if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY1", body); rr.Code != http.StatusOK { + t.Fatalf("own report → %d %s", rr.Code, rr.Body.String()) + } + if r, _ := st.LatestOSReport("h1"); r == nil || r.Outcome != "applied" { + t.Fatalf("report not stored: %+v", r) + } +} + +func storeRelease(id, pkgs string) store.OSRelease { + at, _ := time.Parse(time.RFC3339, "2026-10-04T12:00:00Z") + return store.OSRelease{ID: id, Fingerprint: "fp", ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs} +} diff --git a/hub/internal/api/testdata/desired-state-osupdate.golden.json b/hub/internal/api/testdata/desired-state-osupdate.golden.json new file mode 100644 index 00000000..da28c48c --- /dev/null +++ b/hub/internal/api/testdata/desired-state-osupdate.golden.json @@ -0,0 +1,17 @@ +{ + "generation": 1, + "desired_state": { + "os_update": { + "ring": 1, + "enabled": true, + "release": { + "id": "os-20261004-120000", + "snapshot": "20261004T120000Z", + "packages": [ + {"name": "libc6", "version": "2.41-12+deb13u4", "origin": "Debian"}, + {"name": "openssl", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"} + ] + } + } + } +} diff --git a/hub/internal/i18n/locales/en.json b/hub/internal/i18n/locales/en.json index 3b9e7d43..e47a1f4c 100644 --- a/hub/internal/i18n/locales/en.json +++ b/hub/internal/i18n/locales/en.json @@ -86,5 +86,6 @@ "bind.resend.hint": "If this was your link and your box is not linked yet, we send a fresh link to the e-mail address you gave Felhom.", "bind.resend.button": "Send me a new link", "bind.resent.lead": "Done.", - "bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support." + "bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support.", + "mail.event.os_update_applied": "System security fixes were installed on your box. You do not need to do anything." } diff --git a/hub/internal/i18n/locales/hu.json b/hub/internal/i18n/locales/hu.json index b40efafe..90dee775 100644 --- a/hub/internal/i18n/locales/hu.json +++ b/hub/internal/i18n/locales/hu.json @@ -86,5 +86,6 @@ "bind.resend.hint": "Ha ez a te linked volt, és a dobozod még nincs összekötve, új linket küldünk arra az e-mail címre, amelyet a Felhomnál megadtál.", "bind.resend.button": "Új linket kérek", "bind.resent.lead": "Kész.", - "bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak." + "bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak.", + "mail.event.os_update_applied": "Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned." } diff --git a/hub/internal/notify/dispatcher.go b/hub/internal/notify/dispatcher.go index 3c54954a..d222965a 100644 --- a/hub/internal/notify/dispatcher.go +++ b/hub/internal/notify/dispatcher.go @@ -680,6 +680,13 @@ var operatorOnlyEvents = map[string]bool{ "offsite_prune_guard_refused": true, // R-833 (v0.129.0): the operator raised ONE window's removal cap — an operator act, logged. "offsite_window_large_grant": true, + // OS updates (hub v0.130.0, `11` §8 step 2): run failures, rings, switches and approvals are operator facts. + // os_update_applied is deliberately NOT here — it is the household's one line (info: recorded, never mailed). + "os_update_failed": true, + "os_update_health_failed": true, + "os_release_approved": true, + "os_release_approved_now": true, + "os_update_settings_changed": true, // R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow // blobs. A customer can take no action on it — the remedy is the operator's inspection of the // off-site tier — and the text is operator-grade English naming host ids and retained-blob diff --git a/hub/internal/notify/testdata/mail_goldens/en/customer_event_os_update_applied.txt b/hub/internal/notify/testdata/mail_goldens/en/customer_event_os_update_applied.txt new file mode 100644 index 00000000..6a00c7bc --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/en/customer_event_os_update_applied.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Warning: System security fixes were installed on your box. You do not need to do anything. +--- +Dear Customer, + +Your Felhom system sent the following notification: + +System security fixes were installed on your box. You do not need to do anything. + +Details: +- Server: demo-fixture +- Time: 2026-01-15 10:30 +- Level: Warning +- Type: os_update_applied + +If you have any questions, contact your operator. + +Best regards, +Felhom.eu monitoring diff --git a/hub/internal/notify/testdata/mail_goldens/hu/customer_event_os_update_applied.txt b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_os_update_applied.txt new file mode 100644 index 00000000..45cc05b6 --- /dev/null +++ b/hub/internal/notify/testdata/mail_goldens/hu/customer_event_os_update_applied.txt @@ -0,0 +1,18 @@ +SUBJECT: [Felhom] Figyelmeztetés: Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned. +--- +Kedves Ügyfél! + +A Felhom rendszered a következő értesítést küldte: + +Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned. + +Részletek: +- Szerver: demo-fixture +- Időpont: 2026-01-15 10:30 +- Szint: Figyelmeztetés +- Típus: os_update_applied + +Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel. + +Üdvözlettel, +Felhom.eu monitoring diff --git a/hub/internal/osupdates/service.go b/hub/internal/osupdates/service.go new file mode 100644 index 00000000..c0fd1d02 --- /dev/null +++ b/hub/internal/osupdates/service.go @@ -0,0 +1,436 @@ +// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0). +// +// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed +// set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that +// package agree on. A candidate is APPROVED when, since it was first seen: +// - every ring-0 box runs it (its newest report matches the candidate for every package it has), +// - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and +// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs. +// +// The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state +// and installs exactly those versions. Rules pinned by service_test.go. +package osupdates + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "sort" + "strings" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +// Event types — operator-only except EventApplied, the household's line (`11` §5.7). +const ( + EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed" + EventFailed = "os_update_failed" // error, operator: the run failed or was refused + EventHealthFailed = "os_update_health_failed" // error, operator: the guest was not healthy after the run + EventReleaseApprove = "os_release_approved" // info, operator + EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once + EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed +) + +// Package is one name=version with its origin ("Debian" | "Debian-Security"). +type Package struct { + Name string `json:"name"` + Version string `json:"version"` + Origin string `json:"origin"` +} + +// Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict). +type Report struct { + RunID string `json:"run_id"` + Trigger string `json:"trigger"` // night | debug + Mode string `json:"mode"` // apply | inventory + Ring int `json:"ring"` + ReleaseID string `json:"release_id"` + Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed + Healthy bool `json:"healthy"` + HealthReason string `json:"health_reason,omitempty"` + VMID int `json:"vmid"` + Upgraded []Package `json:"upgraded,omitempty"` + Installed []Package `json:"installed,omitempty"` + Pending []PendingPkg `json:"pending,omitempty"` + NotCovered []string `json:"not_covered,omitempty"` + RestartNeeded []string `json:"restart_needed,omitempty"` + DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"` + RebootNeeded bool `json:"reboot_needed,omitempty"` + Refused json.RawMessage `json:"refused,omitempty"` + Log []string `json:"log,omitempty"` +} + +// PendingPkg is one update the guest's sources offer. +type PendingPkg struct { + Name string `json:"name"` + From string `json:"from"` + To string `json:"to"` + Origin []string `json:"origin"` +} + +// Block is what a box receives in its desired state (`os_update`). +type Block struct { + Ring int `json:"ring"` + Enabled bool `json:"enabled"` + Release *ReleaseBlock `json:"release,omitempty"` +} + +// ReleaseBlock is the newest approved release, for ring 1. +type ReleaseBlock struct { + ID string `json:"id"` + Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org) + Packages []Package `json:"packages"` +} + +// Service ties the store, the events and the clock together. +type Service struct { + Store *store.Store + Emit func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests + ApproveAfter time.Duration + NightsRequired int + Logger *log.Logger + Now func() time.Time + Bump func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests +} + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +func (s *Service) logf(f string, a ...any) { + if s.Logger != nil { + s.Logger.Printf(f, a...) + } +} + +func (s *Service) event(customerID, typ, sev, msg string, details any) { + dj := "" + if details != nil { + if b, err := json.Marshal(details); err == nil { + dj = string(b) + } + } + if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil { + s.logf("[WARN] osupdates: save event %s: %v", typ, err) + } + if s.Emit != nil { + s.Emit(customerID, typ, sev, msg, dj, "hub") + } +} + +// Ingest stores a run and raises its events. The household gets one line per run that installed something. +func (s *Service) Ingest(hostID string, r Report) error { + h, err := s.Store.GetHost(hostID) + if err != nil || h == nil { + return fmt.Errorf("osupdates: unknown host %q", hostID) + } + raw, _ := json.Marshal(r) + if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome, + Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil { + return err + } + s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d", + hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)) + details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome, + "upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason} + switch r.Outcome { + case "applied", "health_failed": + s.event(h.CustomerID, EventApplied, "info", + fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details) + if !r.Healthy || r.Outcome == "health_failed" { + s.event(h.CustomerID, EventHealthFailed, "error", + fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.", + hostID, len(r.Upgraded), r.HealthReason), details) + } + case "refused", "failed": + s.event(h.CustomerID, EventFailed, "error", + fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details) + } + return nil +} + +// candidate derives the version set every ring-0 box agrees on, from each box's newest report. +func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) { + latest := map[string]*store.OSReport{} + byPkg := map[string]map[string]Package{} // name -> host -> pkg + for _, h := range ring0 { + rep, err := s.Store.LatestOSReport(h) + if err != nil { + return nil, nil, err + } + if rep == nil { + return nil, nil, nil // a ring-0 box that never reported: no candidate + } + latest[h] = rep + var r Report + if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil { + return nil, nil, err + } + for _, p := range r.Installed { + if p.Origin != "Debian" && p.Origin != "Debian-Security" { + continue + } + if byPkg[p.Name] == nil { + byPkg[p.Name] = map[string]Package{} + } + byPkg[p.Name][h] = p + } + } + cand := map[string]Package{} + for name, hosts := range byPkg { + var v string + agree := true + var pick Package + for _, p := range hosts { + if v == "" { + v, pick = p.Version, p + } else if p.Version != v { + agree = false + } + } + if agree { + cand[name] = pick + } + } + return cand, latest, nil +} + +func fingerprint(c map[string]Package) (string, []Package) { + var list []Package + for _, p := range c { + list = append(list, p) + } + sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name }) + h := sha256.New() + for _, p := range list { + fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version) + } + return hex.EncodeToString(h.Sum(nil))[:16], list +} + +// ring0Hosts lists the ring-0 boxes that have the switch ON. +func (s *Service) ring0Hosts() ([]string, error) { + hosts, err := s.Store.ListHosts() + if err != nil { + return nil, err + } + var out []string + for _, h := range hosts { + if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled { + out = append(out, h.HostID) + } + } + sort.Strings(out) + return out, nil +} + +// Status explains the approval state (for the log and the fleet page). +type Status struct { + Fingerprint string + FirstSeen time.Time + Packages int + Waiting string // why not approved yet ("" = approved or nothing to do) +} + +// Evaluate checks the approval rule and approves when it holds. Called every minute. +func (s *Service) Evaluate() (Status, error) { + ring0, err := s.ring0Hosts() + if err != nil || len(ring0) == 0 { + return Status{Waiting: "no ring-0 box"}, err + } + cand, _, err := s.candidate(ring0) + if err != nil || cand == nil { + return Status{Waiting: "a ring-0 box has not reported"}, err + } + fp, list := fingerprint(cand) + pj, _ := json.Marshal(list) + first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()) + if err != nil { + return Status{}, err + } + st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)} + if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { + st.Waiting = "" + return st, nil // already approved + } + if age := s.now().Sub(first); age < s.ApproveAfter { + st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter) + return st, nil + } + for _, h := range ring0 { + reps, err := s.Store.OSReportsSince(h, first) + if err != nil { + return st, err + } + nights := 0 + for _, r := range reps { + if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" { + st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339)) + return st, nil + } + if r.Trigger == "night" { + nights++ + } + } + if nights < s.NightsRequired { + st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired) + return st, nil + } + } + if err := s.approve(fp, list, "auto"); err != nil { + return st, err + } + return st, nil +} + +// ApproveNow approves the current candidate at once (operator, urgent fix). +func (s *Service) ApproveNow() (string, error) { + ring0, err := s.ring0Hosts() + if err != nil || len(ring0) == 0 { + return "", fmt.Errorf("osupdates: no ring-0 box") + } + cand, _, err := s.candidate(ring0) + if err != nil || cand == nil { + return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet") + } + fp, list := fingerprint(cand) + pj, _ := json.Marshal(list) + if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil { + return "", err + } + if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp { + return rel.ID, nil + } + if err := s.approve(fp, list, "operator"); err != nil { + return "", err + } + rel, _ := s.Store.LatestOSRelease() + s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)), + map[string]any{"release_id": rel.ID, "packages": len(list)}) + return rel.ID, nil +} + +func (s *Service) approve(fp string, list []Package, by string) error { + at := s.now().UTC().Truncate(time.Second) + id := "os-" + at.Format("20060102-150405") + pj, _ := json.Marshal(list) + if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil { + return err + } + s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp) + s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)), + map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp}) + if s.Bump != nil { + hosts, _ := s.Store.ListHosts() + for _, h := range hosts { + if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled { + s.Bump(h.HostID) + } + } + } + return nil +} + +// DesiredBlock is the `os_update` block a box receives. +func (s *Service) DesiredBlock(hostID string) Block { + st := s.Store.GetOSHostSettings(hostID) + b := Block{Ring: st.Ring, Enabled: st.Enabled} + if st.Ring == 1 { + if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil { + var list []Package + if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil { + b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list} + } + } + } + return b +} + +// SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event. +func (s *Service) SetRing(hostID string, ring int) error { + if ring != 0 && ring != 1 { + return fmt.Errorf("osupdates: ring must be 0 or 1") + } + h, err := s.Store.GetHost(hostID) + if err != nil || h == nil { + return fmt.Errorf("osupdates: unknown host %q", hostID) + } + if err := s.Store.SetOSRing(hostID, ring); err != nil { + return err + } + s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring}) + if s.Bump != nil { + s.Bump(hostID) + } + return nil +} + +func (s *Service) SetEnabled(hostID string, on bool) error { + h, err := s.Store.GetHost(hostID) + if err != nil || h == nil { + return fmt.Errorf("osupdates: unknown host %q", hostID) + } + if err := s.Store.SetOSEnabled(hostID, on); err != nil { + return err + } + s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]), + map[string]any{"host_id": hostID, "enabled": on}) + if s.Bump != nil { + s.Bump(hostID) + } + return nil +} + +// FleetLine is one box on the fleet page. +type FleetLine struct { + HostID string + Ring int + Enabled bool + ReleaseID string + LastOutcome string + LastAt time.Time + Pending int + NotCovered int + RestartNeeded int +} + +// Fleet lists every box. +func (s *Service) Fleet() ([]FleetLine, error) { + hosts, err := s.Store.ListHosts() + if err != nil { + return nil, err + } + var out []FleetLine + for _, h := range hosts { + st := s.Store.GetOSHostSettings(h.HostID) + l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled} + if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil { + var r Report + _ = json.Unmarshal([]byte(rep.ReportJSON), &r) + l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt + l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded) + } + out = append(out, l) + } + return out, nil +} + +// FleetJSON is Fleet plus the approval status, for the operator's fleet route. +func (s *Service) FleetJSON() (any, error) { + lines, err := s.Fleet() + if err != nil { + return nil, err + } + rel, _ := s.Store.LatestOSRelease() + out := map[string]any{"boxes": lines} + if rel != nil { + out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy} + } + return out, nil +} diff --git a/hub/internal/osupdates/service_test.go b/hub/internal/osupdates/service_test.go new file mode 100644 index 00000000..0308f6e5 --- /dev/null +++ b/hub/internal/osupdates/service_test.go @@ -0,0 +1,189 @@ +package osupdates + +import ( + "log" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" +) + +type fix struct { + s *Service + now time.Time + events []string + bumps []string +} + +func newFix(t *testing.T) *fix { + t.Helper() + st, err := store.New(filepath.Join(t.TempDir(), "hub.db"), log.New(os.Stderr, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + for _, h := range []struct{ host, cust string }{{"hp", "c-hp"}, {"n100", "c-n100"}, {"cust1", "c-1"}} { + if err := st.UpsertHost(&store.Host{HostID: h.host, CustomerID: h.cust, APIKey: "k-" + h.host}); err != nil { + t.Fatal(err) + } + } + f := &fix{now: time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)} + f.s = &Service{Store: st, ApproveAfter: 24 * time.Hour, NightsRequired: 1, Now: func() time.Time { return f.now }, + Emit: func(_, typ, _, _, _, _ string) { f.events = append(f.events, typ) }, + Bump: func(h string) { f.bumps = append(f.bumps, h) }} + _ = st.SetOSRing("hp", 0) + _ = st.SetOSRing("n100", 0) + return f +} + +func pk(name, ver string) Package { return Package{Name: name, Version: ver, Origin: "Debian"} } + +func (f *fix) report(t *testing.T, host, trigger string, healthy bool, pkgs ...Package) { + t.Helper() + outcome := "applied" + if !healthy { + outcome = "health_failed" + } + if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Trigger: trigger, Mode: "apply", Outcome: outcome, + Healthy: healthy, Installed: pkgs, Upgraded: pkgs[:1]}); err != nil { + t.Fatal(err) + } +} + +// The ruled wait: approved only after every ring-0 box ran the set healthy for ApproveAfter AND through a night run. +// Red-proof: drop the age check, the healthy check or the nights check in Evaluate and a sub-step fails. +func TestApproval_WaitHealthyAndOneNight(t *testing.T) { + f := newFix(t) + set := []Package{pk("libc6", "2.41-12+deb13u4"), pk("openssl", "3.5.7-1~deb13u3")} + f.report(t, "hp", "debug", true, set...) + f.report(t, "n100", "debug", true, set...) + st, _ := f.s.Evaluate() + if !strings.HasPrefix(st.Waiting, "healthy for") { + t.Fatalf("fresh set approved or wrong reason: %+v", st) + } + f.now = f.now.Add(25 * time.Hour) + st, _ = f.s.Evaluate() + if !strings.Contains(st.Waiting, "night run") { + t.Fatalf("approved without a night run: %+v", st) + } + f.report(t, "hp", "night", true, set...) + f.report(t, "n100", "night", true, set...) + if _, err := f.s.Evaluate(); err != nil { + t.Fatal(err) + } + rel, _ := f.s.Store.LatestOSRelease() + if rel == nil || rel.ApprovedBy != "auto" { + t.Fatalf("not approved: %+v", rel) + } + if len(f.bumps) != 1 || f.bumps[0] != "cust1" { + t.Fatalf("only the ring-1 box must be nudged, got %v", f.bumps) + } + b := f.s.DesiredBlock("cust1") + if b.Ring != 1 || !b.Enabled || b.Release == nil || len(b.Release.Packages) != 2 || b.Release.Snapshot != rel.ApprovedAt.UTC().Format("20060102T150405Z") { + t.Fatalf("ring-1 block = %+v", b) + } + if hb := f.s.DesiredBlock("hp"); hb.Ring != 0 || hb.Release != nil { + t.Fatalf("a ring-0 box must not get a release (it installs everything pending): %+v", hb) + } +} + +func TestApproval_UnhealthyRunBlocks(t *testing.T) { + f := newFix(t) + set := []Package{pk("libc6", "2.41-12+deb13u4")} + f.report(t, "hp", "debug", true, set...) + f.report(t, "n100", "debug", true, set...) + f.s.Evaluate() + f.now = f.now.Add(25 * time.Hour) + f.report(t, "hp", "night", false, set...) + f.report(t, "n100", "night", true, set...) + st, _ := f.s.Evaluate() + if rel, _ := f.s.Store.LatestOSRelease(); rel != nil { + t.Fatalf("approved although a ring-0 run was not healthy: %+v", st) + } + if !strings.Contains(st.Waiting, "healthy=false") { + t.Fatalf("reason = %q", st.Waiting) + } + found := false + for _, e := range f.events { + found = found || e == EventHealthFailed + } + if !found { + t.Fatalf("no %s event: %v", EventHealthFailed, f.events) + } +} + +// Ring 0 disagreeing on a package: that package is left out of the candidate (C9 — approve what ALL ring 0 runs). +func TestCandidate_DisagreementLeftOut(t *testing.T) { + f := newFix(t) + f.report(t, "hp", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u5")) + f.report(t, "n100", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u4")) + cand, _, err := f.s.candidate([]string{"hp", "n100"}) + if err != nil { + t.Fatal(err) + } + if _, ok := cand["curl"]; ok || cand["libc6"].Version != "2.41-12+deb13u4" { + t.Fatalf("candidate = %+v", cand) + } +} + +// Non-Debian origins never enter a release (the fast lane is Debian / Debian-Security only, C3). +func TestCandidate_OnlyDebianOrigins(t *testing.T) { + f := newFix(t) + d := Package{Name: "docker-ce", Version: "5:29.8.2", Origin: "Docker"} + f.report(t, "hp", "night", true, pk("libc6", "x1"), d) + f.report(t, "n100", "night", true, pk("libc6", "x1"), d) + cand, _, _ := f.s.candidate([]string{"hp", "n100"}) + if _, ok := cand["docker-ce"]; ok { + t.Fatal("a Docker package entered the candidate") + } +} + +func TestApproveNow_IsAnOperatorEvent(t *testing.T) { + f := newFix(t) + f.report(t, "hp", "debug", true, pk("libc6", "x1")) + f.report(t, "n100", "debug", true, pk("libc6", "x1")) + id, err := f.s.ApproveNow() + if err != nil || id == "" { + t.Fatalf("%q %v", id, err) + } + rel, _ := f.s.Store.LatestOSRelease() + if rel.ApprovedBy != "operator" { + t.Fatalf("approved_by = %q", rel.ApprovedBy) + } + n := 0 + for _, e := range f.events { + if e == EventApprovedNow { + n++ + } + } + if n != 1 { + t.Fatalf("want one %s, got %v", EventApprovedNow, f.events) + } +} + +func TestSwitchAndRing(t *testing.T) { + f := newFix(t) + if err := f.s.SetEnabled("cust1", false); err != nil { + t.Fatal(err) + } + if b := f.s.DesiredBlock("cust1"); b.Enabled { + t.Fatal("switch OFF not delivered") + } + if err := f.s.SetRing("cust1", 2); err == nil { + t.Fatal("ring 2 accepted") + } + if b := f.s.DesiredBlock("nobody-row"); b.Ring != 1 || !b.Enabled { + t.Fatalf("default must be ring 1, ON: %+v", b) + } +} + +func TestIngest_AppliedIsTheHouseholdsLine(t *testing.T) { + f := newFix(t) + f.report(t, "cust1", "night", true, pk("libc6", "x1")) + if len(f.events) != 1 || f.events[0] != EventApplied { + t.Fatalf("events = %v", f.events) + } +} diff --git a/hub/internal/store/os_updates.go b/hub/internal/store/os_updates.go new file mode 100644 index 00000000..15988a22 --- /dev/null +++ b/hub/internal/store/os_updates.go @@ -0,0 +1,211 @@ +package store + +import ( + "database/sql" + "time" +) + +// OS updates, guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0). +// +// Three records: +// - os_host_settings: the operator's per-box ring (0 = demo boxes that take every update first, 1 = every other +// box) and switch (ON by default — decision 12's shape). A host with no row is ring 1, ON. +// - os_reports: every run the agent reports (the full installed set rides in report_json — C9: what ring 0 RUNS). +// - os_candidates / os_releases: the version set ring 0 runs, first seen when, and the approved releases. + +func (s *Store) migrateOSUpdates() error { + _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS os_host_settings ( + host_id TEXT PRIMARY KEY, + ring INTEGER NOT NULL DEFAULT 1, + enabled INTEGER NOT NULL DEFAULT 1, + updated_at DATETIME NOT NULL DEFAULT (datetime('now')) + ); + CREATE TABLE IF NOT EXISTS os_reports ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + host_id TEXT NOT NULL, + received_at DATETIME NOT NULL DEFAULT (datetime('now')), + trigger TEXT NOT NULL DEFAULT '', + mode TEXT NOT NULL DEFAULT '', + outcome TEXT NOT NULL DEFAULT '', + healthy INTEGER NOT NULL DEFAULT 0, + release_id TEXT NOT NULL DEFAULT '', + report_json TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX IF NOT EXISTS idx_os_reports_host ON os_reports(host_id, id); + CREATE TABLE IF NOT EXISTS os_candidates ( + fingerprint TEXT PRIMARY KEY, + first_seen DATETIME NOT NULL, + packages_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS os_releases ( + id TEXT PRIMARY KEY, + fingerprint TEXT NOT NULL, + approved_at DATETIME NOT NULL, + approved_by TEXT NOT NULL, + packages_json TEXT NOT NULL + ); + `) + return err +} + +// OSHostSettings is one box's ring and switch. +type OSHostSettings struct { + HostID string + Ring int + Enabled bool +} + +// GetOSHostSettings returns the box's settings; a box with no row is ring 1, ON. +func (s *Store) GetOSHostSettings(hostID string) OSHostSettings { + st := OSHostSettings{HostID: hostID, Ring: 1, Enabled: true} + var ring, en int + if err := s.db.QueryRow(`SELECT ring, enabled FROM os_host_settings WHERE host_id = ?`, hostID).Scan(&ring, &en); err == nil { + st.Ring, st.Enabled = ring, en == 1 + } + return st +} + +// SetOSRing sets the box's ring (0 or 1). +func (s *Store) SetOSRing(hostID string, ring int) error { + _, err := s.db.Exec(`INSERT INTO os_host_settings (host_id, ring) VALUES (?, ?) + ON CONFLICT(host_id) DO UPDATE SET ring = excluded.ring, updated_at = datetime('now')`, hostID, ring) + return err +} + +// SetOSEnabled sets the box's switch. +func (s *Store) SetOSEnabled(hostID string, on bool) error { + v := 0 + if on { + v = 1 + } + _, err := s.db.Exec(`INSERT INTO os_host_settings (host_id, enabled) VALUES (?, ?) + ON CONFLICT(host_id) DO UPDATE SET enabled = excluded.enabled, updated_at = datetime('now')`, hostID, v) + return err +} + +// OSReport is one stored run. +type OSReport struct { + ID int64 + HostID string + ReceivedAt time.Time + Trigger string + Mode string + Outcome string + Healthy bool + ReleaseID string + ReportJSON string +} + +// SaveOSReport stores one run. +func (s *Store) SaveOSReport(r OSReport) (int64, error) { + h := 0 + if r.Healthy { + h = 1 + } + at := r.ReceivedAt + if at.IsZero() { + at = time.Now() + } + // received_at comes from the CALLER's clock: the approval rule compares it with first_seen, which the + // service stamps with its own clock — two clocks would make "since first seen" miss reports. + res, err := s.db.Exec(`INSERT INTO os_reports (host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + r.HostID, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +func scanOSReports(rows *sql.Rows) ([]OSReport, error) { + defer rows.Close() + var out []OSReport + for rows.Next() { + var r OSReport + var at string + var h int + if err := rows.Scan(&r.ID, &r.HostID, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil { + return nil, err + } + r.ReceivedAt, r.Healthy = parseSQLiteTime(at), h == 1 + out = append(out, r) + } + return out, rows.Err() +} + +const osReportCols = `id, host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json` + +// LatestOSReport returns the box's newest run, or nil. +func (s *Store) LatestOSReport(hostID string) (*OSReport, error) { + rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? ORDER BY id DESC LIMIT 1`, hostID) + if err != nil { + return nil, err + } + rs, err := scanOSReports(rows) + if err != nil || len(rs) == 0 { + return nil, err + } + return &rs[0], nil +} + +// OSReportsSince returns the box's runs received at or after t, oldest first. +func (s *Store) OSReportsSince(hostID string, t time.Time) ([]OSReport, error) { + rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND received_at >= ? ORDER BY id`, + hostID, t.UTC().Format("2006-01-02 15:04:05")) + if err != nil { + return nil, err + } + return scanOSReports(rows) +} + +// OSCandidateFirstSeen records a candidate set the first time it is seen and returns when that was. +func (s *Store) OSCandidateFirstSeen(fingerprint, packagesJSON string, now time.Time) (time.Time, error) { + if _, err := s.db.Exec(`INSERT OR IGNORE INTO os_candidates (fingerprint, first_seen, packages_json) VALUES (?, ?, ?)`, + fingerprint, now.UTC().Format("2006-01-02 15:04:05"), packagesJSON); err != nil { + return time.Time{}, err + } + var at string + if err := s.db.QueryRow(`SELECT first_seen FROM os_candidates WHERE fingerprint = ?`, fingerprint).Scan(&at); err != nil { + return time.Time{}, err + } + return parseSQLiteTime(at), nil +} + +// OSRelease is one approved version set. +type OSRelease struct { + ID string + Fingerprint string + ApprovedAt time.Time + ApprovedBy string + PackagesJSON string +} + +// SaveOSRelease stores an approved release. +func (s *Store) SaveOSRelease(r OSRelease) error { + _, err := s.db.Exec(`INSERT INTO os_releases (id, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?)`, + r.ID, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON) + return err +} + +// LatestOSRelease returns the newest approved release, or nil. +func (s *Store) LatestOSRelease() (*OSRelease, error) { + var r OSRelease + var at string + err := s.db.QueryRow(`SELECT id, fingerprint, approved_at, approved_by, packages_json FROM os_releases ORDER BY approved_at DESC, id DESC LIMIT 1`). + Scan(&r.ID, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + r.ApprovedAt = parseSQLiteTime(at) + return &r, nil +} + +// BackdateOSCandidateForTest moves a candidate's first_seen into the past. TEST-ONLY. +func (s *Store) BackdateOSCandidateForTest(fingerprint string, by time.Duration) error { + _, err := s.db.Exec(`UPDATE os_candidates SET first_seen = ? WHERE fingerprint = ?`, + time.Now().Add(-by).UTC().Format("2006-01-02 15:04:05"), fingerprint) + return err +} diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 2b7ad2cf..1693f106 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -858,6 +858,10 @@ func (s *Store) migrate() error { } // R-833 (v0.129.0): the cap each window was opened with — an operator grant may raise it once. s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN max_remove INTEGER") + // OS updates, guest fast lane (hub v0.130.0, `11` §8 step 2). + if err := s.migrateOSUpdates(); err != nil { + return fmt.Errorf("os_updates: %w", err) + } return nil } diff --git a/hub/internal/web/os_updates.go b/hub/internal/web/os_updates.go new file mode 100644 index 00000000..6e9fb591 --- /dev/null +++ b/hub/internal/web/os_updates.go @@ -0,0 +1,52 @@ +package web + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" +) + +// handleOSAdmin serves the operator's OS-update controls (behind the operator login, like every route here): +// +// POST /os/ring/ ring=0|1 +// POST /os/enabled/ on=1|0 +// POST /os/approve-now approve the current ring-0 set at once (an operator event) +// GET /os/fleet one line per box (JSON) +func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path string) { + if s.osUpdates == nil { + http.Error(w, "os updates not configured", http.StatusServiceUnavailable) + return + } + reply := func(v any, err error) { + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) + } + switch { + case r.Method == http.MethodGet && path == "/os/fleet": + reply(s.osUpdates.FleetJSON()) + case r.Method == http.MethodPost && strings.HasPrefix(path, "/os/ring/"): + n, err := strconv.Atoi(r.FormValue("ring")) + if err != nil { + http.Error(w, "ring must be 0 or 1", http.StatusBadRequest) + return + } + reply(map[string]bool{"ok": true}, s.osUpdates.SetRing(strings.TrimPrefix(path, "/os/ring/"), n)) + case r.Method == http.MethodPost && strings.HasPrefix(path, "/os/enabled/"): + on := r.FormValue("on") + if on != "0" && on != "1" { + http.Error(w, "on must be 0 or 1", http.StatusBadRequest) + return + } + reply(map[string]bool{"ok": true}, s.osUpdates.SetEnabled(strings.TrimPrefix(path, "/os/enabled/"), on == "1")) + case r.Method == http.MethodPost && path == "/os/approve-now": + id, err := s.osUpdates.ApproveNow() + reply(map[string]string{"release_id": id}, err) + default: + http.Error(w, "not found", http.StatusNotFound) + } +} diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index 114bbf10..99fe41e2 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -73,6 +73,7 @@ type Server struct { // offsiteWindowAdmin: operator one-shot grant / weekly switch (decision 68). nil → 503. offsiteWindowGrant func(customerID string) error offsiteWindowGrantMax func(customerID string, maxRemove int) error + osUpdates OSUpdateAdmin offsiteWindowSwitch func(on bool) error // operator key-file clean-up (decision 72) and abandonment cancel (decision 74). nil → 503. offsiteRemoveUnpinned func(ctx context.Context, customerID string) (int, error) @@ -212,6 +213,17 @@ func (s *Server) SetOffsiteKeyAdmin(remove func(context.Context, string) (int, e s.offsiteRemoveUnpinned, s.offsiteAbandonCancel = remove, cancel } +// OSUpdateAdmin is the operator side of the guest fast lane (hub v0.130.0). +type OSUpdateAdmin interface { + SetRing(hostID string, ring int) error + SetEnabled(hostID string, on bool) error + ApproveNow() (string, error) + FleetJSON() (any, error) +} + +// SetOSUpdateAdmin wires the OS-update operator routes. +func (s *Server) SetOSUpdateAdmin(a OSUpdateAdmin) { s.osUpdates = a } + // SetOffsiteWindowLargeGrant wires the operator's one-window raised cap (R-833). func (s *Server) SetOffsiteWindowLargeGrant(fn func(string, int) error) { s.offsiteWindowGrantMax = fn } @@ -681,6 +693,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte("{\"ok\":true}\n")) + case strings.HasPrefix(path, "/os/"): + // Operator (hub v0.130.0, `11` §8 step 2): per-box ring and switch, approve now, the fleet lines. + s.handleOSAdmin(w, r, path) case path == "/offsite/key-audit": // Operator: run the daily off-site key check now (decision 69). Same code path as the 07:10 job. if r.Method != http.MethodPost {