hub v0.130.0: OS updates, guest fast lane — rings, per-box switch, OS releases approved from ring 0, os-report, os_update desired block

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 10:56:39 +02:00
parent 6ed79cd2e9
commit c5f91174f6
25 changed files with 1338 additions and 2 deletions
+6
View File
@@ -50,6 +50,7 @@ type Poker interface {
type Handler struct {
store *store.Store
offsiteKeys OffsiteKeyService // decision 69 key registrar (nil → 503)
osUpdates OSUpdateService // `11` §8 step 2, the guest fast lane (nil → 503, no block merged)
apiKey string
resendAPIKey string
fromEmail string
@@ -288,6 +289,10 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/pbs/consume-token"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/pbs/consume-token")
h.handleConsumePBSToken(w, r, hostID)
// OS updates (hub v0.130.0): the agent's report after every OS-leg run — per-host key, self-scoped.
case r.Method == http.MethodPost && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/os-report"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/os-report")
h.handleOSReport(w, r, hostID)
// Desired-state serving (slice 10A) — per-host-key, self-scoped (a host reads only its own).
case r.Method == http.MethodGet && strings.HasPrefix(path, "/hosts/") && strings.HasSuffix(path, "/desired-state"):
hostID := strings.TrimSuffix(strings.TrimPrefix(path, "/hosts/"), "/desired-state")
@@ -1747,6 +1752,7 @@ func (h *Handler) handleGetDesiredState(w http.ResponseWriter, r *http.Request,
// S2: merge the hub-OWNED wireguard block at read time (no peer → pass-through unchanged;
// the stored operator blob is never modified). See api/wg.go mergeWireguard.
desired = h.mergeWireguard(pathHostID, desired)
desired = h.mergeOSUpdate(pathHostID, desired)
resp := map[string]interface{}{
"generation": host.DesiredGeneration,
"desired_state": json.RawMessage(desired), // opaque to the hub — agent owns the schema
+71
View File
@@ -0,0 +1,71 @@
package api
import (
"encoding/json"
"io"
"net/http"
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
)
// OSUpdateService is the hub half of the guest fast lane (`11` §8 step 2; hub v0.130.0).
type OSUpdateService interface {
Ingest(hostID string, r osupdates.Report) error
DesiredBlock(hostID string) osupdates.Block
}
// SetOSUpdateService wires the OS-update service. nil → the report endpoint answers 503 and no block is merged.
func (h *Handler) SetOSUpdateService(s OSUpdateService) { h.osUpdates = s }
// handleOSReport: POST /api/v1/hosts/{id}/os-report — the agent's report after every OS-leg run. Per-host key,
// SELF-SCOPED (a host reports only for itself).
func (h *Handler) handleOSReport(w http.ResponseWriter, r *http.Request, pathHostID string) {
authHostID, _, isGlobal, ok := h.checkAuthHost(r)
if !ok {
http.Error(w, "Unauthorized", http.StatusUnauthorized)
return
}
if pathHostID == "" || (!isGlobal && authHostID != pathHostID) {
http.Error(w, "Forbidden: host_id mismatch", http.StatusForbidden)
return
}
if h.osUpdates == nil {
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
return
}
body, err := io.ReadAll(io.LimitReader(r.Body, 4<<20))
if err != nil {
http.Error(w, "read error", http.StatusBadRequest)
return
}
var rep osupdates.Report
if err := json.Unmarshal(body, &rep); err != nil || rep.RunID == "" {
http.Error(w, "body must be an os report with run_id", http.StatusBadRequest)
return
}
if err := h.osUpdates.Ingest(pathHostID, rep); err != nil {
h.logger.Printf("[WARN] os-report from %s: %v", pathHostID, err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
}
// mergeOSUpdate adds the hub-OWNED `os_update` block to a host's desired state at read time (like
// mergeWireguard): the stored operator blob is never modified. No service → pass-through unchanged.
func (h *Handler) mergeOSUpdate(hostID, desired string) string {
if h.osUpdates == nil {
return desired
}
var doc map[string]interface{}
if err := json.Unmarshal([]byte(desired), &doc); err != nil {
h.logger.Printf("[ERROR] os_update merge %s: stored desired_json unparsable: %v (serving unmerged)", hostID, err)
return desired
}
doc["os_update"] = h.osUpdates.DesiredBlock(hostID)
out, err := json.Marshal(doc)
if err != nil {
return desired
}
return string(out)
}
+83
View File
@@ -0,0 +1,83 @@
package api
import (
"encoding/json"
"log"
"net/http"
"os"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/osupdates"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// The os_update block a box receives is a contract DUPLICATED with felhom-agent:
// testdata/desired-state-osupdate.golden.json MUST stay byte-identical with the agent's
// internal/hub/testdata copy (the agent's test decodes it). This test proves the hub SERVES exactly that shape.
func TestOSUpdate_DesiredBlockMatchesTheGolden(t *testing.T) {
h, st, _ := newTestHandler(t)
seedHost(t, st, "h1", "c1", "HKEY1")
svc := &osupdates.Service{Store: st, ApproveAfter: 0, NightsRequired: 0, Logger: log.New(os.Stderr, "", 0)}
h.SetOSUpdateService(svc)
rel := `[{"name":"libc6","version":"2.41-12+deb13u4","origin":"Debian"},{"name":"openssl","version":"3.5.7-1~deb13u3","origin":"Debian-Security"}]`
if err := st.SaveOSRelease(storeRelease("os-20261004-120000", rel)); err != nil {
t.Fatal(err)
}
rr := do(h, http.MethodGet, "/hosts/h1/desired-state", "HKEY1", "")
if rr.Code != 200 {
t.Fatalf("GET desired-state: %d", rr.Code)
}
var got struct {
DesiredState struct {
OSUpdate json.RawMessage `json:"os_update"`
} `json:"desired_state"`
}
json.Unmarshal(rr.Body.Bytes(), &got)
raw, err := os.ReadFile("testdata/desired-state-osupdate.golden.json")
if err != nil {
t.Fatal(err)
}
var golden struct {
DesiredState struct {
OSUpdate json.RawMessage `json:"os_update"`
} `json:"desired_state"`
}
if err := json.Unmarshal(raw, &golden); err != nil {
t.Fatal(err)
}
var a, b any
json.Unmarshal(got.DesiredState.OSUpdate, &a)
json.Unmarshal(golden.DesiredState.OSUpdate, &b)
ab, _ := json.Marshal(a)
bb, _ := json.Marshal(b)
if string(ab) != string(bb) {
t.Fatalf("served os_update diverged from the golden:\n served: %s\n golden: %s", ab, bb)
}
}
// A box reports only for itself; another box's key is refused and nothing is stored.
func TestOSReport_SelfScoped(t *testing.T) {
h, st, _ := newTestHandler(t)
seedHost(t, st, "h1", "c1", "HKEY1")
seedHost(t, st, "h2", "c2", "HKEY2")
h.SetOSUpdateService(&osupdates.Service{Store: st})
body := `{"run_id":"r1","trigger":"night","mode":"apply","outcome":"applied","healthy":true,"upgraded":[{"name":"libc6","version":"x","origin":"Debian"}]}`
if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY2", body); rr.Code != http.StatusForbidden {
t.Fatalf("cross-host report → %d, want 403", rr.Code)
}
if r, _ := st.LatestOSReport("h1"); r != nil {
t.Fatal("a refused report was stored")
}
if rr := do(h, http.MethodPost, "/hosts/h1/os-report", "HKEY1", body); rr.Code != http.StatusOK {
t.Fatalf("own report → %d %s", rr.Code, rr.Body.String())
}
if r, _ := st.LatestOSReport("h1"); r == nil || r.Outcome != "applied" {
t.Fatalf("report not stored: %+v", r)
}
}
func storeRelease(id, pkgs string) store.OSRelease {
at, _ := time.Parse(time.RFC3339, "2026-10-04T12:00:00Z")
return store.OSRelease{ID: id, Fingerprint: "fp", ApprovedAt: at, ApprovedBy: "auto", PackagesJSON: pkgs}
}
@@ -0,0 +1,17 @@
{
"generation": 1,
"desired_state": {
"os_update": {
"ring": 1,
"enabled": true,
"release": {
"id": "os-20261004-120000",
"snapshot": "20261004T120000Z",
"packages": [
{"name": "libc6", "version": "2.41-12+deb13u4", "origin": "Debian"},
{"name": "openssl", "version": "3.5.7-1~deb13u3", "origin": "Debian-Security"}
]
}
}
}
}
+2 -1
View File
@@ -86,5 +86,6 @@
"bind.resend.hint": "If this was your link and your box is not linked yet, we send a fresh link to the e-mail address you gave Felhom.",
"bind.resend.button": "Send me a new link",
"bind.resent.lead": "Done.",
"bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support."
"bind.resent.body": "If this was a real link and your box is not linked yet, a new e-mail reaches your registered address within a few minutes. If it does not, contact support.",
"mail.event.os_update_applied": "System security fixes were installed on your box. You do not need to do anything."
}
+2 -1
View File
@@ -86,5 +86,6 @@
"bind.resend.hint": "Ha ez a te linked volt, és a dobozod még nincs összekötve, új linket küldünk arra az e-mail címre, amelyet a Felhomnál megadtál.",
"bind.resend.button": "Új linket kérek",
"bind.resent.lead": "Kész.",
"bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak."
"bind.resent.body": "Ha ez egy valódi hivatkozás volt, és a dobozod még nincs összekötve, néhány percen belül új e-mailt kapsz a regisztrált címedre. Ha nem jön, szólj az ügyfélszolgálatnak.",
"mail.event.os_update_applied": "Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned."
}
+7
View File
@@ -680,6 +680,13 @@ var operatorOnlyEvents = map[string]bool{
"offsite_prune_guard_refused": true,
// R-833 (v0.129.0): the operator raised ONE window's removal cap — an operator act, logged.
"offsite_window_large_grant": true,
// OS updates (hub v0.130.0, `11` §8 step 2): run failures, rings, switches and approvals are operator facts.
// os_update_applied is deliberately NOT here — it is the household's one line (info: recorded, never mailed).
"os_update_failed": true,
"os_update_health_failed": true,
"os_release_approved": true,
"os_release_approved_now": true,
"os_update_settings_changed": true,
// R-197 (v0.93.0). "The sealed offsite repository key changed" is a custody fact about escrow
// blobs. A customer can take no action on it — the remedy is the operator's inspection of the
// off-site tier — and the text is operator-grade English naming host ids and retained-blob
@@ -0,0 +1,18 @@
SUBJECT: [Felhom] Warning: System security fixes were installed on your box. You do not need to do anything.
---
Dear Customer,
Your Felhom system sent the following notification:
System security fixes were installed on your box. You do not need to do anything.
Details:
- Server: demo-fixture
- Time: 2026-01-15 10:30
- Level: Warning
- Type: os_update_applied
If you have any questions, contact your operator.
Best regards,
Felhom.eu monitoring
@@ -0,0 +1,18 @@
SUBJECT: [Felhom] Figyelmeztetés: Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned.
---
Kedves Ügyfél!
A Felhom rendszered a következő értesítést küldte:
Rendszerbiztonsági javítások telepítve a dobozodra. Ehhez nem kell semmit tenned.
Részletek:
- Szerver: demo-fixture
- Időpont: 2026-01-15 10:30
- Szint: Figyelmeztetés
- Típus: os_update_applied
Ha kérdésed van, vedd fel a kapcsolatot az üzemeltetővel.
Üdvözlettel,
Felhom.eu monitoring
+436
View File
@@ -0,0 +1,436 @@
// Package osupdates is the hub half of the guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0).
//
// Ring 0 (the demo boxes) installs every pending Debian / Debian-Security fix each night and reports the FULL installed
// set (C9). The hub derives the CANDIDATE: every Debian-origin package=version that all ring-0 boxes having that
// package agree on. A candidate is APPROVED when, since it was first seen:
// - every ring-0 box runs it (its newest report matches the candidate for every package it has),
// - ApproveAfter (default 24 h) has passed with every ring-0 report healthy, and
// - every ring-0 box has completed NightsRequired (default 1) post-backup night runs.
//
// The operator can approve at once ("approve now", an urgent fix). Ring 1 then gets the release in its desired state
// and installs exactly those versions. Rules pinned by service_test.go.
package osupdates
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"sort"
"strings"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
// Event types — operator-only except EventApplied, the household's line (`11` §5.7).
const (
EventApplied = "os_update_applied" // info, CUSTOMER: "system security fixes installed"
EventFailed = "os_update_failed" // error, operator: the run failed or was refused
EventHealthFailed = "os_update_health_failed" // error, operator: the guest was not healthy after the run
EventReleaseApprove = "os_release_approved" // info, operator
EventApprovedNow = "os_release_approved_now" // warning, operator: an operator approved at once
EventSettings = "os_update_settings_changed" // info, operator: ring or switch changed
)
// Package is one name=version with its origin ("Debian" | "Debian-Security").
type Package struct {
Name string `json:"name"`
Version string `json:"version"`
Origin string `json:"origin"`
}
// Report is what the agent POSTs after every run (the wrapper's report plus the leg's verdict).
type Report struct {
RunID string `json:"run_id"`
Trigger string `json:"trigger"` // night | debug
Mode string `json:"mode"` // apply | inventory
Ring int `json:"ring"`
ReleaseID string `json:"release_id"`
Outcome string `json:"outcome"` // applied | nothing | inventory | refused | failed | health_failed
Healthy bool `json:"healthy"`
HealthReason string `json:"health_reason,omitempty"`
VMID int `json:"vmid"`
Upgraded []Package `json:"upgraded,omitempty"`
Installed []Package `json:"installed,omitempty"`
Pending []PendingPkg `json:"pending,omitempty"`
NotCovered []string `json:"not_covered,omitempty"`
RestartNeeded []string `json:"restart_needed,omitempty"`
DockerRestartNeeded bool `json:"docker_restart_needed,omitempty"`
RebootNeeded bool `json:"reboot_needed,omitempty"`
Refused json.RawMessage `json:"refused,omitempty"`
Log []string `json:"log,omitempty"`
}
// PendingPkg is one update the guest's sources offer.
type PendingPkg struct {
Name string `json:"name"`
From string `json:"from"`
To string `json:"to"`
Origin []string `json:"origin"`
}
// Block is what a box receives in its desired state (`os_update`).
type Block struct {
Ring int `json:"ring"`
Enabled bool `json:"enabled"`
Release *ReleaseBlock `json:"release,omitempty"`
}
// ReleaseBlock is the newest approved release, for ring 1.
type ReleaseBlock struct {
ID string `json:"id"`
Snapshot string `json:"snapshot"` // approval time as YYYYMMDDTHHMMSSZ (decision 79: snapshot.debian.org)
Packages []Package `json:"packages"`
}
// Service ties the store, the events and the clock together.
type Service struct {
Store *store.Store
Emit func(customerID, eventType, severity, message, details, source string) // dispatcher; nil in tests
ApproveAfter time.Duration
NightsRequired int
Logger *log.Logger
Now func() time.Time
Bump func(hostID string) // bump a host's desired generation (store.BumpHostDesired); nil in tests
}
func (s *Service) now() time.Time {
if s.Now != nil {
return s.Now()
}
return time.Now()
}
func (s *Service) logf(f string, a ...any) {
if s.Logger != nil {
s.Logger.Printf(f, a...)
}
}
func (s *Service) event(customerID, typ, sev, msg string, details any) {
dj := ""
if details != nil {
if b, err := json.Marshal(details); err == nil {
dj = string(b)
}
}
if _, err := s.Store.SaveEvent(customerID, typ, sev, msg, dj, "hub"); err != nil {
s.logf("[WARN] osupdates: save event %s: %v", typ, err)
}
if s.Emit != nil {
s.Emit(customerID, typ, sev, msg, dj, "hub")
}
}
// Ingest stores a run and raises its events. The household gets one line per run that installed something.
func (s *Service) Ingest(hostID string, r Report) error {
h, err := s.Store.GetHost(hostID)
if err != nil || h == nil {
return fmt.Errorf("osupdates: unknown host %q", hostID)
}
raw, _ := json.Marshal(r)
if _, err := s.Store.SaveOSReport(store.OSReport{HostID: hostID, ReceivedAt: s.now(), Trigger: r.Trigger, Mode: r.Mode, Outcome: r.Outcome,
Healthy: r.Healthy, ReleaseID: r.ReleaseID, ReportJSON: string(raw)}); err != nil {
return err
}
s.logf("[INFO] osupdates: %s reported run %s: ring=%d mode=%s outcome=%s healthy=%v upgraded=%d pending=%d not-covered=%d restart-needed=%d",
hostID, r.RunID, r.Ring, r.Mode, r.Outcome, r.Healthy, len(r.Upgraded), len(r.Pending), len(r.NotCovered), len(r.RestartNeeded))
details := map[string]any{"host_id": hostID, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
switch r.Outcome {
case "applied", "health_failed":
s.event(h.CustomerID, EventApplied, "info",
fmt.Sprintf("System security fixes installed (%d package(s)).", len(r.Upgraded)), details)
if !r.Healthy || r.Outcome == "health_failed" {
s.event(h.CustomerID, EventHealthFailed, "error",
fmt.Sprintf("OS update on %s: the guest was NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically (no guest snapshot is possible, R-837); last night's whole-guest backup is the undo.",
hostID, len(r.Upgraded), r.HealthReason), details)
}
case "refused", "failed":
s.event(h.CustomerID, EventFailed, "error",
fmt.Sprintf("OS update on %s %s: %s", hostID, r.Outcome, strings.TrimSpace(string(r.Refused)+" "+r.HealthReason)), details)
}
return nil
}
// candidate derives the version set every ring-0 box agrees on, from each box's newest report.
func (s *Service) candidate(ring0 []string) (map[string]Package, map[string]*store.OSReport, error) {
latest := map[string]*store.OSReport{}
byPkg := map[string]map[string]Package{} // name -> host -> pkg
for _, h := range ring0 {
rep, err := s.Store.LatestOSReport(h)
if err != nil {
return nil, nil, err
}
if rep == nil {
return nil, nil, nil // a ring-0 box that never reported: no candidate
}
latest[h] = rep
var r Report
if err := json.Unmarshal([]byte(rep.ReportJSON), &r); err != nil {
return nil, nil, err
}
for _, p := range r.Installed {
if p.Origin != "Debian" && p.Origin != "Debian-Security" {
continue
}
if byPkg[p.Name] == nil {
byPkg[p.Name] = map[string]Package{}
}
byPkg[p.Name][h] = p
}
}
cand := map[string]Package{}
for name, hosts := range byPkg {
var v string
agree := true
var pick Package
for _, p := range hosts {
if v == "" {
v, pick = p.Version, p
} else if p.Version != v {
agree = false
}
}
if agree {
cand[name] = pick
}
}
return cand, latest, nil
}
func fingerprint(c map[string]Package) (string, []Package) {
var list []Package
for _, p := range c {
list = append(list, p)
}
sort.Slice(list, func(i, j int) bool { return list[i].Name < list[j].Name })
h := sha256.New()
for _, p := range list {
fmt.Fprintf(h, "%s=%s\n", p.Name, p.Version)
}
return hex.EncodeToString(h.Sum(nil))[:16], list
}
// ring0Hosts lists the ring-0 boxes that have the switch ON.
func (s *Service) ring0Hosts() ([]string, error) {
hosts, err := s.Store.ListHosts()
if err != nil {
return nil, err
}
var out []string
for _, h := range hosts {
if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 0 && st.Enabled {
out = append(out, h.HostID)
}
}
sort.Strings(out)
return out, nil
}
// Status explains the approval state (for the log and the fleet page).
type Status struct {
Fingerprint string
FirstSeen time.Time
Packages int
Waiting string // why not approved yet ("" = approved or nothing to do)
}
// Evaluate checks the approval rule and approves when it holds. Called every minute.
func (s *Service) Evaluate() (Status, error) {
ring0, err := s.ring0Hosts()
if err != nil || len(ring0) == 0 {
return Status{Waiting: "no ring-0 box"}, err
}
cand, _, err := s.candidate(ring0)
if err != nil || cand == nil {
return Status{Waiting: "a ring-0 box has not reported"}, err
}
fp, list := fingerprint(cand)
pj, _ := json.Marshal(list)
first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
if err != nil {
return Status{}, err
}
st := Status{Fingerprint: fp, FirstSeen: first, Packages: len(list)}
if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
st.Waiting = ""
return st, nil // already approved
}
if age := s.now().Sub(first); age < s.ApproveAfter {
st.Waiting = fmt.Sprintf("healthy for %s of %s", age.Round(time.Minute), s.ApproveAfter)
return st, nil
}
for _, h := range ring0 {
reps, err := s.Store.OSReportsSince(h, first)
if err != nil {
return st, err
}
nights := 0
for _, r := range reps {
if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
st.Waiting = fmt.Sprintf("%s reported %s (healthy=%v) at %s since the set was first seen", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
return st, nil
}
if r.Trigger == "night" {
nights++
}
}
if nights < s.NightsRequired {
st.Waiting = fmt.Sprintf("%s has %d of %d night run(s) since the set was first seen", h, nights, s.NightsRequired)
return st, nil
}
}
if err := s.approve(fp, list, "auto"); err != nil {
return st, err
}
return st, nil
}
// ApproveNow approves the current candidate at once (operator, urgent fix).
func (s *Service) ApproveNow() (string, error) {
ring0, err := s.ring0Hosts()
if err != nil || len(ring0) == 0 {
return "", fmt.Errorf("osupdates: no ring-0 box")
}
cand, _, err := s.candidate(ring0)
if err != nil || cand == nil {
return "", fmt.Errorf("osupdates: a ring-0 box has not reported yet")
}
fp, list := fingerprint(cand)
pj, _ := json.Marshal(list)
if _, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now()); err != nil {
return "", err
}
if rel, _ := s.Store.LatestOSRelease(); rel != nil && rel.Fingerprint == fp {
return rel.ID, nil
}
if err := s.approve(fp, list, "operator"); err != nil {
return "", err
}
rel, _ := s.Store.LatestOSRelease()
s.event("", EventApprovedNow, "warning", fmt.Sprintf("The operator approved OS release %s at once (%d packages), without the wait.", rel.ID, len(list)),
map[string]any{"release_id": rel.ID, "packages": len(list)})
return rel.ID, nil
}
func (s *Service) approve(fp string, list []Package, by string) error {
at := s.now().UTC().Truncate(time.Second)
id := "os-" + at.Format("20060102-150405")
pj, _ := json.Marshal(list)
if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
return err
}
s.logf("[WARN] osupdates: OS release %s APPROVED by %s (%d packages, fingerprint %s)", id, by, len(list), fp)
s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s approved (%s, %d packages).", id, by, len(list)),
map[string]any{"release_id": id, "approved_by": by, "packages": len(list), "fingerprint": fp})
if s.Bump != nil {
hosts, _ := s.Store.ListHosts()
for _, h := range hosts {
if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
s.Bump(h.HostID)
}
}
}
return nil
}
// DesiredBlock is the `os_update` block a box receives.
func (s *Service) DesiredBlock(hostID string) Block {
st := s.Store.GetOSHostSettings(hostID)
b := Block{Ring: st.Ring, Enabled: st.Enabled}
if st.Ring == 1 {
if rel, err := s.Store.LatestOSRelease(); err == nil && rel != nil {
var list []Package
if json.Unmarshal([]byte(rel.PackagesJSON), &list) == nil {
b.Release = &ReleaseBlock{ID: rel.ID, Snapshot: rel.ApprovedAt.UTC().Format("20060102T150405Z"), Packages: list}
}
}
}
return b
}
// SetRing / SetEnabled are operator acts; each bumps the box's desired generation and is an operator event.
func (s *Service) SetRing(hostID string, ring int) error {
if ring != 0 && ring != 1 {
return fmt.Errorf("osupdates: ring must be 0 or 1")
}
h, err := s.Store.GetHost(hostID)
if err != nil || h == nil {
return fmt.Errorf("osupdates: unknown host %q", hostID)
}
if err := s.Store.SetOSRing(hostID, ring); err != nil {
return err
}
s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates: %s is now ring %d.", hostID, ring), map[string]any{"host_id": hostID, "ring": ring})
if s.Bump != nil {
s.Bump(hostID)
}
return nil
}
func (s *Service) SetEnabled(hostID string, on bool) error {
h, err := s.Store.GetHost(hostID)
if err != nil || h == nil {
return fmt.Errorf("osupdates: unknown host %q", hostID)
}
if err := s.Store.SetOSEnabled(hostID, on); err != nil {
return err
}
s.event(h.CustomerID, EventSettings, "info", fmt.Sprintf("OS updates on %s switched %s.", hostID, map[bool]string{true: "ON", false: "OFF"}[on]),
map[string]any{"host_id": hostID, "enabled": on})
if s.Bump != nil {
s.Bump(hostID)
}
return nil
}
// FleetLine is one box on the fleet page.
type FleetLine struct {
HostID string
Ring int
Enabled bool
ReleaseID string
LastOutcome string
LastAt time.Time
Pending int
NotCovered int
RestartNeeded int
}
// Fleet lists every box.
func (s *Service) Fleet() ([]FleetLine, error) {
hosts, err := s.Store.ListHosts()
if err != nil {
return nil, err
}
var out []FleetLine
for _, h := range hosts {
st := s.Store.GetOSHostSettings(h.HostID)
l := FleetLine{HostID: h.HostID, Ring: st.Ring, Enabled: st.Enabled}
if rep, _ := s.Store.LatestOSReport(h.HostID); rep != nil {
var r Report
_ = json.Unmarshal([]byte(rep.ReportJSON), &r)
l.ReleaseID, l.LastOutcome, l.LastAt = r.ReleaseID, r.Outcome, rep.ReceivedAt
l.Pending, l.NotCovered, l.RestartNeeded = len(r.Pending), len(r.NotCovered), len(r.RestartNeeded)
}
out = append(out, l)
}
return out, nil
}
// FleetJSON is Fleet plus the approval status, for the operator's fleet route.
func (s *Service) FleetJSON() (any, error) {
lines, err := s.Fleet()
if err != nil {
return nil, err
}
rel, _ := s.Store.LatestOSRelease()
out := map[string]any{"boxes": lines}
if rel != nil {
out["latest_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
}
return out, nil
}
+189
View File
@@ -0,0 +1,189 @@
package osupdates
import (
"log"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)
type fix struct {
s *Service
now time.Time
events []string
bumps []string
}
func newFix(t *testing.T) *fix {
t.Helper()
st, err := store.New(filepath.Join(t.TempDir(), "hub.db"), log.New(os.Stderr, "", 0))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { st.Close() })
for _, h := range []struct{ host, cust string }{{"hp", "c-hp"}, {"n100", "c-n100"}, {"cust1", "c-1"}} {
if err := st.UpsertHost(&store.Host{HostID: h.host, CustomerID: h.cust, APIKey: "k-" + h.host}); err != nil {
t.Fatal(err)
}
}
f := &fix{now: time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC)}
f.s = &Service{Store: st, ApproveAfter: 24 * time.Hour, NightsRequired: 1, Now: func() time.Time { return f.now },
Emit: func(_, typ, _, _, _, _ string) { f.events = append(f.events, typ) },
Bump: func(h string) { f.bumps = append(f.bumps, h) }}
_ = st.SetOSRing("hp", 0)
_ = st.SetOSRing("n100", 0)
return f
}
func pk(name, ver string) Package { return Package{Name: name, Version: ver, Origin: "Debian"} }
func (f *fix) report(t *testing.T, host, trigger string, healthy bool, pkgs ...Package) {
t.Helper()
outcome := "applied"
if !healthy {
outcome = "health_failed"
}
if err := f.s.Ingest(host, Report{RunID: host + trigger + f.now.String(), Trigger: trigger, Mode: "apply", Outcome: outcome,
Healthy: healthy, Installed: pkgs, Upgraded: pkgs[:1]}); err != nil {
t.Fatal(err)
}
}
// The ruled wait: approved only after every ring-0 box ran the set healthy for ApproveAfter AND through a night run.
// Red-proof: drop the age check, the healthy check or the nights check in Evaluate and a sub-step fails.
func TestApproval_WaitHealthyAndOneNight(t *testing.T) {
f := newFix(t)
set := []Package{pk("libc6", "2.41-12+deb13u4"), pk("openssl", "3.5.7-1~deb13u3")}
f.report(t, "hp", "debug", true, set...)
f.report(t, "n100", "debug", true, set...)
st, _ := f.s.Evaluate()
if !strings.HasPrefix(st.Waiting, "healthy for") {
t.Fatalf("fresh set approved or wrong reason: %+v", st)
}
f.now = f.now.Add(25 * time.Hour)
st, _ = f.s.Evaluate()
if !strings.Contains(st.Waiting, "night run") {
t.Fatalf("approved without a night run: %+v", st)
}
f.report(t, "hp", "night", true, set...)
f.report(t, "n100", "night", true, set...)
if _, err := f.s.Evaluate(); err != nil {
t.Fatal(err)
}
rel, _ := f.s.Store.LatestOSRelease()
if rel == nil || rel.ApprovedBy != "auto" {
t.Fatalf("not approved: %+v", rel)
}
if len(f.bumps) != 1 || f.bumps[0] != "cust1" {
t.Fatalf("only the ring-1 box must be nudged, got %v", f.bumps)
}
b := f.s.DesiredBlock("cust1")
if b.Ring != 1 || !b.Enabled || b.Release == nil || len(b.Release.Packages) != 2 || b.Release.Snapshot != rel.ApprovedAt.UTC().Format("20060102T150405Z") {
t.Fatalf("ring-1 block = %+v", b)
}
if hb := f.s.DesiredBlock("hp"); hb.Ring != 0 || hb.Release != nil {
t.Fatalf("a ring-0 box must not get a release (it installs everything pending): %+v", hb)
}
}
func TestApproval_UnhealthyRunBlocks(t *testing.T) {
f := newFix(t)
set := []Package{pk("libc6", "2.41-12+deb13u4")}
f.report(t, "hp", "debug", true, set...)
f.report(t, "n100", "debug", true, set...)
f.s.Evaluate()
f.now = f.now.Add(25 * time.Hour)
f.report(t, "hp", "night", false, set...)
f.report(t, "n100", "night", true, set...)
st, _ := f.s.Evaluate()
if rel, _ := f.s.Store.LatestOSRelease(); rel != nil {
t.Fatalf("approved although a ring-0 run was not healthy: %+v", st)
}
if !strings.Contains(st.Waiting, "healthy=false") {
t.Fatalf("reason = %q", st.Waiting)
}
found := false
for _, e := range f.events {
found = found || e == EventHealthFailed
}
if !found {
t.Fatalf("no %s event: %v", EventHealthFailed, f.events)
}
}
// Ring 0 disagreeing on a package: that package is left out of the candidate (C9 — approve what ALL ring 0 runs).
func TestCandidate_DisagreementLeftOut(t *testing.T) {
f := newFix(t)
f.report(t, "hp", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u5"))
f.report(t, "n100", "night", true, pk("libc6", "2.41-12+deb13u4"), pk("curl", "8.14.1-2+deb13u4"))
cand, _, err := f.s.candidate([]string{"hp", "n100"})
if err != nil {
t.Fatal(err)
}
if _, ok := cand["curl"]; ok || cand["libc6"].Version != "2.41-12+deb13u4" {
t.Fatalf("candidate = %+v", cand)
}
}
// Non-Debian origins never enter a release (the fast lane is Debian / Debian-Security only, C3).
func TestCandidate_OnlyDebianOrigins(t *testing.T) {
f := newFix(t)
d := Package{Name: "docker-ce", Version: "5:29.8.2", Origin: "Docker"}
f.report(t, "hp", "night", true, pk("libc6", "x1"), d)
f.report(t, "n100", "night", true, pk("libc6", "x1"), d)
cand, _, _ := f.s.candidate([]string{"hp", "n100"})
if _, ok := cand["docker-ce"]; ok {
t.Fatal("a Docker package entered the candidate")
}
}
func TestApproveNow_IsAnOperatorEvent(t *testing.T) {
f := newFix(t)
f.report(t, "hp", "debug", true, pk("libc6", "x1"))
f.report(t, "n100", "debug", true, pk("libc6", "x1"))
id, err := f.s.ApproveNow()
if err != nil || id == "" {
t.Fatalf("%q %v", id, err)
}
rel, _ := f.s.Store.LatestOSRelease()
if rel.ApprovedBy != "operator" {
t.Fatalf("approved_by = %q", rel.ApprovedBy)
}
n := 0
for _, e := range f.events {
if e == EventApprovedNow {
n++
}
}
if n != 1 {
t.Fatalf("want one %s, got %v", EventApprovedNow, f.events)
}
}
func TestSwitchAndRing(t *testing.T) {
f := newFix(t)
if err := f.s.SetEnabled("cust1", false); err != nil {
t.Fatal(err)
}
if b := f.s.DesiredBlock("cust1"); b.Enabled {
t.Fatal("switch OFF not delivered")
}
if err := f.s.SetRing("cust1", 2); err == nil {
t.Fatal("ring 2 accepted")
}
if b := f.s.DesiredBlock("nobody-row"); b.Ring != 1 || !b.Enabled {
t.Fatalf("default must be ring 1, ON: %+v", b)
}
}
func TestIngest_AppliedIsTheHouseholdsLine(t *testing.T) {
f := newFix(t)
f.report(t, "cust1", "night", true, pk("libc6", "x1"))
if len(f.events) != 1 || f.events[0] != EventApplied {
t.Fatalf("events = %v", f.events)
}
}
+211
View File
@@ -0,0 +1,211 @@
package store
import (
"database/sql"
"time"
)
// OS updates, guest fast lane (`11-os-updates.md` §5.3, §8 step 2; hub v0.130.0).
//
// Three records:
// - os_host_settings: the operator's per-box ring (0 = demo boxes that take every update first, 1 = every other
// box) and switch (ON by default — decision 12's shape). A host with no row is ring 1, ON.
// - os_reports: every run the agent reports (the full installed set rides in report_json — C9: what ring 0 RUNS).
// - os_candidates / os_releases: the version set ring 0 runs, first seen when, and the approved releases.
func (s *Store) migrateOSUpdates() error {
_, err := s.db.Exec(`
CREATE TABLE IF NOT EXISTS os_host_settings (
host_id TEXT PRIMARY KEY,
ring INTEGER NOT NULL DEFAULT 1,
enabled INTEGER NOT NULL DEFAULT 1,
updated_at DATETIME NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS os_reports (
id INTEGER PRIMARY KEY AUTOINCREMENT,
host_id TEXT NOT NULL,
received_at DATETIME NOT NULL DEFAULT (datetime('now')),
trigger TEXT NOT NULL DEFAULT '',
mode TEXT NOT NULL DEFAULT '',
outcome TEXT NOT NULL DEFAULT '',
healthy INTEGER NOT NULL DEFAULT 0,
release_id TEXT NOT NULL DEFAULT '',
report_json TEXT NOT NULL DEFAULT '{}'
);
CREATE INDEX IF NOT EXISTS idx_os_reports_host ON os_reports(host_id, id);
CREATE TABLE IF NOT EXISTS os_candidates (
fingerprint TEXT PRIMARY KEY,
first_seen DATETIME NOT NULL,
packages_json TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS os_releases (
id TEXT PRIMARY KEY,
fingerprint TEXT NOT NULL,
approved_at DATETIME NOT NULL,
approved_by TEXT NOT NULL,
packages_json TEXT NOT NULL
);
`)
return err
}
// OSHostSettings is one box's ring and switch.
type OSHostSettings struct {
HostID string
Ring int
Enabled bool
}
// GetOSHostSettings returns the box's settings; a box with no row is ring 1, ON.
func (s *Store) GetOSHostSettings(hostID string) OSHostSettings {
st := OSHostSettings{HostID: hostID, Ring: 1, Enabled: true}
var ring, en int
if err := s.db.QueryRow(`SELECT ring, enabled FROM os_host_settings WHERE host_id = ?`, hostID).Scan(&ring, &en); err == nil {
st.Ring, st.Enabled = ring, en == 1
}
return st
}
// SetOSRing sets the box's ring (0 or 1).
func (s *Store) SetOSRing(hostID string, ring int) error {
_, err := s.db.Exec(`INSERT INTO os_host_settings (host_id, ring) VALUES (?, ?)
ON CONFLICT(host_id) DO UPDATE SET ring = excluded.ring, updated_at = datetime('now')`, hostID, ring)
return err
}
// SetOSEnabled sets the box's switch.
func (s *Store) SetOSEnabled(hostID string, on bool) error {
v := 0
if on {
v = 1
}
_, err := s.db.Exec(`INSERT INTO os_host_settings (host_id, enabled) VALUES (?, ?)
ON CONFLICT(host_id) DO UPDATE SET enabled = excluded.enabled, updated_at = datetime('now')`, hostID, v)
return err
}
// OSReport is one stored run.
type OSReport struct {
ID int64
HostID string
ReceivedAt time.Time
Trigger string
Mode string
Outcome string
Healthy bool
ReleaseID string
ReportJSON string
}
// SaveOSReport stores one run.
func (s *Store) SaveOSReport(r OSReport) (int64, error) {
h := 0
if r.Healthy {
h = 1
}
at := r.ReceivedAt
if at.IsZero() {
at = time.Now()
}
// received_at comes from the CALLER's clock: the approval rule compares it with first_seen, which the
// service stamps with its own clock — two clocks would make "since first seen" miss reports.
res, err := s.db.Exec(`INSERT INTO os_reports (host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
r.HostID, at.UTC().Format("2006-01-02 15:04:05"), r.Trigger, r.Mode, r.Outcome, h, r.ReleaseID, r.ReportJSON)
if err != nil {
return 0, err
}
return res.LastInsertId()
}
func scanOSReports(rows *sql.Rows) ([]OSReport, error) {
defer rows.Close()
var out []OSReport
for rows.Next() {
var r OSReport
var at string
var h int
if err := rows.Scan(&r.ID, &r.HostID, &at, &r.Trigger, &r.Mode, &r.Outcome, &h, &r.ReleaseID, &r.ReportJSON); err != nil {
return nil, err
}
r.ReceivedAt, r.Healthy = parseSQLiteTime(at), h == 1
out = append(out, r)
}
return out, rows.Err()
}
const osReportCols = `id, host_id, received_at, trigger, mode, outcome, healthy, release_id, report_json`
// LatestOSReport returns the box's newest run, or nil.
func (s *Store) LatestOSReport(hostID string) (*OSReport, error) {
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? ORDER BY id DESC LIMIT 1`, hostID)
if err != nil {
return nil, err
}
rs, err := scanOSReports(rows)
if err != nil || len(rs) == 0 {
return nil, err
}
return &rs[0], nil
}
// OSReportsSince returns the box's runs received at or after t, oldest first.
func (s *Store) OSReportsSince(hostID string, t time.Time) ([]OSReport, error) {
rows, err := s.db.Query(`SELECT `+osReportCols+` FROM os_reports WHERE host_id = ? AND received_at >= ? ORDER BY id`,
hostID, t.UTC().Format("2006-01-02 15:04:05"))
if err != nil {
return nil, err
}
return scanOSReports(rows)
}
// OSCandidateFirstSeen records a candidate set the first time it is seen and returns when that was.
func (s *Store) OSCandidateFirstSeen(fingerprint, packagesJSON string, now time.Time) (time.Time, error) {
if _, err := s.db.Exec(`INSERT OR IGNORE INTO os_candidates (fingerprint, first_seen, packages_json) VALUES (?, ?, ?)`,
fingerprint, now.UTC().Format("2006-01-02 15:04:05"), packagesJSON); err != nil {
return time.Time{}, err
}
var at string
if err := s.db.QueryRow(`SELECT first_seen FROM os_candidates WHERE fingerprint = ?`, fingerprint).Scan(&at); err != nil {
return time.Time{}, err
}
return parseSQLiteTime(at), nil
}
// OSRelease is one approved version set.
type OSRelease struct {
ID string
Fingerprint string
ApprovedAt time.Time
ApprovedBy string
PackagesJSON string
}
// SaveOSRelease stores an approved release.
func (s *Store) SaveOSRelease(r OSRelease) error {
_, err := s.db.Exec(`INSERT INTO os_releases (id, fingerprint, approved_at, approved_by, packages_json) VALUES (?, ?, ?, ?, ?)`,
r.ID, r.Fingerprint, r.ApprovedAt.UTC().Format("2006-01-02 15:04:05"), r.ApprovedBy, r.PackagesJSON)
return err
}
// LatestOSRelease returns the newest approved release, or nil.
func (s *Store) LatestOSRelease() (*OSRelease, error) {
var r OSRelease
var at string
err := s.db.QueryRow(`SELECT id, fingerprint, approved_at, approved_by, packages_json FROM os_releases ORDER BY approved_at DESC, id DESC LIMIT 1`).
Scan(&r.ID, &r.Fingerprint, &at, &r.ApprovedBy, &r.PackagesJSON)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
r.ApprovedAt = parseSQLiteTime(at)
return &r, nil
}
// BackdateOSCandidateForTest moves a candidate's first_seen into the past. TEST-ONLY.
func (s *Store) BackdateOSCandidateForTest(fingerprint string, by time.Duration) error {
_, err := s.db.Exec(`UPDATE os_candidates SET first_seen = ? WHERE fingerprint = ?`,
time.Now().Add(-by).UTC().Format("2006-01-02 15:04:05"), fingerprint)
return err
}
+4
View File
@@ -858,6 +858,10 @@ func (s *Store) migrate() error {
}
// R-833 (v0.129.0): the cap each window was opened with — an operator grant may raise it once.
s.db.Exec("ALTER TABLE offsite_windows ADD COLUMN max_remove INTEGER")
// OS updates, guest fast lane (hub v0.130.0, `11` §8 step 2).
if err := s.migrateOSUpdates(); err != nil {
return fmt.Errorf("os_updates: %w", err)
}
return nil
}
+52
View File
@@ -0,0 +1,52 @@
package web
import (
"encoding/json"
"net/http"
"strconv"
"strings"
)
// handleOSAdmin serves the operator's OS-update controls (behind the operator login, like every route here):
//
// POST /os/ring/<host_id> ring=0|1
// POST /os/enabled/<host_id> on=1|0
// POST /os/approve-now approve the current ring-0 set at once (an operator event)
// GET /os/fleet one line per box (JSON)
func (s *Server) handleOSAdmin(w http.ResponseWriter, r *http.Request, path string) {
if s.osUpdates == nil {
http.Error(w, "os updates not configured", http.StatusServiceUnavailable)
return
}
reply := func(v any, err error) {
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(v)
}
switch {
case r.Method == http.MethodGet && path == "/os/fleet":
reply(s.osUpdates.FleetJSON())
case r.Method == http.MethodPost && strings.HasPrefix(path, "/os/ring/"):
n, err := strconv.Atoi(r.FormValue("ring"))
if err != nil {
http.Error(w, "ring must be 0 or 1", http.StatusBadRequest)
return
}
reply(map[string]bool{"ok": true}, s.osUpdates.SetRing(strings.TrimPrefix(path, "/os/ring/"), n))
case r.Method == http.MethodPost && strings.HasPrefix(path, "/os/enabled/"):
on := r.FormValue("on")
if on != "0" && on != "1" {
http.Error(w, "on must be 0 or 1", http.StatusBadRequest)
return
}
reply(map[string]bool{"ok": true}, s.osUpdates.SetEnabled(strings.TrimPrefix(path, "/os/enabled/"), on == "1"))
case r.Method == http.MethodPost && path == "/os/approve-now":
id, err := s.osUpdates.ApproveNow()
reply(map[string]string{"release_id": id}, err)
default:
http.Error(w, "not found", http.StatusNotFound)
}
}
+15
View File
@@ -73,6 +73,7 @@ type Server struct {
// offsiteWindowAdmin: operator one-shot grant / weekly switch (decision 68). nil → 503.
offsiteWindowGrant func(customerID string) error
offsiteWindowGrantMax func(customerID string, maxRemove int) error
osUpdates OSUpdateAdmin
offsiteWindowSwitch func(on bool) error
// operator key-file clean-up (decision 72) and abandonment cancel (decision 74). nil → 503.
offsiteRemoveUnpinned func(ctx context.Context, customerID string) (int, error)
@@ -212,6 +213,17 @@ func (s *Server) SetOffsiteKeyAdmin(remove func(context.Context, string) (int, e
s.offsiteRemoveUnpinned, s.offsiteAbandonCancel = remove, cancel
}
// OSUpdateAdmin is the operator side of the guest fast lane (hub v0.130.0).
type OSUpdateAdmin interface {
SetRing(hostID string, ring int) error
SetEnabled(hostID string, on bool) error
ApproveNow() (string, error)
FleetJSON() (any, error)
}
// SetOSUpdateAdmin wires the OS-update operator routes.
func (s *Server) SetOSUpdateAdmin(a OSUpdateAdmin) { s.osUpdates = a }
// SetOffsiteWindowLargeGrant wires the operator's one-window raised cap (R-833).
func (s *Server) SetOffsiteWindowLargeGrant(fn func(string, int) error) { s.offsiteWindowGrantMax = fn }
@@ -681,6 +693,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("{\"ok\":true}\n"))
case strings.HasPrefix(path, "/os/"):
// Operator (hub v0.130.0, `11` §8 step 2): per-box ring and switch, approve now, the fleet lines.
s.handleOSAdmin(w, r, path)
case path == "/offsite/key-audit":
// Operator: run the daily off-site key check now (decision 69). Same code path as the 07:10 job.
if r.Method != http.MethodPost {