hub v0.133.0 (R-859 test approvals end with the test; R-840 bundle on the System page, manifest, alarm); installer 1.31.0 (root files from the config bundle); bundle bootstrap script; golden 0.293.0 evidence; rulings 96–99; drill-r50 removed (evidence)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-04 20:18:37 +02:00
parent 6b820143f8
commit ff1db11db4
45 changed files with 1707 additions and 42 deletions
@@ -0,0 +1,90 @@
+package osupdates
+
+import (
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+)
+
+func bundleReport(t *testing.T, f *fix, host, version, sha string) {
+	t.Helper()
+	body := `{"system":{"pve_version":"pve-manager/9.2.2/x","config_bundle":{"version":"` + version + `","bundle_sha256":"` + sha + `"}}}`
+	h, err := f.s.Store.GetHost(host)
+	if err != nil || h == nil {
+		t.Fatalf("no host %s", host)
+	}
+	if err := f.s.Store.SaveHostReport(host, h.CustomerID, []byte(body), store.HostReportDenorm{AgentVersion: "0.143.0"}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func vouch(t *testing.T, f *fix, sha string) {
+	t.Helper()
+	if err := f.s.Store.SetArtifactManifest(store.ArtifactManifest{AgentVersion: "0.143.0", AgentSHA256: "x", BundleSHA256: sha}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func count(sent []string, typ string) int {
+	n := 0
+	for _, s := range sent {
+		if s == typ {
+			n++
+		}
+	}
+	return n
+}
+
+// R-840: a box behind the vouched bundle is told to the operator after 7 days — not before, once, and the clock
+// restarts when the box catches up. "none" (no bundle ever) counts; "unknown" never does.
+func TestBundleAlarm_AfterSevenDaysBehind(t *testing.T) {
+	f := newFix(t)
+	vouch(t, f, "new")
+	bundleReport(t, f, "cust1", "none", "")
+	sent, _ := f.s.Alarms()
+	if count(sent, EventBundleBehind) != 0 {
+		t.Fatal("alarm on the first sight")
+	}
+	f.now = f.now.Add(6 * 24 * time.Hour)
+	if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 0 {
+		t.Fatal("alarm before 7 days")
+	}
+	f.now = f.now.Add(25 * time.Hour)
+	if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 1 {
+		t.Fatalf("no alarm after 7 days: %v", sent)
+	}
+	if sent, _ = f.s.Alarms(); count(sent, EventBundleBehind) != 0 {
+		t.Fatal("the alarm must not repeat at once")
+	}
+	bundleReport(t, f, "cust1", "0.143.0", "new")
+	f.s.Alarms()
+	if !f.s.Store.BundleBehindSince("cust1").IsZero() {
+		t.Fatal("caught up: the clock must clear")
+	}
+}
+
+func TestBundleAlarm_UnknownAndNoVouchedBundleSayNothing(t *testing.T) {
+	f := newFix(t)
+	vouch(t, f, "new")
+	bundleReport(t, f, "cust1", "unknown", "")
+	f.s.Alarms()
+	f.now = f.now.Add(30 * 24 * time.Hour)
+	if sent, _ := f.s.Alarms(); count(sent, EventBundleBehind) != 0 || !f.s.Store.BundleBehindSince("cust1").IsZero() {
+		t.Fatalf("unknown is not a fact: %v", sent)
+	}
+	// control: the same box saying "none" IS behind (proves the report above was read at all)
+	bundleReport(t, f, "cust1", "none", "")
+	f.s.Alarms()
+	if f.s.Store.BundleBehindSince("cust1").IsZero() {
+		t.Fatal("control: a box saying none must start the clock")
+	}
+	g := newFix(t)
+	vouch(t, g, "")
+	bundleReport(t, g, "cust1", "0.143.0", "some-bundle")
+	g.s.Alarms()
+	g.now = g.now.Add(30 * 24 * time.Hour)
+	if sent, _ := g.s.Alarms(); count(sent, EventBundleBehind) != 0 {
+		t.Fatalf("nothing vouched, nothing behind: %v", sent)
+	}
+}
@@ -29,6 +29,7 @@ import (

 	"gitea.dooplex.hu/admin/felhom-hub/internal/semver"
 	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	"gitea.dooplex.hu/admin/felhom-hub/internal/sysfacts"
 )

 // Layers.
@@ -68,6 +69,10 @@ const (
 	EventRebootNeeded   = "os_reboot_needed"           // warning, operator: reboot needed for RebootAfter
 	EventRing0Stalled   = "os_ring0_stalled"           // error, operator: ring 0 approved nothing for Ring0StallAfter
 	EventNotCovered     = "os_not_covered"             // warning, operator: fast-lane packages no release covers
+	// EventCancelled: a TEST approval was cancelled because the hub started without the TEST override (`11` §5.3.1).
+	EventCancelled = "os_release_cancelled" // warning, operator
+	// EventBundleBehind: a box's root-owned config bundle has differed from the vouched one for BundleBehindAfter (R-840).
+	EventBundleBehind = "os_config_bundle_behind" // warning, operator
 )

 // Package is one name=version with its origin ("Debian" | "Debian-Security").
@@ -165,9 +170,15 @@ type Service struct {
 	RebootAfter     time.Duration // 14 d
 	Ring0StallAfter time.Duration // 7 d
 	NotCoveredAfter time.Duration // 14 d
-	Logger          *log.Logger
-	Now             func() time.Time
-	Bump            func(hostID string)
+	// BundleBehindAfter: a box's config bundle differs from the vouched one this long → an operator alarm (R-840;
+	// decided by CC unattended — operator may reverse). Zero = 7 d.
+	BundleBehindAfter time.Duration
+	Logger            *log.Logger
+	Now               func() time.Time
+	Bump              func(hostID string)
+	// TestOverride names the TEST overrides active at start ("" = none, the ruled waits). Every approval made while it
+	// is set carries the `test` mark; CancelTestReleases cancels them at a start without it (`11` §5.3.1).
+	TestOverride string
 }

 func (s *Service) now() time.Time {
@@ -427,6 +438,8 @@ type ReleaseInfo struct {
 	ApprovedAt time.Time
 	ApprovedBy string
 	Packages   int
+	Test       bool   // a TEST approval still in force: amber on the System page
+	Cancelled  string // set on a cancelled one (the page lists the last 7 days')
 }

 // Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent.
@@ -439,7 +452,20 @@ func (s *Service) Releases() []ReleaseInfo {
 		}
 		var list []Package
 		_ = json.Unmarshal([]byte(rel.PackagesJSON), &list)
-		out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)})
+		out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list), Test: rel.Test})
+	}
+	return out
+}
+
+// CancelledReleases lists the approvals cancelled in the last 7 days (the System page says what stopped being served).
+func (s *Service) CancelledReleases() []ReleaseInfo {
+	rels, _ := s.Store.CancelledOSReleasesSince(s.now().Add(-7 * 24 * time.Hour))
+	var out []ReleaseInfo
+	for _, r := range rels {
+		var list []Package
+		_ = json.Unmarshal([]byte(r.PackagesJSON), &list)
+		out = append(out, ReleaseInfo{Layer: r.Layer, ID: r.ID, ApprovedAt: r.ApprovedAt, ApprovedBy: r.ApprovedBy, Packages: len(list),
+			Test: r.Test, Cancelled: r.CancelledAt})
 	}
 	return out
 }
@@ -455,6 +481,9 @@ func (s *Service) Candidates() []Status {
 	return append(out, d)
 }

+// BundleThreshold is the config-bundle alarm's wait (the System page turns the cell red at it).
+func (s *Service) BundleThreshold() time.Duration { return dflt(s.BundleBehindAfter, 7*24*time.Hour) }
+
 // Thresholds are the alarm numbers the System page colours by (the same values the alarms use).
 func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) {
 	return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
@@ -577,12 +606,18 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error {
 	at := s.now().UTC().Truncate(time.Second)
 	id := "os-" + layer + "-" + at.Format("20060102-150405")
 	pj, _ := json.Marshal(list)
-	if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by, PackagesJSON: string(pj)}); err != nil {
+	test := s.TestOverride != ""
+	if err := s.Store.SaveOSRelease(store.OSRelease{ID: id, Layer: layer, Fingerprint: fp, ApprovedAt: at, ApprovedBy: by,
+		PackagesJSON: string(pj), Test: test}); err != nil {
 		return err
 	}
-	s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
-	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
-		map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
+	mark := ""
+	if test {
+		mark = " — TEST approval (" + s.TestOverride + "); cancelled when the hub starts without the override"
+	}
+	s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)%s", id, layer, by, len(list), fp, mark)
+	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages)%s.", id, layer, by, len(list), mark),
+		map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp, "test": test})
 	if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state
 		hosts, _ := s.Store.ListHosts()
 		for _, h := range hosts {
@@ -594,6 +629,45 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error {
 	return nil
 }

+// CancelTestReleases runs at every hub start. Without a TEST override it cancels every test-marked approval that no
+// real (non-test) approval has superseded: no ring-1 box installs it from then on; what boxes already installed stays.
+// Each cancellation is an operator event; ring-1 boxes are bumped so their next plan has no cancelled release. With the
+// override still active it does nothing (the test is still running). Returns the cancelled ids.
+func (s *Service) CancelTestReleases() ([]string, error) {
+	if s.TestOverride != "" {
+		s.logf("[INFO] osupdates: TEST override active (%s) — test approvals stay in force", s.TestOverride)
+		return nil, nil
+	}
+	var ids []string
+	for _, layer := range AllLayers {
+		rels, err := s.Store.UnsupersededTestReleases(layer)
+		if err != nil {
+			return ids, err
+		}
+		for _, r := range rels {
+			reason := "approved under a TEST override; the hub started without it"
+			if err := s.Store.CancelOSRelease(r.ID, reason, s.now()); err != nil {
+				return ids, err
+			}
+			ids = append(ids, r.ID)
+			s.logf("[WARN] osupdates: TEST approval %s (%s, approved %s by %s) CANCELLED — no ring-1 box installs it from now on",
+				r.ID, layer, r.ApprovedAt.UTC().Format(time.RFC3339), r.ApprovedBy)
+			s.event("", EventCancelled, "warning", fmt.Sprintf("OS release %s (%s) was a TEST approval and is cancelled: "+
+				"no further box installs it. Boxes that already installed it keep it.", r.ID, layer),
+				map[string]any{"release_id": r.ID, "layer": layer, "approved_at": r.ApprovedAt.UTC().Format(time.RFC3339), "approved_by": r.ApprovedBy})
+		}
+	}
+	if len(ids) > 0 && s.Bump != nil {
+		hosts, _ := s.Store.ListHosts()
+		for _, h := range hosts {
+			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
+				s.Bump(h.HostID)
+			}
+		}
+	}
+	return ids, nil
+}
+
 func (s *Service) releaseBlock(layer string) *ReleaseBlock {
 	rel, err := s.Store.LatestOSRelease(layer)
 	if err != nil || rel == nil {
@@ -860,6 +934,38 @@ func (s *Service) Alarms() ([]string, error) {
 			}
 		}
 	}
+	// 5. R-840: the box's ROOT-OWNED config bundle (sudoers, wrappers, units) differs from the vouched agent's for
+	//    BundleBehindAfter. "none" (no bundle ever reached the box) counts as behind; "unknown" (the box could not say)
+	//    counts as nothing — never a guess. Only when the vouched agent carries a bundle at all.
+	man := s.Store.GetArtifactManifest()
+	for _, h := range hosts {
+		if man.BundleSHA256 == "" {
+			break
+		}
+		rj, _ := s.Store.GetLatestHostReportJSON(h.CustomerID)
+		sys := sysfacts.Parse(rj)
+		if !sys.Present || sys.Bundle.Version == sysfacts.Unknown {
+			continue
+		}
+		behind := sys.Bundle.BundleSHA256 != man.BundleSHA256
+		since := s.Store.BundleBehindSince(h.HostID)
+		switch {
+		case !behind && !since.IsZero():
+			_ = s.Store.SetBundleBehindSince(h.HostID, time.Time{})
+			since = time.Time{}
+		case behind && since.IsZero():
+			since = now
+			_ = s.Store.SetBundleBehindSince(h.HostID, since)
+		}
+		after := dflt(s.BundleBehindAfter, 7*24*time.Hour)
+		if s.raise("bundle:"+h.HostID, behind && now.Sub(since) >= after, h.CustomerID, EventBundleBehind, "warning",
+			fmt.Sprintf("Root files: %s still runs config bundle %s; the vouched agent %s carries a newer one (since %s). "+
+				"Send it with a signed agent_config_update (`11` §5.4.2).", h.HostID, sys.Bundle.Version, man.AgentVersion,
+				since.UTC().Format("2006-01-02")),
+			map[string]any{"host_id": h.HostID, "box_bundle": sys.Bundle.Version, "vouched_agent": man.AgentVersion, "since": since}) {
+			sent = append(sent, EventBundleBehind)
+		}
+	}
 	// 3. Ring 0 approved nothing for `stall` while ring 0 has pending fast-lane updates: the whole fleet stopped
 	//    getting fixes.
 	ring0, _ := s.ring0Hosts()
@@ -0,0 +1,173 @@
+package osupdates
+
+import (
+	"database/sql"
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
+	_ "modernc.org/sqlite"
+)
+
+// `11` §5.3.1 (hub v0.133.0): test approvals end with the test.
+
+// approveUnderTest makes ring 0 run a set and approves it at once under a TEST override (OS_APPROVE_AFTER=0 shape).
+func approveUnderTest(t *testing.T, f *fix, override string, set ...Package) string {
+	t.Helper()
+	f.s.TestOverride = override
+	f.s.ApproveAfter, f.s.NightsRequired = 0, 0
+	f.report(t, "hp", "debug", true, set...)
+	f.report(t, "n100", "debug", true, set...)
+	if _, err := f.s.Evaluate(); err != nil {
+		t.Fatal(err)
+	}
+	rel, _ := f.s.Store.LatestOSRelease(LayerGuest)
+	if rel == nil {
+		t.Fatal("not approved")
+	}
+	return rel.ID
+}
+
+// An approval made under the override carries the mark; one made without it does not.
+func TestTestApproval_IsMarked(t *testing.T) {
+	f := newFix(t)
+	approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4"))
+	rel, _ := f.s.Store.LatestOSRelease(LayerGuest)
+	if !rel.Test {
+		t.Fatalf("an approval under a TEST override must be marked: %+v", rel)
+	}
+	if info := f.s.Releases(); len(info) != 1 || !info[0].Test {
+		t.Fatalf("the System page must see the mark: %+v", info)
+	}
+	f.s.TestOverride = ""
+	f.now = f.now.Add(time.Hour)
+	f.report(t, "hp", "debug", true, pk("libc6", "u5"))
+	f.report(t, "n100", "debug", true, pk("libc6", "u5"))
+	f.s.Evaluate()
+	rel, _ = f.s.Store.LatestOSRelease(LayerGuest)
+	if rel.Test {
+		t.Fatalf("an approval without the override must not be marked: %+v", rel)
+	}
+}
+
+// The consequence: after a restart without the override, a ring-1 box gets NO plan from the test approval; the
+// cancellation is an operator event and ring-1 boxes are bumped.
+func TestTestApproval_CancelledAtAStartWithoutTheOverride(t *testing.T) {
+	f := newFix(t)
+	id := approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4"))
+	if b := f.s.DesiredBlock("cust1"); b.Release == nil || b.Release.ID != id {
+		t.Fatalf("before the restart the ring-1 box is served the release: %+v", b)
+	}
+	f.events, f.bumps = nil, nil
+	f.s.TestOverride = "" // the hub restarts without the override
+	ids, err := f.s.CancelTestReleases()
+	if err != nil || len(ids) != 1 || ids[0] != id {
+		t.Fatalf("cancelled %v, %v", ids, err)
+	}
+	if b := f.s.DesiredBlock("cust1"); b.Release != nil {
+		t.Fatalf("a ring-1 box must get no plan from a cancelled test approval: %+v", b.Release)
+	}
+	if len(f.events) != 1 || f.events[0] != EventCancelled {
+		t.Fatalf("events = %v", f.events)
+	}
+	if len(f.bumps) != 1 || f.bumps[0] != "cust1" {
+		t.Fatalf("ring-1 boxes must be bumped: %v", f.bumps)
+	}
+	if c := f.s.CancelledReleases(); len(c) != 1 || c[0].ID != id {
+		t.Fatalf("the page must list the cancellation: %+v", c)
+	}
+	// once is enough: a second start cancels nothing more
+	if again, _ := f.s.CancelTestReleases(); len(again) != 0 {
+		t.Fatalf("second start cancelled %v", again)
+	}
+}
+
+func TestTestApproval_StaysWhileTheOverrideIsStillOn(t *testing.T) {
+	f := newFix(t)
+	approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4"))
+	if ids, _ := f.s.CancelTestReleases(); len(ids) != 0 {
+		t.Fatalf("the test is still running; nothing to cancel: %v", ids)
+	}
+	if b := f.s.DesiredBlock("cust1"); b.Release == nil {
+		t.Fatal("still served while the override is on")
+	}
+}
+
+// A test approval that a REAL approval has superseded is history, not cancelled; the real one stays served.
+func TestTestApproval_SupersededIsLeftAlone(t *testing.T) {
+	f := newFix(t)
+	old := approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", pk("libc6", "u4"))
+	f.s.TestOverride = ""
+	f.now = f.now.Add(time.Hour)
+	f.report(t, "hp", "debug", true, pk("libc6", "u5"))
+	f.report(t, "n100", "debug", true, pk("libc6", "u5"))
+	f.s.Evaluate()
+	real, _ := f.s.Store.LatestOSRelease(LayerGuest)
+	if real.ID == old || real.Test {
+		t.Fatalf("setup: %+v", real)
+	}
+	if ids, _ := f.s.CancelTestReleases(); len(ids) != 0 {
+		t.Fatalf("a superseded test approval must not be cancelled: %v", ids)
+	}
+	if b := f.s.DesiredBlock("cust1"); b.Release == nil || b.Release.ID != real.ID {
+		t.Fatalf("the real release stays served: %+v", b.Release)
+	}
+}
+
+// After a cancellation the SAME set is approved again by the ruled wait (a real release) — the cancel is not a ban.
+func TestTestApproval_TheSetIsApprovedAgainByTheRuledWait(t *testing.T) {
+	f := newFix(t)
+	set := []Package{pk("libc6", "u4")}
+	approveUnderTest(t, f, "OS_APPROVE_AFTER=0s", set...)
+	f.s.TestOverride = ""
+	f.s.ApproveAfter, f.s.NightsRequired = 24*time.Hour, 1
+	f.s.CancelTestReleases()
+	f.now = f.now.Add(25 * time.Hour)
+	f.report(t, "hp", "night", true, set...)
+	f.report(t, "n100", "night", true, set...)
+	f.s.Evaluate()
+	rel, _ := f.s.Store.LatestOSRelease(LayerGuest)
+	if rel == nil || rel.Test {
+		t.Fatalf("the ruled wait must approve the set again, unmarked: %+v", rel)
+	}
+}
+
+// The one-time backfill: on a database from before the mark, an approval earlier than 24 h after its set was first
+// seen (the 2026-10-04 shape: 1.5 h) is marked test; one after the ruled wait is not.
+func TestTestApproval_BackfillMarksTheEarlyApprovals(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "hub.db")
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, q := range []string{
+		`CREATE TABLE os_candidates (fingerprint TEXT PRIMARY KEY, first_seen DATETIME NOT NULL, packages_json TEXT NOT NULL)`,
+		`CREATE TABLE os_releases (id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL, approved_at DATETIME NOT NULL, approved_by TEXT NOT NULL, packages_json TEXT NOT NULL, layer TEXT NOT NULL DEFAULT 'guest')`,
+		`INSERT INTO os_candidates VALUES ('fpA', '2026-10-04 11:07:00', '[]'), ('fpB', '2026-10-02 08:00:00', '[]')`,
+		`INSERT INTO os_candidates VALUES ('fpD', '2026-10-04 14:28:00', '[]')`,
+		`INSERT INTO os_releases VALUES ('os-guest-early', 'fpA', '2026-10-04 12:39:33', 'auto', '[]', 'guest'),
+		                                ('os-guest-ruled', 'fpB', '2026-10-03 09:00:00', 'auto', '[]', 'guest'),
+		                                ('os-docker-button', 'fpD', '2026-10-04 14:28:42', 'operator', '[]', 'docker')`,
+	} {
+		if _, err := db.Exec(q); err != nil {
+			t.Fatal(err)
+		}
+	}
+	db.Close()
+	st, err := store.New(path, log.New(os.Stderr, "", 0))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer st.Close()
+	rels, _ := st.UnsupersededTestReleases(LayerGuest)
+	if len(rels) != 1 || rels[0].ID != "os-guest-early" {
+		t.Fatalf("backfill: unsuperseded test releases = %+v", rels)
+	}
+	// the operator's own button approval is not backfilled (a person approved it)
+	if d, _ := st.UnsupersededTestReleases(LayerDocker); len(d) != 0 {
+		t.Fatalf("backfill marked the operator's Docker approval: %+v", d)
+	}
+}