From 5f31da04bc16b9582ae5260ccbb3a6aaab3690b0 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Mon, 19 Oct 2026 20:27:01 +0200 Subject: [PATCH] hub v0.162.0 (built, not deployed): no repeat mail after a restart for a known-down host or an already-reported stale restore test (R-1001); the Day-0 agent list leaves out packages without the felhom-agent file, a vouch of one is refused (R-1011); an absent tunnel reads 'none set up' (R-1070) --- hub/CHANGELOG.md | 29 +++ hub/internal/api/handler.go | 2 +- hub/internal/gitea/gitea.go | 21 +- hub/internal/gitea/gitea_test.go | 42 ++++ hub/internal/monitor/host_staleness.go | 47 ++++- hub/internal/monitor/r1001_restart_test.go | 187 ++++++++++++++++++ hub/internal/monitor/restoretest.go | 24 +++ hub/internal/store/store.go | 24 +++ hub/internal/web/artifact_choices_test.go | 33 +++- .../web/artifact_installability_test.go | 6 +- hub/internal/web/configs.go | 3 + hub/internal/web/r292_artifact_flash_test.go | 7 +- hub/internal/web/server.go | 12 ++ hub/internal/web/system.go | 4 + hub/internal/web/system_test.go | 16 ++ hub/internal/web/templates/configuration.html | 3 + 16 files changed, 443 insertions(+), 17 deletions(-) create mode 100644 hub/internal/monitor/r1001_restart_test.go diff --git a/hub/CHANGELOG.md b/hub/CHANGELOG.md index 78a148f1..ffc73dbe 100644 --- a/hub/CHANGELOG.md +++ b/hub/CHANGELOG.md @@ -1,3 +1,32 @@ +## v0.162.0 — a hub restart no longer re-mails a known-down box (R-1001); the Day-0 agent list offers only real agent packages (R-1011); a box with no tunnel reads „none set up" (R-1070) (2026-10-19) + +Built, NOT deployed (the night brief of 2026-10-19: the hub is never deployed at night; the deploy is the operator's +word in the morning). + +- **No repeat mail after a restart (R-1001).** Measured 2026-10-13: every hub restart re-mailed `host_down` and + `restore_test_stale` for Tester 2, silent for days — the staleness checker left a down host unseeded so the first + sweep emits, and its comment said the dispatcher's 1-hour cooldown would swallow the repeat (host liveness events + bypass it); the restore-test edge lived in memory. Now: at start a host whose newest `host_stale`/`host_down` event + is newer than its last report is seeded with that state silently (`store.LatestHostLivenessEvent`, the „Host :" + messages); a seeded „stale" still escalates to `host_down`; a host that reports again recovers once and its next + silence alarms. `restore_test_stale` reads its edge from the DB on the first sweep after a start: a stale signal for + the tier inside the read window and newer than the tier's last proof means the run was reported. A failed lookup, or + a signal older than the window, lets the alarm through. Tests `TestHostStaleness_R1001_RestartDoesNotReMail`, + `TestHostStaleness_R1001_NewSilenceStillMails`, `TestRestoreTestStale_R1001_RestartDoesNotReMail` — each restart is + a NEW checker over the SAME database; red-proved (seeding removed → `the restarted hub re-sent [host_down]`; the DB + check removed → `re-sent 2 restore_test_stale signal(s)`). +- **The Day-0 agent list (R-1011).** `gitea.FileSHA256` with a file named now answers exactly that file: a version + without it is `ErrNoFile` (it wraps `ErrNoSHA`), never the first file's sha. The agent dropdown leaves out the step + bundles (`0.156.1-step1` … hold only `felhom-config-bundle.json`; read from the registry 2026-10-19: 6 such + versions, every release version holds `felhom-agent`, every golden `golden.tar.zst`), one INFO line per resolve; a + vouch of one is refused with the new flash `artifact_file_missing`. Tests `TestFileSHA256_R1011_NeverAnotherFilesSHA`, + `TestArtifactChoices_R1011_StepBundleIsNotAnAgentVersion`, a row in `TestArtifactSave_FlashNamesTheCause`; + red-proved (the old fallback → `the list offers 0.156.1-step1 (sha "bundle-…")`). Two test fakes named the golden's + file after its package; they now serve `golden.tar.zst`. +- **The agent's new tunnel state `absent`** (R-1070, agent Unreleased): the System page shows „none set up" (plain, no + amber) for a box with no cloudflared container; `not_running` stays red, other states amber. The tunnel alarm counts + `not_running` only, unchanged. Test `TestSystemRows_R1070_AbsentTunnelIsNotAWarning`. + ## v0.161.0 — the passphrase gives keys only until the box uses its own (R-1159, operator decision D19); an alarm for an ep0 household with no prune job (R-1165) (2026-10-19) - **The retrieval passphrase's keys** (R-1159, D19 = „box key first, then lock"; `05` §8 [DESIGN R-1159]). Measured the diff --git a/hub/internal/api/handler.go b/hub/internal/api/handler.go index cf90aaef..e4c4e2bf 100644 --- a/hub/internal/api/handler.go +++ b/hub/internal/api/handler.go @@ -771,7 +771,7 @@ type hostReportPayload struct { Tiers []hostForeignKeyArchives `json:"tiers"` } `json:"foreign_key_archives"` Cloudflared struct { - Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown (older: active | inactive | …) + Status string `json:"status"` // agent >= 0.141.0: running | not_running | unknown; absent since R-1070 (never alarms) (older: active | inactive | …) Detail string `json:"detail,omitempty"` } `json:"cloudflared"` // DR recipe — the agent's storage/guest/PBS half (secret-free). RawMessage = stored verbatim, diff --git a/hub/internal/gitea/gitea.go b/hub/internal/gitea/gitea.go index 6252ba00..ea8ec8db 100644 --- a/hub/internal/gitea/gitea.go +++ b/hub/internal/gitea/gitea.go @@ -70,6 +70,11 @@ func IsNotFound(err error) bool { // ErrNoSHA marks a version whose metadata answered but carried no usable sha256 (R-292). var ErrNoSHA = errors.New("registry listed no sha256") +// ErrNoFile marks a version that has files but not the one asked for (R-1011, hub 0.162.0): an agent step bundle +// (`0.156.1-step1`) holds only felhom-config-bundle.json. It wraps ErrNoSHA, so a caller that only knows ErrNoSHA +// still refuses. +var ErrNoFile = fmt.Errorf("the package version holds no such file: %w", ErrNoSHA) + type pkgEntry struct { Name string `json:"name"` Version string `json:"version"` @@ -100,9 +105,11 @@ type pkgFile struct { SHA256 string `json:"sha256"` } -// FileSHA256 returns the sha256 of a file inside a generic package version. When the version holds -// multiple files it prefers an exact match to preferredFile (e.g. "golden.tar.zst"); otherwise it uses -// the first file. This is a cheap metadata call — the artifact bytes are never downloaded. +// FileSHA256 returns the sha256 of a file inside a generic package version. With preferredFile set it is EXACTLY that +// file: a version without it is ErrNoFile, never another file's sha (R-1011 — the Day-0 list offered agent step +// bundles, whose only file is the config bundle, and showed that file's sha as the agent's). Only an empty +// preferredFile takes the first file. This is a cheap metadata call — the artifact bytes are never downloaded. +// Pinned by TestFileSHA256_R1011_NeverAnotherFilesSHA. func (c *Client) FileSHA256(ctx context.Context, pkgName, version, preferredFile string) (string, error) { u := fmt.Sprintf("%s/api/v1/packages/%s/generic/%s/%s/files", c.baseURL, c.owner, pkgName, version) @@ -114,11 +121,17 @@ func (c *Client) FileSHA256(ctx context.Context, pkgName, version, preferredFile return "", fmt.Errorf("no files listed for %s/%s: %w", pkgName, version, ErrNoSHA) } if preferredFile != "" { + names := make([]string, 0, len(files)) for _, f := range files { - if f.Name == preferredFile && f.SHA256 != "" { + if f.Name == preferredFile { + if f.SHA256 == "" { + return "", fmt.Errorf("no sha256 for %s/%s file %q: %w", pkgName, version, f.Name, ErrNoSHA) + } return f.SHA256, nil } + names = append(names, f.Name) } + return "", fmt.Errorf("%s/%s has no file %q (it holds %s): %w", pkgName, version, preferredFile, strings.Join(names, ", "), ErrNoFile) } if files[0].SHA256 == "" { return "", fmt.Errorf("no sha256 for %s/%s file %q: %w", pkgName, version, files[0].Name, ErrNoSHA) diff --git a/hub/internal/gitea/gitea_test.go b/hub/internal/gitea/gitea_test.go index 6d8ddfdf..813a218b 100644 --- a/hub/internal/gitea/gitea_test.go +++ b/hub/internal/gitea/gitea_test.go @@ -2,6 +2,7 @@ package gitea import ( "context" + "errors" "net/http" "net/http/httptest" "strings" @@ -86,3 +87,44 @@ func TestGetJSON_ErrorOnNon200(t *testing.T) { t.Fatal("expected an error on HTTP 500") } } + +// R-1011: with a file named, a version without it is ErrNoFile (which is also ErrNoSHA) — never the first file's sha. +// Seen 2026-10-14: the Day-0 list showed agent `0.156.1-step1` (only felhom-config-bundle.json) with the bundle's sha. +// +// COMPANION RED-PROOF (observed 2026-10-19): with the pre-fix fallback to files[0], it fails with +// +// gitea_test.go: step bundle: sha "bbbb" err , want ErrNoFile and no sha +// +// Restored. +func TestFileSHA256_R1011_NeverAnotherFilesSHA(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case strings.Contains(r.URL.Path, "/felhom-agent/0.156.1-step1/files"): + w.Write([]byte(`[{"name":"felhom-config-bundle.json","sha256":"bbbb"}]`)) + case strings.Contains(r.URL.Path, "/felhom-agent/0.156.1/files"): + w.Write([]byte(`[{"name":"felhom-config-bundle.json","sha256":"bbbb"},{"name":"felhom-agent","sha256":"aaaa"}]`)) + case strings.Contains(r.URL.Path, "/felhom-agent/0.1.0/files"): + w.Write([]byte(`[{"name":"felhom-agent","sha256":""}]`)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + c := New(srv.URL, "admin", "u", "t") + ctx := context.Background() + + sha, err := c.FileSHA256(ctx, "felhom-agent", "0.156.1-step1", "felhom-agent") + if sha != "" || !errors.Is(err, ErrNoFile) || !errors.Is(err, ErrNoSHA) { + t.Fatalf("step bundle: sha %q err %v, want ErrNoFile and no sha", sha, err) + } + if sha, err = c.FileSHA256(ctx, "felhom-agent", "0.156.1", "felhom-agent"); err != nil || sha != "aaaa" { + t.Fatalf("the release beside its bundle: sha %q err %v, want the binary's aaaa", sha, err) + } + if sha, err = c.FileSHA256(ctx, "felhom-agent", "0.1.0", "felhom-agent"); sha != "" || !errors.Is(err, ErrNoSHA) || errors.Is(err, ErrNoFile) { + t.Fatalf("the file without a sha: sha %q err %v, want ErrNoSHA (not ErrNoFile)", sha, err) + } + if sha, err = c.FileSHA256(ctx, "felhom-agent", "0.156.1-step1", ""); err != nil || sha != "bbbb" { + t.Fatalf("no file named: the first file stays the answer, got %q %v", sha, err) + } +} diff --git a/hub/internal/monitor/host_staleness.go b/hub/internal/monitor/host_staleness.go index 14d117d8..86211abf 100644 --- a/hub/internal/monitor/host_staleness.go +++ b/hub/internal/monitor/host_staleness.go @@ -49,16 +49,31 @@ func NewHostStalenessChecker(s *store.Store, threshold time.Duration, onEvent Ev logger.Printf("[WARN] Host staleness checker: failed to seed states: %v", err) return sc } - var okCount, staleCount, downCount int + var okCount, staleCount, downCount, knownCount int for _, row := range rows { if s.IsCustomerBlocked(row.CustomerID) { continue } sc.customerOf[row.HostID] = row.CustomerID age := time.Since(row.LastReportAt) - // F2: seed only HEALTHY hosts. A host already stale/down at (re)start is left UNSEEDED so the - // first Check() observes oldState=="" and emits once (a box that went silent while the hub was - // down would otherwise stay silent). The dispatcher's 1h cooldown dedups the re-emit. + // F2: a host already stale/down at (re)start is left UNSEEDED so the first Check() observes oldState=="" + // and emits once (a box that went silent while the hub was down would otherwise stay silent). + // + // R-1001 (hub 0.162.0): UNLESS the alarm was already raised for this silence — the newest host_stale / + // host_down event for the host is newer than its last report. Then the host is seeded with the state that + // event recorded, silently. The old comment said „the dispatcher's 1h cooldown dedups the re-emit"; it did + // not (host liveness events bypass that cooldown, `dispatcher.go` nodeLivenessEvents), so every hub restart + // re-mailed the operator about Tester 2, silent for days (measured 2026-10-13). A host seeded „stale" still + // emits host_down when it crosses downAfter — a new transition. A failed lookup leaves it unseeded (the + // alarm, never the silence). Pinned by TestHostStaleness_R1001_RestartDoesNotReMail. + if age > sc.threshold { + if known := sc.alreadyAlarmed(row); known != "" { + sc.states[row.HostID] = known + sc.downtimeStart[row.HostID] = row.LastReportAt + knownCount++ + continue + } + } switch { case age > sc.downAfter: downCount++ @@ -69,10 +84,30 @@ func NewHostStalenessChecker(s *store.Store, threshold time.Duration, onEvent Ev okCount++ } } - logger.Printf("[INFO] Host staleness checker initialized: %d ok, %d stale, %d down (stale/down left unseeded → first Check emits; host_stale after %s, host_down after %s)", okCount, staleCount, downCount, threshold, 2*threshold) + logger.Printf("[INFO] Host staleness checker initialized: %d ok, %d already alarmed (seeded silently, R-1001), %d stale, %d down (left unseeded → first Check emits; host_stale after %s, host_down after %s)", okCount, knownCount, staleCount, downCount, threshold, 2*threshold) return sc } +// alreadyAlarmed returns "stale" or "down" when the newest liveness event for the host is that alarm and is not older +// than the host's last report (the operator was told about THIS silence), else "". +func (sc *HostStalenessChecker) alreadyAlarmed(row store.HostStaleRow) string { + ev, err := sc.store.LatestHostLivenessEvent(row.CustomerID, row.HostID) + if err != nil { + sc.logger.Printf("[WARN] Host staleness: could not read the last alarm for %s (%v) — the first Check alarms (R-1001)", row.HostID, err) + return "" + } + if ev == nil || ev.CreatedAt.Before(row.LastReportAt) { + return "" + } + switch ev.EventType { + case "host_down": + return "down" + case "host_stale": + return "stale" + } + return "" +} + // Check evaluates all hosts and emits events on state transitions. Call every 60s. func (sc *HostStalenessChecker) Check() { rows, err := sc.store.GetHostStaleness() @@ -125,7 +160,7 @@ func (sc *HostStalenessChecker) Check() { if oldState == "" { // F2: first observation (incl. a hub restart). Healthy seeds silently; a host already // stale/down at first sight emits once (else a box that went silent while the hub was - // down would never alert). The dispatcher's 1h cooldown dedups a re-emit on hub restart. + // down would never alert). A host whose alarm was already raised was seeded at start (R-1001). sc.states[row.HostID] = newState if newState != "ok" { sc.emitTransition(row.HostID, row.CustomerID, "unknown", newState, age) diff --git a/hub/internal/monitor/r1001_restart_test.go b/hub/internal/monitor/r1001_restart_test.go new file mode 100644 index 00000000..574fe134 --- /dev/null +++ b/hub/internal/monitor/r1001_restart_test.go @@ -0,0 +1,187 @@ +package monitor + +import ( + "database/sql" + "io" + "log" + "path/filepath" + "testing" + "time" + + "gitea.dooplex.hu/admin/felhom-hub/internal/store" + _ "modernc.org/sqlite" +) + +// R-1001 (measured 2026-10-13 18:33 UTC, hub v0.149.0 rollout): Tester 2, silent 216 h, got a fresh host_down and +// restore_test_stale mail on a hub restart. A restart is a NEW checker over the SAME database — that is what these +// tests build: alarm with checker 1, then a second checker on the same DB (the restarted hub) must not mail again, +// and a host that goes silent after the start must still mail once. + +func r1001Store(t *testing.T) (*store.Store, *sql.DB) { + t.Helper() + path := filepath.Join(t.TempDir(), "r1001.db") + st, err := store.New(path, log.New(io.Discard, "", 0)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + return st, db +} + +func r1001Host(t *testing.T, st *store.Store, db *sql.DB, cid, hid string, minutesAgo int) { + t.Helper() + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: cid, APIKey: "k-" + cid, RetrievalPassword: "p"}); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: hid, CustomerID: cid, APIKey: "hk-" + hid}); err != nil { + t.Fatal(err) + } + if err := st.SaveHostReport(hid, cid, []byte(`{}`), store.HostReportDenorm{}); err != nil { + t.Fatal(err) + } + backdate(t, db, hid, minutesAgo) +} + +// COMPANION RED-PROOF (observed 2026-10-19): with the R-1001 seeding removed from NewHostStalenessChecker (the pre-fix +// F2 shape), the restart case fails with +// +// r1001_restart_test.go: the restarted hub re-sent [host_down] for a host whose silence was already alarmed +// +// Restored. +func TestHostStaleness_R1001_RestartDoesNotReMail(t *testing.T) { + st, db := r1001Store(t) + r1001Host(t, st, db, "tester-2", "Tester-2-be8404", 216*60) // down for 9 days + var got []string + on := func(_, et, _, _, _, _ string) { got = append(got, et) } + quiet := log.New(io.Discard, "", 0) + + first := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) + first.Check() + if len(got) != 1 || got[0] != "host_down" { + t.Fatalf("the first hub ever to see the silence must alarm once: %v", got) + } + + got = nil + restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) + restarted.Check() + restarted.Check() + if len(got) != 0 { + t.Fatalf("the restarted hub re-sent %v for a host whose silence was already alarmed", got) + } + if s := restarted.GetState("Tester-2-be8404"); s != "down" { + t.Fatalf("the already-alarmed host must be seeded with its alarm's state, got %q", s) + } + + // The box reports again → host_recovered once, and its next silence alarms again (a new run). + backdate(t, db, "Tester-2-be8404", 0) + restarted.Check() + if len(got) != 1 || got[0] != "host_recovered" { + t.Fatalf("a seeded host that reports again must recover once: %v", got) + } + got = nil + backdate(t, db, "Tester-2-be8404", 70) + again := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) // a restart in the middle of a NEW silence + again.Check() + if len(got) != 1 || got[0] != "host_down" { + t.Fatalf("a new silence after a report must alarm (the old alarm is older than the report): %v", got) + } +} + +// The other half of the rule: a host that goes down AFTER the start mails once; an alarm seeded „stale" still +// escalates to host_down when it crosses the down threshold (a new transition, not a repeat). +func TestHostStaleness_R1001_NewSilenceStillMails(t *testing.T) { + st, db := r1001Host2(t) + var got []string + on := func(_, et, _, _, _, _ string) { got = append(got, et) } + quiet := log.New(io.Discard, "", 0) + + sc := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) + sc.Check() + if len(got) != 0 { + t.Fatalf("a healthy host at start: no mail, got %v", got) + } + backdate(t, db, "h-new", 70) // down after the start + sc.Check() + if len(got) != 1 || got[0] != "host_down" { + t.Fatalf("a host down after the start must mail once: %v", got) + } + + // stale alarmed, then a restart, then down → host_down (new transition), once. + got = nil + backdate(t, db, "h-new", 0) + sc.Check() // recovered + backdate(t, db, "h-new", 40) + sc.Check() // host_stale + got = nil + restarted := NewHostStalenessChecker(st, 30*time.Minute, on, quiet) + restarted.Check() + if len(got) != 0 { + t.Fatalf("restart during an alarmed stale: no mail, got %v", got) + } + backdate(t, db, "h-new", 70) + restarted.Check() + if len(got) != 1 || got[0] != "host_down" { + t.Fatalf("stale → down after the restart must mail host_down once: %v", got) + } +} + +func r1001Host2(t *testing.T) (*store.Store, *sql.DB) { + st, db := r1001Store(t) + r1001Host(t, st, db, "c-new", "h-new", 0) + return st, db +} + +// restore_test_stale across a restart: the same DB, a second checker. Both tiers unproven for 20 days. +// +// COMPANION RED-PROOF (observed 2026-10-19): with the staleAlreadyReported case removed from checkStaleness, it fails +// with +// +// r1001_restart_test.go: the restarted hub re-sent 2 restore_test_stale signal(s) +// +// Restored. +func TestRestoreTestStale_R1001_RestartDoesNotReMail(t *testing.T) { + st, db := r1001Store(t) + if err := st.SaveCustomerConfig(&store.CustomerConfig{CustomerID: "c1", APIKey: "k", RetrievalPassword: "p"}); err != nil { + t.Fatal(err) + } + if err := st.UpsertHost(&store.Host{HostID: "h1", CustomerID: "c1", APIKey: "k1"}); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if err := st.SaveHostReport("h1", "c1", []byte(rtReport(t, "", false, now, "")), store.HostReportDenorm{}); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`UPDATE host_reports SET received_at = datetime('now', '-20 days')`); err != nil { + t.Fatal(err) + } + var got []capturedEvent + first := rtChecker(st, &got, nil) + first.Check() + if len(got) != 2 { + t.Fatalf("two tiers unproven for 20 days: want 2 stale signals, got %v", got) + } + + got = nil + restarted := rtChecker(st, &got, nil) + restarted.Check() + restarted.Check() + if len(got) != 0 { + t.Fatalf("the restarted hub re-sent %d restore_test_stale signal(s)", len(got)) + } + + // An earlier signal outside the read window does not hold the next one back. + if _, err := db.Exec(`UPDATE events SET created_at = datetime('now', '-15 days') WHERE event_type = ?`, EventRestoreTestStale); err != nil { + t.Fatal(err) + } + got = nil + later := rtChecker(st, &got, nil) + later.Check() + if len(got) != 2 { + t.Fatalf("a signal older than the read window must not silence the tier: got %v", got) + } +} diff --git a/hub/internal/monitor/restoretest.go b/hub/internal/monitor/restoretest.go index 673c96ea..14f32be3 100644 --- a/hub/internal/monitor/restoretest.go +++ b/hub/internal/monitor/restoretest.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "log" + "strings" "sync" "time" @@ -277,6 +278,9 @@ func (c *RestoreTestChecker) checkStaleness(customerID, latestJSON string, now t c.mu.Unlock() switch { + case state == "stale" && prev == "" && c.staleAlreadyReported(customerID, tier, proven[tier], now): + // R-1001: the first sweep after a hub restart — this tier's stale run was already reported. + c.logger.Printf("[INFO] restore-test staleness: %s %s tier still unproven, already reported before the restart — not re-sent (R-1001)", customerID, tier) case state == "stale" && prev != "stale": c.emit(customerID, EventRestoreTestStale, "warning", v.reason) case v.verdict == verdictUnknown && prev == "": @@ -287,6 +291,26 @@ func (c *RestoreTestChecker) checkStaleness(customerID, latestJSON string, now t } } +// staleAlreadyReported (R-1001, hub 0.162.0): the edge state lives in memory, so a hub restart forgot that a tier's +// stale signal was already sent and the first sweep re-sent it (measured 2026-10-13, Tester 2). The DB holds the +// answer: a restore_test_stale event for this tier („ tier: …") inside the read window and newer than the +// tier's last proof means this run of staleness was reported. The window bound keeps it honest — a pass between that +// event and now would be inside the window and newer than it. An event older than the window, or a failed read, lets +// the signal through (the alarm, never the silence). Pinned by TestRestoreTestStale_R1001_RestartDoesNotReMail. +func (c *RestoreTestChecker) staleAlreadyReported(customerID, tier string, provenAt, now time.Time) bool { + evs, err := c.store.GetEventsByType(customerID, EventRestoreTestStale, now.Add(-restoreWindowRead)) + if err != nil { + c.logger.Printf("[WARN] restore-test staleness: could not read the past signals for %s (%v) — sending (R-1001)", customerID, err) + return false + } + for _, e := range evs { + if strings.HasPrefix(e.Message, tier+" tier:") && e.CreatedAt.After(provenAt) { + return true + } + } + return false +} + // assessRestoreProven is the per-tier verdict. PURE (now and the window injected) so the policy is // unit-tested — the property that made R-81 provable, kept deliberately. // diff --git a/hub/internal/store/store.go b/hub/internal/store/store.go index 0c39bf83..45a4f66f 100644 --- a/hub/internal/store/store.go +++ b/hub/internal/store/store.go @@ -2704,6 +2704,30 @@ func (s *Store) GetLatestEventByType(customerID, eventType string) (*Event, erro return &e, nil } +// LatestHostLivenessEvent returns the newest host_stale / host_down / host_recovered event for hostID, or nil (R-1001). +// The host staleness checker writes every one of them as „Host : …" under the host's customer, so the prefix names +// the host (substr, not LIKE: a host id may hold `_`). +func (s *Store) LatestHostLivenessEvent(customerID, hostID string) (*Event, error) { + prefix := "Host " + hostID + ":" + var e Event + var createdAt string + err := s.db.QueryRow(` + SELECT id, customer_id, event_type, severity, message, details_json, source, created_at + FROM events + WHERE customer_id = ? AND event_type IN ('host_stale', 'host_down', 'host_recovered') AND substr(message, 1, ?) = ? + ORDER BY created_at DESC, id DESC + LIMIT 1`, customerID, len(prefix), prefix, + ).Scan(&e.ID, &e.CustomerID, &e.EventType, &e.Severity, &e.Message, &e.DetailsJSON, &e.Source, &createdAt) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, err + } + e.CreatedAt = parseSQLiteTime(createdAt) + return &e, nil +} + // GetAllRecentEvents returns the most recent events across all customers. func (s *Store) GetAllRecentEvents(limit int) ([]Event, error) { rows, err := s.db.Query(` diff --git a/hub/internal/web/artifact_choices_test.go b/hub/internal/web/artifact_choices_test.go index b4459db8..0955ce24 100644 --- a/hub/internal/web/artifact_choices_test.go +++ b/hub/internal/web/artifact_choices_test.go @@ -23,9 +23,10 @@ import ( // fakeGitea serves the two endpoints the client uses, with a controllable per-request delay so // "concurrent" is measurable rather than asserted. type fakeGitea struct { - versions []string - delay time.Duration - failVer string // this version's /files call 500s + versions []string + delay time.Duration + failVer string // this version's /files call 500s + bundleOnly string // R-1011: this version's package holds only the config bundle (an agent step bundle) inFlight int32 maxSeen int32 @@ -58,6 +59,10 @@ func (f *fakeGitea) start(t *testing.T) *gitea.Client { } } ver := versionFromFilesPath(r.URL.Path) + if f.bundleOnly != "" && ver == f.bundleOnly { + fmt.Fprintf(w, `[{"name":"felhom-config-bundle.json","sha256":"bundle-%s"}]`, ver) + return + } fmt.Fprintf(w, `[{"name":"felhom-agent","sha256":"sha-%s"}]`, ver) return } @@ -246,3 +251,25 @@ func TestArtifactChoices_CacheIsPerPackage(t *testing.T) { t.Error("precondition") } } + +// R-1011: the Day-0 agent list shows only versions whose package holds the felhom-agent file. A step bundle +// (`0.156.1-step1`, only felhom-config-bundle.json) is left out — never listed with the bundle's sha. +// +// COMPANION RED-PROOF (observed 2026-10-19): with FileSHA256's pre-fix fallback to the first file, it fails with +// +// artifact_choices_test.go: the list offers 0.156.1-step1 (sha "bundle-0.156.1-step1"), a package with no agent binary +// +// Restored. +func TestArtifactChoices_R1011_StepBundleIsNotAnAgentVersion(t *testing.T) { + f := &fakeGitea{versions: []string{"0.157.0", "0.156.1-step1", "0.156.1"}, bundleOnly: "0.156.1-step1"} + s := newTestWebServer(t, f.start(t)) + got := s.artifactChoices(context.Background(), "felhom-agent", "felhom-agent") + for _, c := range got { + if c.Version == "0.156.1-step1" { + t.Fatalf("the list offers %s (sha %q), a package with no agent binary", c.Version, c.SHA256) + } + } + if len(got) != 2 || got[0].SHA256 != "sha-0.157.0" || got[1].Version != "0.156.1" || got[1].SHA256 != "sha-0.156.1" { + t.Fatalf("the release versions must stay, each with its own binary's sha: %+v", got) + } +} diff --git a/hub/internal/web/artifact_installability_test.go b/hub/internal/web/artifact_installability_test.go index a5948d66..c1fb0a7c 100644 --- a/hub/internal/web/artifact_installability_test.go +++ b/hub/internal/web/artifact_installability_test.go @@ -51,8 +51,12 @@ func (f *fakeRegistry) start(t *testing.T) *gitea.Client { if f.shaless[seg[5]+"/"+seg[6]] { sha = "" } + name := seg[5] // the agent's file is named after its package … + if name == pkgGolden { + name = fileGolden // … the golden's is not (R-1011: the hub now asks for exactly that file) + } w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`[{"name":"` + seg[5] + `","sha256":"` + sha + `"}]`)) + _, _ = w.Write([]byte(`[{"name":"` + name + `","sha256":"` + sha + `"}]`)) return } w.WriteHeader(http.StatusNotFound) diff --git a/hub/internal/web/configs.go b/hub/internal/web/configs.go index 2b46ad4e..5f735b8b 100644 --- a/hub/internal/web/configs.go +++ b/hub/internal/web/configs.go @@ -1589,6 +1589,7 @@ func (s *Server) handleSetArtifacts(w http.ResponseWriter, r *http.Request) { // // artifact_version_missing — the registry answered 404 for that version // artifact_sha_missing — the version exists but the registry listed no sha256 +// artifact_file_missing — the version exists but its package holds no such file (an agent step bundle, R-1011) // artifact_unverifiable — the registry did not answer usefully (network error, 5xx) // artifact_sha_invalid — a hand-typed sha is not 64 hex characters (manual path only) func (s *Server) resolveArtifactSHA(ctx context.Context, pkg, file, version, submittedSHA string) (string, string) { @@ -1602,6 +1603,8 @@ func (s *Server) resolveArtifactSHA(ctx context.Context, pkg, file, version, sub switch { case gitea.IsNotFound(err): flash = "artifact_version_missing" + case errors.Is(err, gitea.ErrNoFile): + flash = "artifact_file_missing" case errors.Is(err, gitea.ErrNoSHA): flash = "artifact_sha_missing" } diff --git a/hub/internal/web/r292_artifact_flash_test.go b/hub/internal/web/r292_artifact_flash_test.go index 30895fdf..df4bc742 100644 --- a/hub/internal/web/r292_artifact_flash_test.go +++ b/hub/internal/web/r292_artifact_flash_test.go @@ -47,8 +47,10 @@ func TestArtifactSave_FlashNamesTheCause(t *testing.T) { }{ {"version not found", http.StatusNotFound, "", "artifact_version_missing"}, {"registry failing", http.StatusBadGateway, "", "artifact_unverifiable"}, - {"no sha listed", http.StatusOK, `[{"name":"x","sha256":""}]`, "artifact_sha_missing"}, - {"healthy", http.StatusOK, `[{"name":"x","sha256":"` + sha + `"}]`, "artifacts_set"}, + {"no sha listed", http.StatusOK, `[{"name":"felhom-agent","sha256":""},{"name":"golden.tar.zst","sha256":""}]`, "artifact_sha_missing"}, + // R-1011: a step bundle — files, but not the agent binary — is refused by name, never another file's sha. + {"no agent binary (a step bundle)", http.StatusOK, `[{"name":"felhom-config-bundle.json","sha256":"` + sha + `"}]`, "artifact_file_missing"}, + {"healthy", http.StatusOK, `[{"name":"felhom-agent","sha256":"` + sha + `"},{"name":"golden.tar.zst","sha256":"` + sha + `"}]`, "artifacts_set"}, } { t.Run(tc.name, func(t *testing.T) { s, _ := newTestServer(t) @@ -81,6 +83,7 @@ func TestArtifactFlashes_EachRendersItsOwnText(t *testing.T) { for flash, want := range map[string]string{ "artifact_version_missing": "the registry has no such version", "artifact_sha_missing": "no checksum for a chosen version", + "artifact_file_missing": "holds no artifact", "artifact_unverifiable": "could not verify", "artifact_sha_invalid": "a checksum you typed is not valid", } { diff --git a/hub/internal/web/server.go b/hub/internal/web/server.go index 2ba60202..c994fa83 100644 --- a/hub/internal/web/server.go +++ b/hub/internal/web/server.go @@ -6,6 +6,7 @@ import ( "crypto/subtle" "encoding/hex" "encoding/json" + "errors" "fmt" "html/template" "log" @@ -385,7 +386,14 @@ func (s *Server) artifactChoices(ctx context.Context, pkg, file string) []artifa wg.Wait() out := make([]artifactChoice, 0, len(vers)) + var noFile []string for i, v := range vers { + if errors.Is(errs[i], gitea.ErrNoFile) { + // R-1011: a version whose package does not hold the artifact (an agent step bundle) is not an agent + // version — left out of the dropdown, one INFO line per resolve rather than a WARN per version. + noFile = append(noFile, v) + continue + } if errs[i] != nil { // unchanged behaviour: a failed lookup drops THAT version, never the whole list s.logger.Printf("[WARN] artifact sha (%s/%s): %v", pkg, v, errs[i]) @@ -394,6 +402,10 @@ func (s *Server) artifactChoices(ctx context.Context, pkg, file string) []artifa out = append(out, artifactChoice{Version: v, SHA256: shas[i]}) } + if len(noFile) > 0 { + s.logger.Printf("[INFO] artifact choices (%s): %d version(s) without %s left out (R-1011): %s", pkg, len(noFile), file, strings.Join(noFile, ", ")) + } + // Only a SUCCESSFUL resolve is cached. A Gitea blip must not pin an empty list in front of the // operator for a minute — the earlier returns leave the cache untouched, and so does this one // when every version's sha lookup failed. diff --git a/hub/internal/web/system.go b/hub/internal/web/system.go index 910470a9..a530b00e 100644 --- a/hub/internal/web/system.go +++ b/hub/internal/web/system.go @@ -298,6 +298,10 @@ func buildSystemRows(lines []osupdates.FleetLine, facts map[string]sysfacts.Syst switch l.Tunnel { case "running": r.Tunnel = plain("running") + case "absent": + // R-1070 (agent with the `absent` state): no cloudflared container — a box with no tunnel set up (a test + // or LAN-only box). Not a fault; the hub's tunnel alarm counts not_running only. + r.Tunnel = cell{Text: "none set up", Title: "no cloudflared container in the guest (agent state: absent)"} case "not_running", "inactive": r.Tunnel = cell{Text: l.Tunnel, Class: "bad"} default: diff --git a/hub/internal/web/system_test.go b/hub/internal/web/system_test.go index d5c152eb..b5f381ef 100644 --- a/hub/internal/web/system_test.go +++ b/hub/internal/web/system_test.go @@ -272,3 +272,19 @@ func TestSystemPage_KernelCells(t *testing.T) { t.Fatalf("an older agent: %+v %+v", d, step) } } + +// R-1070: the agent's new `absent` tunnel state (no cloudflared container — no tunnel set up) reads as a plain fact, +// not as a warning; not_running stays red, an unknown state stays amber. +func TestSystemRows_R1070_AbsentTunnelIsNotAWarning(t *testing.T) { + lines := []osupdates.FleetLine{{HostID: "a", Tunnel: "absent"}, {HostID: "b", Tunnel: "not_running"}, {HostID: "c", Tunnel: "unknown"}} + rows := buildSystemRows(lines, nil, nil, time.Hour, time.Hour, time.Hour, time.Now()) + if len(rows) != 3 { + t.Fatalf("rows: %d", len(rows)) + } + if rows[0].Tunnel.Class != "" || rows[0].Tunnel.Text != "none set up" { + t.Errorf("absent: %+v, want a plain „none set up\"", rows[0].Tunnel) + } + if rows[1].Tunnel.Class != "bad" || rows[2].Tunnel.Class != "warn" { + t.Errorf("not_running %+v must stay bad, unknown %+v warn", rows[1].Tunnel, rows[2].Tunnel) + } +} diff --git a/hub/internal/web/templates/configuration.html b/hub/internal/web/templates/configuration.html index c616faeb..838bb882 100644 --- a/hub/internal/web/templates/configuration.html +++ b/hub/internal/web/templates/configuration.html @@ -89,6 +89,9 @@ {{if eq .Flash "artifact_unverifiable"}}
Refused: could not verify — this does not mean anything is missing. The registry did not answer, so the hub cannot tell whether that version is installable. It refuses rather than saving with a warning, because a warning beside a success reads as a success. Manifest unchanged. Check that Gitea is up, then try again. There is deliberately no override: the registry is on your own server, so if it is unreachable the vouch can wait.
{{end}} + {{if eq .Flash "artifact_file_missing"}} +
Refused: that version's package holds no artifact. It has files, but not the agent binary (felhom-agent) or the golden (golden.tar.zst) — an agent step bundle such as 0.156.1-step1 carries only the config bundle. Vouching it would point fresh installs at a package with no binary. Manifest unchanged. Choose the release version.
+ {{end}} {{if eq .Flash "artifact_sha_missing"}}
Refused: no checksum for a chosen version. The hub records the sha256 it reads from Gitea, and that lookup came back empty — so saving would vouch bytes nobody verified. Manifest unchanged.
{{end}}